diff --git a/.github/workflows/terraform.yml b/.github/workflows/terraform.yml index 122ed88025..7bfbf2930a 100644 --- a/.github/workflows/terraform.yml +++ b/.github/workflows/terraform.yml @@ -85,7 +85,15 @@ jobs: "download-lambda", "lambda", "multi-runner", + "compute-providers/ec2", + "compute-providers/ec2/trust-policy", + "compute-providers/microvm", + "compute-providers/microvm/trust-policy", "runner-binaries-syncer", + "runner-stack", + "runner-stack/job-retry", + "runner-stack/scale-runners", + "runner-stack/ssm-housekeeper", "runners", "setup-iam-permissions", "ssm", @@ -214,6 +222,16 @@ jobs: matrix: module: - modules/runners + - modules/multi-runner + - modules/runner-stack + - modules/runner-stack/job-retry + - modules/runner-stack/pool + - modules/runner-stack/scale-runners + - modules/runner-stack/ssm-housekeeper + - modules/compute-providers/ec2 + - modules/compute-providers/ec2/trust-policy + - modules/compute-providers/microvm + - modules/compute-providers/microvm/trust-policy defaults: run: working-directory: ${{ matrix.module }} diff --git a/docs/index.md b/docs/index.md index 7a7d0f70c6..ae2713ff48 100644 --- a/docs/index.md +++ b/docs/index.md @@ -101,7 +101,7 @@ Besides these permissions, the lambdas also need permission to CloudWatch (for l ## Terraform main modules -Currently we support two main modules. The `runners` module is the main module for creating runners. And the 'multi-runner' module is a wrapper around the `runners` module to create multiple runners in one go. The `multi-runner` module is useful for creating runners for multiple repositories or organizations. +Currently we support two main modules. The existing `runners` module remains the stable EC2 implementation, and the `multi-runner` module creates multiple runner configurations in one deployment. Stable `multi_runner_config` entries continue to use the unchanged `runners` module. Entries under `experimental.multi_runner_config_v2` use the new provider-oriented `runner-stack`. That stack coordinates internal provider-neutral modules for scale-up and scale-down, pool, retry, and SSM housekeeping, and owns the common runner role and attachments. The EC2 provider supplies EC2-specific policy requirements and owns the instance profile, launch template, bootstrap resources, and runner log groups. These child modules are implementation details of the experimental stack and are not standalone public entry points. Phase 1 exposes both contracts but requires callers to populate only one runner configuration map per module instance; later releases will translate v1, ship state migration, and only then remove the v1 interface. See the [experimental compute-provider refactor](modules/internal/compute-provider-refactor.md) and [multi-runner v2 migration roadmap](modules/public/multi-runner.md#multi-runner-v2-migration-roadmap). EC2 is the only active Terraform-managed provider; microVM, CodeBuild, and other provider modules are future work. Both modules are built on top of the same base modules. When using the multi-runner module you can deploy different runners with only one deployment. diff --git a/docs/modules/internal/compute-provider-refactor.md b/docs/modules/internal/compute-provider-refactor.md new file mode 100644 index 0000000000..791d247d14 --- /dev/null +++ b/docs/modules/internal/compute-provider-refactor.md @@ -0,0 +1,164 @@ +# Experimental compute-provider refactor + +!!! warning "Experimental opt-in" + + The provider-oriented Terraform interface is experimental. Its schema can change before it becomes stable. To enable it for the whole module instance, leave `multi_runner_config` empty and populate `experimental.multi_runner_config_v2`. When the v2 map is empty, existing `multi_runner_config` deployments continue to use the unchanged legacy implementation. Populating both maps is unsupported. + +## Why this refactor exists + +The scale-up, scale-down, pool, job-retry, queue, SSM housekeeping, and GitHub registration workflows are not inherently EC2-specific. The legacy `runners` module combines that common control plane with EC2 launch templates, instance profiles, bootstrap parameters, log groups, IAM permissions, and Lambda environment variables. Adding another compute provider in that structure would require copying common behavior or adding provider conditionals throughout the module. + +The refactor introduces a provider boundary so MicroVM and other backends can reuse the control plane. Only the policy statements, environment variables, and resources required by the selected compute provider should change. + +## Ownership model + +The implementation is split into orchestration, provider-neutral control-plane components, and compute-provider implementations: + +| Layer | Owns | +| --- | --- | +| `multi-runner` | Module-level v1/v2 mode selection, canonical normalization, configuration keys, build queues, webhook matching, and runner-binary discovery. | +| `runner-stack` | Provider dispatch, internal component wiring, shared runner configuration in SSM, and the common runner role and policy attachments. | +| `runner-stack/scale-runners` | Provider-neutral scale-up and scale-down Lambdas, schedules and queue integration, and their execution roles and policies. | +| `runner-stack/pool` | Optional scheduled runner-pool resources and their Lambda and IAM wiring. | +| `runner-stack/job-retry` | Optional queued-job retry resources and their Lambda and IAM wiring. | +| `runner-stack/ssm-housekeeper` | Parameter Store cleanup Lambda, schedule, logging, and IAM resources. | +| `compute-providers//trust-policy` | Provider-specific default runner-role trust, merged with the optional caller-provided trust document before the common role is created. | +| `compute-providers/` | Provider-specific resources, permission requirements, and the IAM and environment-variable fragments consumed by the common control plane after the runner role is resolved. | + +The EC2 provider owns the instance profile, launch template, security group, AMI and bootstrap parameters, runner log groups, EC2 policy statements, and EC2 Lambda environment variables. The MicroVM provider owns the Lambda MicroVM runtime configuration, execution-role policy, and MicroVM Lambda environment variables. Terraform does not manage MicroVM lifecycle resources directly; the runtime control plane creates and terminates MicroVM runners. + +The modules below `runner-stack` are internal implementation boundaries, not standalone public modules. Callers opt into the experimental interface through `experimental.multi_runner_config_v2`; `multi-runner` calls `runner-stack`, which composes the internal modules. Their direct input and output contracts may change while v2 remains experimental. + +`runner-stack` selects a compute provider from the single populated typed block under `compute_provider`. For example, `compute_provider = { ec2 = { ... } }` selects EC2 and `compute_provider = { microvm = { ... } }` selects MicroVM; there is no separate `type` input that can disagree with the populated block. Exactly one provider block must be populated, and its presence must be known during planning because it determines the module graph. Native input validation enforces this common selection rule, while each compute-provider module owns its provider-specific semantic validation. The stack passes `compute_provider.` to the selected provider module as one nested `config` object. It also passes the provider-neutral `runner`, `github`, `ssm`, and `observability` objects without expanding them back into prefixed scalar inputs. This keeps ownership visible at the module boundary and gives future compute providers an equivalent contract to implement. + +The common stack creates or selects the runner IAM role, but the selected provider owns the role's default trust-policy document. Each provider implements a small `trust-policy` submodule that accepts `additional_trust_policy_json` and returns the final `assume_role_policy`. The full provider separately returns its nested `provider` contract containing `policies.runner`, `policies.scale_up`, `policies.scale_down`, and `policies.pool`, component environment variables, and provider resources. The common stack uses the isolated trust-policy output when it creates the runner role and attaches the full provider's permission documents to the roles owned by the corresponding common components. A provider never creates or attaches a common IAM role. + +The trust relationship is deliberately rendered by an isolated provider submodule: + +1. `runner-stack` selects the provider from the populated typed block. +2. `compute-providers//trust-policy` combines the provider default with `runner.iam.additional_trust_policy_json` without referencing the runner-role input. +3. `runner-stack` creates or selects the common runner role from the returned `assume_role_policy`. +4. The full compute provider receives the resolved role so it can create resources such as the EC2 instance profile and render `iam:PassRole` statements. +5. The provider returns its nested policy, environment-variable, and resource contract. +6. The common components attach the returned policies to the runner, scale-up, scale-down, and pool roles they own. + +The trust-policy output depends only on its input documents, not on the full provider resources that consume the runner role. This preserves provider ownership of the trust relationship while keeping the dependency graph one-way. + +## Phase 1 dispatch and compatibility + +Phase 1 exposes both contracts but requires callers to populate only one runner configuration map per module instance. An empty `experimental.multi_runner_config_v2` selects the stable v1 path. To select the experimental v2 path, `multi_runner_config` must be empty and the v2 map must be populated. The maps are never merged, and supplying both is unsupported. + +```mermaid +flowchart TD + Stable["multi_runner_config"] --> Select{"Is experimental.multi_runner_config_v2 non-empty?"} + Experimental["experimental.multi_runner_config_v2"] --> Select + Select -->|No| V1["Select and normalize v1"] + Select -->|Yes, with v1 empty| V2["Select v2"] + V1 --> Shared["Queues, webhook matching, binary discovery"] + V2 --> Shared + V1 --> Legacy["module.runners[configuration]"] + V2 --> Stack["module.runner_stacks[configuration]"] + Stack --> Scaling["runner-stack/scale-runners"] + Stack --> Pool["runner-stack/pool"] + Stack --> Retry["runner-stack/job-retry"] + Stack --> Housekeeper["runner-stack/ssm-housekeeper"] + Stack --> Trust["compute-providers/provider/trust-policy"] + Trust --> Role["common runner role"] + Role --> Provider + Stack --> Provider["compute-providers/"] + Provider --> Scaling + Provider --> Pool +``` + +The selected input is normalized once so shared resources can consume one representation. Stable normalization does not change stable runner dispatch: + +- When `experimental.multi_runner_config_v2` is empty, every key in `multi_runner_config` continues to call `modules/runners` at its historical `module.runners["configuration"]` address. +- The stable module call receives the original v1 values for compatibility-sensitive inputs. +- Stable queue tagging and the flat `runners_map` output remain unchanged. +- When `multi_runner_config` is empty and `experimental.multi_runner_config_v2` is non-empty, every key in the v2 map calls `modules/runner-stack` at `module.runner_stacks["configuration"]`. +- Experimental resources are exposed separately through the nested `runners_map_v2` output. +- The maps are not combined, and there is no precedence rule between them. Populating both maps is unsupported. + +No state move is included in phase 1. Enabling v2 for a module instance that already manages v1 runners changes its implementation addresses; phase 1 does not migrate that state. Existing deployments should keep v2 empty until the documented state-migration phase. The current v2 path is intended for new or explicitly experimental deployments. + +## Opting in + +Set the complete runner configuration map inside the nested experimental object to use the provider-oriented stack: + +```hcl +module "multi_runner" { + source = "github-aws-runners/github-runner/aws//modules/multi-runner" + + # A non-empty v2 map is the module-level experimental opt-in. Leave + # multi_runner_config empty when using it. + experimental = { + multi_runner_config_v2 = { + arm = { + runner = { + os = "linux" + architecture = "arm64" + maximum_count = 2 + } + + compute_provider = { + ec2 = { + instance_types = ["m7g.large"] + } + } + + matcherConfig = { + labelMatchers = [["self-hosted", "linux", "arm64"]] + } + } + } + } +} +``` + +## Inputs, tags, and outputs + +The v2 object groups provider-neutral settings by owner: `runner`, `github`, `queue`, `lambda`, `scale_up`, `scale_down`, `pool`, `job_retry`, `ssm`, and `observability`. Backend settings live only under `compute_provider.`. Exactly one typed provider block must be populated; that block selects the provider without a second discriminator field. + +Tags follow the same ownership model. Module tags are defaults; shared Lambda, queue, and log-group tags override those defaults; component and subcomponent tags are applied last. EC2 runtime tags belong under `compute_provider.ec2.tags`. The EC2 bootstrap tags required by the runner are protected inside the provider and are not propagated to common resources. + +Application logging settings stay together under `observability.logs`, including `level`, retention, encryption, class, and shared log-group tags. + +In v1 mode, entries remain exclusively in `runners_map` and retain their flat output fields; `runners_map_v2` is empty. In v2 mode, entries are exposed exclusively through `runners_map_v2` and `runners_map` is empty. Common resources are grouped under `runner`, `scale_up`, `scale_down`, and `pool`, while provider-specific resources remain under `provider.`. The provider key is derived dynamically from the selected input block and therefore also identifies the compute provider. For example, the common runner role is available at `runners_map_v2["configuration"].runner.role`, an EC2 selection places launch-template and runner-log artifacts under `runners_map_v2["configuration"].provider.ec2`, and a MicroVM selection places image and execution-role references under `runners_map_v2["configuration"].provider.microvm`. The `pool` value is null when no pool configuration is supplied. + +## Plan-time provider selection and ownership wrappers + +Terraform must know resource and dynamic-block shape during planning, even when an ARN is produced by another resource and remains unknown until apply. Optional inputs that enable IAM policies therefore use a caller-known object as the discriminator and keep the computed value in an `arn` leaf. The relevant configuration fragments are: + +```hcl +ssm = { + kms_key = { + arn = aws_kms_key.runner_parameters.arn + } +} + +compute_provider = { + ec2 = { + ami = { + id_ssm_parameter = { + arn = aws_ssm_parameter.runner_ami.arn + } + kms_key = { + arn = aws_kms_key.runner_ami.arn + } + } + } +} +``` + +The populated `ec2` block tells Terraform which provider module exists and must therefore be known during planning. Within that block, each ownership-wrapper object tells Terraform that the corresponding policy exists; its `arn` may safely be computed. Values such as `observability.logs.kms_key_id`, which configure an existing resource without changing graph shape, remain nullable scalar inputs. + +For experimental multi-runner entries, set `ssm.kms_key` to the key that encrypts the shared GitHub App and runner parameters. The stable root `kms_key_arn` input continues to serve v1 and is not used as a graph-shape discriminator for v2. + +## Migration phases + +1. **Phase 1 — experimental opt-in:** Keep v1 unchanged when the v2 map is empty, or select v2 for the whole module instance when the v2 map is non-empty. Existing v1 deployments do not move and should not use the v2 switch as an in-place migration mechanism. +2. **Phase 2 — translate and migrate:** Deprecate the stable input, dispatch its translated representation through `runner-stack`, and provide tested `moved` blocks plus commands for addresses Terraform cannot move declaratively. +3. **Phase 3 — remove v1:** After a release window in which phase 2 is available, remove the stable input and flat output adapter in a breaking release. +4. **Future — retire `modules/runners`:** Handle direct consumers of the legacy module in a separate deprecation and migration effort. + +A future compute provider must add a typed input block and return the same nested environment-variable, policy, and resource contract before it can be selected in Terraform. Populating more than one provider block, or selecting a block whose resources are not implemented, is intentionally rejected. diff --git a/lambdas/functions/control-plane/package.json b/lambdas/functions/control-plane/package.json index e886920b1f..dc74e770a9 100644 --- a/lambdas/functions/control-plane/package.json +++ b/lambdas/functions/control-plane/package.json @@ -32,7 +32,7 @@ "dependencies": { "@aws-github-runner/aws-powertools-util": "*", "@aws-github-runner/aws-ssm-util": "*", - "@aws-github-runner/runner-providers": "*", + "@aws-github-runner/compute-providers": "*", "@aws-lambda-powertools/parameters": "^2.31.0", "@aws-sdk/client-ec2": "^3.1009.0", "@aws-sdk/client-sqs": "^3.1009.0", diff --git a/lambdas/functions/control-plane/src/control-plane-providers.ts b/lambdas/functions/control-plane/src/control-plane-providers.ts index d3dfbeeeee..6c507692cd 100644 --- a/lambdas/functions/control-plane/src/control-plane-providers.ts +++ b/lambdas/functions/control-plane/src/control-plane-providers.ts @@ -1,8 +1,8 @@ -import { createControlPlaneProviderRegistry } from '@aws-github-runner/runner-providers/control-plane'; -import { runnerProviderTypes } from '@aws-github-runner/runner-providers/provider-types'; +import { createControlPlaneProviderRegistry } from '@aws-github-runner/compute-providers/control-plane'; +import { computeProviderTypes } from '@aws-github-runner/compute-providers/provider-types'; import { createStartRunnerConfig } from './scale-runners/github-runner'; export const controlPlaneProviderRegistry = createControlPlaneProviderRegistry(createStartRunnerConfig); -export { runnerProviderTypes }; +export { computeProviderTypes }; diff --git a/lambdas/functions/control-plane/src/modules.d.ts b/lambdas/functions/control-plane/src/modules.d.ts index 53247cf6c6..d5157ccb37 100644 --- a/lambdas/functions/control-plane/src/modules.d.ts +++ b/lambdas/functions/control-plane/src/modules.d.ts @@ -16,7 +16,7 @@ declare namespace NodeJS { PARAMETER_GITHUB_APP_ID_NAME: string; PARAMETER_GITHUB_APP_KEY_BASE64_NAME: string; RUNNER_OWNER: string; - RUNNER_PROVIDER_TYPE?: string; + COMPUTE_PROVIDER_TYPE?: string; SCALE_DOWN_CONFIG: string; SSM_TOKEN_PATH: string; SSM_CLEANUP_CONFIG: string; diff --git a/lambdas/functions/control-plane/src/pool/pool-contract.test.ts b/lambdas/functions/control-plane/src/pool/pool-contract.test.ts index e7c5eee21a..551d26e757 100644 --- a/lambdas/functions/control-plane/src/pool/pool-contract.test.ts +++ b/lambdas/functions/control-plane/src/pool/pool-contract.test.ts @@ -1,14 +1,14 @@ import type { Octokit } from '@octokit/rest'; -import type { RunnerProviderType } from '@aws-github-runner/runner-providers/provider-types'; +import type { ComputeProviderType } from '@aws-github-runner/compute-providers/provider-types'; import { beforeEach, vi } from 'vitest'; -import { definePoolContractTests } from '../test/runner-provider-contracts/pool'; -import { providerTypes } from '../test/runner-provider-contracts/provider-types'; +import { definePoolContractTests } from '../test/compute-provider-contracts/pool'; +import { providerTypes } from '../test/compute-provider-contracts/provider-types'; import * as ghAuth from '../github/auth'; import { controlPlaneProviderRegistry } from '../control-plane-providers'; import * as githubRunner from '../scale-runners/github-runner'; import { adjust } from './pool'; -import type { PoolRunnerProvider } from './pool-provider'; +import type { PoolComputeProvider } from './pool-provider'; vi.mock('../github/auth', () => ({ createGithubAppAuth: vi.fn(), @@ -35,13 +35,13 @@ const githubClient = { const cleanEnv = process.env; -const lanes = providerTypes.map((type) => ({ +const computeProviders = providerTypes.map((type) => ({ provider: { type, listRunners: vi.fn(), countAvailableRunners: vi.fn(), createRunners: vi.fn(), - } satisfies PoolRunnerProvider, + } satisfies PoolComputeProvider, })); beforeEach(() => { @@ -66,9 +66,9 @@ beforeEach(() => { vi.mocked(githubClient.paginate).mockResolvedValue([]); }); -definePoolContractTests({ +definePoolContractTests({ adjust, + computeProviders, githubInstallationClient: githubClient, - lanes, resolveCapability: mockedResolveCapability, }); diff --git a/lambdas/functions/control-plane/src/pool/pool-provider.ts b/lambdas/functions/control-plane/src/pool/pool-provider.ts index 9c5638a158..1c4012f208 100644 --- a/lambdas/functions/control-plane/src/pool/pool-provider.ts +++ b/lambdas/functions/control-plane/src/pool/pool-provider.ts @@ -1,6 +1,6 @@ export type { CreatePoolRunnersInput, ListPoolRunnersInput, - PoolRunnerProvider, + PoolComputeProvider, RunnerStatus, -} from '@aws-github-runner/runner-providers/core'; +} from '@aws-github-runner/compute-providers/core'; diff --git a/lambdas/functions/control-plane/src/pool/pool.test.ts b/lambdas/functions/control-plane/src/pool/pool.test.ts index 6481e2ce8c..31c0d98653 100644 --- a/lambdas/functions/control-plane/src/pool/pool.test.ts +++ b/lambdas/functions/control-plane/src/pool/pool.test.ts @@ -2,8 +2,8 @@ import { Octokit } from '@octokit/rest'; import moment from 'moment-timezone'; import * as nock from 'nock'; -import { createRunners } from '@aws-github-runner/runner-providers/aws/ec2/control-plane/runner-config'; -import { listEC2Runners } from '@aws-github-runner/runner-providers/aws/ec2/control-plane/runners'; +import { createRunners } from '@aws-github-runner/compute-providers/aws/ec2/control-plane/runner-config'; +import { listEC2Runners } from '@aws-github-runner/compute-providers/aws/ec2/control-plane/runners'; import * as ghAuth from '../github/auth'; import { getGitHubEnterpriseApiUrl } from '../scale-runners/github-runner'; import { adjust } from './pool'; @@ -26,7 +26,7 @@ vi.mock('@octokit/rest', () => ({ }), })); -vi.mock('@aws-github-runner/runner-providers/aws/ec2/control-plane/runners', async () => ({ +vi.mock('@aws-github-runner/compute-providers/aws/ec2/control-plane/runners', async () => ({ listEC2Runners: vi.fn(), // Include any other functions from the module that might be used bootTimeExceeded: vi.fn(), @@ -37,8 +37,10 @@ vi.mock('./../github/auth', async () => ({ createOctokitClient: vi.fn(), })); -vi.mock('@aws-github-runner/runner-providers/aws/ec2/control-plane/runner-config', async (importOriginal) => ({ - ...(await importOriginal()), +vi.mock('@aws-github-runner/compute-providers/aws/ec2/control-plane/runner-config', async (importOriginal) => ({ + ...(await importOriginal< + typeof import('@aws-github-runner/compute-providers/aws/ec2/control-plane/runner-config') + >()), createRunners: vi.fn(), })); @@ -243,8 +245,8 @@ describe('Test simple pool.', () => { }); it('Rejects unsupported pool provider types.', async () => { - await expect(adjust({ poolSize: 10, type: 'microvm' })).rejects.toThrow( - "Unsupported runner provider type 'microvm'", + await expect(adjust({ poolSize: 10, type: 'unsupported-provider' })).rejects.toThrow( + "Unsupported compute provider type 'unsupported-provider'", ); expect(mockListRunners).not.toHaveBeenCalled(); }); diff --git a/lambdas/functions/control-plane/src/pool/pool.ts b/lambdas/functions/control-plane/src/pool/pool.ts index 3a6ed45be9..355c7ed5d1 100644 --- a/lambdas/functions/control-plane/src/pool/pool.ts +++ b/lambdas/functions/control-plane/src/pool/pool.ts @@ -1,6 +1,6 @@ import { Octokit } from '@octokit/rest'; import { createChildLogger } from '@aws-github-runner/aws-powertools-util'; -import { resolveRunnerProviderType } from '@aws-github-runner/runner-providers/provider-types'; +import { resolveComputeProviderType } from '@aws-github-runner/compute-providers/provider-types'; import yn from 'yn'; import { createGithubAppAuth, createGithubInstallationAuth, createOctokitClient } from '../github/auth'; @@ -16,12 +16,12 @@ export interface PoolEvent { } export async function adjust(event: PoolEvent): Promise { - const runnerProviderType = resolveRunnerProviderType(event.type); - const runnerProvider = { - ...controlPlaneProviderRegistry.capability(runnerProviderType, 'pool')(), - type: runnerProviderType, + const computeProviderType = resolveComputeProviderType(event.type); + const computeProvider = { + ...controlPlaneProviderRegistry.capability(computeProviderType, 'pool')(), + type: computeProviderType, }; - logger.info(`Checking current ${runnerProvider.type} pool size against pool of size: ${event.poolSize}`); + logger.info(`Checking current ${computeProvider.type} pool size against pool of size: ${event.poolSize}`); const runnerLabels = process.env.RUNNER_LABELS || ''; const runnerGroup = process.env.RUNNER_GROUP_NAME || ''; const runnerNamePrefix = process.env.RUNNER_NAME_PREFIX || ''; @@ -55,13 +55,13 @@ export async function adjust(event: PoolEvent): Promise { ); // Look up the managed provider runners, but running does not mean idle. - const poolRunners = await runnerProvider.listRunners({ + const poolRunners = await computeProvider.listRunners({ environment, runnerOwner, runnerType: 'Org', }); - const numberOfRunnersInPool = runnerProvider.countAvailableRunners(poolRunners, runnerStatusses, includeBusyRunners); + const numberOfRunnersInPool = computeProvider.countAvailableRunners(poolRunners, runnerStatusses, includeBusyRunners); let topUp = event.poolSize - numberOfRunnersInPool; // The pool must never push the total number of runners (busy + idle) past the configured maximum. @@ -81,7 +81,7 @@ export async function adjust(event: PoolEvent): Promise { if (topUp > 0) { logger.info(`The pool will be topped up with ${topUp} runners.`); - await runnerProvider.createRunners({ + await computeProvider.createRunners({ githubRunnerConfig: { ephemeral, enableJitConfig, diff --git a/lambdas/functions/control-plane/src/scale-runners/scale-down-contract.test.ts b/lambdas/functions/control-plane/src/scale-runners/scale-down-contract.test.ts index ade06747e4..afd25211da 100644 --- a/lambdas/functions/control-plane/src/scale-runners/scale-down-contract.test.ts +++ b/lambdas/functions/control-plane/src/scale-runners/scale-down-contract.test.ts @@ -1,17 +1,17 @@ -import type { RunnerProviderType } from '@aws-github-runner/runner-providers/provider-types'; +import type { ComputeProviderType } from '@aws-github-runner/compute-providers/provider-types'; import { beforeEach, vi } from 'vitest'; -import { providerTypes } from '../test/runner-provider-contracts/provider-types'; -import { defineScaleDownContractTests } from '../test/runner-provider-contracts/scale-down'; +import { providerTypes } from '../test/compute-provider-contracts/provider-types'; +import { defineScaleDownContractTests } from '../test/compute-provider-contracts/scale-down'; import { controlPlaneProviderRegistry } from '../control-plane-providers'; import { scaleDown } from './scale-down'; -import type { ScaleDownRunnerProvider } from './types'; +import type { ScaleDownComputeProvider } from './types'; const mockedResolveCapability = vi.spyOn(controlPlaneProviderRegistry, 'capability'); const cleanEnv = process.env; -const lanes = providerTypes.map((type) => ({ +const computeProviders = providerTypes.map((type) => ({ provider: { type, list: vi.fn(), @@ -19,7 +19,7 @@ const lanes = providerTypes.map((type) => ({ markOrphan: vi.fn(), unmarkOrphan: vi.fn(), terminate: vi.fn(), - } satisfies ScaleDownRunnerProvider, + } satisfies ScaleDownComputeProvider, })); beforeEach(() => { @@ -27,8 +27,8 @@ beforeEach(() => { process.env = { ...cleanEnv }; }); -defineScaleDownContractTests({ - lanes, +defineScaleDownContractTests({ + computeProviders, resolveCapability: mockedResolveCapability, scaleDown, }); diff --git a/lambdas/functions/control-plane/src/scale-runners/scale-down.test.ts b/lambdas/functions/control-plane/src/scale-runners/scale-down.test.ts index 8bf15910f1..f4624c647b 100644 --- a/lambdas/functions/control-plane/src/scale-runners/scale-down.test.ts +++ b/lambdas/functions/control-plane/src/scale-runners/scale-down.test.ts @@ -7,7 +7,7 @@ import { controlPlaneProviderRegistry } from '../control-plane-providers'; import * as ghAuth from '../github/auth'; import { githubCache } from './cache'; import { newestFirstStrategy, oldestFirstStrategy, scaleDown } from './scale-down'; -import type { RunnerInfo, RunnerType, ScaleDownRunnerProvider } from './types'; +import type { RunnerInfo, RunnerType, ScaleDownComputeProvider } from './types'; vi.mock('../github/auth', () => ({ createGithubAppAuth: vi.fn(), @@ -31,24 +31,24 @@ const mockOctokit = { paginate: vi.fn(), }; -const mockRunnerProvider = { +const mockComputeProvider = { type: 'ec2', list: vi.fn(), bootTimeExceeded: vi.fn(), markOrphan: vi.fn(), unmarkOrphan: vi.fn(), terminate: vi.fn(), -} satisfies ScaleDownRunnerProvider; +} satisfies ScaleDownComputeProvider; const mockedResolveCapability = vi.spyOn(controlPlaneProviderRegistry, 'capability'); const mockedAppAuth = vi.mocked(ghAuth.createGithubAppAuth); const mockedInstallationAuth = vi.mocked(ghAuth.createGithubInstallationAuth); const mockCreateClient = vi.mocked(ghAuth.createOctokitClient); -const mockListRunners = vi.mocked(mockRunnerProvider.list); -const mockBootTimeExceeded = vi.mocked(mockRunnerProvider.bootTimeExceeded); -const mockMarkOrphan = vi.mocked(mockRunnerProvider.markOrphan); -const mockUnmarkOrphan = vi.mocked(mockRunnerProvider.unmarkOrphan); -const mockTerminateRunners = vi.mocked(mockRunnerProvider.terminate); +const mockListRunners = vi.mocked(mockComputeProvider.list); +const mockBootTimeExceeded = vi.mocked(mockComputeProvider.bootTimeExceeded); +const mockMarkOrphan = vi.mocked(mockComputeProvider.markOrphan); +const mockUnmarkOrphan = vi.mocked(mockComputeProvider.unmarkOrphan); +const mockTerminateRunners = vi.mocked(mockComputeProvider.terminate); const cleanEnv = process.env; @@ -176,13 +176,13 @@ describe('Scale down runners', () => { process.env.ENVIRONMENT = ENVIRONMENT; process.env.MINIMUM_RUNNING_TIME_IN_MINUTES = MINIMUM_TIME_RUNNING_IN_MINUTES.toString(); process.env.RUNNER_BOOT_TIME_IN_MINUTES = MINIMUM_BOOT_TIME.toString(); - process.env.RUNNER_PROVIDER_TYPE = mockRunnerProvider.type; + process.env.COMPUTE_PROVIDER_TYPE = mockComputeProvider.type; vi.clearAllMocks(); githubCache.clients.clear(); githubCache.runners.clear(); - mockedResolveCapability.mockReturnValue(() => mockRunnerProvider); + mockedResolveCapability.mockReturnValue(() => mockComputeProvider); mockBootTimeExceeded.mockImplementation((runner) => { const launchTimePlusBootTime = moment(runner.launchTime).utc().add(MINIMUM_BOOT_TIME, 'minutes'); return launchTimePlusBootTime < moment(new Date()).utc(); diff --git a/lambdas/functions/control-plane/src/scale-runners/scale-down.ts b/lambdas/functions/control-plane/src/scale-runners/scale-down.ts index 4f449fece1..34c9d6de6f 100644 --- a/lambdas/functions/control-plane/src/scale-runners/scale-down.ts +++ b/lambdas/functions/control-plane/src/scale-runners/scale-down.ts @@ -2,7 +2,7 @@ import { Octokit } from '@octokit/rest'; import { Endpoints } from '@octokit/types'; import { RequestError } from '@octokit/request-error'; import { createChildLogger } from '@aws-github-runner/aws-powertools-util'; -import { resolveRunnerProviderType } from '@aws-github-runner/runner-providers/provider-types'; +import { resolveComputeProviderType } from '@aws-github-runner/compute-providers/provider-types'; import moment from 'moment'; import { createGithubAppAuth, createGithubInstallationAuth, createOctokitClient } from '../github/auth'; @@ -11,7 +11,7 @@ import { GhRunners, githubCache } from './cache'; import { ScalingDownConfigList, getEvictionStrategy, getIdleRunnerCount } from './scale-down-config'; import { metricGitHubAppRateLimit } from '../github/rate-limit'; import { getGitHubEnterpriseApiUrl } from './github-runner'; -import type { RunnerInfo, ScaleDownRunnerProvider } from './types'; +import type { RunnerInfo, ScaleDownComputeProvider } from './types'; const logger = createChildLogger('scale-down'); @@ -163,7 +163,7 @@ async function deleteGitHubRunner( async function removeRunner( runner: RunnerInfo, ghRunnerIds: number[], - runnerProvider: ScaleDownRunnerProvider, + computeProvider: ScaleDownComputeProvider, ): Promise { const githubInstallationClient = await getOrCreateOctokit(runner); try { @@ -190,9 +190,9 @@ async function removeRunner( const failedRunners = results.filter((r) => !r.success); if (allSucceeded) { - await runnerProvider.terminate(runner.id); + await computeProvider.terminate(runner.id); logger.info( - `${runnerProvider.type.toUpperCase()} runner '${runner.id}' is terminated and GitHub runner is de-registered.`, + `${computeProvider.type.toUpperCase()} runner '${runner.id}' is terminated and GitHub runner is de-registered.`, ); } else { // Only terminate the provider runner if it was successfully de-registered from GitHub. @@ -216,7 +216,7 @@ async function removeRunner( async function evaluateAndRemoveRunners( runners: RunnerInfo[], scaleDownConfigs: ScalingDownConfigList, - runnerProvider: ScaleDownRunnerProvider, + computeProvider: ScaleDownComputeProvider, ): Promise { let idleCounter = getIdleRunnerCount(scaleDownConfigs); const evictionStrategy = getEvictionStrategy(scaleDownConfigs); @@ -247,12 +247,12 @@ async function evaluateAndRemoveRunners( await removeRunner( runner, ghRunnersFiltered.map((runner: { id: number }) => runner.id), - runnerProvider, + computeProvider, ); } } - } else if (runnerProvider.bootTimeExceeded(runner)) { - await markOrphan(runner.id, runnerProvider); + } else if (computeProvider.bootTimeExceeded(runner)) { + await markOrphan(runner.id, computeProvider); } else { logger.debug(`Runner ${runner.id} has not yet booted.`); } @@ -260,18 +260,18 @@ async function evaluateAndRemoveRunners( } } -async function markOrphan(id: string, runnerProvider: ScaleDownRunnerProvider): Promise { +async function markOrphan(id: string, computeProvider: ScaleDownComputeProvider): Promise { try { - await runnerProvider.markOrphan(id); + await computeProvider.markOrphan(id); logger.info(`Runner '${id}' tagged as orphan.`); } catch (e) { logger.error(`Failed to tag runner '${id}' as orphan.`, { error: e }); } } -async function unMarkOrphan(id: string, runnerProvider: ScaleDownRunnerProvider): Promise { +async function unMarkOrphan(id: string, computeProvider: ScaleDownComputeProvider): Promise { try { - await runnerProvider.unmarkOrphan(id); + await computeProvider.unmarkOrphan(id); logger.info(`Runner '${id}' untagged as orphan.`); } catch (e) { logger.error(`Failed to un-tag runner '${id}' as orphan.`, { error: e }); @@ -298,9 +298,9 @@ async function lastChanceCheckOrphanRunner(runner: RunnerInfo): Promise return isOrphan; } -async function terminateOrphan(environment: string, runnerProvider: ScaleDownRunnerProvider): Promise { +async function terminateOrphan(environment: string, computeProvider: ScaleDownComputeProvider): Promise { try { - const orphanRunners = await runnerProvider.list(environment, true); + const orphanRunners = await computeProvider.list(environment, true); for (const runner of orphanRunners) { if (runner.bypassRemoval) { @@ -310,13 +310,13 @@ async function terminateOrphan(environment: string, runnerProvider: ScaleDownRun if (runner.githubRunnerId) { const isOrphan = await lastChanceCheckOrphanRunner(runner); if (isOrphan) { - await runnerProvider.terminate(runner.id); + await computeProvider.terminate(runner.id); } else { - await unMarkOrphan(runner.id, runnerProvider); + await unMarkOrphan(runner.id, computeProvider); } } else { logger.info(`Terminating orphan runner '${runner.id}'`); - await runnerProvider.terminate(runner.id).catch((e) => { + await computeProvider.terminate(runner.id).catch((e) => { logger.error(`Failed to terminate orphan runner '${runner.id}'`, { error: e }); }); } @@ -338,8 +338,8 @@ export function newestFirstStrategy(a: RunnerInfo, b: RunnerInfo): number { return oldestFirstStrategy(a, b) * -1; } -async function listRunners(environment: string, runnerProvider: ScaleDownRunnerProvider) { - return await runnerProvider.list(environment); +async function listRunners(environment: string, computeProvider: ScaleDownComputeProvider) { + return await computeProvider.list(environment); } function filterRunners(runners: RunnerInfo[]): RunnerInfo[] { @@ -352,22 +352,22 @@ export async function scaleDown(): Promise { githubCache.reset(); const environment = process.env.ENVIRONMENT; const scaleDownConfigs = JSON.parse(process.env.SCALE_DOWN_CONFIG) as ScalingDownConfigList; - const runnerProviderType = resolveRunnerProviderType(process.env.RUNNER_PROVIDER_TYPE); - const runnerProvider = { - ...controlPlaneProviderRegistry.capability(runnerProviderType, 'scaleDown')(), - type: runnerProviderType, + const computeProviderType = resolveComputeProviderType(process.env.COMPUTE_PROVIDER_TYPE); + const computeProvider = { + ...controlPlaneProviderRegistry.capability(computeProviderType, 'scaleDown')(), + type: computeProviderType, }; // first runners marked to be orphan. - await terminateOrphan(environment, runnerProvider); + await terminateOrphan(environment, computeProvider); // next scale down idle runners with respect to config and mark potential orphans - const providerRunners = await listRunners(environment, runnerProvider); + const providerRunners = await listRunners(environment, computeProvider); const activeProviderRunnersCount = providerRunners.length; logger.info( - `Found: '${activeProviderRunnersCount}' active ${runnerProvider.type.toUpperCase()} runners before clean-up.`, + `Found: '${activeProviderRunnersCount}' active ${computeProvider.type.toUpperCase()} runners before clean-up.`, ); - logger.debug(`Active ${runnerProvider.type.toUpperCase()} runners: ${JSON.stringify(providerRunners)}`); + logger.debug(`Active ${computeProvider.type.toUpperCase()} runners: ${JSON.stringify(providerRunners)}`); if (activeProviderRunnersCount === 0) { logger.debug(`No active runners found for environment: '${environment}'`); @@ -375,10 +375,10 @@ export async function scaleDown(): Promise { } const runners = filterRunners(providerRunners); - await evaluateAndRemoveRunners(runners, scaleDownConfigs, runnerProvider); + await evaluateAndRemoveRunners(runners, scaleDownConfigs, computeProvider); - const activeProviderRunnersCountAfter = (await listRunners(environment, runnerProvider)).length; + const activeProviderRunnersCountAfter = (await listRunners(environment, computeProvider)).length; logger.info( - `Found: '${activeProviderRunnersCountAfter}' active ${runnerProvider.type.toUpperCase()} runners after clean-up.`, + `Found: '${activeProviderRunnersCountAfter}' active ${computeProvider.type.toUpperCase()} runners after clean-up.`, ); } diff --git a/lambdas/functions/control-plane/src/scale-runners/scale-up-contract.test.ts b/lambdas/functions/control-plane/src/scale-runners/scale-up-contract.test.ts index 257f9907ca..3c1a0362bb 100644 --- a/lambdas/functions/control-plane/src/scale-runners/scale-up-contract.test.ts +++ b/lambdas/functions/control-plane/src/scale-runners/scale-up-contract.test.ts @@ -1,13 +1,13 @@ import type { Octokit } from '@octokit/rest'; import { beforeEach, vi } from 'vitest'; -import { providerTypes } from '../test/runner-provider-contracts/provider-types'; -import { defineScaleUpContractTests } from '../test/runner-provider-contracts/scale-up'; +import { providerTypes } from '../test/compute-provider-contracts/provider-types'; +import { defineScaleUpContractTests } from '../test/compute-provider-contracts/scale-up'; import * as ghAuth from '../github/auth'; import { controlPlaneProviderRegistry } from '../control-plane-providers'; import * as githubRunner from './github-runner'; import { scaleUp } from './scale-up'; -import type { ActionRequestMessageSQS, ScaleUpRunnerProvider } from './types'; +import type { ActionRequestMessageSQS, ScaleUpComputeProvider } from './types'; vi.mock('../github/auth', () => ({ createGithubAppAuth: vi.fn(), @@ -43,14 +43,14 @@ const payloads: ActionRequestMessageSQS[] = [ const cleanEnv = process.env; -const lanes = providerTypes.map((type) => ({ +const computeProviders = providerTypes.map((type) => ({ provider: { type, resolveLabelsForRunners: vi.fn(), getCurrentRunners: vi.fn(), createRunners: vi.fn(), - } satisfies ScaleUpRunnerProvider, - state: { lane: type }, + } satisfies ScaleUpComputeProvider, + state: { computeProvider: type }, })); beforeEach(() => { @@ -75,9 +75,9 @@ beforeEach(() => { }); defineScaleUpContractTests({ + computeProviders, createPayloads: () => structuredClone(payloads), githubInstallationClient: githubClient, - lanes, resolveCapability: mockedResolveCapability, scaleUp, }); diff --git a/lambdas/functions/control-plane/src/scale-runners/scale-up.test.ts b/lambdas/functions/control-plane/src/scale-runners/scale-up.test.ts index 6b6d0b5b76..eadec6706a 100644 --- a/lambdas/functions/control-plane/src/scale-runners/scale-up.test.ts +++ b/lambdas/functions/control-plane/src/scale-runners/scale-up.test.ts @@ -14,7 +14,7 @@ import type { ActionRequestMessageSQS, CreateRunnerResult, CreateScaleUpRunnersInput, - ScaleUpRunnerProvider, + ScaleUpComputeProvider, } from './types'; import { getParameter } from '@aws-github-runner/aws-ssm-util'; import { beforeEach, describe, expect, it, vi } from 'vitest'; @@ -57,15 +57,15 @@ const mockSSMClient = mockClient(SSMClient); const mockSSMgetParameter = vi.mocked(getParameter); const mockPublishRetryMessage = vi.mocked(publishRetryMessage); const testProviderState = { provider: 'test' }; -const mockRunnerProvider: ScaleUpRunnerProvider = { +const mockComputeProvider: ScaleUpComputeProvider = { type: 'ec2', resolveLabelsForRunners: vi.fn(), getCurrentRunners: vi.fn(), createRunners: vi.fn(), }; -const mockResolveLabelsForRunners = vi.mocked(mockRunnerProvider.resolveLabelsForRunners); -const mockGetCurrentRunners = vi.mocked(mockRunnerProvider.getCurrentRunners); -const mockCreateRunners = vi.mocked(mockRunnerProvider.createRunners); +const mockResolveLabelsForRunners = vi.mocked(mockComputeProvider.resolveLabelsForRunners); +const mockGetCurrentRunners = vi.mocked(mockComputeProvider.getCurrentRunners); +const mockCreateRunners = vi.mocked(mockComputeProvider.createRunners); const mockedResolveCapability = vi.spyOn(controlPlaneProviderRegistry, 'capability'); function createRunnerResult(instances: string[], retryableErrorCount = 0, nonRetryableErrorCount = 0) { @@ -189,7 +189,7 @@ beforeEach(() => { defaultSSMGetParameterMockImpl(); defaultOctokitMockImpl(); - mockedResolveCapability.mockReturnValue(() => mockRunnerProvider); + mockedResolveCapability.mockReturnValue(() => mockComputeProvider); mockResolveLabelsForRunners.mockImplementation(async (labels) => ({ runnerLabels: labels.filter((label) => label.startsWith('ghr-')), state: testProviderState, @@ -2174,11 +2174,21 @@ describe('Retry mechanism tests', () => { }); }); -describe('runner provider selection', () => { +describe('compute provider selection', () => { + it('defaults scale-up to EC2 when no compute provider is configured', async () => { + delete process.env.COMPUTE_PROVIDER_TYPE; + + await scaleUpModule.scaleUp(TEST_DATA); + + expect(mockedResolveCapability).toHaveBeenCalledWith('ec2', 'scaleUp'); + }); + it('rejects unsupported scale-up provider types', async () => { - process.env.RUNNER_PROVIDER_TYPE = 'microvm'; + process.env.COMPUTE_PROVIDER_TYPE = 'unsupported-provider'; - await expect(scaleUpModule.scaleUp(TEST_DATA)).rejects.toThrow("Unsupported runner provider type 'microvm'"); + await expect(scaleUpModule.scaleUp(TEST_DATA)).rejects.toThrow( + "Unsupported compute provider type 'unsupported-provider'", + ); expect(mockedAppAuth).not.toHaveBeenCalled(); }); }); diff --git a/lambdas/functions/control-plane/src/scale-runners/scale-up.ts b/lambdas/functions/control-plane/src/scale-runners/scale-up.ts index 4f06d5a1ff..3cd829a2fb 100644 --- a/lambdas/functions/control-plane/src/scale-runners/scale-up.ts +++ b/lambdas/functions/control-plane/src/scale-runners/scale-up.ts @@ -1,5 +1,5 @@ import { addPersistentContextToChildLogger, createChildLogger } from '@aws-github-runner/aws-powertools-util'; -import { resolveRunnerProviderType } from '@aws-github-runner/runner-providers/provider-types'; +import { resolveComputeProviderType } from '@aws-github-runner/compute-providers/provider-types'; import { Octokit } from '@octokit/rest'; import yn from 'yn'; @@ -86,10 +86,10 @@ export async function scaleUp(payloads: ActionRequestMessageSQS[]): Promise 0 ? (messages[0].labels ?? []) : []; - const runnerLabelResolution = await runnerProvider.resolveLabelsForRunners(messageLabels); + const runnerLabelResolution = await computeProvider.resolveLabelsForRunners(messageLabels); const resolvedRunnerLabels = runnerLabelResolution.runnerLabels; if (resolvedRunnerLabels.length > 0) { @@ -251,7 +251,7 @@ export async function scaleUp(payloads: ActionRequestMessageSQS[]): Promise = Omit & { type: TType }; +type TestPoolProvider = Omit & { type: TType }; -export interface PoolContractLane { +export interface PoolContractProvider { provider: TestPoolProvider; } interface PoolContractOptions { adjust: (event: PoolEvent) => Promise; githubInstallationClient: Octokit; - lanes: readonly PoolContractLane[]; + computeProviders: readonly PoolContractProvider[]; resolveCapability: MockInstance<(type: TType, capability: 'pool') => () => Omit, 'type'>>; } export function definePoolContractTests({ adjust, + computeProviders, githubInstallationClient, - lanes, resolveCapability, }: PoolContractOptions): void { - describe.each(lanes.map((lane) => [lane.provider.type, lane] as const))( + describe.each(computeProviders.map((computeProvider) => [computeProvider.provider.type, computeProvider] as const))( '%s pool orchestration contract', (_, { provider }) => { beforeEach(() => { diff --git a/lambdas/functions/control-plane/src/test/compute-provider-contracts/provider-types.ts b/lambdas/functions/control-plane/src/test/compute-provider-contracts/provider-types.ts new file mode 100644 index 0000000000..ac63b71d5c --- /dev/null +++ b/lambdas/functions/control-plane/src/test/compute-provider-contracts/provider-types.ts @@ -0,0 +1,3 @@ +import { computeProviderTypes } from '../../control-plane-providers'; + +export const providerTypes = computeProviderTypes; diff --git a/lambdas/functions/control-plane/src/test/runner-provider-contracts/scale-down.ts b/lambdas/functions/control-plane/src/test/compute-provider-contracts/scale-down.ts similarity index 83% rename from lambdas/functions/control-plane/src/test/runner-provider-contracts/scale-down.ts rename to lambdas/functions/control-plane/src/test/compute-provider-contracts/scale-down.ts index 451e33f9e1..97f6674c9a 100644 --- a/lambdas/functions/control-plane/src/test/runner-provider-contracts/scale-down.ts +++ b/lambdas/functions/control-plane/src/test/compute-provider-contracts/scale-down.ts @@ -1,15 +1,15 @@ import { beforeEach, describe, expect, it, type MockInstance, vi } from 'vitest'; -import type { ScaleDownRunnerProvider } from '../../scale-runners/types'; +import type { ScaleDownComputeProvider } from '../../scale-runners/types'; -type TestScaleDownProvider = Omit & { type: TType }; +type TestScaleDownProvider = Omit & { type: TType }; -export interface ScaleDownContractLane { +export interface ScaleDownContractProvider { provider: TestScaleDownProvider; } interface ScaleDownContractOptions { - lanes: readonly ScaleDownContractLane[]; + computeProviders: readonly ScaleDownContractProvider[]; resolveCapability: MockInstance< (type: TType, capability: 'scaleDown') => () => Omit, 'type'> >; @@ -17,16 +17,16 @@ interface ScaleDownContractOptions { } export function defineScaleDownContractTests({ - lanes, + computeProviders, resolveCapability, scaleDown, }: ScaleDownContractOptions): void { - describe.each(lanes.map((lane) => [lane.provider.type, lane] as const))( + describe.each(computeProviders.map((computeProvider) => [computeProvider.provider.type, computeProvider] as const))( '%s scale-down orchestration contract', (_, { provider }) => { beforeEach(() => { process.env.ENVIRONMENT = 'test-environment'; - process.env.RUNNER_PROVIDER_TYPE = provider.type; + process.env.COMPUTE_PROVIDER_TYPE = provider.type; process.env.SCALE_DOWN_CONFIG = '[]'; resolveCapability.mockReturnValue(() => provider); diff --git a/lambdas/functions/control-plane/src/test/runner-provider-contracts/scale-up.ts b/lambdas/functions/control-plane/src/test/compute-provider-contracts/scale-up.ts similarity index 73% rename from lambdas/functions/control-plane/src/test/runner-provider-contracts/scale-up.ts rename to lambdas/functions/control-plane/src/test/compute-provider-contracts/scale-up.ts index 804c8a2fbb..ecab384118 100644 --- a/lambdas/functions/control-plane/src/test/runner-provider-contracts/scale-up.ts +++ b/lambdas/functions/control-plane/src/test/compute-provider-contracts/scale-up.ts @@ -1,19 +1,19 @@ import type { Octokit } from '@octokit/rest'; import { beforeEach, describe, expect, it, type MockInstance, vi } from 'vitest'; -import type { ActionRequestMessageSQS, ScaleUpRunnerProvider } from '../../scale-runners/types'; +import type { ActionRequestMessageSQS, ScaleUpComputeProvider } from '../../scale-runners/types'; -type TestScaleUpProvider = Omit & { type: TType }; +type TestScaleUpProvider = Omit & { type: TType }; -export interface ScaleUpContractLane { +export interface ScaleUpContractProvider { provider: TestScaleUpProvider; state: unknown; } interface ScaleUpContractOptions { createPayloads: () => ActionRequestMessageSQS[]; + computeProviders: readonly ScaleUpContractProvider[]; githubInstallationClient: Octokit; - lanes: readonly ScaleUpContractLane[]; resolveCapability: MockInstance< (type: TType, capability: 'scaleUp') => () => Omit, 'type'> >; @@ -27,19 +27,19 @@ const createResult = { }; export function defineScaleUpContractTests({ + computeProviders, createPayloads, githubInstallationClient, - lanes, resolveCapability, scaleUp, }: ScaleUpContractOptions): void { - describe.each(lanes.map((lane) => [lane.provider.type, lane] as const))( + describe.each(computeProviders.map((computeProvider) => [computeProvider.provider.type, computeProvider] as const))( '%s scale-up orchestration contract', (_, { provider, state }) => { beforeEach(() => { process.env.ENABLE_ORGANIZATION_RUNNERS = 'true'; process.env.RUNNERS_MAXIMUM_COUNT = '3'; - process.env.RUNNER_PROVIDER_TYPE = provider.type; + process.env.COMPUTE_PROVIDER_TYPE = provider.type; resolveCapability.mockReturnValue(() => provider); vi.mocked(provider.resolveLabelsForRunners).mockResolvedValue({ runnerLabels: [], state }); @@ -47,14 +47,14 @@ export function defineScaleUpContractTests({ vi.mocked(provider.createRunners).mockResolvedValue(createResult); }); - it('forwards the prepared lane state through runner lookup and creation', async () => { + it('forwards the prepared compute-provider state through runner lookup and creation', async () => { const payloads = createPayloads(); - payloads[0].labels = ['lane-label']; + payloads[0].labels = ['compute-provider-label']; await scaleUp(payloads); expect(resolveCapability).toHaveBeenCalledWith(provider.type, 'scaleUp'); - expect(provider.resolveLabelsForRunners).toHaveBeenCalledWith(['lane-label']); + expect(provider.resolveLabelsForRunners).toHaveBeenCalledWith(['compute-provider-label']); expect(provider.getCurrentRunners).toHaveBeenCalledWith(state, { runnerOwner: payloads[0].repositoryOwner, runnerType: 'Org', @@ -68,7 +68,7 @@ export function defineScaleUpContractTests({ ); }); - it('does not query current runners when the lane has unlimited capacity', async () => { + it('does not query current runners when the compute provider has unlimited capacity', async () => { process.env.RUNNERS_MAXIMUM_COUNT = '-1'; const payloads = createPayloads(); payloads.push({ ...payloads[0], id: 2, messageId: 'message-2' }); @@ -79,7 +79,7 @@ export function defineScaleUpContractTests({ expect(provider.createRunners).toHaveBeenCalledWith(expect.objectContaining({ numberOfRunners: 2 })); }); - it('does not create runners when the lane has reached maximum capacity', async () => { + it('does not create runners when the compute provider has reached maximum capacity', async () => { process.env.RUNNERS_MAXIMUM_COUNT = '1'; vi.mocked(provider.getCurrentRunners).mockResolvedValue(1); diff --git a/lambdas/functions/control-plane/src/test/runner-provider-contracts/provider-types.ts b/lambdas/functions/control-plane/src/test/runner-provider-contracts/provider-types.ts deleted file mode 100644 index 6a97a2cc2b..0000000000 --- a/lambdas/functions/control-plane/src/test/runner-provider-contracts/provider-types.ts +++ /dev/null @@ -1,3 +0,0 @@ -import { runnerProviderTypes } from '../../control-plane-providers'; - -export const providerTypes = runnerProviderTypes; diff --git a/lambdas/functions/webhook/package.json b/lambdas/functions/webhook/package.json index f6bd1898ba..34f4ef3de9 100644 --- a/lambdas/functions/webhook/package.json +++ b/lambdas/functions/webhook/package.json @@ -30,7 +30,7 @@ "dependencies": { "@aws-github-runner/aws-powertools-util": "*", "@aws-github-runner/aws-ssm-util": "*", - "@aws-github-runner/runner-providers": "*", + "@aws-github-runner/compute-providers": "*", "@aws-sdk/client-sqs": "^3.1009.0", "@middy/core": "^6.4.5", "@octokit/rest": "22.0.1", diff --git a/lambdas/functions/webhook/src/runners/aws-dynamic-labels-policy.ts b/lambdas/functions/webhook/src/runners/aws-dynamic-labels-policy.ts deleted file mode 100644 index c23e7f66ad..0000000000 --- a/lambdas/functions/webhook/src/runners/aws-dynamic-labels-policy.ts +++ /dev/null @@ -1 +0,0 @@ -export type { AwsDynamicLabelsPolicy, AwsDynamicLabelsValueRule } from '@aws-github-runner/runner-providers'; diff --git a/lambdas/functions/webhook/src/runners/aws-dynamic-labels.test.ts b/lambdas/functions/webhook/src/runners/aws-dynamic-labels.test.ts deleted file mode 100644 index 870aae8f65..0000000000 --- a/lambdas/functions/webhook/src/runners/aws-dynamic-labels.test.ts +++ /dev/null @@ -1,65 +0,0 @@ -import type { RunnerProviderType } from '@aws-github-runner/runner-providers/provider-types'; -import { describe, expect, it } from 'vitest'; - -import type { RunnerMatcherConfig } from '../sqs'; -import { selectAwsDynamicLabelQueue } from './aws-dynamic-labels'; - -describe('selectAwsDynamicLabelQueue', () => { - it('defaults queues without a provider to EC2 dynamic label handling', () => { - const queue = runnerQueue('default-ec2'); - - expect(selectAwsDynamicLabelQueue([queue], ['self-hosted', 'linux'], ['ghr-ec2-instance-type:t3.large'])).toEqual({ - queue, - labels: ['self-hosted', 'linux', 'ghr-ec2-instance-type:t3.large'], - }); - }); - - it('normalizes runner provider casing and surrounding whitespace', () => { - const queue = runnerQueue('normalized-ec2'); - (queue as unknown as { runnerProvider: string }).runnerProvider = ' EC2 '; - - expect(selectAwsDynamicLabelQueue([queue], ['self-hosted', 'linux'], ['ghr-ec2-instance-type:t3.large'])).toEqual({ - queue, - labels: ['self-hosted', 'linux', 'ghr-ec2-instance-type:t3.large'], - }); - }); - - it('skips an unsupported provider strategy and selects the next supported queue', () => { - const unsupportedQueue = runnerQueue('unsupported-provider'); - (unsupportedQueue as unknown as { runnerProvider: string }).runnerProvider = 'unsupported'; - const ec2Queue = runnerQueue('ec2'); - - expect( - selectAwsDynamicLabelQueue( - [unsupportedQueue, ec2Queue], - ['self-hosted', 'linux'], - ['ghr-ec2-instance-type:t3.large'], - ), - ).toEqual({ - queue: ec2Queue, - labels: ['self-hosted', 'linux', 'ghr-ec2-instance-type:t3.large'], - }); - }); - - it('rejects a malformed non-string runner provider without throwing', () => { - const queue = runnerQueue('malformed-provider'); - (queue as unknown as { runnerProvider: number }).runnerProvider = 42; - - expect( - selectAwsDynamicLabelQueue([queue], ['self-hosted', 'linux'], ['ghr-ec2-instance-type:t3.large']), - ).toBeUndefined(); - }); -}); - -function runnerQueue(id: string, runnerProvider?: RunnerProviderType): RunnerMatcherConfig { - return { - id, - arn: `arn:${id}`, - runnerProvider, - matcherConfig: { - labelMatchers: [['self-hosted', 'linux']], - exactMatch: true, - enableDynamicLabels: true, - }, - }; -} diff --git a/lambdas/functions/webhook/src/runners/aws-dynamic-labels.ts b/lambdas/functions/webhook/src/runners/aws-dynamic-labels.ts deleted file mode 100644 index 7a05cd8cc8..0000000000 --- a/lambdas/functions/webhook/src/runners/aws-dynamic-labels.ts +++ /dev/null @@ -1,29 +0,0 @@ -import { createChildLogger } from '@aws-github-runner/aws-powertools-util'; -import type { DynamicLabelDispatchTarget } from '@aws-github-runner/runner-providers'; -import { normalizeRunnerProviderType } from '@aws-github-runner/runner-providers/provider-types'; -import { webhookProviderRegistry } from '@aws-github-runner/runner-providers/webhook'; - -import type { RunnerMatcherConfig } from '../sqs'; - -const logger = createChildLogger('handler'); - -export function selectAwsDynamicLabelQueue( - matches: RunnerMatcherConfig[], - nonGhrLabels: string[], - sanitizedGhrLabels: string[], -): DynamicLabelDispatchTarget | undefined { - for (const queue of matches) { - const provider = normalizeRunnerProviderType(queue.runnerProvider); - const dynamicLabels = provider ? webhookProviderRegistry.capability(provider, 'dynamicLabels') : undefined; - - if (!dynamicLabels) { - logger.warn(`Queue ${queue.id} has unsupported runner provider '${provider ?? String(queue.runnerProvider)}'`); - continue; - } - - const target = dynamicLabels.selectQueue({ queue, nonGhrLabels, sanitizedGhrLabels }); - if (target) return target; - } - - return undefined; -} diff --git a/lambdas/functions/webhook/src/runners/dispatch.test.ts b/lambdas/functions/webhook/src/runners/dispatch.test.ts index ae571da9d8..bb2cdc7cce 100644 --- a/lambdas/functions/webhook/src/runners/dispatch.test.ts +++ b/lambdas/functions/webhook/src/runners/dispatch.test.ts @@ -1,4 +1,5 @@ import { getParameter } from '@aws-github-runner/aws-ssm-util'; +import { selectDynamicLabelQueue } from '@aws-github-runner/compute-providers/webhook'; import nock from 'nock'; import { WorkflowJobEvent } from '@octokit/webhooks-types'; @@ -14,6 +15,9 @@ import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest'; vi.mock('../sqs'); vi.mock('@aws-github-runner/aws-ssm-util'); +vi.mock('@aws-github-runner/compute-providers/webhook', () => ({ + selectDynamicLabelQueue: vi.fn(), +})); const GITHUB_APP_WEBHOOK_SECRET = 'TEST_SECRET'; @@ -246,7 +250,14 @@ describe('Dispatcher', () => { describe('per-matcher dynamic labels handling', () => { const baseRunner = runnerConfig[0]; - it('strips invalid ghr- labels (too long, bad chars) before policy and dispatch', async () => { + beforeEach(() => { + vi.mocked(selectDynamicLabelQueue).mockImplementation((matches, nonGhrLabels, sanitizedGhrLabels) => ({ + queue: matches[0], + labels: [...nonGhrLabels, ...sanitizedGhrLabels], + })); + }); + + it('strips invalid ghr- labels before provider selection and dispatch', async () => { const longLabel = 'ghr-' + 'a'.repeat(125); // 129 chars config = await createConfig(undefined, [ { @@ -276,19 +287,25 @@ describe('Dispatcher', () => { } as unknown as WorkflowJobEvent; const resp = await dispatch(event, 'workflow_job', config); expect(resp.statusCode).toBe(201); + expect(selectDynamicLabelQueue).toHaveBeenCalledWith( + [expect.objectContaining({ id: baseRunner.id })], + ['self-hosted', 'linux'], + ['ghr-valid:value', 'ghr-list:value;another'], + ); expect(sendActionRequest).toHaveBeenCalledWith( expect.objectContaining({ labels: ['self-hosted', 'linux', 'ghr-valid:value', 'ghr-list:value;another'] }), ); }); - it('rejects the job (202) when the only matching runner has enableDynamicLabels=false', async () => { + it('rejects the job when no provider accepts the dynamic labels', async () => { + vi.mocked(selectDynamicLabelQueue).mockReturnValue(undefined); config = await createConfig(undefined, [ { ...baseRunner, matcherConfig: { labelMatchers: [['self-hosted', 'linux']], exactMatch: true, - enableDynamicLabels: false, + enableDynamicLabels: true, }, }, ]); @@ -296,7 +313,7 @@ describe('Dispatcher', () => { ...workFlowJobEvent, workflow_job: { ...workFlowJobEvent.workflow_job, - labels: ['self-hosted', 'linux', 'ghr-ec2-instance-type:t3.large'], + labels: ['self-hosted', 'linux', 'ghr-provider-setting:value'], }, } as unknown as WorkflowJobEvent; const resp = await dispatch(event, 'workflow_job', config); @@ -304,50 +321,20 @@ describe('Dispatcher', () => { expect(sendActionRequest).not.toHaveBeenCalled(); }); - it('keeps dynamic labels when the matched runner enables them and has no policy', async () => { + it('dispatches to the queue and labels returned by the provider selector', async () => { config = await createConfig(undefined, [ { ...baseRunner, + id: 'first', matcherConfig: { labelMatchers: [['self-hosted', 'linux']], exactMatch: true, enableDynamicLabels: true, }, }, - ]); - const event = { - ...workFlowJobEvent, - workflow_job: { - ...workFlowJobEvent.workflow_job, - labels: ['self-hosted', 'linux', 'ghr-ec2-instance-type:t3.large'], - }, - } as unknown as WorkflowJobEvent; - const resp = await dispatch(event, 'workflow_job', config); - expect(resp.statusCode).toBe(201); - expect(sendActionRequest).toHaveBeenCalledWith( - expect.objectContaining({ labels: ['self-hosted', 'linux', 'ghr-ec2-instance-type:t3.large'] }), - ); - }); - - it('skips a matching runner whose policy rejects the dynamic labels and uses the next compliant one', async () => { - config = await createConfig(undefined, [ - { - ...baseRunner, - id: 'strict', - matcherConfig: { - labelMatchers: [['self-hosted', 'linux']], - exactMatch: true, - enableDynamicLabels: true, - awsDynamicLabelsPolicy: { - restricted_keys: { - 'instance-type': { allowed: ['m5.*'] }, - }, - }, - }, - }, { ...baseRunner, - id: 'permissive', + id: 'selected', matcherConfig: { labelMatchers: [['self-hosted', 'linux']], exactMatch: true, @@ -355,61 +342,29 @@ describe('Dispatcher', () => { }, }, ]); + + vi.mocked(selectDynamicLabelQueue).mockImplementation((matches) => ({ + queue: matches[1], + labels: ['self-hosted', 'linux', 'ghr-provider-setting:normalized'], + })); + const event = { ...workFlowJobEvent, workflow_job: { ...workFlowJobEvent.workflow_job, - labels: ['self-hosted', 'linux', 'ghr-ec2-instance-type:t3.large'], + labels: ['self-hosted', 'linux', 'ghr-provider-setting:requested'], }, } as unknown as WorkflowJobEvent; const resp = await dispatch(event, 'workflow_job', config); expect(resp.statusCode).toBe(201); expect(sendActionRequest).toHaveBeenCalledWith( expect.objectContaining({ - queueId: 'permissive', - labels: ['self-hosted', 'linux', 'ghr-ec2-instance-type:t3.large'], + queueId: 'selected', + labels: ['self-hosted', 'linux', 'ghr-provider-setting:normalized'], }), ); }); - it('rejects the job (202) when no runner accepts the policy', async () => { - config = await createConfig(undefined, [ - { - ...baseRunner, - id: 'first', - matcherConfig: { - labelMatchers: [['self-hosted', 'linux']], - exactMatch: true, - enableDynamicLabels: true, - awsDynamicLabelsPolicy: { - restricted_keys: { - 'instance-type': { allowed: ['m5.*'] }, - }, - }, - }, - }, - { - ...baseRunner, - id: 'second', - matcherConfig: { - labelMatchers: [['self-hosted', 'linux']], - exactMatch: true, - enableDynamicLabels: false, - }, - }, - ]); - const event = { - ...workFlowJobEvent, - workflow_job: { - ...workFlowJobEvent.workflow_job, - labels: ['self-hosted', 'linux', 'ghr-ec2-instance-type:t3.large'], - }, - } as unknown as WorkflowJobEvent; - const resp = await dispatch(event, 'workflow_job', config); - expect(resp.statusCode).toBe(202); - expect(sendActionRequest).not.toHaveBeenCalled(); - }); - it('forwards non-dynamic jobs as-is to the first match', async () => { config = await createConfig(undefined, [ { @@ -419,7 +374,6 @@ describe('Dispatcher', () => { labelMatchers: [['self-hosted', 'linux']], exactMatch: true, enableDynamicLabels: true, - awsDynamicLabelsPolicy: {}, }, }, ]); @@ -435,6 +389,7 @@ describe('Dispatcher', () => { expect(sendActionRequest).toHaveBeenCalledWith( expect.objectContaining({ queueId: 'first', labels: ['self-hosted', 'linux'] }), ); + expect(selectDynamicLabelQueue).not.toHaveBeenCalled(); }); }); }); diff --git a/lambdas/functions/webhook/src/runners/dispatch.ts b/lambdas/functions/webhook/src/runners/dispatch.ts index 47c1f1bfc0..da6dc01221 100644 --- a/lambdas/functions/webhook/src/runners/dispatch.ts +++ b/lambdas/functions/webhook/src/runners/dispatch.ts @@ -1,11 +1,11 @@ import { createChildLogger } from '@aws-github-runner/aws-powertools-util'; +import { selectDynamicLabelQueue } from '@aws-github-runner/compute-providers/webhook'; import { WorkflowJobEvent } from '@octokit/webhooks-types'; import { Response } from '../lambda'; import { RunnerMatcherConfig, sendActionRequest } from '../sqs'; import ValidationError from '../ValidationError'; import { ConfigDispatcher, ConfigWebhook, QueueSelectionStrategy } from '../ConfigLoader'; -import { selectAwsDynamicLabelQueue } from './aws-dynamic-labels'; import { canRunJob, splitWorkflowJobLabels } from './labels'; const logger = createChildLogger('handler'); @@ -84,7 +84,7 @@ async function handleWorkflowJob( // Dynamic labels present: prefer the first provider-compliant queue. The // queue selection strategy applies to standard jobs only; dynamic-label jobs // always use the first compliant queue. - const dynamicTarget = selectAwsDynamicLabelQueue(matches, nonGhrLabels, sanitizedGhrLabels); + const dynamicTarget = selectDynamicLabelQueue(matches, nonGhrLabels, sanitizedGhrLabels); if (dynamicTarget) { targets = [dynamicTarget.queue]; diff --git a/lambdas/functions/webhook/src/sqs/index.ts b/lambdas/functions/webhook/src/sqs/index.ts index 40fe9f1e75..2ac1a5b6a7 100644 --- a/lambdas/functions/webhook/src/sqs/index.ts +++ b/lambdas/functions/webhook/src/sqs/index.ts @@ -2,7 +2,7 @@ import { SQS, SendMessageCommandInput } from '@aws-sdk/client-sqs'; import { WorkflowJobEvent } from '@octokit/webhooks-types'; import { createChildLogger, getTracedAWSV3Client } from '@aws-github-runner/aws-powertools-util'; -export type { MatcherConfig, RunnerConfig, RunnerMatcherConfig } from '@aws-github-runner/runner-providers'; +export type { MatcherConfig, RunnerConfig, RunnerMatcherConfig } from '@aws-github-runner/compute-providers'; const logger = createChildLogger('sqs'); diff --git a/lambdas/libs/compute-providers/aws/dynamic-labels-policy.ts b/lambdas/libs/compute-providers/aws/dynamic-labels-policy.ts new file mode 100644 index 0000000000..64b7507add --- /dev/null +++ b/lambdas/libs/compute-providers/aws/dynamic-labels-policy.ts @@ -0,0 +1,61 @@ +import type { AwsDynamicLabelsPolicy } from '../contracts'; + +function globToRegExp(glob: string): RegExp { + const escaped = glob.replace(/[.+^${}()|[\]\\]/g, '\\$&'); + const pattern = escaped.replace(/\*/g, '.*').replace(/\?/g, '.'); + return new RegExp(`^${pattern}$`); +} + +function matchesAny(value: string, patterns: string[] | undefined): boolean { + if (!patterns || patterns.length === 0) return false; + return patterns.some((pattern) => globToRegExp(pattern).test(value)); +} + +function evaluateLabel(label: string, policy: AwsDynamicLabelsPolicy, labelPrefix: string): string | null { + const stripped = label.slice(labelPrefix.length); + const colonIndex = stripped.indexOf(':'); + const key = colonIndex === -1 ? stripped : stripped.slice(0, colonIndex); + const value = colonIndex === -1 ? undefined : stripped.slice(colonIndex + 1); + + if (policy.blocked_keys?.includes(key)) { + return `key '${key}' is in blocked_keys`; + } + + const rule = policy.restricted_keys?.[key]; + if (!rule || value === undefined) return null; + + if (rule.allowed && rule.allowed.length > 0 && !matchesAny(value, rule.allowed)) { + return `value '${value}' not in allowed list`; + } + if (rule.denied && matchesAny(value, rule.denied)) { + return `value '${value}' in denied list`; + } + if (rule.max !== undefined && rule.max !== null) { + const valueNumber = Number(value); + const maximum = Number(rule.max); + if (!Number.isFinite(valueNumber) || !Number.isFinite(maximum)) { + return `max set but value '${value}' or max '${rule.max}' is not numeric`; + } + if (valueNumber > maximum) { + return `value '${value}' exceeds max '${rule.max}'`; + } + } + + return null; +} + +export function violationsAgainstAwsDynamicLabelsPolicy( + labels: string[], + policy: AwsDynamicLabelsPolicy | null | undefined, + labelPrefix: string, +): { label: string; reason: string }[] { + if (!policy) return []; + + const violations: { label: string; reason: string }[] = []; + for (const label of labels) { + if (!label.startsWith(labelPrefix)) continue; + const reason = evaluateLabel(label, policy, labelPrefix); + if (reason) violations.push({ label, reason }); + } + return violations; +} diff --git a/lambdas/libs/runner-providers/aws/ec2/control-plane.ts b/lambdas/libs/compute-providers/aws/ec2/control-plane.ts similarity index 84% rename from lambdas/libs/runner-providers/aws/ec2/control-plane.ts rename to lambdas/libs/compute-providers/aws/ec2/control-plane.ts index ef07fec861..6b4fb7c0bf 100644 --- a/lambdas/libs/runner-providers/aws/ec2/control-plane.ts +++ b/lambdas/libs/compute-providers/aws/ec2/control-plane.ts @@ -1,4 +1,4 @@ -import type { CreateStartRunnerConfig, RunnerProviderPlugin } from '../../core'; +import type { CreateStartRunnerConfig, ComputeProviderPlugin } from '../../core'; import type { ControlPlaneProviderCapabilities, ControlPlaneProviderModule } from '../../contracts'; import type {} from './src/environment'; @@ -8,7 +8,7 @@ import { createEc2ScaleUpProvider } from './src/control-plane/scale-up'; export function createEc2ControlPlanePlugin( createStartRunnerConfig: CreateStartRunnerConfig, -): RunnerProviderPlugin { +): ComputeProviderPlugin { return { type: 'ec2', capabilities: { diff --git a/lambdas/libs/runner-providers/aws/ec2/src/control-plane/dynamic-labels.ts b/lambdas/libs/compute-providers/aws/ec2/src/control-plane/dynamic-labels.ts similarity index 100% rename from lambdas/libs/runner-providers/aws/ec2/src/control-plane/dynamic-labels.ts rename to lambdas/libs/compute-providers/aws/ec2/src/control-plane/dynamic-labels.ts diff --git a/lambdas/libs/runner-providers/aws/ec2/src/control-plane/pool.test.ts b/lambdas/libs/compute-providers/aws/ec2/src/control-plane/pool.test.ts similarity index 100% rename from lambdas/libs/runner-providers/aws/ec2/src/control-plane/pool.test.ts rename to lambdas/libs/compute-providers/aws/ec2/src/control-plane/pool.test.ts diff --git a/lambdas/libs/runner-providers/aws/ec2/src/control-plane/pool.ts b/lambdas/libs/compute-providers/aws/ec2/src/control-plane/pool.ts similarity index 97% rename from lambdas/libs/runner-providers/aws/ec2/src/control-plane/pool.ts rename to lambdas/libs/compute-providers/aws/ec2/src/control-plane/pool.ts index f08a70d05e..4e8e813a13 100644 --- a/lambdas/libs/runner-providers/aws/ec2/src/control-plane/pool.ts +++ b/lambdas/libs/compute-providers/aws/ec2/src/control-plane/pool.ts @@ -3,7 +3,7 @@ import type { CreateStartRunnerConfig, CreatePoolRunnersInput, ListPoolRunnersInput, - PoolRunnerProvider, + PoolComputeProvider, RunnerInfo, RunnerStatus, } from '../../../../core'; @@ -53,7 +53,7 @@ async function createEc2PoolRunners( export function createEc2PoolProvider( createStartRunnerConfig: CreateStartRunnerConfig, -): Omit, 'type'> { +): Omit, 'type'> { return { listRunners: listEc2PoolRunners, countAvailableRunners: calculateEc2PoolSize, diff --git a/lambdas/libs/runner-providers/aws/ec2/src/control-plane/runner-config.ts b/lambdas/libs/compute-providers/aws/ec2/src/control-plane/runner-config.ts similarity index 100% rename from lambdas/libs/runner-providers/aws/ec2/src/control-plane/runner-config.ts rename to lambdas/libs/compute-providers/aws/ec2/src/control-plane/runner-config.ts diff --git a/lambdas/libs/runner-providers/aws/ec2/src/control-plane/runners.d.ts b/lambdas/libs/compute-providers/aws/ec2/src/control-plane/runners.d.ts similarity index 100% rename from lambdas/libs/runner-providers/aws/ec2/src/control-plane/runners.d.ts rename to lambdas/libs/compute-providers/aws/ec2/src/control-plane/runners.d.ts diff --git a/lambdas/libs/runner-providers/aws/ec2/src/control-plane/runners.test.ts b/lambdas/libs/compute-providers/aws/ec2/src/control-plane/runners.test.ts similarity index 100% rename from lambdas/libs/runner-providers/aws/ec2/src/control-plane/runners.test.ts rename to lambdas/libs/compute-providers/aws/ec2/src/control-plane/runners.test.ts diff --git a/lambdas/libs/runner-providers/aws/ec2/src/control-plane/runners.ts b/lambdas/libs/compute-providers/aws/ec2/src/control-plane/runners.ts similarity index 100% rename from lambdas/libs/runner-providers/aws/ec2/src/control-plane/runners.ts rename to lambdas/libs/compute-providers/aws/ec2/src/control-plane/runners.ts diff --git a/lambdas/libs/runner-providers/aws/ec2/src/control-plane/scale-down.test.ts b/lambdas/libs/compute-providers/aws/ec2/src/control-plane/scale-down.test.ts similarity index 100% rename from lambdas/libs/runner-providers/aws/ec2/src/control-plane/scale-down.test.ts rename to lambdas/libs/compute-providers/aws/ec2/src/control-plane/scale-down.test.ts diff --git a/lambdas/libs/runner-providers/aws/ec2/src/control-plane/scale-down.ts b/lambdas/libs/compute-providers/aws/ec2/src/control-plane/scale-down.ts similarity index 80% rename from lambdas/libs/runner-providers/aws/ec2/src/control-plane/scale-down.ts rename to lambdas/libs/compute-providers/aws/ec2/src/control-plane/scale-down.ts index 2609deb95b..55d52ac298 100644 --- a/lambdas/libs/runner-providers/aws/ec2/src/control-plane/scale-down.ts +++ b/lambdas/libs/compute-providers/aws/ec2/src/control-plane/scale-down.ts @@ -1,4 +1,4 @@ -import type { RunnerInfo, ScaleDownRunnerProvider } from '../../../../core'; +import type { RunnerInfo, ScaleDownComputeProvider } from '../../../../core'; import { bootTimeExceeded, listEC2Runners, tag, terminateRunner, untag } from './runners'; async function listEc2ScaleDownRunners(environment: string, orphan?: boolean): Promise { @@ -13,7 +13,7 @@ async function unmarkEc2RunnerOrphan(id: string): Promise { await untag(id, [{ Key: 'ghr:orphan', Value: 'true' }]); } -export function createEc2ScaleDownProvider(): Omit { +export function createEc2ScaleDownProvider(): Omit { return { list: listEc2ScaleDownRunners, bootTimeExceeded, diff --git a/lambdas/libs/runner-providers/aws/ec2/src/control-plane/scale-up.test.ts b/lambdas/libs/compute-providers/aws/ec2/src/control-plane/scale-up.test.ts similarity index 100% rename from lambdas/libs/runner-providers/aws/ec2/src/control-plane/scale-up.test.ts rename to lambdas/libs/compute-providers/aws/ec2/src/control-plane/scale-up.test.ts diff --git a/lambdas/libs/runner-providers/aws/ec2/src/control-plane/scale-up.ts b/lambdas/libs/compute-providers/aws/ec2/src/control-plane/scale-up.ts similarity index 97% rename from lambdas/libs/runner-providers/aws/ec2/src/control-plane/scale-up.ts rename to lambdas/libs/compute-providers/aws/ec2/src/control-plane/scale-up.ts index d5886a57ce..d72edaf7a4 100644 --- a/lambdas/libs/runner-providers/aws/ec2/src/control-plane/scale-up.ts +++ b/lambdas/libs/compute-providers/aws/ec2/src/control-plane/scale-up.ts @@ -5,7 +5,7 @@ import type { CreateStartRunnerConfig, CurrentRunnersInput, RunnerLabelResolution, - ScaleUpRunnerProvider, + ScaleUpComputeProvider, } from '../../../../core'; import yn from 'yn'; @@ -83,7 +83,7 @@ async function createEc2ScaleUpRunners( export function createEc2ScaleUpProvider( createStartRunnerConfig: CreateStartRunnerConfig, -): Omit, 'type'> { +): Omit, 'type'> { return { resolveLabelsForRunners: resolveEc2LabelsForRunners, getCurrentRunners: getCurrentEc2Runners, diff --git a/lambdas/libs/runner-providers/aws/ec2/src/environment.d.ts b/lambdas/libs/compute-providers/aws/ec2/src/environment.d.ts similarity index 100% rename from lambdas/libs/runner-providers/aws/ec2/src/environment.d.ts rename to lambdas/libs/compute-providers/aws/ec2/src/environment.d.ts diff --git a/lambdas/libs/runner-providers/aws/ec2/src/webhook/dynamic-labels-policy.test.ts b/lambdas/libs/compute-providers/aws/ec2/src/webhook/dynamic-labels-policy.test.ts similarity index 100% rename from lambdas/libs/runner-providers/aws/ec2/src/webhook/dynamic-labels-policy.test.ts rename to lambdas/libs/compute-providers/aws/ec2/src/webhook/dynamic-labels-policy.test.ts diff --git a/lambdas/libs/compute-providers/aws/ec2/src/webhook/dynamic-labels-policy.ts b/lambdas/libs/compute-providers/aws/ec2/src/webhook/dynamic-labels-policy.ts new file mode 100644 index 0000000000..8babbadd55 --- /dev/null +++ b/lambdas/libs/compute-providers/aws/ec2/src/webhook/dynamic-labels-policy.ts @@ -0,0 +1,23 @@ +import type { AwsDynamicLabelsPolicy, AwsDynamicLabelsValueRule } from '../../../../contracts'; +import { violationsAgainstAwsDynamicLabelsPolicy } from '../../../dynamic-labels-policy'; + +export type Ec2DynamicLabelsValueRule = AwsDynamicLabelsValueRule; + +/** + * EC2 dynamic labels policy schema. `blocked_keys` rejects keys outright; + * `restricted_keys` applies optional per-key value rules. Keys use the + * `` segment of a `ghr-ec2-:` label in the same hyphenated + * form as the labels themselves (e.g. `instance-type`). + */ +export type Ec2DynamicLabelsPolicy = AwsDynamicLabelsPolicy; + +/** + * Inspects the labels and returns the rejection reasons for any `ghr-ec2-*` + * label that violates the policy. Non-`ghr-ec2-*` labels are ignored. + */ +export function violationsAgainstPolicy( + labels: string[], + policy: Ec2DynamicLabelsPolicy | null | undefined, +): { label: string; reason: string }[] { + return violationsAgainstAwsDynamicLabelsPolicy(labels, policy, 'ghr-ec2-'); +} diff --git a/lambdas/libs/runner-providers/aws/ec2/src/webhook/dynamic-labels.test.ts b/lambdas/libs/compute-providers/aws/ec2/src/webhook/dynamic-labels.test.ts similarity index 53% rename from lambdas/libs/runner-providers/aws/ec2/src/webhook/dynamic-labels.test.ts rename to lambdas/libs/compute-providers/aws/ec2/src/webhook/dynamic-labels.test.ts index d201f4f607..99c1844a5f 100644 --- a/lambdas/libs/runner-providers/aws/ec2/src/webhook/dynamic-labels.test.ts +++ b/lambdas/libs/compute-providers/aws/ec2/src/webhook/dynamic-labels.test.ts @@ -1,18 +1,38 @@ import { describe, expect, it } from 'vitest'; import type { RunnerMatcherConfig } from '../../../../contracts'; -import { selectEc2DynamicLabelQueue } from './dynamic-labels'; +import { ec2DynamicLabelProvider } from './dynamic-labels'; + +describe('ec2DynamicLabelProvider', () => { + it('returns no violations when the queue has no policy', () => { + const queue = runnerQueue('no-policy'); + + expect(getViolations(queue)).toEqual([]); + }); + + it('returns violations for labels rejected by the policy', () => { + const strictQueue = runnerQueue('strict'); + strictQueue.matcherConfig.awsDynamicLabelsPolicy = { + restricted_keys: { + 'instance-type': { allowed: ['m5.*'] }, + }, + }; + + expect(getViolations(strictQueue)).toEqual([ + { + label: 'ghr-ec2-instance-type:t3.large', + reason: "value 't3.large' not in allowed list", + }, + ]); + }); -describe('selectEc2DynamicLabelQueue', () => { it('enforces a legacy EC2 dynamic labels policy when the new key is absent', () => { const queue = runnerQueue('legacy-ec2-policy'); queue.matcherConfig.ec2DynamicLabelsPolicy = { blocked_keys: ['instance-type'], }; - expect( - selectEc2DynamicLabelQueue([queue], ['self-hosted', 'linux'], ['ghr-ec2-instance-type:t3.large']), - ).toBeUndefined(); + expect(getViolations(queue)).toHaveLength(1); }); it('falls back to the legacy EC2 dynamic labels policy when the new policy is null', () => { @@ -22,9 +42,7 @@ describe('selectEc2DynamicLabelQueue', () => { }; queue.matcherConfig.awsDynamicLabelsPolicy = null; - expect( - selectEc2DynamicLabelQueue([queue], ['self-hosted', 'linux'], ['ghr-ec2-instance-type:t3.large']), - ).toBeUndefined(); + expect(getViolations(queue)).toHaveLength(1); }); it('prefers a configured AWS dynamic labels policy over the legacy policy', () => { @@ -36,18 +54,22 @@ describe('selectEc2DynamicLabelQueue', () => { blocked_keys: [], }; - expect(selectEc2DynamicLabelQueue([queue], ['self-hosted', 'linux'], ['ghr-ec2-instance-type:t3.large'])).toEqual({ - queue, - labels: ['self-hosted', 'linux', 'ghr-ec2-instance-type:t3.large'], - }); + expect(getViolations(queue)).toEqual([]); }); }); +function getViolations(queue: RunnerMatcherConfig) { + return ec2DynamicLabelProvider.getViolations({ + queue, + labels: ['ghr-ec2-instance-type:t3.large'], + }); +} + function runnerQueue(id: string): RunnerMatcherConfig { return { id, arn: `arn:${id}`, - runnerProvider: 'ec2', + computeProvider: 'ec2', matcherConfig: { labelMatchers: [['self-hosted', 'linux']], exactMatch: true, diff --git a/lambdas/libs/compute-providers/aws/ec2/src/webhook/dynamic-labels.ts b/lambdas/libs/compute-providers/aws/ec2/src/webhook/dynamic-labels.ts new file mode 100644 index 0000000000..5e671da189 --- /dev/null +++ b/lambdas/libs/compute-providers/aws/ec2/src/webhook/dynamic-labels.ts @@ -0,0 +1,26 @@ +import { createChildLogger } from '@aws-github-runner/aws-powertools-util'; + +import type { DynamicLabelProvider, RunnerMatcherConfig } from '../../../../contracts'; +import { violationsAgainstPolicy } from './dynamic-labels-policy'; + +const logger = createChildLogger('handler'); + +function resolveEc2DynamicLabelsPolicy(queue: RunnerMatcherConfig) { + const hasLegacyEc2DynamicLabelsPolicy = Object.prototype.hasOwnProperty.call( + queue.matcherConfig, + 'ec2DynamicLabelsPolicy', + ); + + if (queue.matcherConfig.awsDynamicLabelsPolicy == null && hasLegacyEc2DynamicLabelsPolicy) { + logger.warn( + `Queue ${queue.id}: using deprecated matcherConfig.ec2DynamicLabelsPolicy; migrate to matcherConfig.awsDynamicLabelsPolicy`, + ); + return queue.matcherConfig.ec2DynamicLabelsPolicy; + } + + return queue.matcherConfig.awsDynamicLabelsPolicy; +} + +export const ec2DynamicLabelProvider: DynamicLabelProvider = { + getViolations: ({ queue, labels }) => violationsAgainstPolicy(labels, resolveEc2DynamicLabelsPolicy(queue)), +}; diff --git a/lambdas/libs/runner-providers/aws/ec2/webhook.ts b/lambdas/libs/compute-providers/aws/ec2/webhook.ts similarity index 70% rename from lambdas/libs/runner-providers/aws/ec2/webhook.ts rename to lambdas/libs/compute-providers/aws/ec2/webhook.ts index 1ad62e00b4..1357c93f06 100644 --- a/lambdas/libs/runner-providers/aws/ec2/webhook.ts +++ b/lambdas/libs/compute-providers/aws/ec2/webhook.ts @@ -1,9 +1,9 @@ -import type { RunnerProviderPlugin } from '../../core'; +import type { ComputeProviderPlugin } from '../../core'; import type { WebhookProviderCapabilities, WebhookProviderModule } from '../../contracts'; import { ec2DynamicLabelProvider } from './src/webhook/dynamic-labels'; -export function createEc2WebhookPlugin(): RunnerProviderPlugin { +export function createEc2WebhookPlugin(): ComputeProviderPlugin { return { type: 'ec2', capabilities: { dynamicLabels: ec2DynamicLabelProvider }, diff --git a/lambdas/libs/compute-providers/aws/microvm/README.md b/lambdas/libs/compute-providers/aws/microvm/README.md new file mode 100644 index 0000000000..e729bd4383 --- /dev/null +++ b/lambdas/libs/compute-providers/aws/microvm/README.md @@ -0,0 +1,70 @@ +# Lambda MicroVM compute provider + +This provider manages a compatible AWS Lambda MicroVM image through the control-plane Lambda. It currently supports ephemeral JIT runners only. + +The MicroVM image `/run` hook receives this `runHookPayload`: + +```json +{ + "version": 1, + "runnerConfigSsmPath": "/github-action-runners/example/token" +} +``` + +Lambda adds `microvmId` beside that payload. The image must poll the SecureString parameter at `/`, start the GitHub runner with its encoded JIT configuration, delete the parameter after reading it, and terminate the MicroVM after the job completes. + +The control-plane Lambda requires these provider environment variables: + +- `MICROVM_IMAGE_ARN` +- `MICROVM_EXECUTION_ROLE_ARN` +- `MICROVM_IMAGE_VERSION` (optional) +- `MICROVM_INGRESS_NETWORK_CONNECTORS` (optional JSON array or comma-separated list) +- `MICROVM_EGRESS_NETWORK_CONNECTORS` (optional JSON array or comma-separated list) +- `MICROVM_MAXIMUM_DURATION_IN_SECONDS` (optional, defaults to 3600) +- `MICROVM_LOG_GROUP` (optional) + +## Dynamic labels + +When a runner matcher enables dynamic labels, workflow jobs can override the +following `RunMicrovm` inputs: + +| Label | Override | +| --------------------------------------------------- | ---------------------------------------------- | +| `ghr-microvm-egress-network-connectors:` | One egress network connector ARN | +| `ghr-microvm-image-arn:` | MicroVM image ARN | +| `ghr-microvm-image-version:` | MicroVM image version | +| `ghr-microvm-maximum-duration-in-seconds:` | Maximum lifetime from 1 through 28,800 seconds | + +Repeat `ghr-microvm-egress-network-connectors:` to attach multiple +connectors. Specify one ARN per label; `RunMicrovm` accepts at most 10. These +labels replace the compute provider's configured +`MICROVM_EGRESS_NETWORK_CONNECTORS` value for that job. + +Lambda MicroVM does not expose CPU or memory as `RunMicrovm` inputs. Select an +image and version with the required resources instead. Labels such as +`ghr-microvm-memory` are rejected. + +Execution roles, ingress network connectors, logging, idle policy, run hook +payloads, and client tokens remain deployment-controlled. Egress connector +overrides change the runner's network boundary and should be restricted to +approved connector ARNs with `awsDynamicLabelsPolicy`. + +Use the matcher's `awsDynamicLabelsPolicy` to restrict values accepted from +workflow jobs. The MicroVM policy keys are `egress-network-connectors`, +`image-arn`, `image-version`, and `maximum-duration-in-seconds`. For example: + +```json +{ + "restricted_keys": { + "egress-network-connectors": { + "allowed": ["arn:aws:lambda:eu-west-1:123456789012:network-connector:github-runner-*"] + }, + "image-arn": { + "allowed": ["arn:aws:lambda:eu-west-1:123456789012:microvm-image:github-runner-*"] + }, + "maximum-duration-in-seconds": { + "max": 3600 + } + } +} +``` diff --git a/lambdas/libs/compute-providers/aws/microvm/control-plane.ts b/lambdas/libs/compute-providers/aws/microvm/control-plane.ts new file mode 100644 index 0000000000..d6287ca1e1 --- /dev/null +++ b/lambdas/libs/compute-providers/aws/microvm/control-plane.ts @@ -0,0 +1,25 @@ +import type { ComputeProviderPlugin, CreateStartRunnerConfig } from '../../core'; + +import type { ControlPlaneProviderCapabilities, ControlPlaneProviderModule } from '../../contracts'; +import type {} from './src/environment'; +import { createMicrovmPoolProvider } from './src/control-plane/pool'; +import { createMicrovmScaleDownProvider } from './src/control-plane/scale-down'; +import { createMicrovmScaleUpProvider } from './src/control-plane/scale-up'; + +export function createMicrovmControlPlanePlugin( + createStartRunnerConfig: CreateStartRunnerConfig, +): ComputeProviderPlugin { + return { + type: 'microvm', + capabilities: { + pool: () => createMicrovmPoolProvider(createStartRunnerConfig), + scaleUp: () => createMicrovmScaleUpProvider(createStartRunnerConfig), + scaleDown: createMicrovmScaleDownProvider, + }, + }; +} + +export const provider = { + type: 'microvm', + createPlugin: createMicrovmControlPlanePlugin, +} satisfies ControlPlaneProviderModule<'microvm'>; diff --git a/lambdas/libs/compute-providers/aws/microvm/provider.test.ts b/lambdas/libs/compute-providers/aws/microvm/provider.test.ts new file mode 100644 index 0000000000..44d7d55a76 --- /dev/null +++ b/lambdas/libs/compute-providers/aws/microvm/provider.test.ts @@ -0,0 +1,30 @@ +import { expect, it, vi } from 'vitest'; + +import { provider as controlPlaneProvider } from './control-plane'; +import { provider as webhookProvider } from './webhook'; + +it('exposes every MicroVM compute provider capability from its compute-provider entry point', () => { + const controlPlanePlugin = controlPlaneProvider.createPlugin(vi.fn(async () => [])); + const webhookPlugin = webhookProvider.createPlugin(); + + expect(controlPlanePlugin.type).toBe('microvm'); + expect(controlPlanePlugin.capabilities.pool()).toEqual({ + listRunners: expect.any(Function), + countAvailableRunners: expect.any(Function), + createRunners: expect.any(Function), + }); + expect(controlPlanePlugin.capabilities.scaleUp()).toEqual({ + resolveLabelsForRunners: expect.any(Function), + getCurrentRunners: expect.any(Function), + createRunners: expect.any(Function), + }); + expect(controlPlanePlugin.capabilities.scaleDown()).toEqual({ + list: expect.any(Function), + bootTimeExceeded: expect.any(Function), + markOrphan: expect.any(Function), + unmarkOrphan: expect.any(Function), + terminate: expect.any(Function), + }); + expect(webhookPlugin.type).toBe('microvm'); + expect(webhookPlugin.capabilities.dynamicLabels.getViolations).toEqual(expect.any(Function)); +}); diff --git a/lambdas/libs/compute-providers/aws/microvm/src/control-plane/config.test.ts b/lambdas/libs/compute-providers/aws/microvm/src/control-plane/config.test.ts new file mode 100644 index 0000000000..ce692a1ab4 --- /dev/null +++ b/lambdas/libs/compute-providers/aws/microvm/src/control-plane/config.test.ts @@ -0,0 +1,71 @@ +import { beforeEach, describe, expect, it } from 'vitest'; + +import { loadMicrovmProviderConfig } from './config'; + +const cleanEnv = process.env; + +beforeEach(() => { + process.env = { ...cleanEnv }; + process.env.MICROVM_IMAGE_ARN = 'arn:aws:lambda:eu-west-1:123456789012:microvm-image:runner'; + process.env.MICROVM_EXECUTION_ROLE_ARN = 'arn:aws:iam::123456789012:role/microvm-runner'; + delete process.env.MICROVM_IMAGE_VERSION; + delete process.env.MICROVM_INGRESS_NETWORK_CONNECTORS; + delete process.env.MICROVM_EGRESS_NETWORK_CONNECTORS; + delete process.env.MICROVM_MAXIMUM_DURATION_IN_SECONDS; + delete process.env.MICROVM_LOG_GROUP; +}); + +describe('loadMicrovmProviderConfig', () => { + it('loads required values and applies optional defaults', () => { + expect(loadMicrovmProviderConfig()).toEqual({ + imageIdentifier: process.env.MICROVM_IMAGE_ARN, + imageVersion: undefined, + executionRoleArn: process.env.MICROVM_EXECUTION_ROLE_ARN, + ingressNetworkConnectors: undefined, + egressNetworkConnectors: undefined, + maximumDurationInSeconds: 3600, + logging: undefined, + }); + }); + + it('loads versions, logging, duration, and either connector list format', () => { + process.env.MICROVM_IMAGE_VERSION = ' 3.0 '; + process.env.MICROVM_INGRESS_NETWORK_CONNECTORS = '["arn:ingress:one","arn:ingress:two"]'; + process.env.MICROVM_EGRESS_NETWORK_CONNECTORS = 'arn:egress:one, arn:egress:two'; + process.env.MICROVM_MAXIMUM_DURATION_IN_SECONDS = '1200'; + process.env.MICROVM_LOG_GROUP = ' /aws/lambda-microvms/runner '; + + expect(loadMicrovmProviderConfig()).toMatchObject({ + imageVersion: '3.0', + ingressNetworkConnectors: ['arn:ingress:one', 'arn:ingress:two'], + egressNetworkConnectors: ['arn:egress:one', 'arn:egress:two'], + maximumDurationInSeconds: 1200, + logging: { cloudWatch: { logGroup: '/aws/lambda-microvms/runner' } }, + }); + }); + + it.each([ + ['MICROVM_IMAGE_ARN', 'MICROVM_IMAGE_ARN'], + ['MICROVM_EXECUTION_ROLE_ARN', 'MICROVM_EXECUTION_ROLE_ARN'], + ])('requires %s', (environmentVariable, expectedName) => { + delete process.env[environmentVariable]; + + expect(() => loadMicrovmProviderConfig()).toThrow( + `${expectedName} must be configured for the MicroVM compute provider`, + ); + }); + + it.each(['0', '28801', '1.5', 'invalid'])('rejects invalid maximum duration %s', (duration) => { + process.env.MICROVM_MAXIMUM_DURATION_IN_SECONDS = duration; + + expect(() => loadMicrovmProviderConfig()).toThrow( + 'MICROVM_MAXIMUM_DURATION_IN_SECONDS must be an integer between 1 and 28800', + ); + }); + + it.each(['[not-json', '[]', '["valid", 2]', 'first,'])('rejects malformed connector lists %s', (connectors) => { + process.env.MICROVM_EGRESS_NETWORK_CONNECTORS = connectors; + + expect(() => loadMicrovmProviderConfig()).toThrow(/MICROVM_EGRESS_NETWORK_CONNECTORS must/); + }); +}); diff --git a/lambdas/libs/compute-providers/aws/microvm/src/control-plane/config.ts b/lambdas/libs/compute-providers/aws/microvm/src/control-plane/config.ts new file mode 100644 index 0000000000..7c0a7662b9 --- /dev/null +++ b/lambdas/libs/compute-providers/aws/microvm/src/control-plane/config.ts @@ -0,0 +1,88 @@ +import type { Logging, RunMicrovmCommandInput } from '@aws-sdk/client-lambda-microvms'; + +const DEFAULT_MAXIMUM_DURATION_IN_SECONDS = 3600; +const MAXIMUM_DURATION_IN_SECONDS = 28800; + +export interface MicrovmProviderConfig { + egressNetworkConnectors?: string[]; + executionRoleArn: string; + imageIdentifier: string; + imageVersion?: string; + ingressNetworkConnectors?: string[]; + logging?: Logging; + maximumDurationInSeconds: number; +} + +function requiredEnvironmentValue(name: string, value: string | undefined): string { + const trimmed = value?.trim(); + if (!trimmed) { + throw new Error(`${name} must be configured for the MicroVM compute provider`); + } + return trimmed; +} + +function optionalEnvironmentValue(value: string | undefined): string | undefined { + const trimmed = value?.trim(); + return trimmed ? trimmed : undefined; +} + +function parseNetworkConnectors(name: string, value: string | undefined): string[] | undefined { + const configuredValue = optionalEnvironmentValue(value); + if (!configuredValue) return undefined; + + let connectors: unknown; + try { + connectors = configuredValue.startsWith('[') + ? JSON.parse(configuredValue) + : configuredValue.split(',').map((connector) => connector.trim()); + } catch (error) { + throw new Error(`${name} must be a JSON array or comma-separated list`, { cause: error }); + } + + if ( + !Array.isArray(connectors) || + connectors.length === 0 || + connectors.some((connector) => typeof connector !== 'string' || connector.trim().length === 0) + ) { + throw new Error(`${name} must contain one or more non-empty connector ARNs`); + } + + return connectors.map((connector) => connector.trim()); +} + +function parseMaximumDuration(value: string | undefined): number { + if (!optionalEnvironmentValue(value)) return DEFAULT_MAXIMUM_DURATION_IN_SECONDS; + + const maximumDurationInSeconds = Number(value); + if ( + !Number.isInteger(maximumDurationInSeconds) || + maximumDurationInSeconds < 1 || + maximumDurationInSeconds > MAXIMUM_DURATION_IN_SECONDS + ) { + throw new Error( + `MICROVM_MAXIMUM_DURATION_IN_SECONDS must be an integer between 1 and ${MAXIMUM_DURATION_IN_SECONDS}`, + ); + } + + return maximumDurationInSeconds; +} + +export function loadMicrovmProviderConfig(): MicrovmProviderConfig { + const logGroup = optionalEnvironmentValue(process.env.MICROVM_LOG_GROUP); + + return { + imageIdentifier: requiredEnvironmentValue('MICROVM_IMAGE_ARN', process.env.MICROVM_IMAGE_ARN), + imageVersion: optionalEnvironmentValue(process.env.MICROVM_IMAGE_VERSION), + executionRoleArn: requiredEnvironmentValue('MICROVM_EXECUTION_ROLE_ARN', process.env.MICROVM_EXECUTION_ROLE_ARN), + ingressNetworkConnectors: parseNetworkConnectors( + 'MICROVM_INGRESS_NETWORK_CONNECTORS', + process.env.MICROVM_INGRESS_NETWORK_CONNECTORS, + ), + egressNetworkConnectors: parseNetworkConnectors( + 'MICROVM_EGRESS_NETWORK_CONNECTORS', + process.env.MICROVM_EGRESS_NETWORK_CONNECTORS, + ), + maximumDurationInSeconds: parseMaximumDuration(process.env.MICROVM_MAXIMUM_DURATION_IN_SECONDS), + logging: logGroup ? ({ cloudWatch: { logGroup } } satisfies RunMicrovmCommandInput['logging']) : undefined, + }; +} diff --git a/lambdas/libs/compute-providers/aws/microvm/src/control-plane/microvms.test.ts b/lambdas/libs/compute-providers/aws/microvm/src/control-plane/microvms.test.ts new file mode 100644 index 0000000000..7d7199a3cc --- /dev/null +++ b/lambdas/libs/compute-providers/aws/microvm/src/control-plane/microvms.test.ts @@ -0,0 +1,311 @@ +import { + LambdaMicrovmsClient, + ListMicrovmsCommand, + ListTagsCommand, + RunMicrovmCommand, + TagResourceCommand, + TerminateMicrovmCommand, + UntagResourceCommand, +} from '@aws-sdk/client-lambda-microvms'; +import { mockClient } from 'aws-sdk-client-mock'; +import 'aws-sdk-client-mock-jest/vitest'; +import { beforeEach, describe, expect, it, vi } from 'vitest'; + +import type { MicrovmProviderConfig } from './config'; +import { + isRetryableMicrovmError, + listMicrovmRunners, + microvmArn, + microvmBootTimeExceeded, + runMicrovmRunner, + tagMicrovm, + terminateMicrovm, + untagMicrovm, +} from './microvms'; + +const mockMicrovmClient = mockClient(LambdaMicrovmsClient); +const imageArn = 'arn:aws:lambda:eu-west-1:123456789012:microvm-image:runner'; +const config: MicrovmProviderConfig = { + imageIdentifier: imageArn, + imageVersion: '3.0', + executionRoleArn: 'arn:aws:iam::123456789012:role/microvm-runner', + egressNetworkConnectors: ['arn:egress'], + maximumDurationInSeconds: 1200, + logging: { cloudWatch: { logGroup: '/aws/lambda-microvms/runner' } }, +}; + +beforeEach(() => { + mockMicrovmClient.reset(); + vi.useRealTimers(); + process.env.AWS_REGION = 'eu-west-1'; + process.env.RUNNER_BOOT_TIME_IN_MINUTES = '5'; +}); + +describe('microvmArn', () => { + it('derives the MicroVM resource ARN from its image ARN', () => { + expect(microvmArn(imageArn, 'mvm-123')).toBe('arn:aws:lambda:eu-west-1:123456789012:microvm:mvm-123'); + expect(microvmArn(imageArn.replace('arn:aws:', 'arn:aws-us-gov:'), 'mvm-456')).toContain('arn:aws-us-gov:lambda:'); + }); + + it('rejects image names that cannot identify a customer MicroVM resource', () => { + expect(() => microvmArn('runner', 'mvm-123')).toThrow( + 'MICROVM_IMAGE_ARN is not a valid customer MicroVM image ARN', + ); + }); +}); + +describe('runMicrovmRunner', () => { + it('launches and tags a managed runner', async () => { + mockMicrovmClient.on(RunMicrovmCommand).resolves({ microvmId: 'mvm-123' }); + mockMicrovmClient.on(TagResourceCommand).resolves({}); + + await expect( + runMicrovmRunner({ + config, + environment: 'unit-test', + runHookPayload: '{"version":1}', + runnerOwner: 'Codertocat', + runnerType: 'Org', + source: 'scale-up-lambda', + }), + ).resolves.toBe('mvm-123'); + + expect(mockMicrovmClient).toHaveReceivedCommandWith(RunMicrovmCommand, { + imageIdentifier: imageArn, + imageVersion: '3.0', + executionRoleArn: config.executionRoleArn, + egressNetworkConnectors: ['arn:egress'], + maximumDurationInSeconds: 1200, + logging: config.logging, + runHookPayload: '{"version":1}', + clientToken: expect.any(String), + }); + expect(mockMicrovmClient).toHaveReceivedCommandWith(TagResourceCommand, { + Resource: microvmArn(imageArn, 'mvm-123'), + Tags: { + 'ghr:Application': 'github-action-runner', + 'ghr:created_by': 'scale-up-lambda', + 'ghr:environment': 'unit-test', + 'ghr:Owner': 'Codertocat', + 'ghr:Type': 'Org', + }, + }); + }); + + it('rejects a launch response without an ID', async () => { + mockMicrovmClient.on(RunMicrovmCommand).resolves({}); + + await expect( + runMicrovmRunner({ + config, + environment: 'unit-test', + runHookPayload: '{}', + runnerOwner: 'Codertocat', + runnerType: 'Org', + source: 'pool-lambda', + }), + ).rejects.toThrow('RunMicrovm returned no microvmId'); + }); + + it('terminates a new runner when required tags cannot be applied', async () => { + const tagError = new Error('tag failed'); + mockMicrovmClient.on(RunMicrovmCommand).resolves({ microvmId: 'mvm-untagged' }); + mockMicrovmClient.on(TagResourceCommand).rejects(tagError); + mockMicrovmClient.on(TerminateMicrovmCommand).resolves({}); + + await expect( + runMicrovmRunner({ + config, + environment: 'unit-test', + runHookPayload: '{}', + runnerOwner: 'Codertocat', + runnerType: 'Org', + source: 'scale-up-lambda', + }), + ).rejects.toThrow('tag failed'); + + expect(mockMicrovmClient).toHaveReceivedCommandWith(TerminateMicrovmCommand, { + microvmIdentifier: 'mvm-untagged', + }); + }); + + it('preserves the tag error when cleanup also fails', async () => { + mockMicrovmClient.on(RunMicrovmCommand).resolves({ microvmId: 'mvm-untagged' }); + mockMicrovmClient.on(TagResourceCommand).rejects(new Error('tag failed')); + mockMicrovmClient.on(TerminateMicrovmCommand).rejects(new Error('terminate failed')); + + await expect( + runMicrovmRunner({ + config, + environment: 'unit-test', + runHookPayload: '{}', + runnerOwner: 'Codertocat', + runnerType: 'Org', + source: 'scale-up-lambda', + }), + ).rejects.toThrow('tag failed'); + }); +}); + +describe('listMicrovmRunners', () => { + it('paginates active MicroVMs and filters them by management tags', async () => { + const startedAt = new Date('2026-08-06T10:00:00.000Z'); + mockMicrovmClient + .on(ListMicrovmsCommand) + .resolvesOnce({ + nextToken: 'page-2', + items: [ + { microvmId: 'mvm-managed', imageArn, imageVersion: '3.0', startedAt, state: 'RUNNING' }, + { microvmId: 'mvm-terminated', imageArn, imageVersion: '3.0', startedAt, state: 'TERMINATED' }, + ], + }) + .resolvesOnce({ + items: [{ microvmId: 'mvm-other', imageArn, imageVersion: '3.0', startedAt, state: 'PENDING' }], + }); + mockMicrovmClient + .on(ListTagsCommand) + .resolvesOnce({ + Tags: { + 'ghr:Application': 'github-action-runner', + 'ghr:environment': 'unit-test', + 'ghr:Owner': 'Codertocat', + 'ghr:Type': 'Org', + 'ghr:github_runner_id': '42', + 'ghr:bypass-removal': 'true', + }, + }) + .resolvesOnce({ Tags: { 'ghr:Application': 'another-application' } }); + + await expect( + listMicrovmRunners({ + environment: 'unit-test', + runnerOwner: 'Codertocat', + runnerType: 'Org', + }), + ).resolves.toEqual([ + { + id: 'mvm-managed', + imageArn, + launchTime: startedAt, + owner: 'Codertocat', + type: 'Org', + orphan: false, + githubRunnerId: '42', + bypassRemoval: true, + state: 'RUNNING', + }, + ]); + + expect(mockMicrovmClient).toHaveReceivedNthCommandWith(2, ListMicrovmsCommand, { + maxResults: 50, + nextToken: 'page-2', + }); + }); + + it('applies environment, owner, type, and orphan filters after loading tags', async () => { + mockMicrovmClient.on(ListMicrovmsCommand).resolves({ + items: [ + { + microvmId: 'mvm-filtered', + imageArn, + imageVersion: '3.0', + startedAt: new Date(), + state: 'SUSPENDED', + }, + ], + }); + mockMicrovmClient.on(ListTagsCommand).resolves({ + Tags: { + 'ghr:Application': 'github-action-runner', + 'ghr:environment': 'other', + 'ghr:Owner': 'Other', + 'ghr:Type': 'Repo', + }, + }); + + await expect(listMicrovmRunners({ environment: 'unit-test' })).resolves.toEqual([]); + await expect(listMicrovmRunners({ runnerOwner: 'Codertocat' })).resolves.toEqual([]); + await expect(listMicrovmRunners({ runnerType: 'Org' })).resolves.toEqual([]); + await expect(listMicrovmRunners({ orphan: true })).resolves.toEqual([]); + }); + + it('skips a MicroVM that terminates before its tags can be read', async () => { + const resourceNotFound = Object.assign(new Error('gone'), { name: 'ResourceNotFoundException' }); + mockMicrovmClient.on(ListMicrovmsCommand).resolves({ + items: [{ microvmId: 'mvm-gone', imageArn, imageVersion: '3.0', startedAt: new Date(), state: 'RUNNING' }], + }); + mockMicrovmClient.on(ListTagsCommand).rejects(resourceNotFound); + + await expect(listMicrovmRunners()).resolves.toEqual([]); + }); + + it('surfaces unexpected tag lookup failures', async () => { + mockMicrovmClient.on(ListMicrovmsCommand).resolves({ + items: [{ microvmId: 'mvm-error', imageArn, imageVersion: '3.0', startedAt: new Date(), state: 'RUNNING' }], + }); + mockMicrovmClient.on(ListTagsCommand).rejects(new Error('list tags failed')); + + await expect(listMicrovmRunners()).rejects.toThrow('list tags failed'); + }); +}); + +describe('MicroVM lifecycle helpers', () => { + it('tags, untags, and terminates a MicroVM', async () => { + mockMicrovmClient.on(TagResourceCommand).resolves({}); + mockMicrovmClient.on(UntagResourceCommand).resolves({}); + mockMicrovmClient.on(TerminateMicrovmCommand).resolves({}); + + await tagMicrovm(imageArn, 'mvm-123', { key: 'value' }); + await untagMicrovm(imageArn, 'mvm-123', ['key']); + await terminateMicrovm('mvm-123'); + + expect(mockMicrovmClient).toHaveReceivedCommandWith(TagResourceCommand, { + Resource: microvmArn(imageArn, 'mvm-123'), + Tags: { key: 'value' }, + }); + expect(mockMicrovmClient).toHaveReceivedCommandWith(UntagResourceCommand, { + Resource: microvmArn(imageArn, 'mvm-123'), + TagKeys: ['key'], + }); + expect(mockMicrovmClient).toHaveReceivedCommandWith(TerminateMicrovmCommand, { + microvmIdentifier: 'mvm-123', + }); + }); + + it('evaluates the configured boot window', () => { + vi.useFakeTimers(); + vi.setSystemTime(new Date('2026-08-06T10:10:00.000Z')); + + expect(microvmBootTimeExceeded({})).toBe(false); + expect(microvmBootTimeExceeded({ launchTime: new Date('2026-08-06T10:06:00.000Z') })).toBe(false); + expect(microvmBootTimeExceeded({ launchTime: new Date('2026-08-06T10:04:00.000Z') })).toBe(true); + }); +}); + +describe('isRetryableMicrovmError', () => { + it.each(['ConflictException', 'InternalServerException', 'ServiceQuotaExceededException', 'ThrottlingException'])( + 'classifies %s as retryable', + (name) => { + expect(isRetryableMicrovmError(Object.assign(new Error(name), { name }))).toBe(true); + }, + ); + + it('classifies server, throttling, network, and nested failures as retryable', () => { + expect(isRetryableMicrovmError(Object.assign(new Error('server'), { $fault: 'server' }))).toBe(true); + expect(isRetryableMicrovmError(Object.assign(new Error('throttle'), { $metadata: { httpStatusCode: 429 } }))).toBe( + true, + ); + expect(isRetryableMicrovmError(Object.assign(new Error('network'), { code: 'ECONNRESET' }))).toBe(true); + expect( + isRetryableMicrovmError( + Object.assign(new Error('outer'), { cause: Object.assign(new Error(), { code: 'ETIMEDOUT' }) }), + ), + ).toBe(true); + }); + + it('does not retry configuration, unknown, or non-error failures', () => { + expect(isRetryableMicrovmError(Object.assign(new Error('invalid'), { name: 'ValidationException' }))).toBe(false); + expect(isRetryableMicrovmError(new Error('unknown'))).toBe(false); + expect(isRetryableMicrovmError('failure')).toBe(false); + }); +}); diff --git a/lambdas/libs/compute-providers/aws/microvm/src/control-plane/microvms.ts b/lambdas/libs/compute-providers/aws/microvm/src/control-plane/microvms.ts new file mode 100644 index 0000000000..edc4a3775b --- /dev/null +++ b/lambdas/libs/compute-providers/aws/microvm/src/control-plane/microvms.ts @@ -0,0 +1,223 @@ +import { randomUUID } from 'node:crypto'; + +import { createChildLogger, getTracedAWSV3Client } from '@aws-github-runner/aws-powertools-util'; +import { + LambdaMicrovmsClient, + ListMicrovmsCommand, + ListTagsCommand, + RunMicrovmCommand, + TagResourceCommand, + TerminateMicrovmCommand, + UntagResourceCommand, +} from '@aws-sdk/client-lambda-microvms'; +import type { MicrovmItem, MicrovmState, RunMicrovmCommandInput } from '@aws-sdk/client-lambda-microvms'; + +import type { LambdaRunnerSource, ListRunnerFilters, RunnerInfo, RunnerType } from '../../../../core'; +import type { MicrovmProviderConfig } from './config'; + +const logger = createChildLogger('microvm-runners'); + +const APPLICATION_TAG = 'ghr:Application'; +const APPLICATION_TAG_VALUE = 'github-action-runner'; +const ACTIVE_STATES = new Set(['PENDING', 'RUNNING', 'SUSPENDING', 'SUSPENDED']); + +export interface MicrovmRunnerInfo extends RunnerInfo { + imageArn?: string; + state?: MicrovmState; +} + +export interface RunMicrovmRunnerInput { + config: MicrovmProviderConfig; + environment: string; + runHookPayload: string; + runnerOwner: string; + runnerType: RunnerType; + source: LambdaRunnerSource; +} + +interface AwsErrorLike extends Error { + cause?: unknown; + code?: string; + $fault?: 'client' | 'server'; + $metadata?: { httpStatusCode?: number }; +} + +const RETRYABLE_ERROR_NAMES = new Set([ + 'ConflictException', + 'InternalServerException', + 'RequestTimeout', + 'RequestTimeoutException', + 'ResourceConflictException', + 'ServiceException', + 'ServiceQuotaExceededException', + 'Throttling', + 'ThrottlingException', + 'TooManyRequestsException', +]); + +const RETRYABLE_NETWORK_ERROR_CODES = new Set([ + 'EAI_AGAIN', + 'ECONNREFUSED', + 'ECONNRESET', + 'ENETUNREACH', + 'ENOTFOUND', + 'ETIMEDOUT', +]); + +function microvmClient(): LambdaMicrovmsClient { + return getTracedAWSV3Client(new LambdaMicrovmsClient({ region: process.env.AWS_REGION })); +} + +export function microvmArn(imageArn: string, microvmId: string): string { + const match = /^arn:([^:]+):lambda:([^:]+):([0-9]{12}):microvm-image:.+$/.exec(imageArn); + if (!match) { + throw new Error(`MICROVM_IMAGE_ARN is not a valid customer MicroVM image ARN: ${imageArn}`); + } + + const [, partition, region, accountId] = match; + return `arn:${partition}:lambda:${region}:${accountId}:microvm:${microvmId}`; +} + +export async function runMicrovmRunner(input: RunMicrovmRunnerInput): Promise { + const commandInput: RunMicrovmCommandInput = { + imageIdentifier: input.config.imageIdentifier, + imageVersion: input.config.imageVersion, + executionRoleArn: input.config.executionRoleArn, + ingressNetworkConnectors: input.config.ingressNetworkConnectors, + egressNetworkConnectors: input.config.egressNetworkConnectors, + maximumDurationInSeconds: input.config.maximumDurationInSeconds, + logging: input.config.logging, + runHookPayload: input.runHookPayload, + clientToken: randomUUID(), + }; + + logger.debug('Launching Lambda MicroVM runner', { + imageIdentifier: commandInput.imageIdentifier, + imageVersion: commandInput.imageVersion, + maximumDurationInSeconds: commandInput.maximumDurationInSeconds, + }); + + const response = await microvmClient().send(new RunMicrovmCommand(commandInput)); + if (!response.microvmId) { + throw new Error('RunMicrovm returned no microvmId'); + } + + try { + await tagMicrovm(input.config.imageIdentifier, response.microvmId, { + [APPLICATION_TAG]: APPLICATION_TAG_VALUE, + 'ghr:created_by': input.source, + 'ghr:environment': input.environment, + 'ghr:Owner': input.runnerOwner, + 'ghr:Type': input.runnerType, + }); + } catch (error) { + logger.error(`Failed to tag new MicroVM runner '${response.microvmId}', terminating it`, { error }); + await terminateMicrovm(response.microvmId).catch((terminationError) => { + logger.error(`Failed to terminate untagged MicroVM runner '${response.microvmId}'`, { + error: terminationError, + }); + }); + throw error; + } + + return response.microvmId; +} + +export async function listMicrovmRunners(filters: ListRunnerFilters = {}): Promise { + const client = microvmClient(); + const items: MicrovmItem[] = []; + let nextToken: string | undefined; + + do { + const response = await client.send( + new ListMicrovmsCommand({ + maxResults: 50, + nextToken, + }), + ); + items.push(...(response.items ?? [])); + nextToken = response.nextToken; + } while (nextToken); + + const runners: MicrovmRunnerInfo[] = []; + for (const item of items) { + if (!item.microvmId || !item.imageArn || !item.state || !ACTIVE_STATES.has(item.state)) continue; + + let tags: Record; + try { + tags = + (await client.send(new ListTagsCommand({ Resource: microvmArn(item.imageArn, item.microvmId) }))).Tags ?? {}; + } catch (error) { + if (error instanceof Error && error.name === 'ResourceNotFoundException') continue; + throw error; + } + + if (tags[APPLICATION_TAG] !== APPLICATION_TAG_VALUE) continue; + if (filters.environment !== undefined && tags['ghr:environment'] !== filters.environment) continue; + if (filters.runnerType !== undefined && tags['ghr:Type'] !== filters.runnerType) continue; + if (filters.runnerOwner !== undefined && tags['ghr:Owner'] !== filters.runnerOwner) continue; + if (filters.orphan && tags['ghr:orphan'] !== 'true') continue; + + runners.push({ + id: item.microvmId, + imageArn: item.imageArn, + launchTime: item.startedAt, + owner: tags['ghr:Owner'], + type: tags['ghr:Type'] as RunnerInfo['type'], + orphan: tags['ghr:orphan'] === 'true', + githubRunnerId: tags['ghr:github_runner_id'], + bypassRemoval: tags['ghr:bypass-removal'] === 'true', + state: item.state, + }); + } + + return runners; +} + +export async function tagMicrovm(imageArn: string, microvmId: string, tags: Record): Promise { + await microvmClient().send( + new TagResourceCommand({ + Resource: microvmArn(imageArn, microvmId), + Tags: tags, + }), + ); +} + +export async function untagMicrovm(imageArn: string, microvmId: string, tagKeys: string[]): Promise { + await microvmClient().send( + new UntagResourceCommand({ + Resource: microvmArn(imageArn, microvmId), + TagKeys: tagKeys, + }), + ); +} + +export async function terminateMicrovm(microvmId: string): Promise { + await microvmClient().send(new TerminateMicrovmCommand({ microvmIdentifier: microvmId })); +} + +export function microvmBootTimeExceeded(runner: { launchTime?: Date }): boolean { + if (!runner.launchTime) return false; + + const bootTimeInMinutes = Number(process.env.RUNNER_BOOT_TIME_IN_MINUTES || '5'); + return runner.launchTime.getTime() + bootTimeInMinutes * 60_000 < Date.now(); +} + +export function isRetryableMicrovmError(error: unknown): boolean { + if (!(error instanceof Error)) return false; + + const awsError = error as AwsErrorLike; + if (RETRYABLE_ERROR_NAMES.has(awsError.name)) return true; + + const statusCode = awsError.$metadata?.httpStatusCode; + if ( + awsError.$fault === 'server' || + statusCode === 429 || + (statusCode !== undefined && statusCode >= 500) || + (awsError.code !== undefined && RETRYABLE_NETWORK_ERROR_CODES.has(awsError.code)) + ) { + return true; + } + + return awsError.cause !== undefined && awsError.cause !== error ? isRetryableMicrovmError(awsError.cause) : false; +} diff --git a/lambdas/libs/compute-providers/aws/microvm/src/control-plane/pool.test.ts b/lambdas/libs/compute-providers/aws/microvm/src/control-plane/pool.test.ts new file mode 100644 index 0000000000..8f46818b50 --- /dev/null +++ b/lambdas/libs/compute-providers/aws/microvm/src/control-plane/pool.test.ts @@ -0,0 +1,112 @@ +import type { Octokit } from '@octokit/rest'; +import { beforeEach, describe, expect, it, vi } from 'vitest'; + +import type { CreateGitHubRunnerConfig, CreateStartRunnerConfig } from '../../../../core'; +import { listMicrovmRunners, microvmBootTimeExceeded } from './microvms'; +import type { MicrovmRunnerInfo } from './microvms'; +import { calculateMicrovmPoolSize, createMicrovmPoolProvider } from './pool'; +import { createMicrovmRunners } from './runner-config'; + +vi.mock('./microvms', () => ({ + listMicrovmRunners: vi.fn(), + microvmBootTimeExceeded: vi.fn(), +})); +vi.mock('./runner-config', () => ({ createMicrovmRunners: vi.fn() })); + +const createStartRunnerConfig = vi.fn(); +const githubClient = {} as Octokit; +function runner(id: string, state: MicrovmRunnerInfo['state']): MicrovmRunnerInfo { + return { id, state, owner: 'Codertocat', type: 'Org' }; +} + +function githubRunnerConfig(): CreateGitHubRunnerConfig { + return { + ephemeral: true, + enableJitConfig: true, + runnerLabels: 'self-hosted,microvm', + runnerGroup: 'Default', + runnerNamePrefix: '', + runnerOwner: 'Codertocat', + runnerType: 'Org', + disableAutoUpdate: true, + ssmTokenPath: '/runner/token', + ssmConfigPath: '/runner/config', + ssmParameterStoreTags: [], + }; +} + +beforeEach(() => { + vi.clearAllMocks(); + vi.mocked(listMicrovmRunners).mockResolvedValue([]); + vi.mocked(microvmBootTimeExceeded).mockReturnValue(false); + vi.mocked(createMicrovmRunners).mockResolvedValue({ + instances: ['mvm-1'], + retryableErrorCount: 0, + nonRetryableErrorCount: 0, + }); +}); + +describe('calculateMicrovmPoolSize', () => { + it('counts online idle running runners', () => { + expect( + calculateMicrovmPoolSize( + [runner('mvm-idle', 'RUNNING')], + new Map([['mvm-idle', { busy: false, status: 'online' }]]), + ), + ).toBe(1); + }); + + it('optionally counts online busy runners', () => { + const runners = [runner('mvm-busy', 'RUNNING')]; + const statuses = new Map([['mvm-busy', { busy: true, status: 'online' }]]); + + expect(calculateMicrovmPoolSize(runners, statuses)).toBe(0); + expect(calculateMicrovmPoolSize(runners, statuses, true)).toBe(1); + }); + + it('counts pending runners only during their boot window', () => { + const runners = [runner('mvm-pending', 'PENDING')]; + vi.mocked(microvmBootTimeExceeded).mockReturnValueOnce(false).mockReturnValueOnce(true); + + expect(calculateMicrovmPoolSize(runners, new Map())).toBe(1); + expect(calculateMicrovmPoolSize(runners, new Map())).toBe(0); + }); + + it('does not count suspended or offline runners', () => { + expect( + calculateMicrovmPoolSize( + [runner('mvm-suspended', 'SUSPENDED'), runner('mvm-offline', 'RUNNING')], + new Map([['mvm-offline', { busy: false, status: 'offline' }]]), + ), + ).toBe(0); + }); +}); + +describe('createMicrovmPoolProvider', () => { + it('lists managed MicroVMs and returns successfully created IDs', async () => { + const provider = createMicrovmPoolProvider(createStartRunnerConfig); + const input = { + environment: 'unit-test', + runnerOwner: 'Codertocat', + runnerType: 'Org' as const, + }; + + await expect(provider.listRunners(input)).resolves.toEqual([]); + expect(listMicrovmRunners).toHaveBeenCalledWith(input); + + await expect( + provider.createRunners({ + githubRunnerConfig: githubRunnerConfig(), + numberOfRunners: 1, + githubInstallationClient: githubClient, + }), + ).resolves.toEqual(['mvm-1']); + expect(createMicrovmRunners).toHaveBeenCalledWith( + expect.any(Object), + 1, + githubClient, + createStartRunnerConfig, + 'pool-lambda', + ); + }); +}); diff --git a/lambdas/libs/compute-providers/aws/microvm/src/control-plane/pool.ts b/lambdas/libs/compute-providers/aws/microvm/src/control-plane/pool.ts new file mode 100644 index 0000000000..8deed5562d --- /dev/null +++ b/lambdas/libs/compute-providers/aws/microvm/src/control-plane/pool.ts @@ -0,0 +1,65 @@ +import { createChildLogger } from '@aws-github-runner/aws-powertools-util'; + +import type { + CreatePoolRunnersInput, + CreateStartRunnerConfig, + ListPoolRunnersInput, + PoolComputeProvider, + RunnerStatus, +} from '../../../../core'; +import type { MicrovmRunnerInfo } from './microvms'; +import { listMicrovmRunners, microvmBootTimeExceeded } from './microvms'; +import { createMicrovmRunners } from './runner-config'; + +const logger = createChildLogger('microvm-pool'); + +async function listMicrovmPoolRunners(input: ListPoolRunnersInput): Promise { + return await listMicrovmRunners(input); +} + +async function createMicrovmPoolRunners( + { githubRunnerConfig, numberOfRunners, githubInstallationClient }: CreatePoolRunnersInput, + createStartRunnerConfig: CreateStartRunnerConfig, +): Promise { + const result = await createMicrovmRunners( + githubRunnerConfig, + numberOfRunners, + githubInstallationClient, + createStartRunnerConfig, + 'pool-lambda', + ); + return result.instances; +} + +export function calculateMicrovmPoolSize( + runners: MicrovmRunnerInfo[], + runnerStatus: Map, + includeBusyRunners = false, +): number { + let availableRunners = 0; + + for (const runner of runners) { + const status = runnerStatus.get(runner.id); + if (runner.state === 'RUNNING' && status?.status === 'online' && (!status.busy || includeBusyRunners)) { + availableRunners++; + logger.debug(`MicroVM runner ${runner.id} is online and counted as part of the pool`); + } else if (runner.state === 'PENDING' && !microvmBootTimeExceeded(runner)) { + availableRunners++; + logger.info(`MicroVM runner ${runner.id} is still booting and counted as part of the pool`); + } else { + logger.debug(`MicroVM runner ${runner.id} is not available and is not counted as part of the pool`); + } + } + + return availableRunners; +} + +export function createMicrovmPoolProvider( + createStartRunnerConfig: CreateStartRunnerConfig, +): Omit, 'type'> { + return { + listRunners: listMicrovmPoolRunners, + countAvailableRunners: calculateMicrovmPoolSize, + createRunners: (input) => createMicrovmPoolRunners(input, createStartRunnerConfig), + }; +} diff --git a/lambdas/libs/compute-providers/aws/microvm/src/control-plane/runner-config.test.ts b/lambdas/libs/compute-providers/aws/microvm/src/control-plane/runner-config.test.ts new file mode 100644 index 0000000000..84afb5be3b --- /dev/null +++ b/lambdas/libs/compute-providers/aws/microvm/src/control-plane/runner-config.test.ts @@ -0,0 +1,191 @@ +import type { Octokit } from '@octokit/rest'; +import { beforeEach, describe, expect, it, vi } from 'vitest'; + +import type { CreateGitHubRunnerConfig, CreateStartRunnerConfig } from '../../../../core'; +import { loadMicrovmProviderConfig } from './config'; +import { isRetryableMicrovmError, runMicrovmRunner, tagMicrovm, terminateMicrovm } from './microvms'; +import { createMicrovmRunHookPayload, createMicrovmRunners } from './runner-config'; + +vi.mock('./config', () => ({ loadMicrovmProviderConfig: vi.fn() })); +vi.mock('./microvms', () => ({ + isRetryableMicrovmError: vi.fn(), + runMicrovmRunner: vi.fn(), + tagMicrovm: vi.fn(), + terminateMicrovm: vi.fn(), +})); + +const imageArn = 'arn:aws:lambda:eu-west-1:123456789012:microvm-image:runner'; +const githubClient = {} as Octokit; +const createStartRunnerConfig = vi.fn(); + +function runnerConfig(overrides: Partial = {}): CreateGitHubRunnerConfig { + return { + ephemeral: true, + enableJitConfig: true, + runnerLabels: 'self-hosted,linux,arm64,microvm', + runnerGroup: 'Default', + runnerNamePrefix: 'unit-test-', + runnerOwner: 'Codertocat', + runnerType: 'Org', + disableAutoUpdate: true, + ssmTokenPath: '/github-action-runners/unit-test/token', + ssmConfigPath: '/github-action-runners/unit-test/config', + ssmParameterStoreTags: [], + ...overrides, + }; +} + +beforeEach(() => { + vi.clearAllMocks(); + process.env.ENVIRONMENT = 'unit-test'; + vi.mocked(loadMicrovmProviderConfig).mockReturnValue({ + imageIdentifier: imageArn, + executionRoleArn: 'arn:aws:iam::123456789012:role/microvm-runner', + maximumDurationInSeconds: 1200, + }); + vi.mocked(runMicrovmRunner).mockResolvedValue('mvm-1'); + vi.mocked(tagMicrovm).mockResolvedValue(); + vi.mocked(terminateMicrovm).mockResolvedValue(); + vi.mocked(isRetryableMicrovmError).mockReturnValue(false); + createStartRunnerConfig.mockResolvedValue([]); +}); + +describe('createMicrovmRunHookPayload', () => { + it('contains only the versioned SSM prefix contract', () => { + expect(JSON.parse(createMicrovmRunHookPayload('/runner/token'))).toEqual({ + version: 1, + runnerConfigSsmPath: '/runner/token', + }); + }); +}); + +describe('createMicrovmRunners', () => { + it.each([{ ephemeral: false }, { enableJitConfig: false }])( + 'rejects unsupported runner configuration %j', + async (overrides) => { + await expect( + createMicrovmRunners(runnerConfig(overrides), 2, githubClient, createStartRunnerConfig, 'scale-up-lambda'), + ).resolves.toEqual({ instances: [], retryableErrorCount: 0, nonRetryableErrorCount: 2 }); + + expect(runMicrovmRunner).not.toHaveBeenCalled(); + }, + ); + + it('requires an SSM token path', async () => { + await expect( + createMicrovmRunners( + runnerConfig({ ssmTokenPath: '' }), + 1, + githubClient, + createStartRunnerConfig, + 'scale-up-lambda', + ), + ).resolves.toEqual({ instances: [], retryableErrorCount: 0, nonRetryableErrorCount: 1 }); + }); + + it('classifies invalid provider configuration as non-retryable', async () => { + vi.mocked(loadMicrovmProviderConfig).mockImplementation(() => { + throw new Error('missing image'); + }); + + await expect( + createMicrovmRunners(runnerConfig(), 3, githubClient, createStartRunnerConfig, 'scale-up-lambda'), + ).resolves.toEqual({ instances: [], retryableErrorCount: 0, nonRetryableErrorCount: 3 }); + }); + + it('launches each MicroVM and delivers its JIT configuration', async () => { + vi.mocked(runMicrovmRunner).mockResolvedValueOnce('mvm-1').mockResolvedValueOnce('mvm-2'); + createStartRunnerConfig.mockImplementation(async (_config, runnerIds, _client, options) => { + await options?.onJitConfigCreated?.(runnerIds[0], { githubRunnerId: `github-${runnerIds[0]}`, runnerLabels: [] }); + return []; + }); + + await expect( + createMicrovmRunners(runnerConfig(), 2, githubClient, createStartRunnerConfig, 'pool-lambda'), + ).resolves.toEqual({ instances: ['mvm-1', 'mvm-2'], retryableErrorCount: 0, nonRetryableErrorCount: 0 }); + + expect(runMicrovmRunner).toHaveBeenNthCalledWith(1, { + config: expect.objectContaining({ imageIdentifier: imageArn }), + environment: 'unit-test', + runHookPayload: createMicrovmRunHookPayload('/github-action-runners/unit-test/token'), + runnerOwner: 'Codertocat', + runnerType: 'Org', + source: 'pool-lambda', + }); + expect(createStartRunnerConfig).toHaveBeenCalledTimes(2); + const options = createStartRunnerConfig.mock.calls[0][3]; + expect(options?.getSsmParameterTags?.('mvm-1')).toEqual([{ Key: 'MicrovmId', Value: 'mvm-1' }]); + expect(tagMicrovm).toHaveBeenNthCalledWith(1, imageArn, 'mvm-1', { + 'ghr:github_runner_id': 'github-mvm-1', + }); + }); + + it('applies dynamic labels to the RunMicrovm configuration and metadata tags', async () => { + const overrideImageArn = 'arn:aws:lambda:eu-west-1:123456789012:microvm-image:runner-large'; + const overrideEgressConnectorArn = + 'arn:aws:lambda:eu-west-1:123456789012:network-connector:github-runner-private-egress'; + createStartRunnerConfig.mockImplementation(async (_config, runnerIds, _client, options) => { + await options?.onJitConfigCreated?.(runnerIds[0], { githubRunnerId: 'github-mvm-1', runnerLabels: [] }); + return []; + }); + + await createMicrovmRunners(runnerConfig(), 1, githubClient, createStartRunnerConfig, 'scale-up-lambda', { + egressNetworkConnectors: [overrideEgressConnectorArn], + imageIdentifier: overrideImageArn, + imageVersion: '3.0', + maximumDurationInSeconds: 7200, + }); + + expect(runMicrovmRunner).toHaveBeenCalledWith({ + config: { + egressNetworkConnectors: [overrideEgressConnectorArn], + imageIdentifier: overrideImageArn, + imageVersion: '3.0', + executionRoleArn: 'arn:aws:iam::123456789012:role/microvm-runner', + maximumDurationInSeconds: 7200, + }, + environment: 'unit-test', + runHookPayload: createMicrovmRunHookPayload('/github-action-runners/unit-test/token'), + runnerOwner: 'Codertocat', + runnerType: 'Org', + source: 'scale-up-lambda', + }); + expect(tagMicrovm).toHaveBeenCalledWith(overrideImageArn, 'mvm-1', { + 'ghr:github_runner_id': 'github-mvm-1', + }); + }); + + it('retries a JIT setup failure even when runner cleanup fails', async () => { + createStartRunnerConfig.mockResolvedValue(['mvm-1']); + vi.mocked(terminateMicrovm).mockRejectedValue(new Error('cleanup failed')); + + await expect( + createMicrovmRunners(runnerConfig(), 1, githubClient, createStartRunnerConfig, 'scale-up-lambda'), + ).resolves.toEqual({ instances: [], retryableErrorCount: 1, nonRetryableErrorCount: 0 }); + + expect(terminateMicrovm).toHaveBeenCalledWith('mvm-1'); + }); + + it.each([ + [true, { instances: [], retryableErrorCount: 1, nonRetryableErrorCount: 0 }], + [false, { instances: [], retryableErrorCount: 0, nonRetryableErrorCount: 1 }], + ])('classifies launch failures with retryable=%s', async (retryable, expected) => { + vi.mocked(runMicrovmRunner).mockRejectedValue(new Error('launch failed')); + vi.mocked(isRetryableMicrovmError).mockReturnValue(retryable); + + await expect( + createMicrovmRunners(runnerConfig(), 1, githubClient, createStartRunnerConfig, 'scale-up-lambda'), + ).resolves.toEqual(expected); + }); + + it('attempts cleanup when setup throws after launch', async () => { + createStartRunnerConfig.mockRejectedValue(new Error('JIT setup failed')); + vi.mocked(terminateMicrovm).mockRejectedValue(new Error('cleanup failed')); + + await expect( + createMicrovmRunners(runnerConfig(), 1, githubClient, createStartRunnerConfig, 'scale-up-lambda'), + ).resolves.toEqual({ instances: [], retryableErrorCount: 0, nonRetryableErrorCount: 1 }); + + expect(terminateMicrovm).toHaveBeenCalledWith('mvm-1'); + }); +}); diff --git a/lambdas/libs/compute-providers/aws/microvm/src/control-plane/runner-config.ts b/lambdas/libs/compute-providers/aws/microvm/src/control-plane/runner-config.ts new file mode 100644 index 0000000000..ca3497dd0e --- /dev/null +++ b/lambdas/libs/compute-providers/aws/microvm/src/control-plane/runner-config.ts @@ -0,0 +1,109 @@ +import { createChildLogger } from '@aws-github-runner/aws-powertools-util'; +import type { Octokit } from '@octokit/rest'; + +import type { + CreateGitHubRunnerConfig, + CreateRunnerResult, + CreateStartRunnerConfig, + LambdaRunnerSource, +} from '../../../../core'; +import type { MicrovmDynamicLabelOverrides } from '../dynamic-labels'; +import { loadMicrovmProviderConfig } from './config'; +import { isRetryableMicrovmError, runMicrovmRunner, tagMicrovm, terminateMicrovm } from './microvms'; + +const logger = createChildLogger('microvm-runner-config'); + +export interface MicrovmRunHookPayloadV1 { + runnerConfigSsmPath: string; + version: 1; +} + +export function createMicrovmRunHookPayload(ssmTokenPath: string): string { + return JSON.stringify({ + version: 1, + runnerConfigSsmPath: ssmTokenPath, + } satisfies MicrovmRunHookPayloadV1); +} + +export async function createMicrovmRunners( + githubRunnerConfig: CreateGitHubRunnerConfig, + numberOfRunners: number, + githubInstallationClient: Octokit, + createStartRunnerConfig: CreateStartRunnerConfig, + source: LambdaRunnerSource, + overrides: MicrovmDynamicLabelOverrides = {}, +): Promise { + if (!githubRunnerConfig.ephemeral || !githubRunnerConfig.enableJitConfig) { + logger.error('Lambda MicroVM runners require ephemeral runners with JIT configuration enabled'); + return { instances: [], retryableErrorCount: 0, nonRetryableErrorCount: numberOfRunners }; + } + + if (!githubRunnerConfig.ssmTokenPath?.trim()) { + logger.error('Lambda MicroVM runners require SSM_TOKEN_PATH to deliver JIT configuration'); + return { instances: [], retryableErrorCount: 0, nonRetryableErrorCount: numberOfRunners }; + } + + let config; + try { + config = { ...loadMicrovmProviderConfig(), ...overrides }; + } catch (error) { + logger.error('Invalid Lambda MicroVM provider configuration', { error }); + return { instances: [], retryableErrorCount: 0, nonRetryableErrorCount: numberOfRunners }; + } + + const result: CreateRunnerResult = { + instances: [], + retryableErrorCount: 0, + nonRetryableErrorCount: 0, + }; + const runHookPayload = createMicrovmRunHookPayload(githubRunnerConfig.ssmTokenPath); + + for (let runnerIndex = 0; runnerIndex < numberOfRunners; runnerIndex++) { + let microvmId: string | undefined; + try { + microvmId = await runMicrovmRunner({ + config, + environment: process.env.ENVIRONMENT, + runHookPayload, + runnerOwner: githubRunnerConfig.runnerOwner, + runnerType: githubRunnerConfig.runnerType, + source, + }); + + const failedRunnerIds = await createStartRunnerConfig(githubRunnerConfig, [microvmId], githubInstallationClient, { + getSsmParameterTags: (runnerId) => [{ Key: 'MicrovmId', Value: runnerId }], + onJitConfigCreated: async (runnerId, metadata) => { + await tagMicrovm(config.imageIdentifier, runnerId, { + 'ghr:github_runner_id': metadata.githubRunnerId, + }); + }, + }); + + if (failedRunnerIds.includes(microvmId)) { + await terminateMicrovm(microvmId).catch((terminationError) => { + logger.error(`Failed to terminate MicroVM runner '${microvmId}' after JIT configuration failed`, { + error: terminationError, + }); + }); + result.retryableErrorCount++; + } else { + result.instances.push(microvmId); + } + } catch (error) { + if (microvmId) { + await terminateMicrovm(microvmId).catch((terminationError) => { + logger.error(`Failed to terminate MicroVM runner '${microvmId}' after setup failed`, { + error: terminationError, + }); + }); + } + + const retryable = isRetryableMicrovmError(error); + logger.error('Failed to create Lambda MicroVM runner', { error, retryable }); + if (retryable) result.retryableErrorCount++; + else result.nonRetryableErrorCount++; + } + } + + return result; +} diff --git a/lambdas/libs/compute-providers/aws/microvm/src/control-plane/scale-down.test.ts b/lambdas/libs/compute-providers/aws/microvm/src/control-plane/scale-down.test.ts new file mode 100644 index 0000000000..613364e7d2 --- /dev/null +++ b/lambdas/libs/compute-providers/aws/microvm/src/control-plane/scale-down.test.ts @@ -0,0 +1,74 @@ +import { beforeEach, describe, expect, it, vi } from 'vitest'; + +import { loadMicrovmProviderConfig } from './config'; +import { listMicrovmRunners, microvmBootTimeExceeded, tagMicrovm, terminateMicrovm, untagMicrovm } from './microvms'; +import { createMicrovmScaleDownProvider } from './scale-down'; + +vi.mock('./config', () => ({ loadMicrovmProviderConfig: vi.fn() })); +vi.mock('./microvms', () => ({ + listMicrovmRunners: vi.fn(), + microvmBootTimeExceeded: vi.fn(), + tagMicrovm: vi.fn(), + terminateMicrovm: vi.fn(), + untagMicrovm: vi.fn(), +})); + +const imageArn = 'arn:aws:lambda:eu-west-1:123456789012:microvm-image:runner'; +const overrideImageArn = 'arn:aws:lambda:eu-west-1:123456789012:microvm-image:runner-large'; +const providerConfig = { + imageIdentifier: imageArn, + executionRoleArn: 'arn:aws:iam::123456789012:role/microvm-runner', + maximumDurationInSeconds: 1200, +}; + +beforeEach(() => { + vi.clearAllMocks(); + vi.mocked(loadMicrovmProviderConfig).mockReturnValue(providerConfig); + vi.mocked(listMicrovmRunners).mockResolvedValue([]); + vi.mocked(microvmBootTimeExceeded).mockReturnValue(false); + vi.mocked(tagMicrovm).mockResolvedValue(); + vi.mocked(untagMicrovm).mockResolvedValue(); + vi.mocked(terminateMicrovm).mockResolvedValue(); +}); + +describe('createMicrovmScaleDownProvider', () => { + it('lists active and orphan runners through provider filters', async () => { + const provider = createMicrovmScaleDownProvider(); + + await provider.list('unit-test'); + await provider.list('unit-test', true); + + expect(listMicrovmRunners).toHaveBeenNthCalledWith(1, { + environment: 'unit-test', + orphan: undefined, + }); + expect(listMicrovmRunners).toHaveBeenNthCalledWith(2, { + environment: 'unit-test', + orphan: true, + }); + }); + + it('uses the listed image ARN when marking, unmarking, and terminating runners', async () => { + vi.mocked(listMicrovmRunners).mockResolvedValue([ + { id: 'mvm-1', imageArn: overrideImageArn, owner: 'Codertocat', type: 'Org', state: 'RUNNING' }, + ]); + const provider = createMicrovmScaleDownProvider(); + + await provider.list('unit-test'); + await provider.markOrphan('mvm-1'); + await provider.unmarkOrphan('mvm-1'); + await provider.terminate('mvm-1'); + + expect(tagMicrovm).toHaveBeenCalledWith(overrideImageArn, 'mvm-1', { 'ghr:orphan': 'true' }); + expect(untagMicrovm).toHaveBeenCalledWith(overrideImageArn, 'mvm-1', ['ghr:orphan']); + expect(terminateMicrovm).toHaveBeenCalledWith('mvm-1'); + }); + + it('uses the MicroVM boot-time policy', () => { + const provider = createMicrovmScaleDownProvider(); + const runner = { id: 'mvm-1', owner: 'Codertocat', type: 'Org' as const }; + + expect(provider.bootTimeExceeded(runner)).toBe(false); + expect(microvmBootTimeExceeded).toHaveBeenCalledWith(runner); + }); +}); diff --git a/lambdas/libs/compute-providers/aws/microvm/src/control-plane/scale-down.ts b/lambdas/libs/compute-providers/aws/microvm/src/control-plane/scale-down.ts new file mode 100644 index 0000000000..9ea9dc474a --- /dev/null +++ b/lambdas/libs/compute-providers/aws/microvm/src/control-plane/scale-down.ts @@ -0,0 +1,28 @@ +import type { ScaleDownComputeProvider } from '../../../../core'; +import { loadMicrovmProviderConfig } from './config'; +import type { MicrovmRunnerInfo } from './microvms'; +import { listMicrovmRunners, microvmBootTimeExceeded, tagMicrovm, terminateMicrovm, untagMicrovm } from './microvms'; + +export function createMicrovmScaleDownProvider(): Omit { + const imageArnByRunnerId = new Map(); + + async function list(environment: string, orphan?: boolean): Promise { + const runners = await listMicrovmRunners({ environment, orphan }); + for (const runner of runners) { + if (runner.imageArn) imageArnByRunnerId.set(runner.id, runner.imageArn); + } + return runners; + } + + function imageArnForRunner(id: string): string { + return imageArnByRunnerId.get(id) ?? loadMicrovmProviderConfig().imageIdentifier; + } + + return { + list, + bootTimeExceeded: microvmBootTimeExceeded, + markOrphan: async (id) => await tagMicrovm(imageArnForRunner(id), id, { 'ghr:orphan': 'true' }), + unmarkOrphan: async (id) => await untagMicrovm(imageArnForRunner(id), id, ['ghr:orphan']), + terminate: terminateMicrovm, + }; +} diff --git a/lambdas/libs/compute-providers/aws/microvm/src/control-plane/scale-up.test.ts b/lambdas/libs/compute-providers/aws/microvm/src/control-plane/scale-up.test.ts new file mode 100644 index 0000000000..cfbbda3257 --- /dev/null +++ b/lambdas/libs/compute-providers/aws/microvm/src/control-plane/scale-up.test.ts @@ -0,0 +1,114 @@ +import type { Octokit } from '@octokit/rest'; +import { beforeEach, describe, expect, it, vi } from 'vitest'; + +import type { CreateGitHubRunnerConfig, CreateStartRunnerConfig } from '../../../../core'; +import { listMicrovmRunners } from './microvms'; +import { createMicrovmRunners } from './runner-config'; +import { createMicrovmScaleUpProvider } from './scale-up'; + +vi.mock('./microvms', () => ({ listMicrovmRunners: vi.fn() })); +vi.mock('./runner-config', () => ({ createMicrovmRunners: vi.fn() })); + +const createStartRunnerConfig = vi.fn(); +const githubClient = {} as Octokit; +const overrideImageArn = 'arn:aws:lambda:eu-west-1:123456789012:microvm-image:runner-large'; +const overrideEgressConnectorArn = + 'arn:aws:lambda:eu-west-1:123456789012:network-connector:github-runner-private-egress'; +const githubRunnerConfig: CreateGitHubRunnerConfig = { + ephemeral: true, + enableJitConfig: true, + runnerLabels: 'self-hosted,linux,arm64,microvm', + runnerGroup: 'Default', + runnerNamePrefix: '', + runnerOwner: 'Codertocat', + runnerType: 'Org', + disableAutoUpdate: true, + ssmTokenPath: '/runner/token', + ssmConfigPath: '/runner/config', + ssmParameterStoreTags: [], +}; + +beforeEach(() => { + vi.clearAllMocks(); + process.env.ENVIRONMENT = 'unit-test'; + vi.mocked(listMicrovmRunners).mockResolvedValue([ + { id: 'mvm-current', owner: 'Codertocat', type: 'Org', state: 'RUNNING' }, + ]); + vi.mocked(createMicrovmRunners).mockResolvedValue({ + instances: ['mvm-new'], + retryableErrorCount: 0, + nonRetryableErrorCount: 0, + }); +}); + +describe('createMicrovmScaleUpProvider', () => { + it('resolves supported resource override labels and registers them on the runner', async () => { + const provider = createMicrovmScaleUpProvider(createStartRunnerConfig); + + await expect( + provider.resolveLabelsForRunners([ + `ghr-microvm-egress-network-connectors:${overrideEgressConnectorArn}`, + `ghr-microvm-image-arn:${overrideImageArn}`, + 'ghr-microvm-image-version:3.0', + 'ghr-microvm-maximum-duration-in-seconds:7200', + ]), + ).resolves.toEqual({ + runnerLabels: [ + `ghr-microvm-egress-network-connectors:${overrideEgressConnectorArn}`, + `ghr-microvm-image-arn:${overrideImageArn}`, + 'ghr-microvm-image-version:3.0', + 'ghr-microvm-maximum-duration-in-seconds:7200', + ], + state: { + overrides: { + egressNetworkConnectors: [overrideEgressConnectorArn], + imageIdentifier: overrideImageArn, + imageVersion: '3.0', + maximumDurationInSeconds: 7200, + }, + }, + }); + }); + + it('rejects unsupported MicroVM override labels at the control-plane boundary', async () => { + const provider = createMicrovmScaleUpProvider(createStartRunnerConfig); + + await expect(provider.resolveLabelsForRunners(['ghr-microvm-memory:8192'])).rejects.toThrow( + "key 'memory' is not a supported MicroVM override", + ); + }); + + it('counts managed MicroVMs for the runner owner', async () => { + const provider = createMicrovmScaleUpProvider(createStartRunnerConfig); + + await expect( + provider.getCurrentRunners({ overrides: {} }, { runnerOwner: 'Codertocat', runnerType: 'Org' }), + ).resolves.toBe(1); + expect(listMicrovmRunners).toHaveBeenCalledWith({ + environment: 'unit-test', + runnerOwner: 'Codertocat', + runnerType: 'Org', + }); + }); + + it('delegates runner creation to the shared MicroVM lifecycle', async () => { + const provider = createMicrovmScaleUpProvider(createStartRunnerConfig); + + await expect( + provider.createRunners({ + githubRunnerConfig, + numberOfRunners: 1, + githubInstallationClient: githubClient, + state: { overrides: { imageVersion: '3.0' } }, + }), + ).resolves.toEqual({ instances: ['mvm-new'], retryableErrorCount: 0, nonRetryableErrorCount: 0 }); + expect(createMicrovmRunners).toHaveBeenCalledWith( + githubRunnerConfig, + 1, + githubClient, + createStartRunnerConfig, + 'scale-up-lambda', + { imageVersion: '3.0' }, + ); + }); +}); diff --git a/lambdas/libs/compute-providers/aws/microvm/src/control-plane/scale-up.ts b/lambdas/libs/compute-providers/aws/microvm/src/control-plane/scale-up.ts new file mode 100644 index 0000000000..a3dcf1219e --- /dev/null +++ b/lambdas/libs/compute-providers/aws/microvm/src/control-plane/scale-up.ts @@ -0,0 +1,77 @@ +import type { + CreateRunnerResult, + CreateScaleUpRunnersInput, + CreateStartRunnerConfig, + CurrentRunnersInput, + RunnerLabelResolution, + ScaleUpComputeProvider, +} from '../../../../core'; +import type { MicrovmDynamicLabelOverrides } from '../dynamic-labels'; +import { parseMicrovmDynamicLabels } from '../dynamic-labels'; +import { listMicrovmRunners } from './microvms'; +import { createMicrovmRunners } from './runner-config'; + +interface MicrovmScaleUpState { + overrides: MicrovmDynamicLabelOverrides; +} + +async function resolveMicrovmLabelsForRunners( + messageLabels: string[], +): Promise> { + const trimmedLabels = messageLabels.map((label) => label.trim()); + const parsed = parseMicrovmDynamicLabels(trimmedLabels); + if (parsed.violations.length > 0) { + throw new Error( + `Invalid MicroVM dynamic labels: ${parsed.violations + .map((violation) => `${violation.label} (${violation.reason})`) + .join(', ')}`, + ); + } + + return { + runnerLabels: trimmedLabels.filter((label) => label.startsWith('ghr-')), + state: { overrides: parsed.overrides }, + }; +} + +async function getCurrentMicrovmRunners( + _state: MicrovmScaleUpState, + { runnerType, runnerOwner }: CurrentRunnersInput, +): Promise { + return ( + await listMicrovmRunners({ + environment: process.env.ENVIRONMENT, + runnerType, + runnerOwner, + }) + ).length; +} + +async function createMicrovmScaleUpRunners( + { + githubRunnerConfig, + numberOfRunners, + githubInstallationClient, + state, + }: CreateScaleUpRunnersInput, + createStartRunnerConfig: CreateStartRunnerConfig, +): Promise { + return await createMicrovmRunners( + githubRunnerConfig, + numberOfRunners, + githubInstallationClient, + createStartRunnerConfig, + 'scale-up-lambda', + state.overrides, + ); +} + +export function createMicrovmScaleUpProvider( + createStartRunnerConfig: CreateStartRunnerConfig, +): Omit, 'type'> { + return { + resolveLabelsForRunners: resolveMicrovmLabelsForRunners, + getCurrentRunners: getCurrentMicrovmRunners, + createRunners: (input) => createMicrovmScaleUpRunners(input, createStartRunnerConfig), + }; +} diff --git a/lambdas/libs/compute-providers/aws/microvm/src/dynamic-labels.test.ts b/lambdas/libs/compute-providers/aws/microvm/src/dynamic-labels.test.ts new file mode 100644 index 0000000000..6ea669a143 --- /dev/null +++ b/lambdas/libs/compute-providers/aws/microvm/src/dynamic-labels.test.ts @@ -0,0 +1,73 @@ +import { describe, expect, it } from 'vitest'; + +import { parseMicrovmDynamicLabels } from './dynamic-labels'; + +const imageArn = 'arn:aws:lambda:eu-west-1:123456789012:microvm-image:runner-large'; +const egressConnectorArn = 'arn:aws:lambda:eu-west-1:123456789012:network-connector:github-runner-private-egress'; +const internetEgressConnectorArn = + 'arn:aws:lambda:eu-west-1:aws:network-connector:aws-network-connector:INTERNET_EGRESS'; + +describe('parseMicrovmDynamicLabels', () => { + it('parses every supported RunMicrovm override', () => { + expect( + parseMicrovmDynamicLabels([ + `ghr-microvm-egress-network-connectors:${egressConnectorArn}`, + `ghr-microvm-egress-network-connectors:${internetEgressConnectorArn}`, + `ghr-microvm-image-arn:${imageArn}`, + 'ghr-microvm-image-version:3.0', + 'ghr-microvm-maximum-duration-in-seconds:7200', + ]), + ).toEqual({ + overrides: { + egressNetworkConnectors: [egressConnectorArn, internetEgressConnectorArn], + imageIdentifier: imageArn, + imageVersion: '3.0', + maximumDurationInSeconds: 7200, + }, + violations: [], + }); + }); + + it.each([ + ['ghr-microvm-memory:8192', "key 'memory' is not a supported MicroVM override"], + [ + 'ghr-microvm-egress-network-connectors:not-an-arn', + 'is not a valid Lambda network connector ARN; specify one ARN per label', + ], + [ + `ghr-microvm-egress-network-connectors:${egressConnectorArn};${internetEgressConnectorArn}`, + 'is not a valid Lambda network connector ARN; specify one ARN per label', + ], + ['ghr-microvm-image-arn:not-an-arn', 'is not a valid customer MicroVM image ARN'], + ['ghr-microvm-image-version:', "key 'image-version' requires a value"], + ['ghr-microvm-maximum-duration-in-seconds:0', 'maximum duration must be an integer between 1 and 28800'], + ['ghr-microvm-maximum-duration-in-seconds:28801', 'maximum duration must be an integer between 1 and 28800'], + ])('rejects invalid override %s', (label, reason) => { + const result = parseMicrovmDynamicLabels([label]); + + expect(result.overrides).toEqual({}); + expect(result.violations).toEqual([{ label, reason: expect.stringContaining(reason) }]); + }); + + it('ignores generic dynamic labels', () => { + expect(parseMicrovmDynamicLabels(['ghr-team:platform'])).toEqual({ overrides: {}, violations: [] }); + }); + + it('rejects more than ten egress network connectors', () => { + const labels = Array.from( + { length: 11 }, + (_, index) => + `ghr-microvm-egress-network-connectors:arn:aws:lambda:eu-west-1:123456789012:network-connector:connector-${index}`, + ); + + const result = parseMicrovmDynamicLabels(labels); + + expect(result.overrides.egressNetworkConnectors).toHaveLength(10); + expect(result.violations).toEqual([ + { + label: labels[10], + reason: 'at most 10 egress network connector labels are supported', + }, + ]); + }); +}); diff --git a/lambdas/libs/compute-providers/aws/microvm/src/dynamic-labels.ts b/lambdas/libs/compute-providers/aws/microvm/src/dynamic-labels.ts new file mode 100644 index 0000000000..50c223cfd2 --- /dev/null +++ b/lambdas/libs/compute-providers/aws/microvm/src/dynamic-labels.ts @@ -0,0 +1,90 @@ +export const MICROVM_DYNAMIC_LABEL_PREFIX = 'ghr-microvm-'; + +const MAXIMUM_DURATION_IN_SECONDS = 28_800; +const MAXIMUM_EGRESS_NETWORK_CONNECTORS = 10; +const MICROVM_IMAGE_ARN_PATTERN = /^arn:[^:]+:lambda:[^:]+:[0-9]{12}:microvm-image:.+$/; +const MICROVM_NETWORK_CONNECTOR_ARN_PATTERN = + /^arn:aws[a-zA-Z-]*:lambda:[a-z0-9-]+:(?:[0-9]{12}|aws):network-connector:[a-zA-Z0-9_-]+(?::[a-zA-Z0-9_-]+)?$/; + +export interface MicrovmDynamicLabelOverrides { + egressNetworkConnectors?: string[]; + imageIdentifier?: string; + imageVersion?: string; + maximumDurationInSeconds?: number; +} + +export interface MicrovmDynamicLabelViolation { + label: string; + reason: string; +} + +export function parseMicrovmDynamicLabels(labels: string[]): { + overrides: MicrovmDynamicLabelOverrides; + violations: MicrovmDynamicLabelViolation[]; +} { + const overrides: MicrovmDynamicLabelOverrides = {}; + const violations: MicrovmDynamicLabelViolation[] = []; + + for (const label of labels) { + if (!label.startsWith(MICROVM_DYNAMIC_LABEL_PREFIX)) continue; + + const stripped = label.slice(MICROVM_DYNAMIC_LABEL_PREFIX.length); + const colonIndex = stripped.indexOf(':'); + const key = colonIndex === -1 ? stripped : stripped.slice(0, colonIndex); + const value = colonIndex === -1 ? '' : stripped.slice(colonIndex + 1).trim(); + + if (!value) { + violations.push({ label, reason: `key '${key}' requires a value` }); + continue; + } + + switch (key) { + case 'egress-network-connectors': { + if (!MICROVM_NETWORK_CONNECTOR_ARN_PATTERN.test(value)) { + violations.push({ + label, + reason: `'${value}' is not a valid Lambda network connector ARN; specify one ARN per label`, + }); + break; + } + + const connectors = overrides.egressNetworkConnectors ?? []; + if (connectors.length >= MAXIMUM_EGRESS_NETWORK_CONNECTORS) { + violations.push({ + label, + reason: `at most ${MAXIMUM_EGRESS_NETWORK_CONNECTORS} egress network connector labels are supported`, + }); + } else { + overrides.egressNetworkConnectors = [...connectors, value]; + } + break; + } + case 'image-arn': + if (!MICROVM_IMAGE_ARN_PATTERN.test(value)) { + violations.push({ label, reason: `'${value}' is not a valid customer MicroVM image ARN` }); + } else { + overrides.imageIdentifier = value; + } + break; + case 'image-version': + overrides.imageVersion = value; + break; + case 'maximum-duration-in-seconds': { + const duration = Number(value); + if (!Number.isInteger(duration) || duration < 1 || duration > MAXIMUM_DURATION_IN_SECONDS) { + violations.push({ + label, + reason: `maximum duration must be an integer between 1 and ${MAXIMUM_DURATION_IN_SECONDS}`, + }); + } else { + overrides.maximumDurationInSeconds = duration; + } + break; + } + default: + violations.push({ label, reason: `key '${key}' is not a supported MicroVM override` }); + } + } + + return { overrides, violations }; +} diff --git a/lambdas/libs/compute-providers/aws/microvm/src/environment.d.ts b/lambdas/libs/compute-providers/aws/microvm/src/environment.d.ts new file mode 100644 index 0000000000..91c1931f83 --- /dev/null +++ b/lambdas/libs/compute-providers/aws/microvm/src/environment.d.ts @@ -0,0 +1,15 @@ +export {}; + +declare global { + namespace NodeJS { + interface ProcessEnv { + MICROVM_EGRESS_NETWORK_CONNECTORS: string | undefined; + MICROVM_EXECUTION_ROLE_ARN: string; + MICROVM_IMAGE_ARN: string; + MICROVM_IMAGE_VERSION: string | undefined; + MICROVM_INGRESS_NETWORK_CONNECTORS: string | undefined; + MICROVM_LOG_GROUP: string | undefined; + MICROVM_MAXIMUM_DURATION_IN_SECONDS: string | undefined; + } + } +} diff --git a/lambdas/libs/compute-providers/aws/microvm/src/webhook/dynamic-labels.test.ts b/lambdas/libs/compute-providers/aws/microvm/src/webhook/dynamic-labels.test.ts new file mode 100644 index 0000000000..57a863d74f --- /dev/null +++ b/lambdas/libs/compute-providers/aws/microvm/src/webhook/dynamic-labels.test.ts @@ -0,0 +1,103 @@ +import { describe, expect, it } from 'vitest'; + +import type { RunnerMatcherConfig } from '../../../../contracts'; +import { microvmDynamicLabelProvider } from './dynamic-labels'; + +const imageArn = 'arn:aws:lambda:eu-west-1:123456789012:microvm-image:runner-large'; +const egressConnectorArn = 'arn:aws:lambda:eu-west-1:123456789012:network-connector:github-runner-private-egress'; + +describe('microvmDynamicLabelProvider', () => { + it('accepts supported MicroVM overrides', () => { + const queue = microvmQueue(); + const dynamicLabels = [ + `ghr-microvm-egress-network-connectors:${egressConnectorArn}`, + `ghr-microvm-image-arn:${imageArn}`, + 'ghr-microvm-image-version:3.0', + 'ghr-microvm-maximum-duration-in-seconds:7200', + ]; + + expect(getViolations(queue, dynamicLabels)).toEqual([]); + }); + + it('rejects unsupported MicroVM resource overrides', () => { + expect(getViolations(microvmQueue(), ['ghr-microvm-memory:8192'])).toEqual([ + { + label: 'ghr-microvm-memory:8192', + reason: "key 'memory' is not a supported MicroVM override", + }, + ]); + }); + + it('enforces the AWS dynamic-label policy', () => { + const queue = microvmQueue(); + queue.matcherConfig.awsDynamicLabelsPolicy = { + restricted_keys: { 'maximum-duration-in-seconds': { max: 3600 } }, + }; + + expect(getViolations(queue, ['ghr-microvm-maximum-duration-in-seconds:7200'])).toEqual([ + { + label: 'ghr-microvm-maximum-duration-in-seconds:7200', + reason: "value '7200' exceeds max '3600'", + }, + ]); + }); + + it('applies allowed patterns to the complete image ARN', () => { + const queue = microvmQueue(); + queue.matcherConfig.awsDynamicLabelsPolicy = { + restricted_keys: { + 'image-arn': { + allowed: ['arn:aws:lambda:eu-west-1:123456789012:microvm-image:approved-*'], + }, + }, + }; + + expect( + getViolations(queue, [ + 'ghr-microvm-image-arn:arn:aws:lambda:eu-west-1:123456789012:microvm-image:approved-large', + ]), + ).toEqual([]); + expect( + getViolations(queue, ['ghr-microvm-image-arn:arn:aws:lambda:eu-west-1:123456789012:microvm-image:unapproved']), + ).toHaveLength(1); + }); + + it('applies the policy to each egress connector label', () => { + const queue = microvmQueue(); + queue.matcherConfig.awsDynamicLabelsPolicy = { + restricted_keys: { + 'egress-network-connectors': { + allowed: ['arn:aws:lambda:eu-west-1:123456789012:network-connector:approved-*'], + }, + }, + }; + + expect( + getViolations(queue, [ + 'ghr-microvm-egress-network-connectors:arn:aws:lambda:eu-west-1:123456789012:network-connector:approved-private', + ]), + ).toEqual([]); + expect( + getViolations(queue, [ + 'ghr-microvm-egress-network-connectors:arn:aws:lambda:eu-west-1:123456789012:network-connector:unapproved', + ]), + ).toHaveLength(1); + }); +}); + +function getViolations(queue: RunnerMatcherConfig, labels: string[]) { + return microvmDynamicLabelProvider.getViolations({ queue, labels }); +} + +function microvmQueue(): RunnerMatcherConfig { + return { + id: 'microvm', + arn: 'arn:aws:sqs:eu-west-1:123456789012:microvm', + computeProvider: 'microvm', + matcherConfig: { + labelMatchers: [['self-hosted', 'linux', 'arm64', 'microvm']], + exactMatch: false, + enableDynamicLabels: true, + }, + }; +} diff --git a/lambdas/libs/compute-providers/aws/microvm/src/webhook/dynamic-labels.ts b/lambdas/libs/compute-providers/aws/microvm/src/webhook/dynamic-labels.ts new file mode 100644 index 0000000000..bf57600eef --- /dev/null +++ b/lambdas/libs/compute-providers/aws/microvm/src/webhook/dynamic-labels.ts @@ -0,0 +1,16 @@ +import { violationsAgainstAwsDynamicLabelsPolicy } from '../../../dynamic-labels-policy'; +import type { DynamicLabelProvider } from '../../../../contracts'; +import { MICROVM_DYNAMIC_LABEL_PREFIX, parseMicrovmDynamicLabels } from '../dynamic-labels'; + +export const microvmDynamicLabelProvider: DynamicLabelProvider = { + getViolations: ({ queue, labels }) => { + const parsedLabels = parseMicrovmDynamicLabels(labels); + const policyViolations = violationsAgainstAwsDynamicLabelsPolicy( + labels, + queue.matcherConfig.awsDynamicLabelsPolicy, + MICROVM_DYNAMIC_LABEL_PREFIX, + ); + + return [...parsedLabels.violations, ...policyViolations]; + }, +}; diff --git a/lambdas/libs/compute-providers/aws/microvm/webhook.ts b/lambdas/libs/compute-providers/aws/microvm/webhook.ts new file mode 100644 index 0000000000..48d603e476 --- /dev/null +++ b/lambdas/libs/compute-providers/aws/microvm/webhook.ts @@ -0,0 +1,16 @@ +import type { ComputeProviderPlugin } from '../../core'; + +import type { WebhookProviderCapabilities, WebhookProviderModule } from '../../contracts'; +import { microvmDynamicLabelProvider } from './src/webhook/dynamic-labels'; + +export function createMicrovmWebhookPlugin(): ComputeProviderPlugin { + return { + type: 'microvm', + capabilities: { dynamicLabels: microvmDynamicLabelProvider }, + }; +} + +export const provider = { + type: 'microvm', + createPlugin: createMicrovmWebhookPlugin, +} satisfies WebhookProviderModule<'microvm'>; diff --git a/lambdas/libs/runner-providers/contracts.ts b/lambdas/libs/compute-providers/contracts.ts similarity index 67% rename from lambdas/libs/runner-providers/contracts.ts rename to lambdas/libs/compute-providers/contracts.ts index 1d1fe01b85..85e99f3949 100644 --- a/lambdas/libs/runner-providers/contracts.ts +++ b/lambdas/libs/compute-providers/contracts.ts @@ -1,11 +1,11 @@ import type { CreateStartRunnerConfig, - PoolRunnerProvider, - RunnerProviderPlugin, - ScaleDownRunnerProvider, - ScaleUpRunnerProvider, + PoolComputeProvider, + ComputeProviderPlugin, + ScaleDownComputeProvider, + ScaleUpComputeProvider, } from './core'; -import type { RunnerProviderType } from './provider-types'; +import type { ComputeProviderType } from './provider-types'; export interface AwsDynamicLabelsValueRule { allowed?: string[]; @@ -32,7 +32,7 @@ export interface MatcherConfig { export interface RunnerMatcherConfig { id: string; arn: string; - runnerProvider?: RunnerProviderType; + computeProvider?: ComputeProviderType; matcherConfig: MatcherConfig; } @@ -43,18 +43,19 @@ export interface DynamicLabelDispatchTarget { labels: string[]; } +export interface DynamicLabelViolation { + label: string; + reason: string; +} + export interface DynamicLabelProvider { - selectQueue(input: { - queue: RunnerMatcherConfig; - nonGhrLabels: string[]; - sanitizedGhrLabels: string[]; - }): DynamicLabelDispatchTarget | undefined; + getViolations(input: { queue: RunnerMatcherConfig; labels: string[] }): DynamicLabelViolation[]; } export interface ControlPlaneProviderCapabilities { - pool: () => Omit; - scaleUp: () => Omit; - scaleDown: () => Omit; + pool: () => Omit; + scaleUp: () => Omit; + scaleDown: () => Omit; } export interface WebhookProviderCapabilities { @@ -65,10 +66,10 @@ export interface ControlPlaneProviderModule { type: TType; createPlugin( createStartRunnerConfig: CreateStartRunnerConfig, - ): RunnerProviderPlugin; + ): ComputeProviderPlugin; } export interface WebhookProviderModule { type: TType; - createPlugin(): RunnerProviderPlugin; + createPlugin(): ComputeProviderPlugin; } diff --git a/lambdas/libs/runner-providers/control-plane.ts b/lambdas/libs/compute-providers/control-plane.ts similarity index 76% rename from lambdas/libs/runner-providers/control-plane.ts rename to lambdas/libs/compute-providers/control-plane.ts index 282949fe7e..be2ee494f1 100644 --- a/lambdas/libs/runner-providers/control-plane.ts +++ b/lambdas/libs/compute-providers/control-plane.ts @@ -1,11 +1,11 @@ import type { CreateStartRunnerConfig } from './core'; -import { createRunnerProviderRegistry } from './core'; +import { createComputeProviderRegistry } from './core'; import type { ControlPlaneProviderCapabilities } from './contracts'; import { enabledControlPlaneProviders } from './providers.config.control-plane'; export function createControlPlaneProviderRegistry(createStartRunnerConfig: CreateStartRunnerConfig) { - return createRunnerProviderRegistry( + return createComputeProviderRegistry( enabledControlPlaneProviders.map((provider) => provider.createPlugin(createStartRunnerConfig)), ); } diff --git a/lambdas/libs/runner-providers/core/index.test.ts b/lambdas/libs/compute-providers/core/index.test.ts similarity index 69% rename from lambdas/libs/runner-providers/core/index.test.ts rename to lambdas/libs/compute-providers/core/index.test.ts index b910a9f8ff..8ed95f927e 100644 --- a/lambdas/libs/runner-providers/core/index.test.ts +++ b/lambdas/libs/compute-providers/core/index.test.ts @@ -1,8 +1,8 @@ import { describe, expect, it } from 'vitest'; -import { createRunnerProviderRegistry } from './index'; +import { createComputeProviderRegistry } from './index'; -describe('runner provider registry', () => { +describe('compute provider registry', () => { const plugin = { type: 'ec2' as const, capabilities: { @@ -10,7 +10,7 @@ describe('runner provider registry', () => { pool: () => 'pool', }, }; - const registry = createRunnerProviderRegistry([plugin]); + const registry = createComputeProviderRegistry([plugin]); it('resolves capabilities dynamically', () => { expect(registry.capability('ec2', 'scaleUp')()).toBe('scale-up'); diff --git a/lambdas/libs/runner-providers/core/index.ts b/lambdas/libs/compute-providers/core/index.ts similarity index 81% rename from lambdas/libs/runner-providers/core/index.ts rename to lambdas/libs/compute-providers/core/index.ts index 02e97c694b..8b40ff44cb 100644 --- a/lambdas/libs/runner-providers/core/index.ts +++ b/lambdas/libs/compute-providers/core/index.ts @@ -1,9 +1,9 @@ import type { Octokit } from '@octokit/rest'; -import type { RunnerProviderType } from '../provider-types'; +import type { ComputeProviderType } from '../provider-types'; -export interface RunnerProvider { - type: RunnerProviderType; +export interface ComputeProvider { + type: ComputeProviderType; } export type LambdaRunnerSource = 'scale-up-lambda' | 'pool-lambda'; @@ -64,7 +64,7 @@ export interface CreateRunnerResult { nonRetryableErrorCount: number; } -export interface ScaleUpRunnerProvider extends RunnerProvider { +export interface ScaleUpComputeProvider extends ComputeProvider { resolveLabelsForRunners(messageLabels: string[]): Promise>; getCurrentRunners(state: TState, input: CurrentRunnersInput): Promise; createRunners(input: CreateScaleUpRunnersInput): Promise; @@ -89,7 +89,7 @@ export interface ListRunnerFilters { orphan?: boolean; } -export interface ScaleDownRunnerProvider extends RunnerProvider { +export interface ScaleDownComputeProvider extends ComputeProvider { list(environment: string, orphan?: boolean): Promise; bootTimeExceeded(runner: RunnerInfo): boolean; markOrphan(id: string): Promise; @@ -114,7 +114,7 @@ export interface CreatePoolRunnersInput { githubInstallationClient: Octokit; } -export interface PoolRunnerProvider extends RunnerProvider { +export interface PoolComputeProvider extends ComputeProvider { listRunners(input: ListPoolRunnersInput): Promise; countAvailableRunners( runners: TRunner[], @@ -124,19 +124,19 @@ export interface PoolRunnerProvider extends RunnerProvider { createRunners(input: CreatePoolRunnersInput): Promise; } -export interface RunnerProviderPlugin { +export interface ComputeProviderPlugin { type: TType; capabilities: TCapabilities; } -export function createRunnerProviderRegistry( - plugins: readonly RunnerProviderPlugin[], +export function createComputeProviderRegistry( + plugins: readonly ComputeProviderPlugin[], ) { const pluginsByType = new Map(plugins.map((plugin) => [plugin.type, plugin])); - function get(type: TType): RunnerProviderPlugin { + function get(type: TType): ComputeProviderPlugin { const plugin = pluginsByType.get(type); - if (!plugin) throw new Error(`No runner provider plugin registered for '${type}'`); + if (!plugin) throw new Error(`No compute provider plugin registered for '${type}'`); return plugin; } diff --git a/lambdas/libs/compute-providers/dynamic-labels.test.ts b/lambdas/libs/compute-providers/dynamic-labels.test.ts new file mode 100644 index 0000000000..0eacc9cf62 --- /dev/null +++ b/lambdas/libs/compute-providers/dynamic-labels.test.ts @@ -0,0 +1,12 @@ +import { expect, it } from 'vitest'; + +import { dynamicLabelsForOtherProvider } from './dynamic-labels'; +import { computeProviderTypes } from './provider-types'; + +it.each(computeProviderTypes)('returns labels belonging to providers other than %s', (provider) => { + const providerLabels = computeProviderTypes.map((type) => `ghr-${type}-size:large`); + + expect(dynamicLabelsForOtherProvider(providerLabels, provider)).toEqual( + providerLabels.filter((label) => !label.startsWith(`ghr-${provider}-`)), + ); +}); diff --git a/lambdas/libs/compute-providers/dynamic-labels.ts b/lambdas/libs/compute-providers/dynamic-labels.ts new file mode 100644 index 0000000000..97db9517d3 --- /dev/null +++ b/lambdas/libs/compute-providers/dynamic-labels.ts @@ -0,0 +1,8 @@ +import { computeProviderTypes } from './provider-types'; +import type { ComputeProviderType } from './provider-types'; + +export function dynamicLabelsForOtherProvider(labels: string[], provider: ComputeProviderType): string[] { + return labels.filter((label) => + computeProviderTypes.some((candidate) => candidate !== provider && label.startsWith(`ghr-${candidate}-`)), + ); +} diff --git a/lambdas/libs/runner-providers/package.json b/lambdas/libs/compute-providers/package.json similarity index 83% rename from lambdas/libs/runner-providers/package.json rename to lambdas/libs/compute-providers/package.json index 77c9ea71d2..cd03f897f0 100644 --- a/lambdas/libs/runner-providers/package.json +++ b/lambdas/libs/compute-providers/package.json @@ -1,5 +1,5 @@ { - "name": "@aws-github-runner/runner-providers", + "name": "@aws-github-runner/compute-providers", "version": "1.0.0", "main": "contracts.ts", "exports": { @@ -11,7 +11,9 @@ "./aws/ec2/webhook": "./aws/ec2/webhook.ts", "./aws/ec2/control-plane": "./aws/ec2/control-plane.ts", "./aws/ec2/control-plane/runners": "./aws/ec2/src/control-plane/runners.ts", - "./aws/ec2/control-plane/runner-config": "./aws/ec2/src/control-plane/runner-config.ts" + "./aws/ec2/control-plane/runner-config": "./aws/ec2/src/control-plane/runner-config.ts", + "./aws/microvm/webhook": "./aws/microvm/webhook.ts", + "./aws/microvm/control-plane": "./aws/microvm/control-plane.ts" }, "type": "module", "license": "MIT", @@ -27,6 +29,7 @@ "@aws-github-runner/aws-powertools-util": "*", "@aws-github-runner/aws-ssm-util": "*", "@aws-sdk/client-ec2": "^3.1009.0", + "@aws-sdk/client-lambda-microvms": "^3.1074.0", "@octokit/rest": "22.0.1", "moment": "2.29.4", "yn": "3.1.1" diff --git a/lambdas/libs/compute-providers/provider-types.test.ts b/lambdas/libs/compute-providers/provider-types.test.ts new file mode 100644 index 0000000000..9f6f4a981e --- /dev/null +++ b/lambdas/libs/compute-providers/provider-types.test.ts @@ -0,0 +1,32 @@ +import { describe, expect, it } from 'vitest'; + +import { computeProviderTypes, defaultComputeProvider, resolveComputeProviderType } from './provider-types'; + +const defaultProviderInputs = [undefined, '', ' '] as const; +const supportedProviderCases = computeProviderTypes.flatMap( + (provider) => + [ + [provider, provider], + [` ${provider.toUpperCase()} `, provider], + ] as const, +); + +describe('compute provider configuration', () => { + it('defines an explicit default provider', () => { + expect(computeProviderTypes).toContain(defaultComputeProvider); + }); +}); + +describe('compute provider resolution', () => { + it.each(defaultProviderInputs)('resolves default provider input %j', (type) => { + expect(resolveComputeProviderType(type)).toBe(defaultComputeProvider); + }); + + it.each(supportedProviderCases)('resolves provider type %j to %j', (type, expected) => { + expect(resolveComputeProviderType(type)).toBe(expected); + }); + + it.each([[' Unknown '], ['unsupported-provider'], [null], [1]])('rejects unsupported provider type %j', (type) => { + expect(() => resolveComputeProviderType(type)).toThrow(`Unsupported compute provider type '${String(type)}'`); + }); +}); diff --git a/lambdas/libs/compute-providers/provider-types.ts b/lambdas/libs/compute-providers/provider-types.ts new file mode 100644 index 0000000000..087f61de71 --- /dev/null +++ b/lambdas/libs/compute-providers/provider-types.ts @@ -0,0 +1,22 @@ +export const computeProviderTypes = ['ec2', 'microvm'] as const; + +export type ComputeProviderType = (typeof computeProviderTypes)[number]; + +export const defaultComputeProvider = 'ec2' satisfies ComputeProviderType; + +export function resolveComputeProviderType(type: unknown): ComputeProviderType { + if (type === undefined) return defaultComputeProvider; + if (typeof type !== 'string') { + throw new Error(`Unsupported compute provider type '${String(type)}'`); + } + + const normalizedType = type.trim().toLowerCase(); + if (!normalizedType) return defaultComputeProvider; + + const computeProviderType = computeProviderTypes.find((provider) => provider === normalizedType); + if (!computeProviderType) { + throw new Error(`Unsupported compute provider type '${String(type)}'`); + } + + return computeProviderType; +} diff --git a/lambdas/libs/runner-providers/providers.config.control-plane.ts b/lambdas/libs/compute-providers/providers.config.control-plane.ts similarity index 50% rename from lambdas/libs/runner-providers/providers.config.control-plane.ts rename to lambdas/libs/compute-providers/providers.config.control-plane.ts index 55ebaca95e..45a584bc06 100644 --- a/lambdas/libs/runner-providers/providers.config.control-plane.ts +++ b/lambdas/libs/compute-providers/providers.config.control-plane.ts @@ -1,5 +1,6 @@ import { provider as ec2 } from './aws/ec2/control-plane'; +import { provider as microvm } from './aws/microvm/control-plane'; import type { ControlPlaneProviderModule } from './contracts'; /** Provider plugins included in the control-plane bundle. */ -export const enabledControlPlaneProviders = [ec2] as const satisfies readonly ControlPlaneProviderModule[]; +export const enabledControlPlaneProviders = [ec2, microvm] as const satisfies readonly ControlPlaneProviderModule[]; diff --git a/lambdas/libs/runner-providers/providers.config.webhook.ts b/lambdas/libs/compute-providers/providers.config.webhook.ts similarity index 50% rename from lambdas/libs/runner-providers/providers.config.webhook.ts rename to lambdas/libs/compute-providers/providers.config.webhook.ts index 19c92734da..a4aec0853a 100644 --- a/lambdas/libs/runner-providers/providers.config.webhook.ts +++ b/lambdas/libs/compute-providers/providers.config.webhook.ts @@ -1,5 +1,6 @@ import { provider as ec2 } from './aws/ec2/webhook'; +import { provider as microvm } from './aws/microvm/webhook'; import type { WebhookProviderModule } from './contracts'; /** Provider plugins included in the webhook bundle. */ -export const enabledWebhookProviders = [ec2] as const satisfies readonly WebhookProviderModule[]; +export const enabledWebhookProviders = [ec2, microvm] as const satisfies readonly WebhookProviderModule[]; diff --git a/lambdas/libs/runner-providers/registry.test.ts b/lambdas/libs/compute-providers/registry.test.ts similarity index 84% rename from lambdas/libs/runner-providers/registry.test.ts rename to lambdas/libs/compute-providers/registry.test.ts index 129986119c..93227831cd 100644 --- a/lambdas/libs/runner-providers/registry.test.ts +++ b/lambdas/libs/compute-providers/registry.test.ts @@ -1,7 +1,7 @@ import { expect, it, vi } from 'vitest'; import { createControlPlaneProviderRegistry } from './control-plane'; -import { runnerProviderTypes } from './provider-types'; +import { computeProviderTypes } from './provider-types'; import { enabledControlPlaneProviders } from './providers.config.control-plane'; import { enabledWebhookProviders } from './providers.config.webhook'; import { webhookProviderRegistry } from './webhook'; @@ -12,10 +12,10 @@ it('exposes every configured provider through both capability registries', () => const controlPlaneTypes = enabledControlPlaneProviders.map(({ type }) => type); const webhookTypes = enabledWebhookProviders.map(({ type }) => type); - expect(controlPlaneTypes).toEqual(runnerProviderTypes); - expect(webhookTypes).toEqual(runnerProviderTypes); + expect(controlPlaneTypes).toEqual(computeProviderTypes); + expect(webhookTypes).toEqual(computeProviderTypes); - for (const type of runnerProviderTypes) { + for (const type of computeProviderTypes) { expect(controlPlaneRegistry.capability(type, 'pool')()).toEqual({ listRunners: expect.any(Function), countAvailableRunners: expect.any(Function), @@ -33,6 +33,6 @@ it('exposes every configured provider through both capability registries', () => unmarkOrphan: expect.any(Function), terminate: expect.any(Function), }); - expect(webhookProviderRegistry.capability(type, 'dynamicLabels').selectQueue).toEqual(expect.any(Function)); + expect(webhookProviderRegistry.capability(type, 'dynamicLabels').getViolations).toEqual(expect.any(Function)); } }); diff --git a/lambdas/libs/runner-providers/templates/provider/README.md b/lambdas/libs/compute-providers/templates/provider/README.md similarity index 76% rename from lambdas/libs/runner-providers/templates/provider/README.md rename to lambdas/libs/compute-providers/templates/provider/README.md index 27fbc09086..68a6d7c5ea 100644 --- a/lambdas/libs/runner-providers/templates/provider/README.md +++ b/lambdas/libs/compute-providers/templates/provider/README.md @@ -1,17 +1,17 @@ -# Runner provider template +# Compute provider template Copy this directory to the appropriate provider namespace, for example -`aws/codebuild`, and replace `template` with the new lane type. +`aws/codebuild`, and replace `template` with the new compute-provider type. The template is compile-checked but intentionally not registered. A provider has separate webhook and control-plane entry points so each Lambda bundles only -the code it uses. To enable a completed provider, add its lane type to +the code it uses. To enable a completed provider, add its compute-provider type to `provider-types.ts`, then register each entry point in its matching file: - `providers.config.webhook.ts` - `providers.config.control-plane.ts` -Each entry point exports its module as `provider`. Alias that export to the lane +Each entry point exports its module as `provider`. Alias that export to the compute-provider name when enabling it, for example: ```ts @@ -21,7 +21,7 @@ import { provider as codebuild } from './aws/codebuild/webhook'; Implement every capability before registering the provider: - `pool`: list managed runners, count available runners, and create runners. -- `scaleUp`: prepare lane state, count current runners, and create runners. +- `scaleUp`: prepare compute-provider state, count current runners, and create runners. - `scaleDown`: list, inspect, mark, unmark, and terminate runners. - `dynamicLabels`: select a webhook dispatch target for supported labels. diff --git a/lambdas/libs/runner-providers/templates/provider/control-plane.ts b/lambdas/libs/compute-providers/templates/provider/control-plane.ts similarity index 87% rename from lambdas/libs/runner-providers/templates/provider/control-plane.ts rename to lambdas/libs/compute-providers/templates/provider/control-plane.ts index b58e02e9f5..418ac0dbf9 100644 --- a/lambdas/libs/runner-providers/templates/provider/control-plane.ts +++ b/lambdas/libs/compute-providers/templates/provider/control-plane.ts @@ -1,9 +1,9 @@ import type { CreateStartRunnerConfig, - PoolRunnerProvider, - RunnerProviderPlugin, - ScaleDownRunnerProvider, - ScaleUpRunnerProvider, + PoolComputeProvider, + ComputeProviderPlugin, + ScaleDownComputeProvider, + ScaleUpComputeProvider, } from '../../core'; import type { ControlPlaneProviderCapabilities, ControlPlaneProviderModule } from '../../contracts'; @@ -13,12 +13,12 @@ export interface TemplateScaleUpState { } function notImplemented(operation: string): never { - throw new Error(`Template runner provider must implement ${operation}`); + throw new Error(`Template compute provider must implement ${operation}`); } export function createTemplatePoolProvider( createStartRunnerConfig: CreateStartRunnerConfig, -): Omit { +): Omit { return { listRunners: async () => notImplemented('pool.listRunners'), countAvailableRunners: () => notImplemented('pool.countAvailableRunners'), @@ -33,7 +33,7 @@ export function createTemplatePoolProvider( export function createTemplateScaleUpProvider( createStartRunnerConfig: CreateStartRunnerConfig, -): Omit { +): Omit { return { resolveLabelsForRunners: async (messageLabels) => { void messageLabels; @@ -54,7 +54,7 @@ export function createTemplateScaleUpProvider( }; } -export function createTemplateScaleDownProvider(): Omit { +export function createTemplateScaleDownProvider(): Omit { return { list: async (environment, orphan) => { void environment; @@ -73,7 +73,7 @@ export function createTemplateScaleDownProvider(): Omit { +): ComputeProviderPlugin { return { type: 'template', capabilities: { diff --git a/lambdas/libs/runner-providers/templates/provider/provider.test.ts b/lambdas/libs/compute-providers/templates/provider/provider.test.ts similarity index 86% rename from lambdas/libs/runner-providers/templates/provider/provider.test.ts rename to lambdas/libs/compute-providers/templates/provider/provider.test.ts index d449de3947..2644fc4f2a 100644 --- a/lambdas/libs/runner-providers/templates/provider/provider.test.ts +++ b/lambdas/libs/compute-providers/templates/provider/provider.test.ts @@ -3,7 +3,7 @@ import { expect, it, vi } from 'vitest'; import { provider as controlPlaneProvider } from './control-plane'; import { provider as webhookProvider } from './webhook'; -it('exposes every runner provider capability from its lane entry point', () => { +it('exposes every compute provider capability from its compute-provider entry point', () => { const controlPlanePlugin = controlPlaneProvider.createPlugin(vi.fn(async () => [])); const pool = controlPlanePlugin.capabilities.pool(); const scaleUp = controlPlanePlugin.capabilities.scaleUp(); @@ -29,5 +29,5 @@ it('exposes every runner provider capability from its lane entry point', () => { terminate: expect.any(Function), }); expect(webhookPlugin.type).toBe(webhookProvider.type); - expect(webhookPlugin.capabilities.dynamicLabels.selectQueue).toEqual(expect.any(Function)); + expect(webhookPlugin.capabilities.dynamicLabels.getViolations).toEqual(expect.any(Function)); }); diff --git a/lambdas/libs/runner-providers/templates/provider/webhook.ts b/lambdas/libs/compute-providers/templates/provider/webhook.ts similarity index 59% rename from lambdas/libs/runner-providers/templates/provider/webhook.ts rename to lambdas/libs/compute-providers/templates/provider/webhook.ts index a782ea8477..31c522c588 100644 --- a/lambdas/libs/runner-providers/templates/provider/webhook.ts +++ b/lambdas/libs/compute-providers/templates/provider/webhook.ts @@ -1,16 +1,16 @@ -import type { RunnerProviderPlugin } from '../../core'; +import type { ComputeProviderPlugin } from '../../core'; import type { DynamicLabelProvider, WebhookProviderCapabilities, WebhookProviderModule } from '../../contracts'; export const templateDynamicLabelProvider: DynamicLabelProvider = { - selectQueue: (input) => { + getViolations: (input) => { void input; - // Return a dispatch target when this provider accepts the requested dynamic labels. - return undefined; + // Return violations for dynamic labels this provider does not accept. + return []; }, }; -export function createTemplateWebhookPlugin(): RunnerProviderPlugin { +export function createTemplateWebhookPlugin(): ComputeProviderPlugin { return { type: 'template', capabilities: { dynamicLabels: templateDynamicLabelProvider }, diff --git a/lambdas/libs/runner-providers/tsconfig.json b/lambdas/libs/compute-providers/tsconfig.json similarity index 100% rename from lambdas/libs/runner-providers/tsconfig.json rename to lambdas/libs/compute-providers/tsconfig.json diff --git a/lambdas/libs/runner-providers/vitest.config.ts b/lambdas/libs/compute-providers/vitest.config.ts similarity index 100% rename from lambdas/libs/runner-providers/vitest.config.ts rename to lambdas/libs/compute-providers/vitest.config.ts diff --git a/lambdas/libs/compute-providers/webhook.test.ts b/lambdas/libs/compute-providers/webhook.test.ts new file mode 100644 index 0000000000..3226533f99 --- /dev/null +++ b/lambdas/libs/compute-providers/webhook.test.ts @@ -0,0 +1,107 @@ +import { describe, expect, it, vi } from 'vitest'; + +import type { DynamicLabelProvider, DynamicLabelViolation, RunnerMatcherConfig } from './contracts'; +import { createDynamicLabelQueueSelector } from './webhook'; + +type TestProvider = 'provider-a' | 'provider-b'; + +describe('createDynamicLabelQueueSelector', () => { + it('returns the first queue accepted by its provider', () => { + const queue = runnerQueue('accepted'); + const { selectQueue } = selector(); + + expect(selectQueue([queue], ['self-hosted', 'linux'], ['ghr-test-size:large'])).toEqual({ + queue, + labels: ['self-hosted', 'linux', 'ghr-test-size:large'], + }); + }); + + it('skips queues that disable dynamic labels', () => { + const disabledQueue = runnerQueue('disabled'); + disabledQueue.matcherConfig.enableDynamicLabels = false; + const enabledQueue = runnerQueue('enabled'); + const { getViolations, selectQueue } = selector(); + + expect(selectQueue([disabledQueue, enabledQueue], ['self-hosted'], ['ghr-test-size:large'])).toEqual({ + queue: enabledQueue, + labels: ['self-hosted', 'ghr-test-size:large'], + }); + expect(getViolations).toHaveBeenCalledOnce(); + expect(getViolations).toHaveBeenCalledWith({ queue: enabledQueue, labels: ['ghr-test-size:large'] }); + }); + + it('skips queues whose provider reports violations', () => { + const rejectedQueue = runnerQueue('rejected'); + const acceptedQueue = runnerQueue('accepted'); + const { selectQueue } = selector({ + violationsByQueue: { + rejected: [{ label: 'ghr-test-size:large', reason: 'size is unavailable' }], + }, + }); + + expect(selectQueue([rejectedQueue, acceptedQueue], ['self-hosted'], ['ghr-test-size:large'])).toEqual({ + queue: acceptedQueue, + labels: ['self-hosted', 'ghr-test-size:large'], + }); + }); + + it('returns undefined when every provider reports violations', () => { + const queue = runnerQueue('rejected'); + const { selectQueue } = selector({ + violationsByQueue: { + rejected: [{ label: 'ghr-test-size:large', reason: 'size is unavailable' }], + }, + }); + + expect(selectQueue([queue], ['self-hosted'], ['ghr-test-size:large'])).toBeUndefined(); + }); + + it('skips queues when labels target another provider', () => { + const firstQueue = runnerQueue('first'); + const secondQueue = runnerQueue('second'); + const { getViolations, selectQueue } = selector({ + providerByQueue: { first: 'provider-a', second: 'provider-b' }, + labelsForOtherProvider: (_labels, provider) => (provider === 'provider-a' ? ['ghr-provider-b-size:large'] : []), + }); + + expect(selectQueue([firstQueue, secondQueue], ['self-hosted'], ['ghr-provider-b-size:large'])).toEqual({ + queue: secondQueue, + labels: ['self-hosted', 'ghr-provider-b-size:large'], + }); + expect(getViolations).toHaveBeenCalledOnce(); + expect(getViolations).toHaveBeenCalledWith({ queue: secondQueue, labels: ['ghr-provider-b-size:large'] }); + }); +}); + +function selector(options?: { + providerByQueue?: Record; + violationsByQueue?: Record; + labelsForOtherProvider?: (labels: string[], provider: TestProvider) => string[]; +}) { + const getViolations = vi.fn(({ queue }) => { + return options?.violationsByQueue?.[queue.id] ?? []; + }); + + return { + getViolations, + selectQueue: createDynamicLabelQueueSelector({ + resolveProvider: (queue) => ({ + type: options?.providerByQueue?.[queue.id] ?? 'provider-a', + dynamicLabels: { getViolations }, + }), + dynamicLabelsForOtherProvider: options?.labelsForOtherProvider ?? (() => []), + }), + }; +} + +function runnerQueue(id: string): RunnerMatcherConfig { + return { + id, + arn: `arn:${id}`, + matcherConfig: { + labelMatchers: [['self-hosted', 'linux']], + exactMatch: true, + enableDynamicLabels: true, + }, + }; +} diff --git a/lambdas/libs/compute-providers/webhook.ts b/lambdas/libs/compute-providers/webhook.ts new file mode 100644 index 0000000000..1aa0a7b5e6 --- /dev/null +++ b/lambdas/libs/compute-providers/webhook.ts @@ -0,0 +1,70 @@ +import { createChildLogger } from '@aws-github-runner/aws-powertools-util'; + +import { createComputeProviderRegistry } from './core'; + +import type { + DynamicLabelDispatchTarget, + DynamicLabelProvider, + RunnerMatcherConfig, + WebhookProviderCapabilities, +} from './contracts'; +import { dynamicLabelsForOtherProvider } from './dynamic-labels'; +import { resolveComputeProviderType } from './provider-types'; +import { enabledWebhookProviders } from './providers.config.webhook'; + +const logger = createChildLogger('handler'); + +export const webhookProviderRegistry = createComputeProviderRegistry( + enabledWebhookProviders.map((provider) => provider.createPlugin()), +); + +export function createDynamicLabelQueueSelector(dependencies: { + resolveProvider(queue: RunnerMatcherConfig): { type: TProvider; dynamicLabels: DynamicLabelProvider }; + dynamicLabelsForOtherProvider(labels: string[], provider: TProvider): string[]; +}) { + return ( + matches: RunnerMatcherConfig[], + nonGhrLabels: string[], + sanitizedGhrLabels: string[], + ): DynamicLabelDispatchTarget | undefined => { + for (const queue of matches) { + const { type: provider, dynamicLabels } = dependencies.resolveProvider(queue); + + if (!queue.matcherConfig.enableDynamicLabels) { + logger.warn( + `Queue ${queue.id} matches non-dynamic labels but does not allow dynamic labels; trying next match`, + ); + continue; + } + + const labelsForOtherProvider = dependencies.dynamicLabelsForOtherProvider(sanitizedGhrLabels, provider); + if (labelsForOtherProvider.length > 0) { + logger.warn(`Queue ${queue.id}: dynamic labels target another compute provider; trying next match`, { + dynamicLabels: labelsForOtherProvider, + }); + continue; + } + + const violations = dynamicLabels.getViolations({ queue, labels: sanitizedGhrLabels }); + if (violations.length === 0) { + return { queue, labels: [...nonGhrLabels, ...sanitizedGhrLabels] }; + } + + for (const violation of violations) { + logger.warn( + `Queue ${queue.id}: dynamic label '${violation.label}' is not accepted (${violation.reason}); trying next match`, + ); + } + } + + return undefined; + }; +} + +export const selectDynamicLabelQueue = createDynamicLabelQueueSelector({ + resolveProvider: (queue) => { + const type = resolveComputeProviderType(queue.computeProvider); + return { type, dynamicLabels: webhookProviderRegistry.capability(type, 'dynamicLabels') }; + }, + dynamicLabelsForOtherProvider, +}); diff --git a/lambdas/libs/runner-providers/aws/ec2/src/webhook/dynamic-labels-policy.ts b/lambdas/libs/runner-providers/aws/ec2/src/webhook/dynamic-labels-policy.ts deleted file mode 100644 index a9b919c7bd..0000000000 --- a/lambdas/libs/runner-providers/aws/ec2/src/webhook/dynamic-labels-policy.ts +++ /dev/null @@ -1,73 +0,0 @@ -import type { AwsDynamicLabelsPolicy, AwsDynamicLabelsValueRule } from '../../../../contracts'; - -export type Ec2DynamicLabelsValueRule = AwsDynamicLabelsValueRule; - -/** - * EC2 dynamic labels policy schema. `blocked_keys` rejects keys outright; - * `restricted_keys` applies optional per-key value rules. Keys use the - * `` segment of a `ghr-ec2-:` label in the same hyphenated - * form as the labels themselves (e.g. `instance-type`). - */ -export type Ec2DynamicLabelsPolicy = AwsDynamicLabelsPolicy; - -function globToRegExp(glob: string): RegExp { - const escaped = glob.replace(/[.+^${}()|[\]\\]/g, '\\$&'); - const pattern = escaped.replace(/\*/g, '.*').replace(/\?/g, '.'); - return new RegExp(`^${pattern}$`); -} - -function matchesAny(value: string, patterns: string[] | undefined): boolean { - if (!patterns || patterns.length === 0) return false; - return patterns.some((p) => globToRegExp(p).test(value)); -} - -function evaluateLabel(label: string, policy: Ec2DynamicLabelsPolicy): string | null { - const stripped = label.replace(/^ghr-ec2-/, ''); - const colonIdx = stripped.indexOf(':'); - const key = colonIdx === -1 ? stripped : stripped.slice(0, colonIdx); - const value = colonIdx === -1 ? undefined : stripped.slice(colonIdx + 1); - - if (policy.blocked_keys?.includes(key)) { - return `key '${key}' is in blocked_keys`; - } - - const rule = policy.restricted_keys?.[key]; - if (!rule) return null; - if (value === undefined) return null; - - if (rule.allowed && rule.allowed.length > 0 && !matchesAny(value, rule.allowed)) { - return `value '${value}' not in allowed list`; - } - if (rule.denied && matchesAny(value, rule.denied)) { - return `value '${value}' in denied list`; - } - if (rule.max !== undefined && rule.max !== null) { - const valueNum = Number(value); - const maxNum = Number(rule.max); - if (!Number.isFinite(valueNum) || !Number.isFinite(maxNum)) { - return `max set but value '${value}' or max '${rule.max}' is not numeric`; - } - if (valueNum > maxNum) { - return `value '${value}' exceeds max '${rule.max}'`; - } - } - return null; -} - -/** - * Inspects the labels and returns the rejection reasons for any `ghr-ec2-*` - * label that violates the policy. Non-`ghr-ec2-*` labels are ignored. - */ -export function violationsAgainstPolicy( - labels: string[], - policy: Ec2DynamicLabelsPolicy | null | undefined, -): { label: string; reason: string }[] { - if (!policy) return []; - const violations: { label: string; reason: string }[] = []; - for (const label of labels) { - if (!label.startsWith('ghr-ec2-')) continue; - const reason = evaluateLabel(label, policy); - if (reason) violations.push({ label, reason }); - } - return violations; -} diff --git a/lambdas/libs/runner-providers/aws/ec2/src/webhook/dynamic-labels.ts b/lambdas/libs/runner-providers/aws/ec2/src/webhook/dynamic-labels.ts deleted file mode 100644 index 6ddf5b8fbb..0000000000 --- a/lambdas/libs/runner-providers/aws/ec2/src/webhook/dynamic-labels.ts +++ /dev/null @@ -1,58 +0,0 @@ -import { createChildLogger } from '@aws-github-runner/aws-powertools-util'; - -import type { DynamicLabelDispatchTarget, DynamicLabelProvider, RunnerMatcherConfig } from '../../../../contracts'; -import { violationsAgainstPolicy } from './dynamic-labels-policy'; - -const logger = createChildLogger('handler'); - -export type Ec2DynamicLabelDispatchTarget = DynamicLabelDispatchTarget; - -function resolveEc2DynamicLabelsPolicy(queue: RunnerMatcherConfig) { - const hasLegacyEc2DynamicLabelsPolicy = Object.prototype.hasOwnProperty.call( - queue.matcherConfig, - 'ec2DynamicLabelsPolicy', - ); - - if (queue.matcherConfig.awsDynamicLabelsPolicy == null && hasLegacyEc2DynamicLabelsPolicy) { - logger.warn( - `Queue ${queue.id}: using deprecated matcherConfig.ec2DynamicLabelsPolicy; migrate to matcherConfig.awsDynamicLabelsPolicy`, - ); - return queue.matcherConfig.ec2DynamicLabelsPolicy; - } - - return queue.matcherConfig.awsDynamicLabelsPolicy; -} - -export function selectEc2DynamicLabelQueue( - matches: RunnerMatcherConfig[], - nonGhrLabels: string[], - sanitizedGhrLabels: string[], -): Ec2DynamicLabelDispatchTarget | undefined { - for (const queue of matches) { - if (!queue.matcherConfig.enableDynamicLabels) { - logger.warn(`Queue ${queue.id} matches non-dynamic labels but does not allow dynamic labels; trying next match`); - continue; - } - - const violations = violationsAgainstPolicy(sanitizedGhrLabels, resolveEc2DynamicLabelsPolicy(queue)); - if (violations.length === 0) { - return { - queue, - labels: [...nonGhrLabels, ...sanitizedGhrLabels], - }; - } - - for (const violation of violations) { - logger.warn( - `Queue ${queue.id}: dynamic label '${violation.label}' does not match policy (${violation.reason}); trying next match`, - ); - } - } - - return undefined; -} - -export const ec2DynamicLabelProvider: DynamicLabelProvider = { - selectQueue: ({ queue, nonGhrLabels, sanitizedGhrLabels }) => - selectEc2DynamicLabelQueue([queue], nonGhrLabels, sanitizedGhrLabels), -}; diff --git a/lambdas/libs/runner-providers/provider-types.test.ts b/lambdas/libs/runner-providers/provider-types.test.ts deleted file mode 100644 index d87a689716..0000000000 --- a/lambdas/libs/runner-providers/provider-types.test.ts +++ /dev/null @@ -1,44 +0,0 @@ -import { describe, expect, it } from 'vitest'; - -import { - defaultRunnerProvider, - normalizeRunnerProviderType, - resolveRunnerProviderType, - runnerProviderTypes, -} from './provider-types'; - -describe('runner provider configuration', () => { - it('defines an explicit default provider', () => { - expect(runnerProviderTypes).toContain(defaultRunnerProvider); - }); -}); - -describe('runner provider normalization', () => { - it.each([ - [undefined, 'ec2'], - ['', 'ec2'], - [' ', 'ec2'], - [' EC2 ', 'ec2'], - ])('normalizes provider type %j to %j', (type, expected) => { - expect(normalizeRunnerProviderType(type)).toBe(expected); - }); - - it.each([[' Unknown '], ['microvm'], [null], [1]])('returns undefined for unsupported provider type %j', (type) => { - expect(normalizeRunnerProviderType(type)).toBeUndefined(); - }); -}); - -describe('runner provider resolution', () => { - it.each([ - [undefined, 'ec2'], - ['', 'ec2'], - [' ', 'ec2'], - [' EC2 ', 'ec2'], - ])('resolves provider type %j to %j', (type, expected) => { - expect(resolveRunnerProviderType(type)).toBe(expected); - }); - - it.each([[' Unknown '], ['microvm'], [null], [1]])('rejects unsupported provider type %j', (type) => { - expect(() => resolveRunnerProviderType(type)).toThrow(`Unsupported runner provider type '${String(type)}'`); - }); -}); diff --git a/lambdas/libs/runner-providers/provider-types.ts b/lambdas/libs/runner-providers/provider-types.ts deleted file mode 100644 index 236a750544..0000000000 --- a/lambdas/libs/runner-providers/provider-types.ts +++ /dev/null @@ -1,24 +0,0 @@ -export const runnerProviderTypes = ['ec2'] as const; - -export type RunnerProviderType = (typeof runnerProviderTypes)[number]; - -export const defaultRunnerProvider = 'ec2' satisfies RunnerProviderType; - -export function normalizeRunnerProviderType(type: unknown): RunnerProviderType | undefined { - if (type === undefined) return defaultRunnerProvider; - if (typeof type !== 'string') return undefined; - - const normalizedType = type.trim().toLowerCase(); - if (!normalizedType) return defaultRunnerProvider; - - return runnerProviderTypes.find((runnerProviderType) => runnerProviderType === normalizedType); -} - -export function resolveRunnerProviderType(type: unknown): RunnerProviderType { - const normalizedType = normalizeRunnerProviderType(type); - if (!normalizedType) { - throw new Error(`Unsupported runner provider type '${String(type)}'`); - } - - return normalizedType; -} diff --git a/lambdas/libs/runner-providers/webhook.ts b/lambdas/libs/runner-providers/webhook.ts deleted file mode 100644 index e79e4624f9..0000000000 --- a/lambdas/libs/runner-providers/webhook.ts +++ /dev/null @@ -1,8 +0,0 @@ -import { createRunnerProviderRegistry } from './core'; - -import type { WebhookProviderCapabilities } from './contracts'; -import { enabledWebhookProviders } from './providers.config.webhook'; - -export const webhookProviderRegistry = createRunnerProviderRegistry( - enabledWebhookProviders.map((provider) => provider.createPlugin()), -); diff --git a/lambdas/yarn.lock b/lambdas/yarn.lock index 38cce704ba..3bd86d6ada 100644 --- a/lambdas/yarn.lock +++ b/lambdas/yarn.lock @@ -141,13 +141,30 @@ __metadata: languageName: unknown linkType: soft +"@aws-github-runner/compute-providers@npm:*, @aws-github-runner/compute-providers@workspace:libs/compute-providers": + version: 0.0.0-use.local + resolution: "@aws-github-runner/compute-providers@workspace:libs/compute-providers" + dependencies: + "@aws-github-runner/aws-powertools-util": "npm:*" + "@aws-github-runner/aws-ssm-util": "npm:*" + "@aws-sdk/client-ec2": "npm:^3.1009.0" + "@aws-sdk/client-lambda-microvms": "npm:^3.1074.0" + "@octokit/rest": "npm:22.0.1" + aws-sdk-client-mock: "npm:^4.1.0" + aws-sdk-client-mock-jest: "npm:^4.1.0" + moment: "npm:2.29.4" + nock: "npm:^14.0.10" + yn: "npm:3.1.1" + languageName: unknown + linkType: soft + "@aws-github-runner/control-plane@workspace:functions/control-plane": version: 0.0.0-use.local resolution: "@aws-github-runner/control-plane@workspace:functions/control-plane" dependencies: "@aws-github-runner/aws-powertools-util": "npm:*" "@aws-github-runner/aws-ssm-util": "npm:*" - "@aws-github-runner/runner-providers": "npm:*" + "@aws-github-runner/compute-providers": "npm:*" "@aws-lambda-powertools/parameters": "npm:^2.31.0" "@aws-sdk/client-ec2": "npm:^3.1009.0" "@aws-sdk/client-sqs": "npm:^3.1009.0" @@ -193,22 +210,6 @@ __metadata: languageName: unknown linkType: soft -"@aws-github-runner/runner-providers@npm:*, @aws-github-runner/runner-providers@workspace:libs/runner-providers": - version: 0.0.0-use.local - resolution: "@aws-github-runner/runner-providers@workspace:libs/runner-providers" - dependencies: - "@aws-github-runner/aws-powertools-util": "npm:*" - "@aws-github-runner/aws-ssm-util": "npm:*" - "@aws-sdk/client-ec2": "npm:^3.1009.0" - "@octokit/rest": "npm:22.0.1" - aws-sdk-client-mock: "npm:^4.1.0" - aws-sdk-client-mock-jest: "npm:^4.1.0" - moment: "npm:2.29.4" - nock: "npm:^14.0.10" - yn: "npm:3.1.1" - languageName: unknown - linkType: soft - "@aws-github-runner/termination-watcher@workspace:functions/termination-watcher": version: 0.0.0-use.local resolution: "@aws-github-runner/termination-watcher@workspace:functions/termination-watcher" @@ -238,7 +239,7 @@ __metadata: dependencies: "@aws-github-runner/aws-powertools-util": "npm:*" "@aws-github-runner/aws-ssm-util": "npm:*" - "@aws-github-runner/runner-providers": "npm:*" + "@aws-github-runner/compute-providers": "npm:*" "@aws-sdk/client-eventbridge": "npm:^3.1009.0" "@aws-sdk/client-sqs": "npm:^3.1009.0" "@middy/core": "npm:^6.4.5" @@ -439,6 +440,22 @@ __metadata: languageName: node linkType: hard +"@aws-sdk/client-lambda-microvms@npm:^3.1074.0": + version: 3.1104.0 + resolution: "@aws-sdk/client-lambda-microvms@npm:3.1104.0" + dependencies: + "@aws-sdk/core": "npm:^3.977.6" + "@aws-sdk/credential-provider-node": "npm:^3.972.78" + "@aws-sdk/types": "npm:^3.974.2" + "@smithy/core": "npm:^3.31.1" + "@smithy/fetch-http-handler": "npm:^5.6.13" + "@smithy/node-http-handler": "npm:^4.9.13" + "@smithy/types": "npm:^4.16.1" + tslib: "npm:^2.6.2" + checksum: 10c0/219ad52f822def4caa4a20d8d91d46a1b78e6726363a145be86c37cdeef4e4c13653e8a59ada67154146c6c2554e2c12944efad35c689850bf6f72f2d55246f4 + languageName: node + linkType: hard + "@aws-sdk/client-s3@npm:^3.1009.0": version: 3.1014.0 resolution: "@aws-sdk/client-s3@npm:3.1014.0" @@ -620,6 +637,22 @@ __metadata: languageName: node linkType: hard +"@aws-sdk/core@npm:^3.977.6": + version: 3.977.6 + resolution: "@aws-sdk/core@npm:3.977.6" + dependencies: + "@aws-sdk/types": "npm:^3.974.2" + "@aws-sdk/xml-builder": "npm:^3.972.37" + "@aws/lambda-invoke-store": "npm:^0.3.0" + "@smithy/core": "npm:^3.31.1" + "@smithy/signature-v4": "npm:^5.6.12" + "@smithy/types": "npm:^4.16.1" + bowser: "npm:^2.11.0" + tslib: "npm:^2.6.2" + checksum: 10c0/4d743603bb41aeed426e2928be0947202191c341f9fbefe9ea347b0b4b7154b1ea94189d01c8abf3b03b9635449e2e7c268bd67379ea2294b9f49a61b909b9af + languageName: node + linkType: hard + "@aws-sdk/crc64-nvme@npm:^3.972.5": version: 3.972.5 resolution: "@aws-sdk/crc64-nvme@npm:3.972.5" @@ -643,6 +676,19 @@ __metadata: languageName: node linkType: hard +"@aws-sdk/credential-provider-env@npm:^3.972.67": + version: 3.972.67 + resolution: "@aws-sdk/credential-provider-env@npm:3.972.67" + dependencies: + "@aws-sdk/core": "npm:^3.977.6" + "@aws-sdk/types": "npm:^3.974.2" + "@smithy/core": "npm:^3.31.1" + "@smithy/types": "npm:^4.16.1" + tslib: "npm:^2.6.2" + checksum: 10c0/547bcac01ac0912d0e42bb11f7d51bafcf2eaab1db35a098bea2be322211a86457ea60455a5294e58081c32376c240b07e66f81946a9be30e9722f723c6eaac2 + languageName: node + linkType: hard + "@aws-sdk/credential-provider-http@npm:^3.972.23": version: 3.972.23 resolution: "@aws-sdk/credential-provider-http@npm:3.972.23" @@ -661,6 +707,21 @@ __metadata: languageName: node linkType: hard +"@aws-sdk/credential-provider-http@npm:^3.972.69": + version: 3.972.69 + resolution: "@aws-sdk/credential-provider-http@npm:3.972.69" + dependencies: + "@aws-sdk/core": "npm:^3.977.6" + "@aws-sdk/types": "npm:^3.974.2" + "@smithy/core": "npm:^3.31.1" + "@smithy/fetch-http-handler": "npm:^5.6.13" + "@smithy/node-http-handler": "npm:^4.9.13" + "@smithy/types": "npm:^4.16.1" + tslib: "npm:^2.6.2" + checksum: 10c0/6e4cf9628919163a2a9784bf8618bc85a8c0ba7056813bedb9758c04eb3b36663f5099cfad329f89ac86c4e408bb3d0698ee7cff7e4a61c8a0335ab98078d678 + languageName: node + linkType: hard + "@aws-sdk/credential-provider-ini@npm:^3.972.23": version: 3.972.23 resolution: "@aws-sdk/credential-provider-ini@npm:3.972.23" @@ -683,6 +744,27 @@ __metadata: languageName: node linkType: hard +"@aws-sdk/credential-provider-ini@npm:^3.973.12": + version: 3.973.12 + resolution: "@aws-sdk/credential-provider-ini@npm:3.973.12" + dependencies: + "@aws-sdk/core": "npm:^3.977.6" + "@aws-sdk/credential-provider-env": "npm:^3.972.67" + "@aws-sdk/credential-provider-http": "npm:^3.972.69" + "@aws-sdk/credential-provider-login": "npm:^3.972.74" + "@aws-sdk/credential-provider-process": "npm:^3.972.67" + "@aws-sdk/credential-provider-sso": "npm:^3.973.11" + "@aws-sdk/credential-provider-web-identity": "npm:^3.972.73" + "@aws-sdk/nested-clients": "npm:^3.997.41" + "@aws-sdk/types": "npm:^3.974.2" + "@smithy/core": "npm:^3.31.1" + "@smithy/credential-provider-imds": "npm:^4.4.16" + "@smithy/types": "npm:^4.16.1" + tslib: "npm:^2.6.2" + checksum: 10c0/84646fee1c61e31b2052d902559ecf163c1d00558ecdc21d77b396250527348b9ebba324d0bf8ffee4b3e45476c691de502e6faad3d39d1f7420eee5d326c7c5 + languageName: node + linkType: hard + "@aws-sdk/credential-provider-login@npm:^3.972.23": version: 3.972.23 resolution: "@aws-sdk/credential-provider-login@npm:3.972.23" @@ -699,6 +781,20 @@ __metadata: languageName: node linkType: hard +"@aws-sdk/credential-provider-login@npm:^3.972.74": + version: 3.972.74 + resolution: "@aws-sdk/credential-provider-login@npm:3.972.74" + dependencies: + "@aws-sdk/core": "npm:^3.977.6" + "@aws-sdk/nested-clients": "npm:^3.997.41" + "@aws-sdk/types": "npm:^3.974.2" + "@smithy/core": "npm:^3.31.1" + "@smithy/types": "npm:^4.16.1" + tslib: "npm:^2.6.2" + checksum: 10c0/1ab9996accb61bccbdaefae023e9befab9e5062435370a37f672089457dc13c485d9d2fee6926381672bdb32143c00266b1aa5913b18ef4873584907835b3a92 + languageName: node + linkType: hard + "@aws-sdk/credential-provider-node@npm:^3.972.24": version: 3.972.24 resolution: "@aws-sdk/credential-provider-node@npm:3.972.24" @@ -719,6 +815,25 @@ __metadata: languageName: node linkType: hard +"@aws-sdk/credential-provider-node@npm:^3.972.78": + version: 3.972.78 + resolution: "@aws-sdk/credential-provider-node@npm:3.972.78" + dependencies: + "@aws-sdk/credential-provider-env": "npm:^3.972.67" + "@aws-sdk/credential-provider-http": "npm:^3.972.69" + "@aws-sdk/credential-provider-ini": "npm:^3.973.12" + "@aws-sdk/credential-provider-process": "npm:^3.972.67" + "@aws-sdk/credential-provider-sso": "npm:^3.973.11" + "@aws-sdk/credential-provider-web-identity": "npm:^3.972.73" + "@aws-sdk/types": "npm:^3.974.2" + "@smithy/core": "npm:^3.31.1" + "@smithy/credential-provider-imds": "npm:^4.4.16" + "@smithy/types": "npm:^4.16.1" + tslib: "npm:^2.6.2" + checksum: 10c0/2b6e5bd455a3c2b530a884a0c5919bb7d2d91941b655a56351b957de038d318c1d42b86674e20893ee7dab6db6ea32c4653bce9b1d3ca98ac803d6b49a948343 + languageName: node + linkType: hard + "@aws-sdk/credential-provider-process@npm:^3.972.21": version: 3.972.21 resolution: "@aws-sdk/credential-provider-process@npm:3.972.21" @@ -733,6 +848,19 @@ __metadata: languageName: node linkType: hard +"@aws-sdk/credential-provider-process@npm:^3.972.67": + version: 3.972.67 + resolution: "@aws-sdk/credential-provider-process@npm:3.972.67" + dependencies: + "@aws-sdk/core": "npm:^3.977.6" + "@aws-sdk/types": "npm:^3.974.2" + "@smithy/core": "npm:^3.31.1" + "@smithy/types": "npm:^4.16.1" + tslib: "npm:^2.6.2" + checksum: 10c0/0381c39f171df2119791b03647545ab5084f6a8d2c227c5d3c5bfa9db027d0566102b6322bc09075c0779b0f0aa88ae1ca7bbdc773d8415d8dd8f163e69e45ea + languageName: node + linkType: hard + "@aws-sdk/credential-provider-sso@npm:^3.972.23": version: 3.972.23 resolution: "@aws-sdk/credential-provider-sso@npm:3.972.23" @@ -749,6 +877,21 @@ __metadata: languageName: node linkType: hard +"@aws-sdk/credential-provider-sso@npm:^3.973.11": + version: 3.973.11 + resolution: "@aws-sdk/credential-provider-sso@npm:3.973.11" + dependencies: + "@aws-sdk/core": "npm:^3.977.6" + "@aws-sdk/nested-clients": "npm:^3.997.41" + "@aws-sdk/token-providers": "npm:3.1103.0" + "@aws-sdk/types": "npm:^3.974.2" + "@smithy/core": "npm:^3.31.1" + "@smithy/types": "npm:^4.16.1" + tslib: "npm:^2.6.2" + checksum: 10c0/d6df0ae72009c2f74f1c7f12e41c0a7b395ba1860d4f9f1554fd8fbc3b5f0c1be64c83aacd2b329561e27844520ae0b57bb268265f5e7850b1d0fb769455d7a1 + languageName: node + linkType: hard + "@aws-sdk/credential-provider-web-identity@npm:^3.972.23": version: 3.972.23 resolution: "@aws-sdk/credential-provider-web-identity@npm:3.972.23" @@ -764,6 +907,20 @@ __metadata: languageName: node linkType: hard +"@aws-sdk/credential-provider-web-identity@npm:^3.972.73": + version: 3.972.73 + resolution: "@aws-sdk/credential-provider-web-identity@npm:3.972.73" + dependencies: + "@aws-sdk/core": "npm:^3.977.6" + "@aws-sdk/nested-clients": "npm:^3.997.41" + "@aws-sdk/types": "npm:^3.974.2" + "@smithy/core": "npm:^3.31.1" + "@smithy/types": "npm:^4.16.1" + tslib: "npm:^2.6.2" + checksum: 10c0/a7bee06b4200ff04141d4ce07d49d69f24b55b57f3aab929b445a9d9ea70f043d0b09fca8b95872d2b92df6837b771aeb14b03ca784d17ce1ee871b14173558c + languageName: node + linkType: hard + "@aws-sdk/lib-storage@npm:^3.1009.0": version: 3.1014.0 resolution: "@aws-sdk/lib-storage@npm:3.1014.0" @@ -1002,6 +1159,22 @@ __metadata: languageName: node linkType: hard +"@aws-sdk/nested-clients@npm:^3.997.41": + version: 3.997.41 + resolution: "@aws-sdk/nested-clients@npm:3.997.41" + dependencies: + "@aws-sdk/core": "npm:^3.977.6" + "@aws-sdk/signature-v4-multi-region": "npm:^3.996.43" + "@aws-sdk/types": "npm:^3.974.2" + "@smithy/core": "npm:^3.31.1" + "@smithy/fetch-http-handler": "npm:^5.6.13" + "@smithy/node-http-handler": "npm:^4.9.13" + "@smithy/types": "npm:^4.16.1" + tslib: "npm:^2.6.2" + checksum: 10c0/fe1a84bb58675a24ecd0ce3b7bcaf1a456494f10c1a9dd5b55bd268be6713f83bd3c3d3dadee6bb51a53bc24ee18b2b0882d6741bcbabc224feeae97454fdb4a + languageName: node + linkType: hard + "@aws-sdk/region-config-resolver@npm:^3.972.9": version: 3.972.9 resolution: "@aws-sdk/region-config-resolver@npm:3.972.9" @@ -1029,6 +1202,18 @@ __metadata: languageName: node linkType: hard +"@aws-sdk/signature-v4-multi-region@npm:^3.996.43": + version: 3.996.43 + resolution: "@aws-sdk/signature-v4-multi-region@npm:3.996.43" + dependencies: + "@aws-sdk/types": "npm:^3.974.2" + "@smithy/signature-v4": "npm:^5.6.12" + "@smithy/types": "npm:^4.16.1" + tslib: "npm:^2.6.2" + checksum: 10c0/268608dd5624c6377243903d588b9c13b8de3f3f3e6bea68fc684d125bc92a991fd15a67cb178d1a7a599d0415ce5283f44ba6b96d14909b185d7ff26a9d979b + languageName: node + linkType: hard + "@aws-sdk/token-providers@npm:3.1014.0": version: 3.1014.0 resolution: "@aws-sdk/token-providers@npm:3.1014.0" @@ -1044,6 +1229,20 @@ __metadata: languageName: node linkType: hard +"@aws-sdk/token-providers@npm:3.1103.0": + version: 3.1103.0 + resolution: "@aws-sdk/token-providers@npm:3.1103.0" + dependencies: + "@aws-sdk/core": "npm:^3.977.6" + "@aws-sdk/nested-clients": "npm:^3.997.41" + "@aws-sdk/types": "npm:^3.974.2" + "@smithy/core": "npm:^3.31.1" + "@smithy/types": "npm:^4.16.1" + tslib: "npm:^2.6.2" + checksum: 10c0/5f86aa221e537b8a3fd11ed76ac025935f8859cc62b3af293abd759b8ca3aa390c17f6716723c08056b3373997a17bbdee2ff568eab5049bf0a372d35893b48d + languageName: node + linkType: hard + "@aws-sdk/types@npm:^3.222.0, @aws-sdk/types@npm:^3.4.1, @aws-sdk/types@npm:^3.973.6": version: 3.973.6 resolution: "@aws-sdk/types@npm:3.973.6" @@ -1054,6 +1253,16 @@ __metadata: languageName: node linkType: hard +"@aws-sdk/types@npm:^3.974.2": + version: 3.974.2 + resolution: "@aws-sdk/types@npm:3.974.2" + dependencies: + "@smithy/types": "npm:^4.16.1" + tslib: "npm:^2.6.2" + checksum: 10c0/b5ce05e8a4160c545edce1e8527e8ac490be7a6651c736f6811190b5d31d5682699889d51186ab0600df756679bebd2df9d650a17f577523441df803c4fb5777 + languageName: node + linkType: hard + "@aws-sdk/util-arn-parser@npm:^3.972.3": version: 3.972.3 resolution: "@aws-sdk/util-arn-parser@npm:3.972.3" @@ -1139,6 +1348,16 @@ __metadata: languageName: node linkType: hard +"@aws-sdk/xml-builder@npm:^3.972.37": + version: 3.972.37 + resolution: "@aws-sdk/xml-builder@npm:3.972.37" + dependencies: + "@smithy/types": "npm:^4.16.1" + tslib: "npm:^2.6.2" + checksum: 10c0/738f9302f495b3b95602641166a4182244add6e9e079201dba7e8994657dd442df0e4cea3355aa8c7d7f08efb385decaaf0b543f03efdb291c118536f36ac1a1 + languageName: node + linkType: hard + "@aws/lambda-invoke-store@npm:0.2.3, @aws/lambda-invoke-store@npm:^0.2.2": version: 0.2.3 resolution: "@aws/lambda-invoke-store@npm:0.2.3" @@ -1146,6 +1365,13 @@ __metadata: languageName: node linkType: hard +"@aws/lambda-invoke-store@npm:^0.3.0": + version: 0.3.0 + resolution: "@aws/lambda-invoke-store@npm:0.3.0" + checksum: 10c0/b4a2e6b3b5397bc606053e64270d26dc5c886336f88a98cad587b1592eec17058f8fb172f1827a9f0e591f3595cf8f01575c8c9b36cde38c06456f8a65204046 + languageName: node + linkType: hard + "@babel/code-frame@npm:^7.0.0, @babel/code-frame@npm:^7.12.13, @babel/code-frame@npm:^7.23.5, @babel/code-frame@npm:^7.28.6, @babel/code-frame@npm:^7.29.0": version: 7.29.0 resolution: "@babel/code-frame@npm:7.29.0" @@ -4404,6 +4630,16 @@ __metadata: languageName: node linkType: hard +"@smithy/core@npm:^3.31.1": + version: 3.31.1 + resolution: "@smithy/core@npm:3.31.1" + dependencies: + "@smithy/types": "npm:^4.16.1" + tslib: "npm:^2.6.2" + checksum: 10c0/b953c792dea2c13249b58c1799e4d6aaf21eb1a61e203b83e8e3a9156bebe14ca0585f0ca1ffdf65a193294dddff92a06fbe5c3fbd63ff0c174c88130b47a128 + languageName: node + linkType: hard + "@smithy/credential-provider-imds@npm:^4.2.12": version: 4.2.12 resolution: "@smithy/credential-provider-imds@npm:4.2.12" @@ -4417,6 +4653,17 @@ __metadata: languageName: node linkType: hard +"@smithy/credential-provider-imds@npm:^4.4.16": + version: 4.4.16 + resolution: "@smithy/credential-provider-imds@npm:4.4.16" + dependencies: + "@smithy/core": "npm:^3.31.1" + "@smithy/types": "npm:^4.16.1" + tslib: "npm:^2.6.2" + checksum: 10c0/d03687efbbd1f95e77b7dcb639f24f1600671929627cd743f7acf9640238746664e91f955026f22e235603e10537d46e31fa60f231adbdf37457e53720bc80f9 + languageName: node + linkType: hard + "@smithy/eventstream-codec@npm:^4.2.12": version: 4.2.12 resolution: "@smithy/eventstream-codec@npm:4.2.12" @@ -4485,6 +4732,17 @@ __metadata: languageName: node linkType: hard +"@smithy/fetch-http-handler@npm:^5.6.13": + version: 5.6.13 + resolution: "@smithy/fetch-http-handler@npm:5.6.13" + dependencies: + "@smithy/core": "npm:^3.31.1" + "@smithy/types": "npm:^4.16.1" + tslib: "npm:^2.6.2" + checksum: 10c0/028ba8794a6c487ebefae7f40d0124f70e51a1f4e0e465457845c1a44fd607320cd3c64d4a961f159aef59470f0fd43f0d2011b44ee5ef753b7e1dccbdf32ca3 + languageName: node + linkType: hard + "@smithy/hash-blob-browser@npm:^4.2.13": version: 4.2.13 resolution: "@smithy/hash-blob-browser@npm:4.2.13" @@ -4650,6 +4908,17 @@ __metadata: languageName: node linkType: hard +"@smithy/node-http-handler@npm:^4.9.13": + version: 4.9.13 + resolution: "@smithy/node-http-handler@npm:4.9.13" + dependencies: + "@smithy/core": "npm:^3.31.1" + "@smithy/types": "npm:^4.16.1" + tslib: "npm:^2.6.2" + checksum: 10c0/2f1cdef7a300ad49c3bb698c2ca4773af5e9202d291cfcd855c1b21ab08b3c4ddf56f3722d3251db4e9b7ac39ec1ebc551b156abf3fa70f74c5491bec421f6b5 + languageName: node + linkType: hard + "@smithy/property-provider@npm:^4.2.12": version: 4.2.12 resolution: "@smithy/property-provider@npm:4.2.12" @@ -4735,6 +5004,17 @@ __metadata: languageName: node linkType: hard +"@smithy/signature-v4@npm:^5.6.12": + version: 5.6.12 + resolution: "@smithy/signature-v4@npm:5.6.12" + dependencies: + "@smithy/core": "npm:^3.31.1" + "@smithy/types": "npm:^4.16.1" + tslib: "npm:^2.6.2" + checksum: 10c0/33656a41ad61dee16209703cb96b46b29014b3c4fad23bfbb90cdb5415ac06c6577b2bfff958ef9e6c19091364945135a0370b12ddc2daed557c903846e81fe7 + languageName: node + linkType: hard + "@smithy/smithy-client@npm:^4.12.7": version: 4.12.7 resolution: "@smithy/smithy-client@npm:4.12.7" @@ -4768,6 +5048,15 @@ __metadata: languageName: node linkType: hard +"@smithy/types@npm:^4.16.1": + version: 4.16.1 + resolution: "@smithy/types@npm:4.16.1" + dependencies: + tslib: "npm:^2.6.2" + checksum: 10c0/e024d9d148deca7bd21d032a9316db109bbe7cf256ffbb8d3981655b9f4f7695c08ec9b87f5a8cf1442e783ba26cb27e4f09603c5bfa3ba1e526c41b1b3e94d2 + languageName: node + linkType: hard + "@smithy/url-parser@npm:^4.2.12": version: 4.2.12 resolution: "@smithy/url-parser@npm:4.2.12" diff --git a/mkdocs.yaml b/mkdocs.yaml index 9b98e84a36..6ec2922a2c 100644 --- a/mkdocs.yaml +++ b/mkdocs.yaml @@ -65,6 +65,7 @@ nav: - Lambda Downloader: modules/public/download-lambda.md - Setup IAM permissions: modules/public/setup-iam-permissions.md - Submodules (internal): + - Compute provider refactor (experimental): modules/internal/compute-provider-refactor.md - Runners: modules/internal/runners.md - Syncer: modules/internal/runner-binaries-syncer.md - SSM: modules/internal/ssm.md diff --git a/modules/compute-providers/ec2/README.md b/modules/compute-providers/ec2/README.md new file mode 100644 index 0000000000..7ac103d23c --- /dev/null +++ b/modules/compute-providers/ec2/README.md @@ -0,0 +1,80 @@ +# EC2 runner provider + +This internal module owns the EC2 compute implementation used by the common runner stack. It creates the runner launch template, security group, instance profile, EC2 bootstrap parameters, and runner log groups. + +The module returns one nested `provider` contract. It groups EC2-specific Lambda settings under `environment_variables`, permission requirements under `policies.runner`, `policies.scale_up`, `policies.scale_down`, and `policies.pool`, and EC2 artifacts under `resources`. The parent stack owns the shared runner role, provider-policy attachments, Lambda functions, execution roles, schedules, queues, retry flow, and SSM housekeeper. + +EC2 is the only active compute provider. The parent stack selects it when `ec2` is the one populated typed block under `compute_provider`; no separate type input is required. A future provider must add its own typed block and implement the same contracts before it can be selected. + + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 1.4.0 | +| [aws](#requirement\_aws) | >= 6.33 | + +## Providers + +| Name | Version | +|------|---------| +| [aws](#provider\_aws) | >= 6.33 | +| [terraform](#provider\_terraform) | n/a | + +## Modules + +No modules. + +## Resources + +| Name | Type | +|------|------| +| [aws_cloudwatch_log_group.gh_runners](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/cloudwatch_log_group) | resource | +| [aws_iam_instance_profile.runner](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_instance_profile) | resource | +| [aws_iam_policy.ami_id_ssm_parameter_read](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_policy) | resource | +| [aws_launch_template.runner](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/launch_template) | resource | +| [aws_security_group.runner_sg](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/security_group) | resource | +| [aws_ssm_parameter.cloudwatch_agent_config_runner](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/ssm_parameter) | resource | +| [aws_ssm_parameter.runner_ami_id](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/ssm_parameter) | resource | +| [aws_ssm_parameter.runner_config_run_as](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/ssm_parameter) | resource | +| [aws_ssm_parameter.runner_enable_cloudwatch](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/ssm_parameter) | resource | +| [terraform_data.validate_config](https://registry.terraform.io/providers/hashicorp/terraform/latest/docs/resources/data) | resource | +| [terraform_data.validate_runner](https://registry.terraform.io/providers/hashicorp/terraform/latest/docs/resources/data) | resource | +| [aws_ami.runner](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/ami) | data source | +| [aws_caller_identity.current](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/caller_identity) | data source | +| [aws_iam_policy_document.ami_id_ssm_parameter_read](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source | +| [aws_iam_policy_document.cloudwatch](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source | +| [aws_iam_policy_document.create_tags](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source | +| [aws_iam_policy_document.describe_tags](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source | +| [aws_iam_policy_document.distribution_bucket](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source | +| [aws_iam_policy_document.pool](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source | +| [aws_iam_policy_document.scale_down](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source | +| [aws_iam_policy_document.scale_up](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source | +| [aws_iam_policy_document.service_linked_role](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source | +| [aws_iam_policy_document.session_manager](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source | +| [aws_iam_policy_document.ssm_parameters](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source | +| [aws_iam_policy_document.terminate_self](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [aws\_partition](#input\_aws\_partition) | AWS partition used to construct IAM ARNs. | `string` | `"aws"` | no | +| [aws\_region](#input\_aws\_region) | AWS region used by compute-provider resources and policy documents. | `string` | n/a | yes | +| [config](#input\_config) | EC2 compute-provider configuration. Paths match `compute_provider.ec2` in the runner stack.

- `ami`: Optional AMI discovery and encryption configuration. Null selects defaults for `runner.os` and `runner.architecture`.
- `ami.filter`: AMI filter names mapped to accepted values and merged over the provider defaults.
- `ami.owners`: AWS account IDs or aliases allowed to own the selected AMI.
- `ami.id_ssm_parameter`: Optional externally managed SSM parameter containing the AMI ID. Its object presence is the plan-time ownership discriminator.
- `ami.id_ssm_parameter.arn`: ARN of the external AMI-ID parameter. The ARN may remain unknown until apply.
- `ami.kms_key`: Optional customer-managed KMS key required for encrypted AMIs or snapshots. Its object presence is the plan-time policy discriminator.
- `ami.kms_key.arn`: ARN of the AMI KMS key. The ARN may remain unknown until apply.
- `vpc_id`: VPC in which runner networking resources are created.
- `subnet_ids`: Subnets from which the control plane may launch runners.
- `overrides.name_runner`: Optional Name tag override for runner compute resources.
- `overrides.name_sg`: Optional Name tag override for the managed security group.
- `instance_profile`: Optional externally managed instance profile. Its object presence is the plan-time ownership discriminator.
- `instance_profile.name`: Name of the external instance profile. The name may remain unknown until apply.
- `instance_profile_path`: IAM path for the provider-managed instance profile. Null derives the path from `prefix`.
- `binaries_syncer.enabled`: Uses the synchronized runner distribution from S3 during bootstrap.
- `binaries_syncer.s3`: S3 object containing the synchronized runner distribution. Required when synchronization is enabled.
- `binaries_syncer.s3.arn`: Runner-distribution bucket ARN used by IAM policies.
- `binaries_syncer.s3.id`: Runner-distribution bucket name used in the bootstrap URI.
- `binaries_syncer.s3.key`: Runner-distribution object key.
- `block_device_mappings`: EBS mappings added to the launch template.
- `block_device_mappings[].delete_on_termination`: Deletes the volume when its runner terminates.
- `block_device_mappings[].device_name`: Device name exposed to the runner instance.
- `block_device_mappings[].encrypted`: Enables EBS encryption.
- `block_device_mappings[].iops`: Provisioned IOPS for volume types that support configurable IOPS.
- `block_device_mappings[].kms_key_id`: KMS key ID or ARN used to encrypt the volume.
- `block_device_mappings[].snapshot_id`: Snapshot used to initialize the volume.
- `block_device_mappings[].throughput`: Provisioned throughput for volume types that support it.
- `block_device_mappings[].volume_initialization_rate`: Fixed initialization rate for supported snapshot-backed volumes.
- `block_device_mappings[].volume_size`: EBS volume size in GiB.
- `block_device_mappings[].volume_type`: EBS volume type.
- `ebs_optimized`: Requests EBS-optimized instances.
- `instance_target_capacity_type`: Primary capacity type, either `spot` or `on-demand`.
- `instance_allocation_strategy`: EC2 Fleet allocation strategy.
- `instance_type_priorities`: Optional numeric priorities keyed by instance type.
- `instance_max_spot_price`: Optional maximum hourly Spot price.
- `instance_types`: EC2 instance types available to the control plane.
- `user_data`: Runner bootstrap user-data configuration.
- `user_data.enabled`: Enables launch-template user data.
- `user_data.template`: Optional path to a custom user-data template.
- `user_data.content`: Optional complete user-data content used instead of a template.
- `user_data.pre_install`: Script inserted before runner installation.
- `user_data.post_install`: Script inserted after runner installation.
- `user_data.debug_logging_enabled`: Enables verbose user-data tracing, which can expose secrets.
- `ssm_enabled`: Includes Session Manager permissions in the provider's runner policy group.
- `create_service_linked_role_spot`: Allows scale-up to create the EC2 Spot service-linked role.
- `cloudwatch_agent.enabled`: Enables CloudWatch agent configuration for runner instances.
- `cloudwatch_agent.config`: Optional complete CloudWatch agent configuration.
- `managed_security_group_enabled`: Creates and attaches the provider-managed security group.
- `log_files`: Optional files collected by the CloudWatch agent. Null uses provider defaults.
- `log_files[].log_group_name`: CloudWatch log-group name before optional prefixing.
- `log_files[].prefix_log_group`: Prefixes the log-group name with the runner stack path.
- `log_files[].file_path`: File or glob read by the CloudWatch agent.
- `log_files[].log_stream_name`: CloudWatch log-stream name template.
- `log_files[].log_class`: CloudWatch log-group class for the collected file.
- `key_name`: Optional EC2 key-pair name.
- `additional_security_group_ids`: Existing security groups attached to runners.
- `detailed_monitoring_enabled`: Enables detailed EC2 monitoring.
- `egress_rules`: Rules created on the managed security group.
- `egress_rules[].cidr_blocks`: IPv4 CIDR destinations.
- `egress_rules[].ipv6_cidr_blocks`: IPv6 CIDR destinations.
- `egress_rules[].prefix_list_ids`: AWS prefix-list destinations.
- `egress_rules[].from_port`: First destination port in the permitted range.
- `egress_rules[].protocol`: IP protocol name or number. Use `-1` for all protocols.
- `egress_rules[].security_groups`: Destination security-group IDs.
- `egress_rules[].self`: Allows traffic to the managed security group itself.
- `egress_rules[].to_port`: Last destination port in the permitted range.
- `egress_rules[].description`: Optional rule description.
- `tags`: Runner instance, volume, network-interface, and eligible Spot-request tags. Provider-required bootstrap tags take final precedence.
- `metadata_options`: Instance Metadata Service configuration.
- `metadata_options.instance_metadata_tags`: Exposes instance tags through Instance Metadata Service when enabled.
- `metadata_options.http_endpoint`: Enables or disables the Instance Metadata Service endpoint.
- `metadata_options.http_tokens`: Controls whether IMDSv2 session tokens are optional or required.
- `metadata_options.http_put_response_hop_limit`: Network hop limit for Instance Metadata Service token responses.
- `credit_specification`: CPU credit mode for burstable instance types.
- `cpu_options`: CPU topology and processor-feature configuration.
- `cpu_options.core_count`: Number of CPU cores exposed to the runner instance.
- `cpu_options.threads_per_core`: Number of hardware threads exposed per CPU core.
- `cpu_options.amd_sev_snp`: Enables or disables AMD SEV-SNP on supported instance types.
- `cpu_options.nested_virtualization`: Enables or disables nested virtualization on supported instance types.
- `placement`: EC2 placement configuration.
- `placement.affinity`: Dedicated Host affinity setting.
- `placement.availability_zone`: Availability Zone in which runner instances are placed.
- `placement.group_id`: Placement-group ID.
- `placement.group_name`: Placement-group name.
- `placement.host_id`: Dedicated Host ID.
- `placement.host_resource_group_arn`: ARN of the host resource group used for placement.
- `placement.spread_domain`: Spread-domain placement value.
- `placement.tenancy`: Instance tenancy, such as `default`, `dedicated`, or `host`.
- `placement.partition_number`: Placement-group partition number.
- `license_specifications`: License Manager configurations added to the launch template.
- `license_specifications[].license_configuration_arn`: ARN of an AWS License Manager license configuration.
- `associate_public_ipv4_address`: Associates a public IPv4 address with runner network interfaces.
- `enable_on_demand_failover_for_errors`: EC2 errors that trigger on-demand fallback after a Spot failure.
- `scale_errors`: EC2 errors treated as retryable scale-up failures.
- `use_dedicated_host`: Enables the dedicated-host launch path. | 
object({
    ami = optional(object({
      filter = optional(map(list(string)), { state = ["available"] })
      owners = optional(list(string), ["amazon"])
      id_ssm_parameter = optional(object({
        arn = string
      }), null)
      kms_key = optional(object({
        arn = string
      }), null)
    }), null)
    vpc_id     = string
    subnet_ids = list(string)
    overrides = optional(object({
      name_runner = optional(string, "")
      name_sg     = optional(string, "")
    }), {})
    instance_profile = optional(object({
      name = string
    }), null)
    instance_profile_path = optional(string, null)
    binaries_syncer = optional(object({
      enabled = optional(bool, true)
      s3 = optional(object({
        arn = string
        id  = string
        key = string
      }), null)
    }), {})
    block_device_mappings = optional(list(object({
      delete_on_termination      = optional(bool, true)
      device_name                = optional(string, "/dev/xvda")
      encrypted                  = optional(bool, true)
      iops                       = optional(number)
      kms_key_id                 = optional(string)
      snapshot_id                = optional(string)
      throughput                 = optional(number)
      volume_initialization_rate = optional(number)
      volume_size                = number
      volume_type                = optional(string, "gp3")
    })), [{ volume_size = 30 }])
    ebs_optimized                 = optional(bool, false)
    instance_target_capacity_type = optional(string, "spot")
    instance_allocation_strategy  = optional(string, "lowest-price")
    instance_type_priorities      = optional(map(number), null)
    instance_max_spot_price       = optional(string, null)
    instance_types                = list(string)
    user_data = optional(object({
      enabled               = optional(bool, true)
      template              = optional(string, null)
      content               = optional(string, null)
      pre_install           = optional(string, "")
      post_install          = optional(string, "")
      debug_logging_enabled = optional(bool, false)
    }), {})
    ssm_enabled                     = optional(bool, false)
    create_service_linked_role_spot = optional(bool, false)
    cloudwatch_agent = optional(object({
      enabled = optional(bool, true)
      config  = optional(string, null)
    }), {})
    managed_security_group_enabled = optional(bool, true)
    log_files = optional(list(object({
      log_group_name   = string
      prefix_log_group = bool
      file_path        = string
      log_stream_name  = string
      log_class        = optional(string, "STANDARD")
    })), null)
    key_name                      = optional(string, null)
    additional_security_group_ids = optional(list(string), [])
    detailed_monitoring_enabled   = optional(bool, false)
    egress_rules = optional(list(object({
      cidr_blocks      = list(string)
      ipv6_cidr_blocks = list(string)
      prefix_list_ids  = list(string)
      from_port        = number
      protocol         = string
      security_groups  = list(string)
      self             = bool
      to_port          = number
      description      = string
      })), [{
      cidr_blocks      = ["0.0.0.0/0"]
      ipv6_cidr_blocks = ["::/0"]
      prefix_list_ids  = null
      from_port        = 0
      protocol         = "-1"
      security_groups  = null
      self             = null
      to_port          = 0
      description      = null
    }])
    tags = optional(map(string), {})
    metadata_options = optional(object({
      instance_metadata_tags      = optional(string, "enabled")
      http_endpoint               = optional(string, "enabled")
      http_tokens                 = optional(string, "required")
      http_put_response_hop_limit = optional(number, 1)
    }), {})
    credit_specification = optional(string, null)
    cpu_options = optional(object({
      core_count            = optional(number)
      threads_per_core      = optional(number)
      amd_sev_snp           = optional(string)
      nested_virtualization = optional(string)
    }), null)
    placement = optional(object({
      affinity                = optional(string)
      availability_zone       = optional(string)
      group_id                = optional(string)
      group_name              = optional(string)
      host_id                 = optional(string)
      host_resource_group_arn = optional(string)
      spread_domain           = optional(string)
      tenancy                 = optional(string)
      partition_number        = optional(number)
    }), null)
    license_specifications = optional(list(object({
      license_configuration_arn = string
    })), [])
    associate_public_ipv4_address        = optional(bool, false)
    enable_on_demand_failover_for_errors = optional(list(string), [])
    scale_errors = optional(list(string), [
      "UnfulfillableCapacity",
      "MaxSpotInstanceCountExceeded",
      "TargetCapacityLimitExceededException",
      "RequestLimitExceeded",
      "ResourceLimitExceeded",
      "MaxSpotInstanceCountExceeded",
      "MaxSpotFleetRequestCountExceeded",
      "InsufficientInstanceCapacity",
      "InsufficientCapacityOnHost",
    ])
    use_dedicated_host = optional(bool, false)
  })
| n/a | yes | +| [github](#input\_github) | GitHub Enterprise Server settings available to compute-provider bootstrap data.

- `enterprise_server.url`: Optional GitHub Enterprise Server base URL. Null selects GitHub.com.
- `enterprise_server.ssl_verify`: Enables TLS certificate verification for GitHub Enterprise Server. | 
object({
    enterprise_server = optional(object({
      url        = optional(string, null)
      ssl_verify = optional(bool, true)
    }), {})
  })
| `{}` | no | +| [observability](#input\_observability) | CloudWatch Logs settings available to compute-provider runner log groups.

- `logs.retention_in_days`: Retention period for provider-owned runner log groups.
- `logs.kms_key_id`: Optional KMS key ID or ARN used to encrypt runner log groups.
- `logs.tags`: Shared log-group tags that override module-level `tags`. | 
object({
    logs = optional(object({
      retention_in_days = optional(number, 180)
      kms_key_id        = optional(string, null)
      tags              = optional(map(string), {})
    }), {})
  })
| `{}` | no | +| [prefix](#input\_prefix) | Prefix used to identify resources created for the runner stack. | `string` | `"github-actions"` | no | +| [runner](#input\_runner) | Provider-neutral runner settings consumed by compute providers.

- `os`: Runner operating system. Supported values are `linux`, `osx`, and `windows`.
- `architecture`: Runner distribution architecture.
- `boot_time_in_minutes`: Expected boot and registration duration used by scale-down and pool.
- `name_prefix`: Prefix added to registered runner names.
- `run_as_root`: Runs the runner service as root.
- `run_as`: Operating-system user used when `run_as_root` is false.
- `hooks.job_started`: Script installed as the runner job-started hook.
- `hooks.job_completed`: Script installed as the runner job-completed hook.
- `iam.role.arn`: Resolved runner-role ARN referenced by provider policies and resources.
- `iam.role.name`: Resolved runner-role name used by provider resources.
- `iam.role.managed`: Whether runner-stack manages the resolved runner role.
- `iam.managed_policy_arns`: Common managed-policy ARNs returned with the provider-specific runner policies for attachment by runner-stack.
- `iam.path`: IAM path available to provider-managed IAM resources. Null derives the path from `prefix`. | 
object({
    os                   = optional(string, "linux")
    architecture         = optional(string, "x64")
    boot_time_in_minutes = optional(number, 5)
    name_prefix          = optional(string, "")
    run_as_root          = optional(bool, false)
    run_as               = optional(string, "ec2-user")
    hooks = optional(object({
      job_started   = optional(string, "")
      job_completed = optional(string, "")
    }), {})
    iam = object({
      role = object({
        arn     = string
        name    = string
        managed = optional(bool, true)
      })
      managed_policy_arns = optional(map(string), {})
      path                = optional(string, null)
    })
  })
| n/a | yes | +| [ssm](#input\_ssm) | Parameter Store paths and tag scopes available to compute-provider bootstrap resources.

- `paths.root`: Root Parameter Store path for the runner stack.
- `paths.tokens`: Path segment used for registration tokens and just-in-time configuration.
- `paths.config`: Path segment used for persistent runner and provider configuration.
- `tags`: Shared SSM tags that override module-level `tags`.
- `parameters.tags`: Parameter-specific tags that override module-level and shared SSM tags. | 
object({
    paths = object({
      root   = string
      tokens = string
      config = string
    })
    tags = optional(map(string), {})
    parameters = optional(object({
      tags = optional(map(string), {})
    }), {})
  })
| n/a | yes | +| [tags](#input\_tags) | Base tags available to taggable compute-provider resources. Provider-specific tags override this map within their documented scopes. | `map(string)` | `{}` | no | + +## Outputs + +| Name | Description | +|------|-------------| +| [environment\_variables](#output\_environment\_variables) | Provider-specific Lambda environment variable fragments consumed by runner-stack. | +| [policies](#output\_policies) | Provider-specific IAM policy fragments consumed by runner-stack. | +| [provider](#output\_provider) | Nested EC2 compute-provider contract consumed by runner-stack. | +| [resources](#output\_resources) | Provider-specific EC2 resources exposed by runner-stack. | + diff --git a/modules/compute-providers/ec2/control-plane.tf b/modules/compute-providers/ec2/control-plane.tf new file mode 100644 index 0000000000..fd9213d00e --- /dev/null +++ b/modules/compute-providers/ec2/control-plane.tf @@ -0,0 +1,220 @@ +# EC2-specific IAM and environment fragments consumed by the common control +# plane in runner-stack. +data "aws_iam_policy_document" "ami_id_ssm_parameter_read" { + count = local.ami_id_ssm_external ? 1 : 0 + + statement { + effect = "Allow" + actions = ["ssm:GetParameter"] + resources = [local.ami_id_ssm_parameter_arn] + } +} + +resource "aws_iam_policy" "ami_id_ssm_parameter_read" { + count = local.ami_id_ssm_external ? 1 : 0 + name = "${var.prefix}-ami-id-ssm-parameter-read" + path = local.role_path + description = "Allows for reading ${var.prefix} GitHub runner AMI ID from an SSM parameter" + tags = local.provider_tags + policy = data.aws_iam_policy_document.ami_id_ssm_parameter_read[0].json +} + +data "aws_iam_policy_document" "scale_up" { + statement { + effect = "Allow" + actions = [ + "ec2:DescribeInstances", + "ec2:DescribeLaunchTemplateVersions", + "ec2:DescribeTags", + "ec2:RunInstances", + "ec2:CreateFleet", + "ec2:CreateTags", + ] + resources = ["*"] + } + + statement { + effect = "Allow" + actions = ["ec2:TerminateInstances"] + resources = ["*"] + + condition { + test = "StringEquals" + variable = "ec2:ResourceTag/ghr:Application" + values = ["github-action-runner"] + } + } + + statement { + effect = "Allow" + actions = ["ec2:TerminateInstances"] + resources = ["*"] + + condition { + test = "StringEquals" + variable = "ec2:ResourceTag/ghr:environment" + values = [var.prefix] + } + } + + statement { + effect = "Allow" + actions = ["iam:PassRole"] + resources = [var.runner.iam.role.arn] + } + + statement { + effect = "Allow" + actions = ["ssm:GetParameter", "ssm:GetParameters"] + resources = [local.ami_id_ssm_module_managed ? aws_ssm_parameter.runner_ami_id[0].arn : local.ami_id_ssm_parameter_arn] + } + + dynamic "statement" { + for_each = local.ami_kms_key_enabled ? [local.ami_kms_key_arn] : [] + + content { + effect = "Allow" + actions = ["kms:DescribeKey", "kms:ReEncrypt*", "kms:Decrypt"] + resources = [statement.value] + } + } + + dynamic "statement" { + for_each = local.ami_kms_key_enabled ? [local.ami_kms_key_arn] : [] + + content { + effect = "Allow" + actions = ["kms:CreateGrant"] + resources = [statement.value] + + condition { + test = "Bool" + variable = "aws:ViaAWSService" + values = ["true"] + } + } + } +} + +data "aws_iam_policy_document" "scale_down" { + statement { + effect = "Allow" + actions = ["ec2:DescribeInstances", "ec2:DescribeTags"] + resources = ["*"] + } + + statement { + effect = "Allow" + actions = ["ec2:TerminateInstances", "ec2:CreateTags", "ec2:DeleteTags"] + resources = ["*"] + + condition { + test = "StringEquals" + variable = "ec2:ResourceTag/ghr:Application" + values = ["github-action-runner"] + } + } + + statement { + effect = "Allow" + actions = ["ec2:TerminateInstances", "ec2:CreateTags", "ec2:DeleteTags"] + resources = ["*"] + + condition { + test = "StringEquals" + variable = "ec2:ResourceTag/ghr:environment" + values = [var.prefix] + } + } +} + +data "aws_iam_policy_document" "pool" { + statement { + effect = "Allow" + actions = [ + "ec2:DescribeInstances", + "ec2:DescribeTags", + "ec2:RunInstances", + "ec2:CreateFleet", + "ec2:CreateTags", + ] + resources = ["*"] + } + + statement { + effect = "Allow" + actions = ["iam:PassRole"] + resources = [var.runner.iam.role.arn] + } + + statement { + effect = "Allow" + actions = ["ssm:GetParameters"] + resources = [local.ami_id_ssm_module_managed ? aws_ssm_parameter.runner_ami_id[0].arn : local.ami_id_ssm_parameter_arn] + } + + dynamic "statement" { + for_each = local.ami_kms_key_enabled ? [local.ami_kms_key_arn] : [] + + content { + effect = "Allow" + actions = ["kms:DescribeKey", "kms:ReEncrypt*", "kms:Decrypt"] + resources = [statement.value] + } + } + + dynamic "statement" { + for_each = local.ami_kms_key_enabled ? [local.ami_kms_key_arn] : [] + + content { + effect = "Allow" + actions = ["kms:CreateGrant"] + resources = [statement.value] + + condition { + test = "Bool" + variable = "aws:ViaAWSService" + values = ["true"] + } + } + } +} + +data "aws_iam_policy_document" "service_linked_role" { + count = var.config.create_service_linked_role_spot ? 1 : 0 + + statement { + effect = "Allow" + actions = ["iam:CreateServiceLinkedRole"] + resources = ["arn:${var.aws_partition}:iam::*:role/aws-service-role/*"] + } +} + +locals { + scale_up_environment_variables = { + AMI_ID_SSM_PARAMETER_NAME = local.ami_id_ssm_parameter_name + INSTANCE_ALLOCATION_STRATEGY = var.config.instance_allocation_strategy + INSTANCE_MAX_SPOT_PRICE = var.config.instance_max_spot_price + INSTANCE_TARGET_CAPACITY_TYPE = var.config.instance_target_capacity_type + INSTANCE_TYPE_PRIORITIES = var.config.instance_type_priorities != null ? jsonencode(var.config.instance_type_priorities) : "" + INSTANCE_TYPES = join(",", var.config.instance_types) + LAUNCH_TEMPLATE_NAME = aws_launch_template.runner.name + SUBNET_IDS = join(",", var.config.subnet_ids) + ENABLE_ON_DEMAND_FAILOVER_FOR_ERRORS = jsonencode(var.config.enable_on_demand_failover_for_errors) + SCALE_ERRORS = jsonencode(var.config.scale_errors) + USE_DEDICATED_HOST = var.config.use_dedicated_host + } + + scale_down_environment_variables = { + RUNNER_BOOT_TIME_IN_MINUTES = var.runner.boot_time_in_minutes + } + + pool_environment_variables = merge(local.scale_up_environment_variables, { + RUNNER_BOOT_TIME_IN_MINUTES = var.runner.boot_time_in_minutes + }) + + scale_up_iam_policy_json = data.aws_iam_policy_document.scale_up.json + scale_down_iam_policy_json = data.aws_iam_policy_document.scale_down.json + pool_iam_policy_json = data.aws_iam_policy_document.pool.json + service_linked_role_policy_json = var.config.create_service_linked_role_spot ? data.aws_iam_policy_document.service_linked_role[0].json : null +} diff --git a/modules/compute-providers/ec2/instance-profile.tf b/modules/compute-providers/ec2/instance-profile.tf new file mode 100644 index 0000000000..68b8842d2d --- /dev/null +++ b/modules/compute-providers/ec2/instance-profile.tf @@ -0,0 +1,9 @@ +# The common runner stack owns the role; EC2 owns the profile consumed by its +# launch template. +resource "aws_iam_instance_profile" "runner" { + count = var.config.instance_profile == null ? 1 : 0 + name = "${var.prefix}-runner-profile" + role = var.runner.iam.role.name + path = local.instance_profile_path + tags = local.provider_tags +} diff --git a/modules/compute-providers/ec2/logging.tf b/modules/compute-providers/ec2/logging.tf new file mode 100644 index 0000000000..00ae952e4d --- /dev/null +++ b/modules/compute-providers/ec2/logging.tf @@ -0,0 +1,75 @@ +# EC2 runner log collection and CloudWatch resources. +locals { + runner_log_files = ( + var.config.log_files != null + ? var.config.log_files + : [ + { + "prefix_log_group" : true, + "file_path" : "/var/log/messages", + "log_group_name" : "messages", + "log_stream_name" : "{instance_id}", + "log_class" : "STANDARD" + }, + { + "log_group_name" : "user_data", + "prefix_log_group" : true, + "file_path" : var.runner.os == "windows" ? "C:/UserData.log" : "/var/log/user-data.log", + "log_stream_name" : "{instance_id}", + "log_class" : "STANDARD" + }, + { + "log_group_name" : "runner", + "prefix_log_group" : true, + "file_path" : var.runner.os == "windows" ? "C:/actions-runner/_diag/Runner_*.log" : "/opt/actions-runner/_diag/Runner_**.log", + "log_stream_name" : "{instance_id}", + "log_class" : "STANDARD" + }, + { + "log_group_name" : "runner-startup", + "prefix_log_group" : true, + "file_path" : var.runner.os == "windows" ? "C:/runner-startup.log" : "/var/log/runner-startup.log", + "log_stream_name" : "{instance_id}", + "log_class" : "STANDARD" + } + ] + ) + # CloudWatch agent collect_list schema expects log_group_class, not log_class + logfiles = var.config.cloudwatch_agent.enabled ? [for l in local.runner_log_files : { + "log_group_name" : l.prefix_log_group ? "/github-self-hosted-runners/${var.prefix}/${l.log_group_name}" : "/${l.log_group_name}" + "log_stream_name" : l.log_stream_name + "file_path" : l.file_path + "log_group_class" : l.log_class + }] : [] + + loggroups_names = distinct([for l in local.logfiles : l.log_group_name]) + # Create a list of unique log classes corresponding to each log group name + # This maintains the same order as loggroups_names for use with count + loggroups_classes = [ + for name in local.loggroups_names : [ + for l in local.logfiles : l.log_group_class + if l.log_group_name == name + ][0] + ] + +} + + +resource "aws_ssm_parameter" "cloudwatch_agent_config_runner" { + count = var.config.cloudwatch_agent.enabled ? 1 : 0 + name = "${var.ssm.paths.root}/${var.ssm.paths.config}/cloudwatch_agent_config_runner" + type = "String" + value = var.config.cloudwatch_agent.config != null ? var.config.cloudwatch_agent.config : templatefile("${path.module}/templates/cloudwatch_config.json", { + logfiles = jsonencode(local.logfiles) + }) + tags = local.ssm_parameter_tags +} + +resource "aws_cloudwatch_log_group" "gh_runners" { + count = length(local.loggroups_names) + name = local.loggroups_names[count.index] + retention_in_days = var.observability.logs.retention_in_days + kms_key_id = var.observability.logs.kms_key_id + log_group_class = local.loggroups_classes[count.index] + tags = local.log_group_tags +} diff --git a/modules/compute-providers/ec2/outputs.tf b/modules/compute-providers/ec2/outputs.tf new file mode 100644 index 0000000000..4e536fcbbe --- /dev/null +++ b/modules/compute-providers/ec2/outputs.tf @@ -0,0 +1,23 @@ +output "environment_variables" { + description = "Provider-specific Lambda environment variable fragments consumed by runner-stack." + value = local.provider_environment_variables +} + +output "policies" { + description = "Provider-specific IAM policy fragments consumed by runner-stack." + value = local.provider_policies +} + +output "resources" { + description = "Provider-specific EC2 resources exposed by runner-stack." + value = local.provider_resources +} + +output "provider" { + description = "Nested EC2 compute-provider contract consumed by runner-stack." + value = { + environment_variables = local.provider_environment_variables + policies = local.provider_policies + resources = local.provider_resources + } +} diff --git a/modules/compute-providers/ec2/policies-runner.tf b/modules/compute-providers/ec2/policies-runner.tf new file mode 100644 index 0000000000..785180cff3 --- /dev/null +++ b/modules/compute-providers/ec2/policies-runner.tf @@ -0,0 +1,206 @@ +# EC2 runner permission documents returned to runner-stack for attachment to +# the common runner role. +data "aws_caller_identity" "current" {} + +locals { + ssm_parameter_arn_prefix = "arn:${var.aws_partition}:ssm:${var.aws_region}:${data.aws_caller_identity.current.account_id}:parameter" + ssm_config_arn = "${local.ssm_parameter_arn_prefix}${var.ssm.paths.root}/${var.ssm.paths.config}" + cloudwatch_config_arn = "${local.ssm_config_arn}/cloudwatch_agent_config_runner" +} + +data "aws_iam_policy_document" "ssm_parameters" { + statement { + effect = "Allow" + actions = [ + "ssm:DeleteParameter", + "ssm:GetParameters", + "ssm:GetParameter", + ] + resources = [ + "${local.ssm_parameter_arn_prefix}${var.ssm.paths.root}/${var.ssm.paths.tokens}/*", + ] + + condition { + test = "StringLike" + variable = "ec2:SourceInstanceARN" + values = ["*/&{aws:ResourceTag/InstanceId}"] + } + } + + statement { + effect = "Allow" + actions = [ + "ssm:GetParameter", + "ssm:GetParameters", + "ssm:GetParametersByPath", + ] + resources = [ + local.ssm_config_arn, + "${local.ssm_config_arn}/*", + ] + } +} + +data "aws_iam_policy_document" "session_manager" { + statement { + effect = "Allow" + actions = [ + "ssm:DescribeAssociation", + "ssm:GetDeployablePatchSnapshotForInstance", + "ssm:GetDocument", + "ssm:DescribeDocument", + "ssm:GetManifest", + "ssm:ListAssociations", + "ssm:ListInstanceAssociations", + "ssm:PutInventory", + "ssm:PutComplianceItems", + "ssm:PutConfigurePackageResult", + "ssm:UpdateAssociationStatus", + "ssm:UpdateInstanceAssociationStatus", + "ssm:UpdateInstanceInformation", + ] + resources = ["*"] + } + + statement { + effect = "Allow" + actions = [ + "ssmmessages:CreateControlChannel", + "ssmmessages:CreateDataChannel", + "ssmmessages:OpenControlChannel", + "ssmmessages:OpenDataChannel", + ] + resources = ["*"] + } + + statement { + effect = "Allow" + actions = [ + "ec2messages:AcknowledgeMessage", + "ec2messages:DeleteMessage", + "ec2messages:FailMessage", + "ec2messages:GetEndpoint", + "ec2messages:GetMessages", + "ec2messages:SendReply", + ] + resources = ["*"] + } +} + +data "aws_iam_policy_document" "distribution_bucket" { + count = var.config.binaries_syncer.enabled ? 1 : 0 + + statement { + sid = "githubActionDist" + effect = "Allow" + actions = ["s3:GetObject", "s3:GetObjectAcl"] + resources = ["${try(var.config.binaries_syncer.s3.arn, "")}/${try(var.config.binaries_syncer.s3.key, "")}"] + } +} + +data "aws_iam_policy_document" "describe_tags" { + statement { + effect = "Allow" + actions = ["ec2:DescribeTags"] + resources = ["*"] + } +} + +data "aws_iam_policy_document" "create_tags" { + statement { + effect = "Allow" + actions = ["ec2:CreateTags"] + resources = ["arn:*:ec2:*:*:instance/*"] + + condition { + test = "ForAllValues:StringEquals" + variable = "aws:TagKeys" + values = ["ghr:github_runner_id"] + } + + condition { + test = "StringEquals" + variable = "aws:ARN" + values = ["&{ec2:SourceInstanceARN}"] + } + } +} + +data "aws_iam_policy_document" "terminate_self" { + statement { + effect = "Allow" + actions = ["ec2:TerminateInstances"] + resources = ["*"] + + condition { + test = "StringEquals" + variable = "aws:ARN" + values = ["&{ec2:SourceInstanceARN}"] + } + } +} + +data "aws_iam_policy_document" "cloudwatch" { + count = var.config.cloudwatch_agent.enabled ? 1 : 0 + + statement { + effect = "Allow" + actions = [ + "cloudwatch:PutMetricData", + "ec2:DescribeVolumes", + "ec2:DescribeTags", + "logs:PutLogEvents", + "logs:DescribeLogStreams", + "logs:DescribeLogGroups", + "logs:CreateLogStream", + ] + resources = ["*"] + } + + statement { + effect = "Allow" + actions = ["ssm:GetParameter"] + resources = ["${local.cloudwatch_config_arn}/*"] + } +} + +locals { + runner_inline_policies = merge( + { + ssm_parameters = { + name = "runner-ssm-parameters" + policy_json = data.aws_iam_policy_document.ssm_parameters.json + } + describe_tags = { + name = "runner-describe-tags" + policy_json = data.aws_iam_policy_document.describe_tags.json + } + create_tags = { + name = "runner-create-tags" + policy_json = data.aws_iam_policy_document.create_tags.json + } + terminate_self = { + name = "ec2" + policy_json = data.aws_iam_policy_document.terminate_self.json + } + }, + var.config.ssm_enabled ? { + session_manager = { + name = "runner-ssm-session" + policy_json = data.aws_iam_policy_document.session_manager.json + } + } : {}, + var.config.binaries_syncer.enabled ? { + distribution_bucket = { + name = "distribution-bucket" + policy_json = data.aws_iam_policy_document.distribution_bucket[0].json + } + } : {}, + var.config.cloudwatch_agent.enabled ? { + cloudwatch = { + name = "CloudWatchLogginAndMetrics" + policy_json = data.aws_iam_policy_document.cloudwatch[0].json + } + } : {}, + ) +} diff --git a/modules/compute-providers/ec2/provider-contract.tf b/modules/compute-providers/ec2/provider-contract.tf new file mode 100644 index 0000000000..5682496d78 --- /dev/null +++ b/modules/compute-providers/ec2/provider-contract.tf @@ -0,0 +1,34 @@ +locals { + provider_environment_variables = { + scale_up = local.scale_up_environment_variables + scale_down = local.scale_down_environment_variables + pool = local.pool_environment_variables + } + + provider_policies = { + runner = { + inline_policies = local.runner_inline_policies + managed_policy_arns = var.runner.iam.managed_policy_arns + } + scale_up = { + iam_policy_json = local.scale_up_iam_policy_json + additional_iam_policy_json = local.service_linked_role_policy_json + managed_policy_enabled = local.ami_id_ssm_external + managed_policy_arn = local.ami_id_ssm_external ? aws_iam_policy.ami_id_ssm_parameter_read[0].arn : null + } + scale_down = { + iam_policy_json = local.scale_down_iam_policy_json + } + pool = { + iam_policy_json = local.pool_iam_policy_json + managed_policy_enabled = local.ami_id_ssm_external + managed_policy_arn = local.ami_id_ssm_external ? aws_iam_policy.ami_id_ssm_parameter_read[0].arn : null + } + } + + provider_resources = { + launch_template = aws_launch_template.runner + runners_log_groups = try(aws_cloudwatch_log_group.gh_runners, []) + logfiles = local.logfiles + } +} diff --git a/modules/compute-providers/ec2/runner-config.tf b/modules/compute-providers/ec2/runner-config.tf new file mode 100644 index 0000000000..f1d859581c --- /dev/null +++ b/modules/compute-providers/ec2/runner-config.tf @@ -0,0 +1,13 @@ +resource "aws_ssm_parameter" "runner_config_run_as" { + name = "${var.ssm.paths.root}/${var.ssm.paths.config}/run_as" + type = "String" + value = var.runner.run_as_root ? "root" : var.runner.run_as + tags = local.ssm_parameter_tags +} + +resource "aws_ssm_parameter" "runner_enable_cloudwatch" { + name = "${var.ssm.paths.root}/${var.ssm.paths.config}/enable_cloudwatch" + type = "String" + value = var.config.cloudwatch_agent.enabled + tags = local.ssm_parameter_tags +} diff --git a/modules/compute-providers/ec2/runner-instances.tf b/modules/compute-providers/ec2/runner-instances.tf new file mode 100644 index 0000000000..e965d8f66c --- /dev/null +++ b/modules/compute-providers/ec2/runner-instances.tf @@ -0,0 +1,325 @@ +# AMI selection, bootstrap rendering, launch template, and security group for +# EC2 runner instances. +locals { + provider_tags = merge( + { + "Name" = format("%s-action-runner", var.prefix) + }, + var.tags, + ) + + ssm_parameter_tags = merge( + local.provider_tags, + var.ssm.tags, + var.ssm.parameters.tags, + ) + + log_group_tags = merge( + local.provider_tags, + var.observability.logs.tags, + ) + + name_sg = var.config.overrides.name_sg == "" ? local.provider_tags["Name"] : var.config.overrides.name_sg + name_runner = var.config.overrides.name_runner == "" ? local.provider_tags["Name"] : var.config.overrides.name_runner + runner_tags = merge( + local.provider_tags, + { + "Name" = local.name_runner + }, + var.config.tags, + { + "ghr:environment" = var.prefix + "ghr:ssm_config_path" = "${var.ssm.paths.root}/${var.ssm.paths.config}" + "ghr:runner_name_prefix" = var.runner.name_prefix + }, + ) + + role_path = var.runner.iam.path == null ? "/${var.prefix}/" : var.runner.iam.path + instance_profile_path = var.config.instance_profile_path == null ? "/${var.prefix}/" : var.config.instance_profile_path + userdata_template = var.config.user_data.template == null ? local.default_userdata_template[var.runner.os] : var.config.user_data.template + s3_location_runner_distribution = var.config.binaries_syncer.enabled ? "s3://${try(var.config.binaries_syncer.s3.id, "")}/${try(var.config.binaries_syncer.s3.key, "")}" : "" + default_ami = { + "windows" = { name = ["Windows_Server-2022-English-Full-ECS_Optimized-*"] } + "linux" = var.runner.architecture == "arm64" ? { name = ["al2023-ami-2023.*-kernel-6.*-arm64"] } : { name = ["al2023-ami-2023.*-kernel-6.*-x86_64"] } + "osx" = var.runner.architecture == "arm64" ? { name = ["amzn-ec2-macos-15.*-arm64"] } : { name = ["amzn-ec2-macos-15.*"] } + } + + default_userdata_template = { + "windows" = "${path.module}/templates/user-data.ps1" + "linux" = "${path.module}/templates/user-data.sh" + "osx" = "${path.module}/templates/user-data-osx.sh" + } + + userdata_install_runner = { + "windows" = "${path.module}/templates/install-runner.ps1" + "linux" = "${path.module}/templates/install-runner.sh" + "osx" = "${path.module}/templates/install-runner-osx.sh" + } + + userdata_start_runner = { + "windows" = "${path.module}/templates/start-runner.ps1" + "linux" = "${path.module}/templates/start-runner.sh" + "osx" = "${path.module}/templates/start-runner-osx.sh" + } + + # Handle AMI configuration + ami_config = var.config.ami != null ? var.config.ami : { + filter = local.default_ami[var.runner.os] + owners = ["amazon"] + id_ssm_parameter = null + kms_key = null + } + ami_kms_key_enabled = local.ami_config.kms_key != null + ami_kms_key_arn = local.ami_kms_key_enabled ? local.ami_config.kms_key.arn : null + ami_filter = merge(local.default_ami[var.runner.os], local.ami_config.filter) + ami_id_ssm_external = local.ami_config.id_ssm_parameter != null + ami_id_ssm_module_managed = !local.ami_id_ssm_external + ami_id_ssm_parameter_arn = local.ami_id_ssm_external ? local.ami_config.id_ssm_parameter.arn : null + # Extract parameter name from ARN (format: arn:aws:ssm:region:account:parameter/path/to/param) + ami_id_ssm_parameter_name = local.ami_id_ssm_external ? try(regex("parameter(/.+)$", local.ami_id_ssm_parameter_arn)[0], null) : null + + user_data = var.config.user_data.enabled ? (var.config.user_data.content == null ? templatefile(local.userdata_template, { + enable_debug_logging = var.config.user_data.debug_logging_enabled + s3_location_runner_distribution = local.s3_location_runner_distribution + pre_install = var.config.user_data.pre_install + install_runner = templatefile(local.userdata_install_runner[var.runner.os], { + S3_LOCATION_RUNNER_DISTRIBUTION = local.s3_location_runner_distribution + RUNNER_ARCHITECTURE = var.runner.architecture + }) + post_install = var.config.user_data.post_install + hook_job_started = var.runner.hooks.job_started + hook_job_completed = var.runner.hooks.job_completed + start_runner = templatefile(local.userdata_start_runner[var.runner.os], { + metadata_tags = var.config.metadata_options != null ? var.config.metadata_options.instance_metadata_tags : "enabled" + }) + ghes_url = var.github.enterprise_server.url + ghes_ssl_verify = var.github.enterprise_server.ssl_verify + + ## retain these for backwards compatibility + environment = var.prefix + enable_cloudwatch_agent = var.config.cloudwatch_agent.enabled + ssm_key_cloudwatch_agent_config = var.config.cloudwatch_agent.enabled ? aws_ssm_parameter.cloudwatch_agent_config_runner[0].name : "" + }) : var.config.user_data.content) : "" + + encoded_user_data = ( + var.runner.os == "linux" ? base64gzip(local.user_data) : + var.runner.os == "windows" ? base64encode(local.user_data) : + var.runner.os == "osx" ? base64encode(local.user_data) : + null + ) +} + +data "aws_ami" "runner" { + most_recent = "true" + + dynamic "filter" { + for_each = local.ami_filter + content { + name = filter.key + values = filter.value + } + } + + owners = local.ami_config.owners +} + +resource "aws_ssm_parameter" "runner_ami_id" { + count = local.ami_id_ssm_module_managed ? 1 : 0 + name = "${var.ssm.paths.root}/${var.ssm.paths.config}/ami_id" + type = "String" + data_type = "aws:ec2:image" + value = data.aws_ami.runner.id + + tags = merge( + local.provider_tags, + local.ssm_parameter_tags, + { + # Remove parentheses from AMI name to comply with AWS tag constraints + "ghr:ami_name" = replace(data.aws_ami.runner.name, "/[()]/", "") + }, + { + "ghr:ami_creation_date" = data.aws_ami.runner.creation_date + }, + { + "ghr:ami_deprecation_time" = data.aws_ami.runner.deprecation_time + } + ) +} + +resource "aws_launch_template" "runner" { + name = "${var.prefix}-action-runner" + + dynamic "block_device_mappings" { + for_each = var.config.block_device_mappings != null ? var.config.block_device_mappings : [] + content { + device_name = block_device_mappings.value.device_name + + ebs { + delete_on_termination = block_device_mappings.value.delete_on_termination + encrypted = block_device_mappings.value.encrypted + iops = block_device_mappings.value.iops + kms_key_id = block_device_mappings.value.kms_key_id + snapshot_id = block_device_mappings.value.snapshot_id + throughput = block_device_mappings.value.throughput + volume_initialization_rate = block_device_mappings.value.volume_initialization_rate + volume_size = block_device_mappings.value.volume_size + volume_type = block_device_mappings.value.volume_type + } + } + } + + dynamic "metadata_options" { + for_each = var.config.metadata_options != null ? [var.config.metadata_options] : [] + + content { + http_endpoint = metadata_options.value.http_endpoint + http_tokens = metadata_options.value.http_tokens + http_put_response_hop_limit = metadata_options.value.http_put_response_hop_limit + instance_metadata_tags = metadata_options.value.instance_metadata_tags + } + } + + dynamic "metadata_options" { + for_each = var.config.metadata_options != null ? [] : [0] + + content { + instance_metadata_tags = "enabled" + } + } + + dynamic "credit_specification" { + for_each = var.config.credit_specification != null ? [var.config.credit_specification] : [] + content { + cpu_credits = credit_specification.value + } + } + + dynamic "cpu_options" { + for_each = var.config.cpu_options != null ? [var.config.cpu_options] : [] + content { + core_count = try(cpu_options.value.core_count, null) + threads_per_core = try(cpu_options.value.threads_per_core, null) + amd_sev_snp = try(cpu_options.value.amd_sev_snp, null) + nested_virtualization = try(cpu_options.value.nested_virtualization, null) + } + } + + dynamic "placement" { + for_each = var.config.placement != null ? [var.config.placement] : [] + content { + affinity = try(placement.value.affinity, null) + availability_zone = try(placement.value.availability_zone, null) + group_id = try(placement.value.group_id, null) + group_name = try(placement.value.group_name, null) + host_id = try(placement.value.host_id, null) + host_resource_group_arn = try(placement.value.host_resource_group_arn, null) + spread_domain = try(placement.value.spread_domain, null) + tenancy = try(placement.value.tenancy, null) + partition_number = try(placement.value.partition_number, null) + } + } + + dynamic "license_specification" { + for_each = var.config.license_specifications + content { + license_configuration_arn = license_specification.value.license_configuration_arn + } + } + + monitoring { + enabled = var.config.detailed_monitoring_enabled + } + + iam_instance_profile { + name = var.config.instance_profile != null ? var.config.instance_profile.name : aws_iam_instance_profile.runner[0].name + } + + instance_initiated_shutdown_behavior = "terminate" + image_id = "resolve:ssm:${local.ami_id_ssm_module_managed ? aws_ssm_parameter.runner_ami_id[0].arn : local.ami_id_ssm_parameter_arn}" + key_name = var.config.key_name + ebs_optimized = var.config.ebs_optimized + + vpc_security_group_ids = !var.config.associate_public_ipv4_address ? compact(concat( + var.config.managed_security_group_enabled ? [aws_security_group.runner_sg[0].id] : [], + var.config.additional_security_group_ids, + )) : [] + + tag_specifications { + resource_type = "instance" + tags = local.runner_tags + } + + tag_specifications { + resource_type = "volume" + tags = local.runner_tags + } + + # We avoid including the "spot-instances-request" tag_specifications block when on_demand_failover_for_errors is defined, + # because when using on-demand fallback, the spot instance request resource is not created and thus the tags would not apply. + # Additionally, tagging spot requests via the CreateFleetCommand in the Lambda function does not work as expected, + # so we rely on Terraform to manage these tags only when spot is exclusively used without on-demand failover. + dynamic "tag_specifications" { + for_each = var.config.instance_target_capacity_type == "spot" && length(var.config.enable_on_demand_failover_for_errors) == 0 ? [1] : [] # Include the block only if the value is "spot" and on_demand_failover_for_errors is not enabled + content { + resource_type = "spot-instances-request" + tags = local.runner_tags + } + } + + tag_specifications { + resource_type = "network-interface" + tags = local.runner_tags + } + + user_data = local.encoded_user_data + + tags = local.provider_tags + + update_default_version = true + + dynamic "network_interfaces" { + for_each = var.config.associate_public_ipv4_address ? [var.config.associate_public_ipv4_address] : [] + iterator = associate_public_ipv4_address + content { + associate_public_ip_address = associate_public_ipv4_address.value + security_groups = compact(concat( + var.config.managed_security_group_enabled ? [aws_security_group.runner_sg[0].id] : [], + var.config.additional_security_group_ids, + )) + } + } +} + +resource "aws_security_group" "runner_sg" { + count = var.config.managed_security_group_enabled ? 1 : 0 + name_prefix = "${var.prefix}-github-actions-runner-sg" + description = "Github Actions Runner security group" + + vpc_id = var.config.vpc_id + + ingress = [] + + dynamic "egress" { + for_each = var.config.egress_rules + iterator = each + + content { + cidr_blocks = each.value.cidr_blocks + ipv6_cidr_blocks = each.value.ipv6_cidr_blocks + prefix_list_ids = each.value.prefix_list_ids + from_port = each.value.from_port + protocol = each.value.protocol + security_groups = each.value.security_groups + self = each.value.self + to_port = each.value.to_port + description = each.value.description + } + } + + tags = merge( + local.provider_tags, + { + "Name" = format("%s", local.name_sg) + }, + ) +} diff --git a/modules/compute-providers/ec2/templates/cloudwatch_config.json b/modules/compute-providers/ec2/templates/cloudwatch_config.json new file mode 100644 index 0000000000..47b9bede8a --- /dev/null +++ b/modules/compute-providers/ec2/templates/cloudwatch_config.json @@ -0,0 +1,12 @@ +{ + "agent": { + "metrics_collection_interval": 5 + }, + "logs": { + "logs_collected": { + "files": { + "collect_list": ${logfiles} + } + } + } +} diff --git a/modules/compute-providers/ec2/templates/install-runner-osx.sh b/modules/compute-providers/ec2/templates/install-runner-osx.sh new file mode 100644 index 0000000000..ed848dad27 --- /dev/null +++ b/modules/compute-providers/ec2/templates/install-runner-osx.sh @@ -0,0 +1,61 @@ +# shellcheck shell=bash + +set -euo pipefail + +## install the runner (macOS) + +s3_location=${S3_LOCATION_RUNNER_DISTRIBUTION} +architecture=${RUNNER_ARCHITECTURE} + +if [ -z "$RUNNER_TARBALL_URL" ] && [ -z "$s3_location" ]; then + echo "Neither RUNNER_TARBALL_URL or s3_location are set" + exit 1 +fi + +file_name="actions-runner.tar.gz" + +echo "Setting up GH Actions runner tool cache" +mkdir -p /Users/runner/hostedtoolcache + +echo "Creating actions-runner directory for the GH Action installation" +sudo mkdir -p /opt/actions-runner +cd /opt/actions-runner || exit 1 + +if [[ -n "$runner_tarball_url" ]]; then + echo "Downloading the GH Action runner from $runner_tarball_url to $file_name" + curl -s -o "$file_name" -L "$runner_tarball_url" +else + echo "Retrieving REGION from AWS API" + token="$(curl -s -f -X PUT "http://169.254.169.254/latest/api/token" \ + -H "X-aws-ec2-metadata-token-ttl-seconds: 180")" + + region="$(curl -s -f -H "X-aws-ec2-metadata-token: $token" \ + http://169.254.169.254/latest/dynamic/instance-identity/document | jq -r .region)" + echo "Retrieved REGION from AWS API ($region)" + + echo "Downloading the GH Action runner from s3 bucket $s3_location" + aws s3 cp "$s3_location" "$file_name" --region "$region" --no-progress +fi + +echo "Un-tar action runner" +tar xzf "./$file_name" +echo "Delete tar file" +rm -rf "$file_name" + +os_name=$(sw_vers -productName 2>/dev/null || echo "macOS") +os_version=$(sw_vers -productVersion 2>/dev/null || echo "unknown") +arch_name=$(uname -m) + +echo "OS: $os_name $os_version ($arch_name)" + +if ! command -v brew >/dev/null 2>&1; then + echo "Homebrew not found; skipping dependency installation via brew" +else + echo "Homebrew detected; install any macOS-specific dependencies here if needed" + # Example: brew install jq awscli +fi + +echo "Set file ownership of action runner" +sudo chown -R "$user_name":staff /opt/actions-runner +sudo chmod 755 "/Users/runner" +sudo chown -R "$user_name":staff /Users/runner/hostedtoolcache diff --git a/modules/compute-providers/ec2/templates/install-runner.ps1 b/modules/compute-providers/ec2/templates/install-runner.ps1 new file mode 100644 index 0000000000..a13f91a65b --- /dev/null +++ b/modules/compute-providers/ec2/templates/install-runner.ps1 @@ -0,0 +1,13 @@ +## install the runner + +Write-Host "Creating actions-runner directory for the GH Action installation" +New-Item -ItemType Directory -Path C:\actions-runner ; Set-Location C:\actions-runner + +Write-Host "Downloading the GH Action runner from s3 bucket $s3_location" +aws s3 cp ${S3_LOCATION_RUNNER_DISTRIBUTION} actions-runner.zip + +Write-Host "Un-zip action runner" +Expand-Archive -Path actions-runner.zip -DestinationPath . + +Write-Host "Delete zip file" +Remove-Item actions-runner.zip diff --git a/modules/compute-providers/ec2/templates/install-runner.sh b/modules/compute-providers/ec2/templates/install-runner.sh new file mode 100644 index 0000000000..5ed5897e7c --- /dev/null +++ b/modules/compute-providers/ec2/templates/install-runner.sh @@ -0,0 +1,73 @@ +# shellcheck shell=bash + +## install the runner + +s3_location=${S3_LOCATION_RUNNER_DISTRIBUTION} + +if [ -z "$RUNNER_TARBALL_URL" ] && [ -z "$s3_location" ]; then + echo "Neither RUNNER_TARBALL_URL or s3_location are set" + exit 1 +fi + +file_name="actions-runner.tar.gz" + +echo "Setting up GH Actions runner tool cache" +# Required for various */setup-* actions to work, location is also know by various environment +# variable names in the actions/runner software : RUNNER_TOOL_CACHE / RUNNER_TOOLSDIRECTORY / AGENT_TOOLSDIRECTORY +# Warning, not all setup actions support the env vars and so this specific path must be created regardless +mkdir -p /opt/hostedtoolcache + +echo "Creating actions-runner directory for the GH Action installation" +cd /opt/ +mkdir -p actions-runner && cd actions-runner + + +if [[ -n "$RUNNER_TARBALL_URL" ]]; then + echo "Downloading the GH Action runner from $RUNNER_TARBALL_URL to $file_name" + curl -s -o $file_name -L "$RUNNER_TARBALL_URL" +else + echo "Retrieving TOKEN from AWS API" + token="$(curl -s -f -X PUT "http://169.254.169.254/latest/api/token" -H "X-aws-ec2-metadata-token-ttl-seconds: 180")" + + region="$(curl -s -f -H "X-aws-ec2-metadata-token: $token" http://169.254.169.254/latest/dynamic/instance-identity/document | jq -r .region)" + echo "Retrieved REGION from AWS API ($region)" + + echo "Downloading the GH Action runner from s3 bucket $s3_location" + aws s3 cp "$s3_location" "$file_name" --region "$region" --no-progress +fi + +echo "Un-tar action runner" +tar xzf ./$file_name +echo "Delete tar file" +rm -rf $file_name + +os_id=$(awk -F= '/^ID=/{print $2}' /etc/os-release) +echo OS: $os_id + +# Install libicu on non-ubuntu, non-debian +if [[ ! "$os_id" =~ ^(ubuntu|debian).* ]]; then + max_attempts=5 + attempt_count=0 + success=false + while [ $success = false ] && [ $attempt_count -le $max_attempts ]; do + echo "Attempt $attempt_count/$max_attempts: Installing libicu" + dnf install -y libicu + if [ $? -eq 0 ]; then + success=true + else + echo "Failed to install libicu" + attempt_count=$(( attempt_count + 1 )) + sleep 5 + fi + done +fi + +# Install dependencies for ubuntu and debian +if [[ "$os_id" =~ ^(ubuntu|debian).* ]]; then + echo "Installing dependencies" + ./bin/installdependencies.sh +fi + +echo "Set file ownership of action runner" +chown -R "$user_name":"$user_name" /opt/actions-runner +chown -R "$user_name":"$user_name" /opt/hostedtoolcache diff --git a/modules/compute-providers/ec2/templates/start-runner-osx.sh b/modules/compute-providers/ec2/templates/start-runner-osx.sh new file mode 100644 index 0000000000..a6da66116d --- /dev/null +++ b/modules/compute-providers/ec2/templates/start-runner-osx.sh @@ -0,0 +1,185 @@ +#!/bin/bash + +# macOS variant of start-runner.sh + +tag_instance_with_runner_id() { + echo "Checking for .runner file to extract agent ID" + + if [[ ! -f "/opt/actions-runner/.runner" ]]; then + echo "Warning: .runner file not found" + return 0 + fi + + echo "Found .runner file, extracting agent ID" + local agent_id + agent_id=$(jq -r '.agentId' /opt/actions-runner/.runner 2>/dev/null || echo "") + + if [[ -z "$agent_id" || "$agent_id" == "null" ]]; then + echo "Warning: Could not extract agent ID from .runner file" + return 0 + fi + + echo "Tagging instance with GitHub runner agent ID: $agent_id" + if aws ec2 create-tags \ + --region "$region" \ + --resources "$instance_id" \ + --tags Key=ghr:github_runner_id,Value="$agent_id"; then + echo "Successfully tagged instance with agent ID: $agent_id" + return 0 + else + echo "Warning: Failed to tag instance with agent ID" + return 0 + fi +} + +cleanup() { + local exit_code="$1" + + if [ "$exit_code" -ne 0 ]; then + echo "ERROR: runner-start-failed with exit code $exit_code" + fi + + if [ "$agent_mode" = "ephemeral" ] || [ "$exit_code" -ne 0 ]; then + echo "Terminating instance" + aws ec2 terminate-instances \ + --instance-ids "$instance_id" \ + --region "$region" || true + fi +} + +trap 'cleanup $?' EXIT + +echo "Retrieving TOKEN from AWS API" +token=$(curl -f -X PUT "http://169.254.169.254/latest/api/token" \ + -H "X-aws-ec2-metadata-token-ttl-seconds: 180" || true) +if [ -z "$token" ]; then + retrycount=0 + until [ -n "$token" ]; do + echo "Failed to retrieve token. Retrying in 5 seconds." + sleep 5 + token=$(curl -f -X PUT "http://169.254.169.254/latest/api/token" \ + -H "X-aws-ec2-metadata-token-ttl-seconds: 180" || true) + retrycount=$((retrycount + 1)) + if [ $retrycount -gt 40 ]; then + break + fi + done +fi + +region=$(curl -f -H "X-aws-ec2-metadata-token: $token" \ + http://169.254.169.254/latest/dynamic/instance-identity/document | jq -r .region) +echo "Retrieved REGION from AWS API ($region)" + +instance_id=$(curl -f -H "X-aws-ec2-metadata-token: $token" \ + http://169.254.169.254/latest/meta-data/instance-id) +echo "Retrieved INSTANCE_ID from AWS API ($instance_id)" + +availability_zone=$(curl -f -H "X-aws-ec2-metadata-token: $token" \ + http://169.254.169.254/latest/meta-data/placement/availability-zone) + +environment=$(curl -f -H "X-aws-ec2-metadata-token: $token" \ + http://169.254.169.254/latest/meta-data/tags/instance/ghr:environment || echo "") +ssm_config_path=$(curl -f -H "X-aws-ec2-metadata-token: $token" \ + http://169.254.169.254/latest/meta-data/tags/instance/ghr:ssm_config_path || echo "") +runner_name_prefix=$(curl -f -H "X-aws-ec2-metadata-token: $token" \ + http://169.254.169.254/latest/meta-data/tags/instance/ghr:runner_name_prefix || echo "") + +echo "Retrieved ghr:environment tag - ($environment)" +echo "Retrieved ghr:ssm_config_path tag - ($ssm_config_path)" +echo "Retrieved ghr:runner_name_prefix tag - ($runner_name_prefix)" + +parameters=$(aws ssm get-parameters-by-path \ + --path "$ssm_config_path" \ + --region "$region" \ + --query "Parameters[*].{Name:Name,Value:Value}") +echo "Retrieved parameters from AWS SSM ($parameters)" + +run_as=$(echo "$parameters" | jq -r '.[] | select(.Name == "'$ssm_config_path'/run_as") | .Value') +echo "Retrieved /$ssm_config_path/run_as parameter - ($run_as)" + +agent_mode=$(echo "$parameters" | jq -r '.[] | select(.Name == "'$ssm_config_path'/agent_mode") | .Value') +echo "Retrieved /$ssm_config_path/agent_mode parameter - ($agent_mode)" + +disable_default_labels=$(echo "$parameters" | jq -r '.[] | select(.Name == "'$ssm_config_path'/disable_default_labels") | .Value') +echo "Retrieved /$ssm_config_path/disable_default_labels parameter - ($disable_default_labels)" + +enable_jit_config=$(echo "$parameters" | jq -r '.[] | select(.Name == "'$ssm_config_path'/enable_jit_config") | .Value') +echo "Retrieved /$ssm_config_path/enable_jit_config parameter - ($enable_jit_config)" + +token_path=$(echo "$parameters" | jq -r '.[] | select(.Name == "'$ssm_config_path'/token_path") | .Value') +echo "Retrieved /$ssm_config_path/token_path parameter - ($token_path)" + +echo "Get GH Runner config from AWS SSM" +config=$(aws ssm get-parameter --name "$token_path"/"$instance_id" --with-decryption --region "$region" | jq -r ".Parameter | .Value") +while [[ -z "$config" ]]; do + echo "Waiting for GH Runner config to become available in AWS SSM" + sleep 1 + config=$(aws ssm get-parameter --name "$token_path"/"$instance_id" --with-decryption --region "$region" | jq -r ".Parameter | .Value") +done + +echo "Delete GH Runner token from AWS SSM" +aws ssm delete-parameter --name "$token_path"/"$instance_id" --region "$region" + +if [ -z "$run_as" ]; then + echo "No user specified, using default ec2-user account" + run_as="ec2-user" +fi + +if [[ "$run_as" == "root" ]]; then + echo "run_as is set to root - export RUNNER_ALLOW_RUNASROOT=1" + export RUNNER_ALLOW_RUNASROOT=1 +fi + +sudo chown -R "$run_as" /opt/actions-runner + +info_arch=$(uname -m) +info_os=$(sw_vers -productName 2>/dev/null || echo "macOS") +info_ver=$(sw_vers -productVersion 2>/dev/null || echo "unknown") + +tee /opt/actions-runner/.setup_info <&1 + + if ($LASTEXITCODE -eq 0) { + Write-Host "Successfully tagged instance with agent ID: $agentId" + return $true + } else { + Write-Host "Warning: Failed to tag instance with agent ID - $tagResult" + return $true + } + } + catch { + Write-Host "Warning: Error processing .runner file - $($_.Exception.Message)" + return $true + } +} + +## Retrieve instance metadata + +Write-Host "Retrieving TOKEN from AWS API" +$token=Invoke-RestMethod -Method PUT -Uri "http://169.254.169.254/latest/api/token" -Headers @{"X-aws-ec2-metadata-token-ttl-seconds" = "180"} +if ( ! $token ) { + $retrycount=0 + do { + echo "Failed to retrieve token. Retrying in 5 seconds." + Start-Sleep 5 + $token=Invoke-RestMethod -Method PUT -Uri "http://169.254.169.254/latest/api/token" -Headers @{"X-aws-ec2-metadata-token-ttl-seconds" = "180"} + $retrycount=$retrycount + 1 + if ( $retrycount -gt 40 ) + { + break + } + } until ($token) +} + +$ami_id=Invoke-RestMethod -Uri "http://169.254.169.254/latest/meta-data/ami-id" -Headers @{"X-aws-ec2-metadata-token" = $token} + +$metadata=Invoke-RestMethod -Uri "http://169.254.169.254/latest/dynamic/instance-identity/document" -Headers @{"X-aws-ec2-metadata-token" = $token} + +$Region = $metadata.region +Write-Host "Retrieved REGION from AWS API ($Region)" + +$InstanceId = $metadata.instanceId +Write-Host "Retrieved InstanceId from AWS API ($InstanceId)" + +$tags=aws ec2 describe-tags --region "$Region" --filters "Name=resource-id,Values=$InstanceId" | ConvertFrom-Json +Write-Host "Retrieved tags from AWS API" + +$environment=$tags.Tags.where( {$_.Key -eq 'ghr:environment'}).value +Write-Host "Retrieved ghr:environment tag - ($environment)" + +$runner_name_prefix=$tags.Tags.where( {$_.Key -eq 'ghr:runner_name_prefix'}).value +Write-Host "Retrieved ghr:runner_name_prefix tag - ($runner_name_prefix)" + +$ssm_config_path=$tags.Tags.where( {$_.Key -eq 'ghr:ssm_config_path'}).value +Write-Host "Retrieved ghr:ssm_config_path tag - ($ssm_config_path)" + +$parameters=$(aws ssm get-parameters-by-path --path "$ssm_config_path" --region "$Region" --query "Parameters[*].{Name:Name,Value:Value}") | ConvertFrom-Json +Write-Host "Retrieved parameters from AWS SSM" + +$run_as=$parameters.where( {$_.Name -eq "$ssm_config_path/run_as"}).value +Write-Host "Retrieved $ssm_config_path/run_as parameter - ($run_as)" + +$enable_cloudwatch_agent=$parameters.where( {$_.Name -eq "$ssm_config_path/enable_cloudwatch"}).value +Write-Host "Retrieved $ssm_config_path/enable_cloudwatch parameter - ($enable_cloudwatch_agent)" + +$agent_mode=$parameters.where( {$_.Name -eq "$ssm_config_path/agent_mode"}).value +Write-Host "Retrieved $ssm_config_path/agent_mode parameter - ($agent_mode)" + +$disable_default_labels=$parameters.where( {$_.Name -eq "$ssm_config_path/disable_default_labels"}).value +Write-Host "Retrieved $ssm_config_path/disable_default_labels parameter - ($disable_default_labels)" + +$enable_jit_config=$parameters.where( {$_.Name -eq "$ssm_config_path/enable_jit_config"}).value +Write-Host "Retrieved $ssm_config_path/enable_jit_config parameter - ($enable_jit_config)" + +$token_path=$parameters.where( {$_.Name -eq "$ssm_config_path/token_path"}).value +Write-Host "Retrieved $ssm_config_path/token_path parameter - ($token_path)" + + +if ($enable_cloudwatch_agent -eq "true") +{ + Write-Host "Enabling CloudWatch Agent" + & 'C:\Program Files\Amazon\AmazonCloudWatchAgent\amazon-cloudwatch-agent-ctl.ps1' -a fetch-config -m ec2 -s -c "ssm:$ssm_config_path/cloudwatch_agent_config_runner" +} + +## Configure the runner + +Write-Host "Get GH Runner config from AWS SSM" +$config = $null +$i = 0 +do { + $config = (aws ssm get-parameters --names "$token_path/$InstanceId" --with-decryption --region $Region --query "Parameters[*].{Name:Name,Value:Value}" | ConvertFrom-Json)[0].value + Write-Host "Waiting for GH Runner config to become available in AWS SSM ($i/30)" + Start-Sleep 1 + $i++ +} while (($null -eq $config) -and ($i -lt 30)) + +Write-Host "Delete GH Runner token from AWS SSM" +aws ssm delete-parameter --name "$token_path/$InstanceId" --region $Region + +# Create or update user +if (-not($run_as)) { + Write-Host "No user specified, using default ec2-user account" + $run_as="ec2-user" +} +Add-Type -AssemblyName "System.Web" +$password = [System.Web.Security.Membership]::GeneratePassword(24, 4) +$securePassword = ConvertTo-SecureString $password -AsPlainText -Force +$username = $run_as +if (!(Get-LocalUser -Name $username -ErrorAction Ignore)) { + New-LocalUser -Name $username -Password $securePassword + Write-Host "Created new user ($username)" +} +else { + Set-LocalUser -Name $username -Password $securePassword + Write-Host "Changed password for user ($username)" +} +# Add user to groups +foreach ($group in @("Administrators", "docker-users")) { + if ((Get-LocalGroup -Name "$group" -ErrorAction Ignore) -and + !(Get-LocalGroupMember -Group "$group" -Member $username -ErrorAction Ignore)) { + Add-LocalGroupMember -Group "$group" -Member $username + Write-Host "Added $username to $group group" + } +} + +# Disable User Access Control (UAC) +# TODO investigate if this is needed or if its overkill - https://github.com/github-aws-runners/terraform-aws-github-runner/issues/1505 +Set-ItemProperty HKLM:\SOFTWARE\Microsoft\Windows\CurrentVersion\Policies\System -Name ConsentPromptBehaviorAdmin -Value 0 -Force +Write-Host "Disabled User Access Control (UAC)" + +$runnerExtraOptions = "" +if ($disable_default_labels -eq "true") { + $runnerExtraOptions += "--no-default-labels" +} + +if ($enable_jit_config -eq "false" -or $agent_mode -ne "ephemeral") { + $configCmd = ".\config.cmd --unattended --name $runner_name_prefix$InstanceId --work `"_work`" $runnerExtraOptions $config" + Write-Host "Configure GH Runner (non ephmeral / no JIT) as user $run_as" + Invoke-Expression $configCmd + + # Tag instance with GitHub runner agent ID for non-JIT runners + Tag-InstanceWithRunnerId +} + +$jsonBody = @( + @{ + group='Runner Image' + detail="AMI id: $ami_id" + } +) +ConvertTo-Json -InputObject $jsonBody | Set-Content -Path "$pwd\.setup_info" + + +Write-Host "Starting the runner in $agent_mode mode" +Write-Host "Starting runner after $(((get-date) - (gcim Win32_OperatingSystem).LastBootUpTime).tostring("hh':'mm':'ss''"))" + +if ($agent_mode -eq "ephemeral") { + if ($enable_jit_config -eq "true") { + Write-Host "Starting with jit config" + Invoke-Expression ".\run.cmd --jitconfig $${config}" + } + else { + Write-Host "Starting without jit config" + Invoke-Expression ".\run.cmd" + } + Write-Host "Runner has finished" + + if ($enable_cloudwatch_agent) + { + Write-Host "Stopping CloudWatch Agent" + & 'C:\Program Files\Amazon\AmazonCloudWatchAgent\amazon-cloudwatch-agent-ctl.ps1' -a stop + } + + Write-Host "Terminating instance" + aws ec2 terminate-instances --instance-ids "$InstanceId" --region "$Region" +} else { + Write-Host "Installing the runner as a service" + + $action = New-ScheduledTaskAction -WorkingDirectory "$pwd" -Execute "run.cmd" + $trigger = Get-CimClass "MSFT_TaskRegistrationTrigger" -Namespace "Root/Microsoft/Windows/TaskScheduler" + Register-ScheduledTask -TaskName "runnertask" -Action $action -Trigger $trigger -User $username -Password $password -RunLevel Highest -Force + Write-Host "Starting runner after $(((get-date) - (gcim Win32_OperatingSystem).LastBootUpTime).tostring("hh':'mm':'ss''"))" +} diff --git a/modules/compute-providers/ec2/templates/start-runner.sh b/modules/compute-providers/ec2/templates/start-runner.sh new file mode 100644 index 0000000000..7f2c0f82c5 --- /dev/null +++ b/modules/compute-providers/ec2/templates/start-runner.sh @@ -0,0 +1,280 @@ +#!/bin/bash + +# https://docs.aws.amazon.com/xray/latest/devguide/xray-api-sendingdata.html +# https://docs.aws.amazon.com/xray/latest/devguide/scorekeep-scripts.html +create_xray_start_segment() { + START_TIME=$(date -d "$(uptime -s)" +%s) + TRACE_ID=$1 + INSTANCE_ID=$2 + SEGMENT_ID=$(dd if=/dev/random bs=8 count=1 2>/dev/null | od -An -tx1 | tr -d ' \t\n') + SEGMENT_DOC="{\"trace_id\": \"$TRACE_ID\", \"id\": \"$SEGMENT_ID\", \"start_time\": $START_TIME, \"in_progress\": true, \"name\": \"Runner\",\"origin\": \"AWS::EC2::Instance\", \"aws\": {\"ec2\":{\"instance_id\":\"$INSTANCE_ID\"}}}" + HEADER='{"format": "json", "version": 1}' + TRACE_DATA="$HEADER\n$SEGMENT_DOC" + echo "$HEADER" > document.txt + echo "$SEGMENT_DOC" >> document.txt + UDP_IP="127.0.0.1" + UDP_PORT=2000 + cat document.txt > /dev/udp/$UDP_IP/$UDP_PORT + echo "$SEGMENT_DOC" +} + +create_xray_success_segment() { + local SEGMENT_DOC=$1 + if [ -z "$SEGMENT_DOC" ]; then + echo "No segment doc provided" + return + fi + SEGMENT_DOC=$(echo "$SEGMENT_DOC" | jq '. | del(.in_progress)') + END_TIME=$(date +%s) + SEGMENT_DOC=$(echo "$SEGMENT_DOC" | jq -c ". + {\"end_time\": $END_TIME}") + HEADER="{\"format\": \"json\", \"version\": 1}" + TRACE_DATA="$HEADER\n$SEGMENT_DOC" + echo "$HEADER" > document.txt + echo "$SEGMENT_DOC" >> document.txt + UDP_IP="127.0.0.1" + UDP_PORT=2000 + cat document.txt > /dev/udp/$UDP_IP/$UDP_PORT + echo "$SEGMENT_DOC" +} + +create_xray_error_segment() { + local SEGMENT_DOC="$1" + if [ -z "$SEGMENT_DOC" ]; then + echo "No segment doc provided" + return + fi + MESSAGE="$2" + ERROR="{\"exceptions\": [{\"message\": \"$MESSAGE\"}]}" + SEGMENT_DOC=$(echo "$SEGMENT_DOC" | jq '. | del(.in_progress)') + END_TIME=$(date +%s) + SEGMENT_DOC=$(echo "$SEGMENT_DOC" | jq -c ". + {\"end_time\": $END_TIME, \"error\": true, \"cause\": $ERROR }") + HEADER="{\"format\": \"json\", \"version\": 1}" + TRACE_DATA="$HEADER\n$SEGMENT_DOC" + echo "$HEADER" > document.txt + echo "$SEGMENT_DOC" >> document.txt + UDP_IP="127.0.0.1" + UDP_PORT=2000 + cat document.txt > /dev/udp/$UDP_IP/$UDP_PORT + echo "$SEGMENT_DOC" +} + +tag_instance_with_runner_id() { + echo "Checking for .runner file to extract agent ID" + + if [[ ! -f "/opt/actions-runner/.runner" ]]; then + echo "Warning: .runner file not found" + return 0 + fi + + echo "Found .runner file, extracting agent ID" + local agent_id + agent_id=$(jq -r '.agentId' /opt/actions-runner/.runner 2>/dev/null || echo "") + + if [[ -z "$agent_id" || "$agent_id" == "null" ]]; then + echo "Warning: Could not extract agent ID from .runner file" + return 0 + fi + + echo "Tagging instance with GitHub runner agent ID: $agent_id" + if aws ec2 create-tags \ + --region "$region" \ + --resources "$instance_id" \ + --tags Key=ghr:github_runner_id,Value="$agent_id"; then + echo "Successfully tagged instance with agent ID: $agent_id" + return 0 + else + echo "Warning: Failed to tag instance with agent ID" + return 0 + fi +} + +cleanup() { + local exit_code="$1" + local error_location="$2" + local error_lineno="$3" + + if [ "$exit_code" -ne 0 ]; then + echo "ERROR: runner-start-failed with exit code $exit_code occurred on $error_location" + create_xray_error_segment "$SEGMENT" "runner-start-failed with exit code $exit_code occurred on $error_location - $error_lineno" + fi + # allows to flush the cloud watch logs and traces + sleep 10 + if [ "$agent_mode" = "ephemeral" ] || [ "$exit_code" -ne 0 ]; then + echo "Stopping CloudWatch service" + systemctl stop amazon-cloudwatch-agent.service || true + echo "Terminating instance" + aws ec2 terminate-instances \ + --instance-ids "$instance_id" \ + --region "$region" \ + || true + fi +} + +trap 'cleanup $? $LINENO $BASH_LINENO' EXIT + +echo "Retrieving TOKEN from AWS API" +token=$(curl -f -X PUT "http://169.254.169.254/latest/api/token" -H "X-aws-ec2-metadata-token-ttl-seconds: 180" || true) +if [ -z "$token" ]; then + retrycount=0 + until [ -n "$token" ]; do + echo "Failed to retrieve token. Retrying in 5 seconds." + sleep 5 + token=$(curl -f -X PUT "http://169.254.169.254/latest/api/token" -H "X-aws-ec2-metadata-token-ttl-seconds: 180" || true) + retrycount=$((retrycount + 1)) + if [ $retrycount -gt 40 ]; then + break + fi + done +fi + +ami_id=$(curl -f -H "X-aws-ec2-metadata-token: $token" -v http://169.254.169.254/latest/meta-data/ami-id) + +region=$(curl -f -H "X-aws-ec2-metadata-token: $token" -v http://169.254.169.254/latest/dynamic/instance-identity/document | jq -r .region) +echo "Retrieved REGION from AWS API ($region)" + +instance_id=$(curl -f -H "X-aws-ec2-metadata-token: $token" -v http://169.254.169.254/latest/meta-data/instance-id) +echo "Retrieved INSTANCE_ID from AWS API ($instance_id)" + +instance_type=$(curl -f -H "X-aws-ec2-metadata-token: $token" -v http://169.254.169.254/latest/meta-data/instance-type) +availability_zone=$(curl -f -H "X-aws-ec2-metadata-token: $token" -v http://169.254.169.254/latest/meta-data/placement/availability-zone) + +%{ if metadata_tags == "enabled" } +environment=$(curl -f -H "X-aws-ec2-metadata-token: $token" -v http://169.254.169.254/latest/meta-data/tags/instance/ghr:environment) +ssm_config_path=$(curl -f -H "X-aws-ec2-metadata-token: $token" -v http://169.254.169.254/latest/meta-data/tags/instance/ghr:ssm_config_path) +runner_name_prefix=$(curl -f -H "X-aws-ec2-metadata-token: $token" -v http://169.254.169.254/latest/meta-data/tags/instance/ghr:runner_name_prefix || echo "") +xray_trace_id=$(curl -f -H "X-aws-ec2-metadata-token: $token" -v http://169.254.169.254/latest/meta-data/tags/instance/ghr:trace_id || echo "") + +%{ else } +tags=$(aws ec2 describe-tags --region "$region" --filters "Name=resource-id,Values=$instance_id") +echo "Retrieved tags from AWS API ($tags)" + +environment=$(echo "$tags" | jq -r '.Tags[] | select(.Key == "ghr:environment") | .Value') +ssm_config_path=$(echo "$tags" | jq -r '.Tags[] | select(.Key == "ghr:ssm_config_path") | .Value') +runner_name_prefix=$(echo "$tags" | jq -r '.Tags[] | select(.Key == "ghr:runner_name_prefix") | .Value' || echo "") +xray_trace_id=$(echo "$tags" | jq -r '.Tags[] | select(.Key == "ghr:trace_id") | .Value' || echo "") + +%{ endif } + +echo "Retrieved ghr:environment tag - ($environment)" +echo "Retrieved ghr:ssm_config_path tag - ($ssm_config_path)" +echo "Retrieved ghr:runner_name_prefix tag - ($runner_name_prefix)" + +parameters=$(aws ssm get-parameters-by-path --path "$ssm_config_path" --region "$region" --query "Parameters[*].{Name:Name,Value:Value}") +echo "Retrieved parameters from AWS SSM ($parameters)" + +run_as=$(echo "$parameters" | jq -r '.[] | select(.Name == "'$ssm_config_path'/run_as") | .Value') +echo "Retrieved /$ssm_config_path/run_as parameter - ($run_as)" + +enable_cloudwatch_agent=$(echo "$parameters" | jq --arg ssm_config_path "$ssm_config_path" -r '.[] | select(.Name == "'$ssm_config_path'/enable_cloudwatch") | .Value') +echo "Retrieved /$ssm_config_path/enable_cloudwatch parameter - ($enable_cloudwatch_agent)" + +agent_mode=$(echo "$parameters" | jq --arg ssm_config_path "$ssm_config_path" -r '.[] | select(.Name == "'$ssm_config_path'/agent_mode") | .Value') +echo "Retrieved /$ssm_config_path/agent_mode parameter - ($agent_mode)" + +disable_default_labels=$(echo "$parameters" | jq --arg ssm_config_path "$ssm_config_path" -r '.[] | select(.Name == "'$ssm_config_path'/disable_default_labels") | .Value') +echo "Retrieved /$ssm_config_path/disable_default_labels parameter - ($disable_default_labels)" + +enable_jit_config=$(echo "$parameters" | jq --arg ssm_config_path "$ssm_config_path" -r '.[] | select(.Name == "'$ssm_config_path'/enable_jit_config") | .Value') +echo "Retrieved /$ssm_config_path/enable_jit_config parameter - ($enable_jit_config)" + +token_path=$(echo "$parameters" | jq --arg ssm_config_path "$ssm_config_path" -r '.[] | select(.Name == "'$ssm_config_path'/token_path") | .Value') +echo "Retrieved /$ssm_config_path/token_path parameter - ($token_path)" + +if [[ "$xray_trace_id" != "" ]]; then + # run xray service + curl https://s3.us-east-2.amazonaws.com/aws-xray-assets.us-east-2/xray-daemon/aws-xray-daemon-linux-3.x.zip -o aws-xray-daemon-linux-3.x.zip + unzip aws-xray-daemon-linux-3.x.zip -d aws-xray-daemon-linux-3.x + chmod +x ./aws-xray-daemon-linux-3.x/xray + ./aws-xray-daemon-linux-3.x/xray -o -n "$region" & + + + SEGMENT=$(create_xray_start_segment "$xray_trace_id" "$instance_id") + echo "$SEGMENT" +fi + +if [[ "$enable_cloudwatch_agent" == "true" ]]; then + echo "Cloudwatch is enabled" + amazon-cloudwatch-agent-ctl -a fetch-config -m ec2 -s -c "ssm:$ssm_config_path/cloudwatch_agent_config_runner" +fi + +## Configure the runner + +echo "Get GH Runner config from AWS SSM" +config=$(aws ssm get-parameter --name "$token_path"/"$instance_id" --with-decryption --region "$region" | jq -r ".Parameter | .Value") +while [[ -z "$config" ]]; do + echo "Waiting for GH Runner config to become available in AWS SSM" + sleep 1 + config=$(aws ssm get-parameter --name "$token_path"/"$instance_id" --with-decryption --region "$region" | jq -r ".Parameter | .Value") +done + +echo "Delete GH Runner token from AWS SSM" +aws ssm delete-parameter --name "$token_path"/"$instance_id" --region "$region" + +if [ -z "$run_as" ]; then + echo "No user specified, using default ec2-user account" + run_as="ec2-user" +fi + +if [[ "$run_as" == "root" ]]; then + echo "run_as is set to root - export RUNNER_ALLOW_RUNASROOT=1" + export RUNNER_ALLOW_RUNASROOT=1 +fi + +chown -R $run_as /opt/actions-runner + +info_arch=$(uname -p) +info_os=$( ( lsb_release -ds || cat /etc/*release || uname -om ) 2>/dev/null | head -n1 | cut -d "=" -f2- | tr -d '"') + +tee /opt/actions-runner/.setup_info </dev/null 2>&1; then + echo "Homebrew detected; you can install extra dependencies via brew if needed" +fi + +user_name=ec2-user + +${install_runner} + +${post_install} + +# Register runner job hooks +# Ref: https://docs.github.com/en/actions/hosting-your-own-runners/managing-self-hosted-runners/running-scripts-before-or-after-a-job +%{ if hook_job_started != "" } +cat > /opt/actions-runner/hook_job_started.sh <<'EOF' +${hook_job_started} +EOF +echo ACTIONS_RUNNER_HOOK_JOB_STARTED=/opt/actions-runner/hook_job_started.sh | tee -a /opt/actions-runner/.env +%{ endif } + +%{ if hook_job_completed != "" } +cat > /opt/actions-runner/hook_job_completed.sh <<'EOF' +${hook_job_completed} +EOF +echo ACTIONS_RUNNER_HOOK_JOB_COMPLETED=/opt/actions-runner/hook_job_completed.sh | tee -a /opt/actions-runner/.env +%{ endif } + +${start_runner} diff --git a/modules/compute-providers/ec2/templates/user-data.ps1 b/modules/compute-providers/ec2/templates/user-data.ps1 new file mode 100644 index 0000000000..a1e3a4da66 --- /dev/null +++ b/modules/compute-providers/ec2/templates/user-data.ps1 @@ -0,0 +1,47 @@ + +$ErrorActionPreference = "Continue" +$VerbosePreference = "Continue" +Start-Transcript -Path "C:\UserData.log" -Append + +${pre_install} + +# Install Chocolatey +[System.Net.ServicePointManager]::SecurityProtocol = [System.Net.SecurityProtocolType]::Tls12 +$env:chocolateyUseWindowsCompression = 'true' +Invoke-WebRequest https://chocolatey.org/install.ps1 -UseBasicParsing | Invoke-Expression + +# Add Chocolatey to powershell profile +$ChocoProfileValue = @' +$ChocolateyProfile = "$env:ChocolateyInstall\helpers\chocolateyProfile.psm1" +if (Test-Path($ChocolateyProfile)) { + Import-Module "$ChocolateyProfile" +} + +refreshenv +'@ +# Write it to the $profile location +Set-Content -Path "$PsHome\Microsoft.PowerShell_profile.ps1" -Value $ChocoProfileValue -Force +# Source it +. "$PsHome\Microsoft.PowerShell_profile.ps1" + + +refreshenv + +Write-Host "Installing cloudwatch agent..." +Invoke-WebRequest -Uri https://s3.amazonaws.com/amazoncloudwatch-agent/windows/amd64/latest/amazon-cloudwatch-agent.msi -OutFile C:\amazon-cloudwatch-agent.msi +$cloudwatchParams = '/i', 'C:\amazon-cloudwatch-agent.msi', '/qn', '/L*v', 'C:\CloudwatchInstall.log' +Start-Process "msiexec.exe" $cloudwatchParams -Wait -NoNewWindow +Remove-Item C:\amazon-cloudwatch-agent.msi + + +# Install dependent tools +Write-Host "Installing additional development tools" +choco install git awscli -y +refreshenv + +${install_runner} +${post_install} +${start_runner} + +Stop-Transcript + diff --git a/modules/compute-providers/ec2/templates/user-data.sh b/modules/compute-providers/ec2/templates/user-data.sh new file mode 100644 index 0000000000..ca69f26d34 --- /dev/null +++ b/modules/compute-providers/ec2/templates/user-data.sh @@ -0,0 +1,81 @@ +#!/bin/bash -e + +install_with_retry() { + max_attempts=5 + attempt_count=0 + success=false + while [ $success = false ] && [ $attempt_count -le $max_attempts ]; do + echo "Attempting $attempt_count/$max_attempts: Installing $*" + dnf install -y $* + if [ $? -eq 0 ]; then + success=true + else + echo "Failed to install $1 - retrying" + attempt_count=$(( attempt_count + 1 )) + sleep 5 + fi + done +} + +exec > >(tee /var/log/user-data.log | logger -t user-data -s 2>/dev/console) 2>&1 + +# AWS suggest to create a log for debug purpose based on https://aws.amazon.com/premiumsupport/knowledge-center/ec2-linux-log-user-data/ +# As side effect all command, set +x disable debugging explicitly. +# +# An alternative for masking tokens could be: exec > >(sed 's/--token\ [^ ]* /--token\ *** /g' > /var/log/user-data.log) 2>&1 + +set +x + +%{ if enable_debug_logging } +set -x +%{ endif } + +${pre_install} + +max_attempts=5 +attempt_count=0 +success=false +while [ $success = false ] && [ $attempt_count -le $max_attempts ]; do + echo "Attempting $attempt_count/$max_attempts: upgrade-minimal" + dnf upgrade-minimal -y +if [ $? -eq 0 ]; then + success=true + else + echo "Failed to run `dnf upgrad-minimal -y` - retrying" + attempt_count=$(( attempt_count + 1 )) + sleep 5 + fi +done + +# Install docker +install_with_retry docker + +service docker start +usermod -a -G docker ec2-user + +install_with_retry amazon-cloudwatch-agent jq git +install_with_retry --allowerasing curl + +user_name=ec2-user + +${install_runner} + +${post_install} + +# Register runner job hooks +# Ref: https://docs.github.com/en/actions/hosting-your-own-runners/managing-self-hosted-runners/running-scripts-before-or-after-a-job +%{ if hook_job_started != "" } +cat > /opt/actions-runner/hook_job_started.sh <<'EOF' +${hook_job_started} +EOF +echo ACTIONS_RUNNER_HOOK_JOB_STARTED=/opt/actions-runner/hook_job_started.sh | tee -a /opt/actions-runner/.env +%{ endif } + +%{ if hook_job_completed != "" } +cat > /opt/actions-runner/hook_job_completed.sh <<'EOF' +${hook_job_completed} +EOF +echo ACTIONS_RUNNER_HOOK_JOB_COMPLETED=/opt/actions-runner/hook_job_completed.sh | tee -a /opt/actions-runner/.env +%{ endif } + +${start_runner} diff --git a/modules/compute-providers/ec2/tests/provider.tftest.hcl b/modules/compute-providers/ec2/tests/provider.tftest.hcl new file mode 100644 index 0000000000..9e733d2e72 --- /dev/null +++ b/modules/compute-providers/ec2/tests/provider.tftest.hcl @@ -0,0 +1,448 @@ +mock_provider "aws" { + mock_data "aws_iam_policy_document" { + defaults = { + json = "{}" + } + } + + mock_data "aws_ami" { + defaults = { + id = "ami-1234567890abcdef0" + name = "runner-test" + creation_date = "2026-01-01T00:00:00.000Z" + deprecation_time = "" + } + } + + mock_data "aws_caller_identity" { + defaults = { + account_id = "123456789012" + } + } +} + +override_data { + target = data.aws_iam_policy_document.scale_up + values = { + json = "{\"Action\":\"ec2:RunInstances\",\"PassRole\":\"arn:aws:iam::123456789012:role/provider-test-runner\"}" + } +} + +override_data { + target = data.aws_iam_policy_document.pool + values = { + json = "{\"Action\":\"iam:PassRole\"}" + } +} + +variables { + aws_region = "eu-west-1" + prefix = "provider-test" + + config = { + vpc_id = "vpc-12345678" + subnet_ids = ["subnet-12345678"] + instance_types = ["m5.large"] + ami = { + filter = { state = ["available"] } + owners = ["amazon"] + id_ssm_parameter = { + arn = "arn:aws:ssm:eu-west-1:123456789012:parameter/github-runner/ami-id" + } + kms_key = null + } + binaries_syncer = { + enabled = true + s3 = { + arn = "arn:aws:s3:::runner-distribution" + id = "runner-distribution" + key = "runner.zip" + } + } + cloudwatch_agent = { + enabled = true + } + ssm_enabled = true + managed_security_group_enabled = true + } + + runner = { + iam = { + role = { + arn = "arn:aws:iam::123456789012:role/provider-test-runner" + name = "provider-test-runner" + } + managed_policy_arns = { + readonly = "arn:aws:iam::aws:policy/ReadOnlyAccess" + } + } + } + + ssm = { + paths = { + root = "/github-runner/provider-test" + tokens = "tokens" + config = "config" + } + } +} + +run "separates_control_plane_contract_from_ec2_resources" { + command = plan + + assert { + condition = toset(keys(output.provider)) == toset(["environment_variables", "policies", "resources"]) + error_message = "The EC2 provider contract must expose only integration and resource data; its module identity must not be repeated in the output." + } + + assert { + condition = output.provider.environment_variables.scale_up["INSTANCE_TYPES"] == "m5.large" + error_message = "The provider contract must expose EC2 scale-up environment variables." + } + + assert { + condition = output.provider.environment_variables.scale_down["RUNNER_BOOT_TIME_IN_MINUTES"] == 5 + error_message = "The provider contract must expose the EC2 scale-down boot grace period." + } + + assert { + condition = strcontains(output.provider.policies.scale_up.iam_policy_json, "ec2:RunInstances") + error_message = "The EC2 provider must own EC2 scale-up permissions." + } + + assert { + condition = !strcontains(output.provider.policies.scale_up.iam_policy_json, "sqs:ReceiveMessage") + error_message = "The EC2 provider must not own common build-queue permissions." + } + + assert { + condition = strcontains(output.provider.policies.pool.iam_policy_json, "iam:PassRole") + error_message = "The EC2 provider must expose pool permissions for its runner role." + } + + assert { + condition = strcontains(output.provider.policies.scale_up.iam_policy_json, "arn:aws:iam::123456789012:role/provider-test-runner") + error_message = "The EC2 provider must use the common runner role ARN for PassRole." + } + + assert { + condition = output.provider.policies.scale_up.managed_policy_enabled + error_message = "An external AMI SSM parameter must enable the scale-up managed policy attachment at plan time." + } + + assert { + condition = output.provider.policies.pool.managed_policy_enabled + error_message = "An external AMI SSM parameter must enable the pool managed policy attachment at plan time." + } + + assert { + condition = ( + contains(flatten([ + for statement in data.aws_iam_policy_document.scale_up.statement : [ + for condition in statement.condition : condition.variable + ] + ]), "ec2:ResourceTag/ghr:environment") + && contains(flatten([ + for statement in data.aws_iam_policy_document.scale_down.statement : [ + for condition in statement.condition : condition.variable + ] + ]), "ec2:ResourceTag/ghr:environment") + && !contains(flatten([ + for statement in data.aws_iam_policy_document.scale_up.statement : [ + for condition in statement.condition : condition.variable + ] + ]), "ec2:ResourceTag/gh:environment") + && !contains(flatten([ + for statement in data.aws_iam_policy_document.scale_down.statement : [ + for condition in statement.condition : condition.variable + ] + ]), "ec2:ResourceTag/gh:environment") + ) + error_message = "EC2 scale policies must authorize resources by the protected ghr:environment tag." + } + + assert { + condition = toset(keys(output.provider.policies)) == toset(["runner", "scale_up", "scale_down", "pool"]) + error_message = "The EC2 provider must expose policies grouped by their owning common component." + } + + assert { + condition = toset(keys(output.provider.policies.runner.inline_policies)) == toset([ + "ssm_parameters", + "describe_tags", + "create_tags", + "terminate_self", + "session_manager", + "distribution_bucket", + "cloudwatch", + ]) + error_message = "The EC2 provider must return the enabled runner permission documents." + } + + assert { + condition = output.provider.policies.runner.managed_policy_arns["readonly"] == "arn:aws:iam::aws:policy/ReadOnlyAccess" + error_message = "The EC2 provider must return common managed runner policy inputs with its provider policies." + } + + assert { + condition = toset(keys(output.provider.resources)) == toset(["launch_template", "runners_log_groups", "logfiles"]) + error_message = "EC2-specific artifacts must remain nested under provider resources." + } + + assert { + condition = aws_iam_instance_profile.runner[0].role == "provider-test-runner" + error_message = "The EC2 instance profile must use the common runner role name." + } + +} + +run "accepts_partial_typed_compute_options" { + command = plan + + variables { + config = { + vpc_id = "vpc-12345678" + subnet_ids = ["subnet-12345678"] + instance_types = ["m5.large"] + ami = { + filter = { state = ["available"] } + owners = ["amazon"] + id_ssm_parameter = { + arn = "arn:aws:ssm:eu-west-1:123456789012:parameter/github-runner/ami-id" + } + kms_key = null + } + binaries_syncer = { + enabled = false + s3 = null + } + cloudwatch_agent = { + enabled = false + } + managed_security_group_enabled = true + overrides = { + name_runner = "custom-runner" + } + metadata_options = { + http_tokens = "optional" + } + } + } + + assert { + condition = local.name_runner == "custom-runner" && local.name_sg == "provider-test-action-runner" + error_message = "Partial name overrides must retain defaults for omitted attributes." + } + + assert { + condition = ( + aws_launch_template.runner.metadata_options[0].http_tokens == "optional" + && aws_launch_template.runner.metadata_options[0].http_endpoint == "enabled" + && aws_launch_template.runner.metadata_options[0].http_put_response_hop_limit == 1 + && aws_launch_template.runner.metadata_options[0].instance_metadata_tags == "enabled" + ) + error_message = "Partial metadata options must retain typed defaults for omitted attributes." + } + + assert { + condition = toset(keys(output.provider.policies.runner.inline_policies)) == toset([ + "ssm_parameters", + "describe_tags", + "create_tags", + "terminate_self", + ]) + error_message = "Disabled optional EC2 features must remove only their corresponding runner policies." + } +} + +run "separates_provider_runner_and_ssm_tags" { + command = plan + + variables { + config = { + vpc_id = "vpc-12345678" + subnet_ids = ["subnet-12345678"] + instance_types = ["m5.large"] + ami = { + filter = { state = ["available"] } + owners = ["amazon"] + id_ssm_parameter = null + kms_key = null + } + binaries_syncer = { + enabled = false + s3 = null + } + cloudwatch_agent = { + enabled = true + } + managed_security_group_enabled = true + tags = { + Name = "runner-name" + Scope = "runner" + RunnerOnly = "runner" + "ghr:environment" = "runner-override" + "ghr:ssm_config_path" = "/runner/override" + "ghr:runner_name_prefix" = "runner-override" + } + } + tags = { + Name = "provider-name" + Scope = "provider" + } + runner = { + name_prefix = "required-prefix" + iam = { + role = { + arn = "arn:aws:iam::123456789012:role/provider-test-runner" + name = "provider-test-runner" + } + } + } + ssm = { + paths = { + root = "/github-runner/provider-test" + tokens = "tokens" + config = "config" + } + parameters = { + tags = { + Name = "ssm-name" + Scope = "ssm" + SsmOnly = "ssm" + "ghr:ami_name" = "ssm-override" + "ghr:ami_creation_date" = "ssm-override" + "ghr:ami_deprecation_time" = "ssm-override" + } + } + } + observability = { + logs = { + tags = { + Name = "log-name" + Scope = "log" + LogOnly = "log" + } + } + } + } + + assert { + condition = ( + aws_launch_template.runner.tags["Name"] == "provider-name" + && aws_launch_template.runner.tags["Scope"] == "provider" + && !contains(keys(aws_launch_template.runner.tags), "RunnerOnly") + && !contains(keys(aws_launch_template.runner.tags), "SsmOnly") + && !contains(keys(aws_launch_template.runner.tags), "ghr:environment") + && !contains(keys(aws_launch_template.runner.tags), "ghr:ssm_config_path") + && !contains(keys(aws_launch_template.runner.tags), "ghr:runner_name_prefix") + ) + error_message = "Non-runner EC2 resources must use provider tags without runner or SSM component tags." + } + + assert { + condition = toset([ + for tag_specification in aws_launch_template.runner.tag_specifications : tag_specification.resource_type + ]) == toset(["instance", "volume", "network-interface", "spot-instances-request"]) + error_message = "The launch template must define runner tags for every supported runner resource type." + } + + assert { + condition = alltrue([ + for tag_specification in aws_launch_template.runner.tag_specifications : ( + tag_specification.tags["Name"] == "runner-name" + && tag_specification.tags["Scope"] == "runner" + && tag_specification.tags["RunnerOnly"] == "runner" + && !contains(keys(tag_specification.tags), "SsmOnly") + && tag_specification.tags["ghr:environment"] == "provider-test" + && tag_specification.tags["ghr:ssm_config_path"] == "/github-runner/provider-test/config" + && tag_specification.tags["ghr:runner_name_prefix"] == "required-prefix" + ) + ]) + error_message = "Runner resource tags must apply runner overrides while protecting mandatory bootstrap tags." + } + + assert { + condition = ( + aws_ssm_parameter.runner_config_run_as.tags["Name"] == "ssm-name" + && aws_ssm_parameter.runner_config_run_as.tags["Scope"] == "ssm" + && aws_ssm_parameter.runner_config_run_as.tags["SsmOnly"] == "ssm" + && !contains(keys(aws_ssm_parameter.runner_config_run_as.tags), "RunnerOnly") + && !contains(keys(aws_ssm_parameter.runner_config_run_as.tags), "ghr:environment") + ) + error_message = "EC2 SSM parameters must merge SSM component tags over provider tags." + } + + assert { + condition = alltrue([ + for log_group in aws_cloudwatch_log_group.gh_runners : ( + log_group.tags["Name"] == "log-name" + && log_group.tags["Scope"] == "log" + && log_group.tags["LogOnly"] == "log" + && !contains(keys(log_group.tags), "RunnerOnly") + && !contains(keys(log_group.tags), "SsmOnly") + ) + ]) + error_message = "EC2 log groups must merge shared log tags over provider tags without runner or SSM tags." + } + + assert { + condition = ( + aws_ssm_parameter.runner_ami_id[0].tags["Name"] == "ssm-name" + && aws_ssm_parameter.runner_ami_id[0].tags["Scope"] == "ssm" + && aws_ssm_parameter.runner_ami_id[0].tags["SsmOnly"] == "ssm" + && aws_ssm_parameter.runner_ami_id[0].tags["ghr:ami_name"] == "runner-test" + && aws_ssm_parameter.runner_ami_id[0].tags["ghr:ami_creation_date"] == "2026-01-01T00:00:00.000Z" + && aws_ssm_parameter.runner_ami_id[0].tags["ghr:ami_deprecation_time"] == "" + ) + error_message = "The managed AMI parameter must preserve authoritative AMI metadata over SSM component tags." + } +} + +run "rejects_external_instance_profile_with_managed_role" { + command = plan + + variables { + config = { + vpc_id = "vpc-12345678" + subnet_ids = ["subnet-12345678"] + instance_types = ["m5.large"] + instance_profile = { + name = "external-runner-profile" + } + binaries_syncer = { + enabled = false + } + } + + runner = { + iam = { + role = { + arn = "arn:aws:iam::123456789012:role/provider-test-runner" + name = "provider-test-runner" + managed = true + } + } + } + } + + expect_failures = [terraform_data.validate_config] +} + +run "requires_distribution_object_when_sync_is_enabled" { + command = plan + + variables { + config = { + vpc_id = "vpc-12345678" + subnet_ids = ["subnet-12345678"] + instance_types = ["m5.large"] + binaries_syncer = { + enabled = true + s3 = null + } + } + } + + expect_failures = [terraform_data.validate_config] +} diff --git a/modules/compute-providers/ec2/trust-policy/README.md b/modules/compute-providers/ec2/trust-policy/README.md new file mode 100644 index 0000000000..3e0c0ce908 --- /dev/null +++ b/modules/compute-providers/ec2/trust-policy/README.md @@ -0,0 +1,41 @@ +# EC2 runner trust policy + +This internal submodule builds the EC2 runner-role trust policy independently from EC2 resources that consume the runner role. It preserves the default EC2 service trust and optionally merges an additional IAM trust policy document supplied by the common runner stack. + + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 1.4.0 | +| [aws](#requirement\_aws) | >= 6.33 | + +## Providers + +| Name | Version | +|------|---------| +| [aws](#provider\_aws) | >= 6.33 | + +## Modules + +No modules. + +## Resources + +| Name | Type | +|------|------| +| [aws_iam_policy_document.assume_role](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source | +| [aws_iam_policy_document.default](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [additional\_trust\_policy\_json](#input\_additional\_trust\_policy\_json) | Optional IAM policy document merged with the default EC2 runner-role trust policy. | `string` | `null` | no | + +## Outputs + +| Name | Description | +|------|-------------| +| [assume\_role\_policy](#output\_assume\_role\_policy) | EC2 runner-role trust policy with the optional additional trust policy merged into it. | + diff --git a/modules/compute-providers/ec2/trust-policy/assume-role.tf b/modules/compute-providers/ec2/trust-policy/assume-role.tf new file mode 100644 index 0000000000..bea81c1b9e --- /dev/null +++ b/modules/compute-providers/ec2/trust-policy/assume-role.tf @@ -0,0 +1,18 @@ +data "aws_iam_policy_document" "default" { + statement { + effect = "Allow" + actions = ["sts:AssumeRole"] + + principals { + type = "Service" + identifiers = ["ec2.amazonaws.com"] + } + } +} + +data "aws_iam_policy_document" "assume_role" { + source_policy_documents = compact([ + data.aws_iam_policy_document.default.json, + var.additional_trust_policy_json, + ]) +} diff --git a/modules/compute-providers/ec2/trust-policy/outputs.tf b/modules/compute-providers/ec2/trust-policy/outputs.tf new file mode 100644 index 0000000000..0c28bf1661 --- /dev/null +++ b/modules/compute-providers/ec2/trust-policy/outputs.tf @@ -0,0 +1,4 @@ +output "assume_role_policy" { + description = "EC2 runner-role trust policy with the optional additional trust policy merged into it." + value = data.aws_iam_policy_document.assume_role.json +} diff --git a/modules/compute-providers/ec2/trust-policy/tests/trust-policy.tftest.hcl b/modules/compute-providers/ec2/trust-policy/tests/trust-policy.tftest.hcl new file mode 100644 index 0000000000..e764e63fd4 --- /dev/null +++ b/modules/compute-providers/ec2/trust-policy/tests/trust-policy.tftest.hcl @@ -0,0 +1,67 @@ +mock_provider "aws" { + mock_data "aws_iam_policy_document" { + defaults = { + json = "{}" + } + } +} + +run "returns_default_ec2_trust_policy" { + command = plan + + assert { + condition = toset(data.aws_iam_policy_document.default.statement[0].actions) == toset(["sts:AssumeRole"]) + error_message = "The default EC2 runner role trust policy must allow sts:AssumeRole." + } + + assert { + condition = anytrue([ + for principal in data.aws_iam_policy_document.default.statement[0].principals : + principal.type == "Service" && toset(principal.identifiers) == toset(["ec2.amazonaws.com"]) + ]) + error_message = "The default EC2 runner role trust policy must trust the EC2 service principal." + } + + assert { + condition = ( + length(data.aws_iam_policy_document.assume_role.source_policy_documents) == 1 + && output.assume_role_policy == data.aws_iam_policy_document.assume_role.json + ) + error_message = "The submodule must return the final EC2 assume-role policy." + } +} + +run "merges_additional_trust_policy" { + command = plan + + variables { + additional_trust_policy_json = jsonencode({ + Version = "2012-10-17" + Statement = [{ + Sid = "TrustedAccount" + Effect = "Allow" + Action = "sts:AssumeRole" + Principal = { AWS = "arn:aws:iam::123456789012:root" } + }] + }) + } + + assert { + condition = ( + length(data.aws_iam_policy_document.assume_role.source_policy_documents) == 2 + && data.aws_iam_policy_document.assume_role.source_policy_documents[1] == var.additional_trust_policy_json + && output.assume_role_policy == data.aws_iam_policy_document.assume_role.json + ) + error_message = "The submodule must merge the additional trust policy into the final assume-role policy." + } +} + +run "rejects_invalid_additional_trust_policy" { + command = plan + + variables { + additional_trust_policy_json = "not-json" + } + + expect_failures = [var.additional_trust_policy_json] +} diff --git a/modules/compute-providers/ec2/trust-policy/variables.tf b/modules/compute-providers/ec2/trust-policy/variables.tf new file mode 100644 index 0000000000..875fa44f4a --- /dev/null +++ b/modules/compute-providers/ec2/trust-policy/variables.tf @@ -0,0 +1,10 @@ +variable "additional_trust_policy_json" { + description = "Optional IAM policy document merged with the default EC2 runner-role trust policy." + type = string + default = null + + validation { + condition = var.additional_trust_policy_json == null ? true : can(jsondecode(var.additional_trust_policy_json)) + error_message = "additional_trust_policy_json must be valid JSON when set." + } +} diff --git a/modules/compute-providers/ec2/trust-policy/versions.tf b/modules/compute-providers/ec2/trust-policy/versions.tf new file mode 100644 index 0000000000..3ef011ea0a --- /dev/null +++ b/modules/compute-providers/ec2/trust-policy/versions.tf @@ -0,0 +1,10 @@ +terraform { + required_version = ">= 1.4.0" + + required_providers { + aws = { + source = "hashicorp/aws" + version = ">= 6.33" + } + } +} diff --git a/modules/compute-providers/ec2/validations.tf b/modules/compute-providers/ec2/validations.tf new file mode 100644 index 0000000000..ff59bafc13 --- /dev/null +++ b/modules/compute-providers/ec2/validations.tf @@ -0,0 +1,53 @@ +resource "terraform_data" "validate_config" { + lifecycle { + precondition { + condition = contains(["spot", "on-demand"], var.config.instance_target_capacity_type) + error_message = "compute_provider.ec2.instance_target_capacity_type must be spot or on-demand." + } + + precondition { + condition = contains( + ["lowest-price", "diversified", "capacity-optimized", "capacity-optimized-prioritized", "price-capacity-optimized", "prioritized"], + var.config.instance_allocation_strategy, + ) + error_message = "compute_provider.ec2.instance_allocation_strategy is not supported." + } + + precondition { + condition = var.config.credit_specification == null ? true : contains(["standard", "unlimited"], var.config.credit_specification) + error_message = "compute_provider.ec2.credit_specification must be null, standard, or unlimited." + } + + precondition { + condition = var.config.cpu_options == null ? true : ( + (var.config.cpu_options.amd_sev_snp == null ? true : contains(["enabled", "disabled"], var.config.cpu_options.amd_sev_snp)) && + (var.config.cpu_options.nested_virtualization == null ? true : contains(["enabled", "disabled"], var.config.cpu_options.nested_virtualization)) + ) + error_message = "compute_provider.ec2.cpu_options amd_sev_snp and nested_virtualization must be enabled or disabled when set." + } + + precondition { + condition = !var.config.binaries_syncer.enabled || var.config.binaries_syncer.s3 != null + error_message = "compute_provider.ec2.binaries_syncer.s3 must be set when compute_provider.ec2.binaries_syncer.enabled is true." + } + + precondition { + condition = var.config.instance_profile == null || !var.runner.iam.role.managed + error_message = "runner.iam.role must be set when compute_provider.ec2.instance_profile selects an external instance profile." + } + } +} + +resource "terraform_data" "validate_runner" { + lifecycle { + precondition { + condition = contains(["linux", "osx", "windows"], var.runner.os) + error_message = "runner.os must be linux, osx, or windows." + } + + precondition { + condition = length(var.runner.name_prefix) <= 45 + error_message = "runner.name_prefix must be at most 45 characters." + } + } +} diff --git a/modules/compute-providers/ec2/variables.tf b/modules/compute-providers/ec2/variables.tf new file mode 100644 index 0000000000..68d7e2d138 --- /dev/null +++ b/modules/compute-providers/ec2/variables.tf @@ -0,0 +1,367 @@ +variable "aws_partition" { + description = "AWS partition used to construct IAM ARNs." + type = string + default = "aws" +} + +variable "aws_region" { + description = "AWS region used by compute-provider resources and policy documents." + type = string +} + +variable "prefix" { + description = "Prefix used to identify resources created for the runner stack." + type = string + default = "github-actions" +} + +variable "tags" { + description = "Base tags available to taggable compute-provider resources. Provider-specific tags override this map within their documented scopes." + type = map(string) + default = {} +} + +variable "config" { + description = <<-EOT + EC2 compute-provider configuration. Paths match `compute_provider.ec2` in the runner stack. + + - `ami`: Optional AMI discovery and encryption configuration. Null selects defaults for `runner.os` and `runner.architecture`. + - `ami.filter`: AMI filter names mapped to accepted values and merged over the provider defaults. + - `ami.owners`: AWS account IDs or aliases allowed to own the selected AMI. + - `ami.id_ssm_parameter`: Optional externally managed SSM parameter containing the AMI ID. Its object presence is the plan-time ownership discriminator. + - `ami.id_ssm_parameter.arn`: ARN of the external AMI-ID parameter. The ARN may remain unknown until apply. + - `ami.kms_key`: Optional customer-managed KMS key required for encrypted AMIs or snapshots. Its object presence is the plan-time policy discriminator. + - `ami.kms_key.arn`: ARN of the AMI KMS key. The ARN may remain unknown until apply. + - `vpc_id`: VPC in which runner networking resources are created. + - `subnet_ids`: Subnets from which the control plane may launch runners. + - `overrides.name_runner`: Optional Name tag override for runner compute resources. + - `overrides.name_sg`: Optional Name tag override for the managed security group. + - `instance_profile`: Optional externally managed instance profile. Its object presence is the plan-time ownership discriminator. + - `instance_profile.name`: Name of the external instance profile. The name may remain unknown until apply. + - `instance_profile_path`: IAM path for the provider-managed instance profile. Null derives the path from `prefix`. + - `binaries_syncer.enabled`: Uses the synchronized runner distribution from S3 during bootstrap. + - `binaries_syncer.s3`: S3 object containing the synchronized runner distribution. Required when synchronization is enabled. + - `binaries_syncer.s3.arn`: Runner-distribution bucket ARN used by IAM policies. + - `binaries_syncer.s3.id`: Runner-distribution bucket name used in the bootstrap URI. + - `binaries_syncer.s3.key`: Runner-distribution object key. + - `block_device_mappings`: EBS mappings added to the launch template. + - `block_device_mappings[].delete_on_termination`: Deletes the volume when its runner terminates. + - `block_device_mappings[].device_name`: Device name exposed to the runner instance. + - `block_device_mappings[].encrypted`: Enables EBS encryption. + - `block_device_mappings[].iops`: Provisioned IOPS for volume types that support configurable IOPS. + - `block_device_mappings[].kms_key_id`: KMS key ID or ARN used to encrypt the volume. + - `block_device_mappings[].snapshot_id`: Snapshot used to initialize the volume. + - `block_device_mappings[].throughput`: Provisioned throughput for volume types that support it. + - `block_device_mappings[].volume_initialization_rate`: Fixed initialization rate for supported snapshot-backed volumes. + - `block_device_mappings[].volume_size`: EBS volume size in GiB. + - `block_device_mappings[].volume_type`: EBS volume type. + - `ebs_optimized`: Requests EBS-optimized instances. + - `instance_target_capacity_type`: Primary capacity type, either `spot` or `on-demand`. + - `instance_allocation_strategy`: EC2 Fleet allocation strategy. + - `instance_type_priorities`: Optional numeric priorities keyed by instance type. + - `instance_max_spot_price`: Optional maximum hourly Spot price. + - `instance_types`: EC2 instance types available to the control plane. + - `user_data`: Runner bootstrap user-data configuration. + - `user_data.enabled`: Enables launch-template user data. + - `user_data.template`: Optional path to a custom user-data template. + - `user_data.content`: Optional complete user-data content used instead of a template. + - `user_data.pre_install`: Script inserted before runner installation. + - `user_data.post_install`: Script inserted after runner installation. + - `user_data.debug_logging_enabled`: Enables verbose user-data tracing, which can expose secrets. + - `ssm_enabled`: Includes Session Manager permissions in the provider's runner policy group. + - `create_service_linked_role_spot`: Allows scale-up to create the EC2 Spot service-linked role. + - `cloudwatch_agent.enabled`: Enables CloudWatch agent configuration for runner instances. + - `cloudwatch_agent.config`: Optional complete CloudWatch agent configuration. + - `managed_security_group_enabled`: Creates and attaches the provider-managed security group. + - `log_files`: Optional files collected by the CloudWatch agent. Null uses provider defaults. + - `log_files[].log_group_name`: CloudWatch log-group name before optional prefixing. + - `log_files[].prefix_log_group`: Prefixes the log-group name with the runner stack path. + - `log_files[].file_path`: File or glob read by the CloudWatch agent. + - `log_files[].log_stream_name`: CloudWatch log-stream name template. + - `log_files[].log_class`: CloudWatch log-group class for the collected file. + - `key_name`: Optional EC2 key-pair name. + - `additional_security_group_ids`: Existing security groups attached to runners. + - `detailed_monitoring_enabled`: Enables detailed EC2 monitoring. + - `egress_rules`: Rules created on the managed security group. + - `egress_rules[].cidr_blocks`: IPv4 CIDR destinations. + - `egress_rules[].ipv6_cidr_blocks`: IPv6 CIDR destinations. + - `egress_rules[].prefix_list_ids`: AWS prefix-list destinations. + - `egress_rules[].from_port`: First destination port in the permitted range. + - `egress_rules[].protocol`: IP protocol name or number. Use `-1` for all protocols. + - `egress_rules[].security_groups`: Destination security-group IDs. + - `egress_rules[].self`: Allows traffic to the managed security group itself. + - `egress_rules[].to_port`: Last destination port in the permitted range. + - `egress_rules[].description`: Optional rule description. + - `tags`: Runner instance, volume, network-interface, and eligible Spot-request tags. Provider-required bootstrap tags take final precedence. + - `metadata_options`: Instance Metadata Service configuration. + - `metadata_options.instance_metadata_tags`: Exposes instance tags through Instance Metadata Service when enabled. + - `metadata_options.http_endpoint`: Enables or disables the Instance Metadata Service endpoint. + - `metadata_options.http_tokens`: Controls whether IMDSv2 session tokens are optional or required. + - `metadata_options.http_put_response_hop_limit`: Network hop limit for Instance Metadata Service token responses. + - `credit_specification`: CPU credit mode for burstable instance types. + - `cpu_options`: CPU topology and processor-feature configuration. + - `cpu_options.core_count`: Number of CPU cores exposed to the runner instance. + - `cpu_options.threads_per_core`: Number of hardware threads exposed per CPU core. + - `cpu_options.amd_sev_snp`: Enables or disables AMD SEV-SNP on supported instance types. + - `cpu_options.nested_virtualization`: Enables or disables nested virtualization on supported instance types. + - `placement`: EC2 placement configuration. + - `placement.affinity`: Dedicated Host affinity setting. + - `placement.availability_zone`: Availability Zone in which runner instances are placed. + - `placement.group_id`: Placement-group ID. + - `placement.group_name`: Placement-group name. + - `placement.host_id`: Dedicated Host ID. + - `placement.host_resource_group_arn`: ARN of the host resource group used for placement. + - `placement.spread_domain`: Spread-domain placement value. + - `placement.tenancy`: Instance tenancy, such as `default`, `dedicated`, or `host`. + - `placement.partition_number`: Placement-group partition number. + - `license_specifications`: License Manager configurations added to the launch template. + - `license_specifications[].license_configuration_arn`: ARN of an AWS License Manager license configuration. + - `associate_public_ipv4_address`: Associates a public IPv4 address with runner network interfaces. + - `enable_on_demand_failover_for_errors`: EC2 errors that trigger on-demand fallback after a Spot failure. + - `scale_errors`: EC2 errors treated as retryable scale-up failures. + - `use_dedicated_host`: Enables the dedicated-host launch path. + EOT + + type = object({ + ami = optional(object({ + filter = optional(map(list(string)), { state = ["available"] }) + owners = optional(list(string), ["amazon"]) + id_ssm_parameter = optional(object({ + arn = string + }), null) + kms_key = optional(object({ + arn = string + }), null) + }), null) + vpc_id = string + subnet_ids = list(string) + overrides = optional(object({ + name_runner = optional(string, "") + name_sg = optional(string, "") + }), {}) + instance_profile = optional(object({ + name = string + }), null) + instance_profile_path = optional(string, null) + binaries_syncer = optional(object({ + enabled = optional(bool, true) + s3 = optional(object({ + arn = string + id = string + key = string + }), null) + }), {}) + block_device_mappings = optional(list(object({ + delete_on_termination = optional(bool, true) + device_name = optional(string, "/dev/xvda") + encrypted = optional(bool, true) + iops = optional(number) + kms_key_id = optional(string) + snapshot_id = optional(string) + throughput = optional(number) + volume_initialization_rate = optional(number) + volume_size = number + volume_type = optional(string, "gp3") + })), [{ volume_size = 30 }]) + ebs_optimized = optional(bool, false) + instance_target_capacity_type = optional(string, "spot") + instance_allocation_strategy = optional(string, "lowest-price") + instance_type_priorities = optional(map(number), null) + instance_max_spot_price = optional(string, null) + instance_types = list(string) + user_data = optional(object({ + enabled = optional(bool, true) + template = optional(string, null) + content = optional(string, null) + pre_install = optional(string, "") + post_install = optional(string, "") + debug_logging_enabled = optional(bool, false) + }), {}) + ssm_enabled = optional(bool, false) + create_service_linked_role_spot = optional(bool, false) + cloudwatch_agent = optional(object({ + enabled = optional(bool, true) + config = optional(string, null) + }), {}) + managed_security_group_enabled = optional(bool, true) + log_files = optional(list(object({ + log_group_name = string + prefix_log_group = bool + file_path = string + log_stream_name = string + log_class = optional(string, "STANDARD") + })), null) + key_name = optional(string, null) + additional_security_group_ids = optional(list(string), []) + detailed_monitoring_enabled = optional(bool, false) + egress_rules = optional(list(object({ + cidr_blocks = list(string) + ipv6_cidr_blocks = list(string) + prefix_list_ids = list(string) + from_port = number + protocol = string + security_groups = list(string) + self = bool + to_port = number + description = string + })), [{ + cidr_blocks = ["0.0.0.0/0"] + ipv6_cidr_blocks = ["::/0"] + prefix_list_ids = null + from_port = 0 + protocol = "-1" + security_groups = null + self = null + to_port = 0 + description = null + }]) + tags = optional(map(string), {}) + metadata_options = optional(object({ + instance_metadata_tags = optional(string, "enabled") + http_endpoint = optional(string, "enabled") + http_tokens = optional(string, "required") + http_put_response_hop_limit = optional(number, 1) + }), {}) + credit_specification = optional(string, null) + cpu_options = optional(object({ + core_count = optional(number) + threads_per_core = optional(number) + amd_sev_snp = optional(string) + nested_virtualization = optional(string) + }), null) + placement = optional(object({ + affinity = optional(string) + availability_zone = optional(string) + group_id = optional(string) + group_name = optional(string) + host_id = optional(string) + host_resource_group_arn = optional(string) + spread_domain = optional(string) + tenancy = optional(string) + partition_number = optional(number) + }), null) + license_specifications = optional(list(object({ + license_configuration_arn = string + })), []) + associate_public_ipv4_address = optional(bool, false) + enable_on_demand_failover_for_errors = optional(list(string), []) + scale_errors = optional(list(string), [ + "UnfulfillableCapacity", + "MaxSpotInstanceCountExceeded", + "TargetCapacityLimitExceededException", + "RequestLimitExceeded", + "ResourceLimitExceeded", + "MaxSpotInstanceCountExceeded", + "MaxSpotFleetRequestCountExceeded", + "InsufficientInstanceCapacity", + "InsufficientCapacityOnHost", + ]) + use_dedicated_host = optional(bool, false) + }) + + nullable = false +} + +variable "runner" { + description = <<-EOT + Provider-neutral runner settings consumed by compute providers. + + - `os`: Runner operating system. Supported values are `linux`, `osx`, and `windows`. + - `architecture`: Runner distribution architecture. + - `boot_time_in_minutes`: Expected boot and registration duration used by scale-down and pool. + - `name_prefix`: Prefix added to registered runner names. + - `run_as_root`: Runs the runner service as root. + - `run_as`: Operating-system user used when `run_as_root` is false. + - `hooks.job_started`: Script installed as the runner job-started hook. + - `hooks.job_completed`: Script installed as the runner job-completed hook. + - `iam.role.arn`: Resolved runner-role ARN referenced by provider policies and resources. + - `iam.role.name`: Resolved runner-role name used by provider resources. + - `iam.role.managed`: Whether runner-stack manages the resolved runner role. + - `iam.managed_policy_arns`: Common managed-policy ARNs returned with the provider-specific runner policies for attachment by runner-stack. + - `iam.path`: IAM path available to provider-managed IAM resources. Null derives the path from `prefix`. + EOT + type = object({ + os = optional(string, "linux") + architecture = optional(string, "x64") + boot_time_in_minutes = optional(number, 5) + name_prefix = optional(string, "") + run_as_root = optional(bool, false) + run_as = optional(string, "ec2-user") + hooks = optional(object({ + job_started = optional(string, "") + job_completed = optional(string, "") + }), {}) + iam = object({ + role = object({ + arn = string + name = string + managed = optional(bool, true) + }) + managed_policy_arns = optional(map(string), {}) + path = optional(string, null) + }) + }) + + nullable = false +} + +variable "github" { + description = <<-EOT + GitHub Enterprise Server settings available to compute-provider bootstrap data. + + - `enterprise_server.url`: Optional GitHub Enterprise Server base URL. Null selects GitHub.com. + - `enterprise_server.ssl_verify`: Enables TLS certificate verification for GitHub Enterprise Server. + EOT + type = object({ + enterprise_server = optional(object({ + url = optional(string, null) + ssl_verify = optional(bool, true) + }), {}) + }) + default = {} + nullable = false +} + +variable "ssm" { + description = <<-EOT + Parameter Store paths and tag scopes available to compute-provider bootstrap resources. + + - `paths.root`: Root Parameter Store path for the runner stack. + - `paths.tokens`: Path segment used for registration tokens and just-in-time configuration. + - `paths.config`: Path segment used for persistent runner and provider configuration. + - `tags`: Shared SSM tags that override module-level `tags`. + - `parameters.tags`: Parameter-specific tags that override module-level and shared SSM tags. + EOT + type = object({ + paths = object({ + root = string + tokens = string + config = string + }) + tags = optional(map(string), {}) + parameters = optional(object({ + tags = optional(map(string), {}) + }), {}) + }) + + nullable = false +} + +variable "observability" { + description = <<-EOT + CloudWatch Logs settings available to compute-provider runner log groups. + + - `logs.retention_in_days`: Retention period for provider-owned runner log groups. + - `logs.kms_key_id`: Optional KMS key ID or ARN used to encrypt runner log groups. + - `logs.tags`: Shared log-group tags that override module-level `tags`. + EOT + type = object({ + logs = optional(object({ + retention_in_days = optional(number, 180) + kms_key_id = optional(string, null) + tags = optional(map(string), {}) + }), {}) + }) + default = {} + nullable = false +} diff --git a/modules/compute-providers/ec2/versions.tf b/modules/compute-providers/ec2/versions.tf new file mode 100644 index 0000000000..3ef011ea0a --- /dev/null +++ b/modules/compute-providers/ec2/versions.tf @@ -0,0 +1,10 @@ +terraform { + required_version = ">= 1.4.0" + + required_providers { + aws = { + source = "hashicorp/aws" + version = ">= 6.33" + } + } +} diff --git a/modules/compute-providers/microvm/README.md b/modules/compute-providers/microvm/README.md new file mode 100644 index 0000000000..042bd973c5 --- /dev/null +++ b/modules/compute-providers/microvm/README.md @@ -0,0 +1,51 @@ + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 1.4.0 | +| [aws](#requirement\_aws) | >= 6.33 | + +## Providers + +| Name | Version | +|------|---------| +| [aws](#provider\_aws) | >= 6.33 | +| [terraform](#provider\_terraform) | n/a | + +## Modules + +No modules. + +## Resources + +| Name | Type | +|------|------| +| [terraform_data.validate_config](https://registry.terraform.io/providers/hashicorp/terraform/latest/docs/resources/data) | resource | +| [terraform_data.validate_runner](https://registry.terraform.io/providers/hashicorp/terraform/latest/docs/resources/data) | resource | +| [aws_iam_policy_document.scale_down](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source | +| [aws_iam_policy_document.scale_up](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [aws\_partition](#input\_aws\_partition) | AWS partition used to construct IAM ARNs. | `string` | `"aws"` | no | +| [aws\_region](#input\_aws\_region) | AWS region used by compute-provider resources and policy documents. | `string` | n/a | yes | +| [config](#input\_config) | Lambda MicroVM compute-provider configuration. Paths match `compute_provider.microvm` in the runner stack.

- `image_identifier`: ARN or ID of the MicroVM image used to run GitHub runners.
- `image_version`: Optional MicroVM image version.
- `execution_role`: Optional externally managed execution role assumed by MicroVMs. Null uses the common runner role.
- `execution_role.arn`: ARN of the externally managed MicroVM execution role.
- `egress_network_connectors`: Egress network connectors passed to RunMicrovm.
- `idle_policy`: Optional auto-suspend and auto-resume configuration passed to RunMicrovm.
- `idle_policy.max_idle_duration_seconds`: Maximum idle time before MicroVM auto-suspend.
- `idle_policy.suspended_duration_seconds`: Maximum suspended time before MicroVM termination.
- `idle_policy.auto_resume_enabled`: Enables automatic resume on inbound traffic while suspended.
- `logging`: Optional RunMicrovm logging union. Exactly one of `cloud_watch` or `disabled` must be selected when set.
- `logging.cloud_watch.log_group`: Optional CloudWatch Logs log group used by MicroVM runtime logs.
- `logging.cloud_watch.log_stream`: Optional CloudWatch Logs log stream used by MicroVM runtime logs.
- `logging.disabled`: Disables MicroVM runtime logging when true.
- `run_hook_payload`: Optional payload delivered to the MicroVM `/run` hook. Maximum 16,384 characters.
- `maximum_duration_in_seconds`: Optional maximum MicroVM lifetime. Valid range is 1 through 28,800 seconds.
- `environment_variables`: Additional provider-specific Lambda environment variables merged into scale-up, scale-down, and pool.
- `tags`: Tags encoded into the MicroVM runner configuration.
- `iam.resource_arns`: Resource ARNs used by the generated MicroVM control-plane policies. The service is new and some actions may require `*`.
- `iam.actions.scale_up`: MicroVM IAM actions used by scale-up and pool.
- `iam.actions.scale_down`: MicroVM IAM actions used by scale-down.
- `iam.additional_policy_json.scale_up`: Optional additional provider policy attached separately to the scale-up Lambda role.
- `iam.managed_policy_arns.scale_up`: Optional managed policy attached to the scale-up Lambda role.
- `iam.managed_policy_arns.pool`: Optional managed policy attached to the pool Lambda role. | 
object({
    image_identifier = string
    image_version    = optional(string, null)
    execution_role = optional(object({
      arn = string
    }), null)
    egress_network_connectors = optional(list(string), [])
    idle_policy = optional(object({
      max_idle_duration_seconds  = number
      suspended_duration_seconds = number
      auto_resume_enabled        = bool
    }), null)
    logging = optional(object({
      cloud_watch = optional(object({
        log_group  = optional(string, null)
        log_stream = optional(string, null)
      }), null)
      disabled = optional(bool, false)
    }), null)
    run_hook_payload            = optional(string, null)
    maximum_duration_in_seconds = optional(number, null)
    environment_variables       = optional(map(string), {})
    tags                        = optional(map(string), {})
    iam = optional(object({
      resource_arns = optional(list(string), ["*"])
      actions = optional(object({
        scale_up   = optional(list(string), null)
        scale_down = optional(list(string), null)
      }), {})
      additional_policy_json = optional(object({
        scale_up = optional(string, null)
      }), {})
      managed_policy_arns = optional(object({
        scale_up = optional(string, null)
        pool     = optional(string, null)
      }), {})
    }), {})
  })
| n/a | yes | +| [github](#input\_github) | GitHub Enterprise Server settings available to compute-provider bootstrap data.

- `enterprise_server.url`: Optional GitHub Enterprise Server base URL. Null selects GitHub.com.
- `enterprise_server.ssl_verify`: Enables TLS certificate verification for GitHub Enterprise Server. | 
object({
    enterprise_server = optional(object({
      url        = optional(string, null)
      ssl_verify = optional(bool, true)
    }), {})
  })
| `{}` | no | +| [observability](#input\_observability) | CloudWatch Logs settings available to compute-provider runner log groups.

- `logs.retention_in_days`: Retention period for provider-owned runner log groups.
- `logs.kms_key_id`: Optional KMS key ID or ARN used to encrypt runner log groups.
- `logs.tags`: Shared log-group tags that override module-level `tags`. | 
object({
    logs = optional(object({
      retention_in_days = optional(number, 180)
      kms_key_id        = optional(string, null)
      tags              = optional(map(string), {})
    }), {})
  })
| `{}` | no | +| [prefix](#input\_prefix) | Prefix used to identify resources created for the runner stack. | `string` | `"github-actions"` | no | +| [runner](#input\_runner) | Provider-neutral runner settings consumed by compute providers.

- `os`: Runner operating system. Supported values are `linux`, `osx`, and `windows`.
- `architecture`: Runner distribution architecture.
- `boot_time_in_minutes`: Expected boot and registration duration used by scale-down and pool.
- `name_prefix`: Prefix added to registered runner names.
- `run_as_root`: Runs the runner service as root.
- `run_as`: Operating-system user used when `run_as_root` is false.
- `hooks.job_started`: Script installed as the runner job-started hook.
- `hooks.job_completed`: Script installed as the runner job-completed hook.
- `iam.role.arn`: Resolved runner-role ARN referenced by provider policies and resources.
- `iam.role.name`: Resolved runner-role name used by provider resources.
- `iam.role.managed`: Whether runner-stack manages the resolved runner role.
- `iam.managed_policy_arns`: Common managed-policy ARNs returned with the provider-specific runner policies for attachment by runner-stack.
- `iam.path`: IAM path available to provider-managed IAM resources. Null derives the path from `prefix`. | 
object({
    os                   = optional(string, "linux")
    architecture         = optional(string, "x64")
    boot_time_in_minutes = optional(number, 5)
    name_prefix          = optional(string, "")
    run_as_root          = optional(bool, false)
    run_as               = optional(string, "ec2-user")
    hooks = optional(object({
      job_started   = optional(string, "")
      job_completed = optional(string, "")
    }), {})
    iam = object({
      role = object({
        arn     = string
        name    = string
        managed = optional(bool, true)
      })
      managed_policy_arns = optional(map(string), {})
      path                = optional(string, null)
    })
  })
| n/a | yes | +| [ssm](#input\_ssm) | Parameter Store paths and tag scopes available to compute-provider bootstrap resources.

- `paths.root`: Root Parameter Store path for the runner stack.
- `paths.tokens`: Path segment used for registration tokens and just-in-time configuration.
- `paths.config`: Path segment used for persistent runner and provider configuration.
- `tags`: Shared SSM tags that override module-level `tags`.
- `parameters.tags`: Parameter-specific tags that override module-level and shared SSM tags. | 
object({
    paths = object({
      root   = string
      tokens = string
      config = string
    })
    tags = optional(map(string), {})
    parameters = optional(object({
      tags = optional(map(string), {})
    }), {})
  })
| n/a | yes | +| [tags](#input\_tags) | Base tags available to taggable compute-provider resources. Provider-specific tags override this map within their documented scopes. | `map(string)` | `{}` | no | + +## Outputs + +| Name | Description | +|------|-------------| +| [environment\_variables](#output\_environment\_variables) | Provider-specific Lambda environment variable fragments consumed by runner-stack. | +| [policies](#output\_policies) | Provider-specific IAM policy fragments consumed by runner-stack. | +| [provider](#output\_provider) | Nested Lambda MicroVM compute-provider contract consumed by runner-stack. | +| [resources](#output\_resources) | Provider-specific MicroVM resources exposed by runner-stack. | + diff --git a/modules/compute-providers/microvm/control-plane.tf b/modules/compute-providers/microvm/control-plane.tf new file mode 100644 index 0000000000..503ee58d3d --- /dev/null +++ b/modules/compute-providers/microvm/control-plane.tf @@ -0,0 +1,110 @@ +data "aws_iam_policy_document" "scale_up" { + statement { + effect = "Allow" + actions = local.scale_up_actions + resources = var.config.iam.resource_arns + } + + statement { + effect = "Allow" + actions = ["iam:PassRole"] + resources = [local.execution_role_arn] + } +} + +data "aws_iam_policy_document" "scale_down" { + statement { + effect = "Allow" + actions = local.scale_down_actions + resources = var.config.iam.resource_arns + } +} + +locals { + default_scale_up_actions = [ + "lambdamicrovms:CreateMicrovmAuthToken", + "lambdamicrovms:GetMicrovm", + "lambdamicrovms:ListMicrovms", + "lambdamicrovms:RunMicrovm", + "lambdamicrovms:TagResource", + ] + + default_scale_down_actions = [ + "lambdamicrovms:GetMicrovm", + "lambdamicrovms:ListMicrovms", + "lambdamicrovms:ListTags", + "lambdamicrovms:TagResource", + "lambdamicrovms:TerminateMicrovm", + "lambdamicrovms:UntagResource", + ] + + scale_up_actions = coalesce(var.config.iam.actions.scale_up, local.default_scale_up_actions) + scale_down_actions = coalesce(var.config.iam.actions.scale_down, local.default_scale_down_actions) + + execution_role_arn = coalesce(try(var.config.execution_role.arn, null), var.runner.iam.role.arn) + + microvm_tags = merge( + var.tags, + var.config.tags, + { + "ghr:Application" = "github-action-runner" + "ghr:environment" = var.prefix + "ghr:runner_name_prefix" = var.runner.name_prefix + }, + ) + + microvm_idle_policy = var.config.idle_policy == null ? null : { + maxIdleDurationSeconds = var.config.idle_policy.max_idle_duration_seconds + suspendedDurationSeconds = var.config.idle_policy.suspended_duration_seconds + autoResumeEnabled = var.config.idle_policy.auto_resume_enabled + } + + microvm_logging = var.config.logging == null ? null : ( + var.config.logging.disabled ? { + disabled = {} + } : { + cloudWatch = { + logGroup = try(var.config.logging.cloud_watch.log_group, null) + logStream = try(var.config.logging.cloud_watch.log_stream, null) + } + } + ) + + microvm_run_config = { + imageIdentifier = var.config.image_identifier + imageVersion = var.config.image_version + executionRoleArn = local.execution_role_arn + egressNetworkConnectors = var.config.egress_network_connectors + idlePolicy = local.microvm_idle_policy + logging = local.microvm_logging + runHookPayload = var.config.run_hook_payload + maximumDurationInSeconds = var.config.maximum_duration_in_seconds + tags = local.microvm_tags + } + + create_environment_variables = merge(var.config.environment_variables, { + MICROVM_AWS_PARTITION = var.aws_partition + MICROVM_AWS_REGION = var.aws_region + MICROVM_EGRESS_NETWORK_CONNECTORS = jsonencode(var.config.egress_network_connectors) + MICROVM_EXECUTION_ROLE_ARN = local.execution_role_arn + MICROVM_IMAGE_IDENTIFIER = var.config.image_identifier + MICROVM_IMAGE_VERSION = var.config.image_version == null ? "" : var.config.image_version + MICROVM_RUN_CONFIG = jsonencode(local.microvm_run_config) + MICROVM_TAGS = jsonencode(local.microvm_tags) + }) + + scale_up_environment_variables = local.create_environment_variables + + scale_down_environment_variables = merge(var.config.environment_variables, { + MICROVM_AWS_PARTITION = var.aws_partition + MICROVM_AWS_REGION = var.aws_region + MICROVM_IMAGE_IDENTIFIER = var.config.image_identifier + MICROVM_IMAGE_VERSION = var.config.image_version == null ? "" : var.config.image_version + MICROVM_TAGS = jsonencode(local.microvm_tags) + RUNNER_BOOT_TIME_IN_MINUTES = var.runner.boot_time_in_minutes + }) + + pool_environment_variables = merge(local.create_environment_variables, { + RUNNER_BOOT_TIME_IN_MINUTES = var.runner.boot_time_in_minutes + }) +} diff --git a/modules/compute-providers/microvm/outputs.tf b/modules/compute-providers/microvm/outputs.tf new file mode 100644 index 0000000000..fd2f8c3bb1 --- /dev/null +++ b/modules/compute-providers/microvm/outputs.tf @@ -0,0 +1,23 @@ +output "environment_variables" { + description = "Provider-specific Lambda environment variable fragments consumed by runner-stack." + value = local.provider_environment_variables +} + +output "policies" { + description = "Provider-specific IAM policy fragments consumed by runner-stack." + value = local.provider_policies +} + +output "resources" { + description = "Provider-specific MicroVM resources exposed by runner-stack." + value = local.provider_resources +} + +output "provider" { + description = "Nested Lambda MicroVM compute-provider contract consumed by runner-stack." + value = { + environment_variables = local.provider_environment_variables + policies = local.provider_policies + resources = local.provider_resources + } +} diff --git a/modules/compute-providers/microvm/provider-contract.tf b/modules/compute-providers/microvm/provider-contract.tf new file mode 100644 index 0000000000..0aff22a31a --- /dev/null +++ b/modules/compute-providers/microvm/provider-contract.tf @@ -0,0 +1,34 @@ +locals { + provider_environment_variables = { + scale_up = local.scale_up_environment_variables + scale_down = local.scale_down_environment_variables + pool = local.pool_environment_variables + } + + provider_policies = { + runner = { + inline_policies = {} + managed_policy_arns = var.runner.iam.managed_policy_arns + } + scale_up = { + iam_policy_json = data.aws_iam_policy_document.scale_up.json + additional_iam_policy_json = var.config.iam.additional_policy_json.scale_up + managed_policy_enabled = var.config.iam.managed_policy_arns.scale_up != null + managed_policy_arn = var.config.iam.managed_policy_arns.scale_up + } + scale_down = { + iam_policy_json = data.aws_iam_policy_document.scale_down.json + } + pool = { + iam_policy_json = data.aws_iam_policy_document.scale_up.json + managed_policy_enabled = var.config.iam.managed_policy_arns.pool != null + managed_policy_arn = var.config.iam.managed_policy_arns.pool + } + } + + provider_resources = { + image_identifier = var.config.image_identifier + image_version = var.config.image_version + execution_role_arn = local.execution_role_arn + } +} diff --git a/modules/compute-providers/microvm/tests/provider.tftest.hcl b/modules/compute-providers/microvm/tests/provider.tftest.hcl new file mode 100644 index 0000000000..9c4d3988dd --- /dev/null +++ b/modules/compute-providers/microvm/tests/provider.tftest.hcl @@ -0,0 +1,194 @@ +mock_provider "aws" { + mock_data "aws_iam_policy_document" { + defaults = { + json = "{}" + } + } +} + +variables { + aws_region = "eu-west-1" + prefix = "microvm-test" + + tags = { + Module = "runner" + } + + config = { + image_identifier = "arn:aws:lambdamicrovms:eu-west-1:123456789012:image/runner" + image_version = "3" + egress_network_connectors = [ + "egress-connector" + ] + idle_policy = { + max_idle_duration_seconds = 300 + suspended_duration_seconds = 900 + auto_resume_enabled = true + } + logging = { + cloud_watch = { + log_group = "/aws/lambdamicrovms/runner" + log_stream = "runtime" + } + } + run_hook_payload = "{\"runner\":\"test\"}" + maximum_duration_in_seconds = 3600 + environment_variables = { + MICROVM_CLUSTER = "runner-cluster" + } + tags = { + Provider = "microvm" + } + } + + runner = { + boot_time_in_minutes = 7 + name_prefix = "microvm-" + iam = { + role = { + arn = "arn:aws:iam::123456789012:role/microvm-test-runner" + name = "microvm-test-runner" + } + managed_policy_arns = { + readonly = "arn:aws:iam::aws:policy/ReadOnlyAccess" + } + } + } + + ssm = { + paths = { + root = "/github-action-runners" + tokens = "tokens" + config = "config" + } + } +} + +run "exposes_microvm_control_plane_contract" { + command = plan + + assert { + condition = toset(keys(output.provider)) == toset(["environment_variables", "policies", "resources"]) + error_message = "The MicroVM provider contract must expose only integration and resource data." + } + + assert { + condition = ( + output.provider.environment_variables.scale_up["MICROVM_CLUSTER"] == "runner-cluster" + && output.provider.environment_variables.scale_up["MICROVM_IMAGE_IDENTIFIER"] == "arn:aws:lambdamicrovms:eu-west-1:123456789012:image/runner" + && output.provider.environment_variables.scale_up["MICROVM_IMAGE_VERSION"] == "3" + && output.provider.environment_variables.scale_up["MICROVM_EXECUTION_ROLE_ARN"] == "arn:aws:iam::123456789012:role/microvm-test-runner" + && output.provider.environment_variables.scale_down["RUNNER_BOOT_TIME_IN_MINUTES"] == 7 + && output.provider.environment_variables.pool["RUNNER_BOOT_TIME_IN_MINUTES"] == 7 + ) + error_message = "The MicroVM provider must expose scale-up, scale-down, and pool environment fragments." + } + + assert { + condition = ( + jsondecode(output.provider.environment_variables.scale_up["MICROVM_RUN_CONFIG"]).imageIdentifier == "arn:aws:lambdamicrovms:eu-west-1:123456789012:image/runner" + && jsondecode(output.provider.environment_variables.scale_up["MICROVM_RUN_CONFIG"]).executionRoleArn == "arn:aws:iam::123456789012:role/microvm-test-runner" + && jsondecode(output.provider.environment_variables.scale_up["MICROVM_RUN_CONFIG"]).idlePolicy.maxIdleDurationSeconds == 300 + && jsondecode(output.provider.environment_variables.scale_up["MICROVM_RUN_CONFIG"]).logging.cloudWatch.logGroup == "/aws/lambdamicrovms/runner" + && jsondecode(output.provider.environment_variables.scale_up["MICROVM_TAGS"]).Provider == "microvm" + && jsondecode(output.provider.environment_variables.scale_up["MICROVM_TAGS"])["ghr:environment"] == "microvm-test" + ) + error_message = "The MicroVM provider must encode the RunMicrovm request and protected runner tags." + } + + assert { + condition = ( + contains(data.aws_iam_policy_document.scale_up.statement[0].actions, "lambdamicrovms:RunMicrovm") + && contains(data.aws_iam_policy_document.scale_up.statement[0].actions, "lambdamicrovms:CreateMicrovmAuthToken") + && data.aws_iam_policy_document.scale_up.statement[1].actions == toset(["iam:PassRole"]) + && data.aws_iam_policy_document.scale_up.statement[1].resources == toset(["arn:aws:iam::123456789012:role/microvm-test-runner"]) + && contains(data.aws_iam_policy_document.scale_down.statement[0].actions, "lambdamicrovms:TerminateMicrovm") + ) + error_message = "The MicroVM provider must own MicroVM scale-up, scale-down, and PassRole permissions." + } + + assert { + condition = ( + toset(keys(output.provider.policies)) == toset(["runner", "scale_up", "scale_down", "pool"]) + && length(output.provider.policies.runner.inline_policies) == 0 + && output.provider.policies.runner.managed_policy_arns["readonly"] == "arn:aws:iam::aws:policy/ReadOnlyAccess" + && !output.provider.policies.scale_up.managed_policy_enabled + && !output.provider.policies.pool.managed_policy_enabled + ) + error_message = "The MicroVM provider must return policy fragments grouped by common component." + } + + assert { + condition = output.provider.resources == { + image_identifier = "arn:aws:lambdamicrovms:eu-west-1:123456789012:image/runner" + image_version = "3" + execution_role_arn = "arn:aws:iam::123456789012:role/microvm-test-runner" + } + error_message = "The MicroVM provider must expose its selected image and execution role as provider resources." + } +} + +run "accepts_external_execution_role_and_policy_overrides" { + command = plan + + variables { + config = { + image_identifier = "runner-image" + execution_role = { + arn = "arn:aws:iam::123456789012:role/external-microvm-execution" + } + logging = { + disabled = true + } + iam = { + resource_arns = ["arn:aws:lambdamicrovms:eu-west-1:123456789012:microvm/*"] + actions = { + scale_up = ["lambdamicrovms:RunMicrovm"] + scale_down = ["lambdamicrovms:TerminateMicrovm"] + } + additional_policy_json = { + scale_up = "{\"Version\":\"2012-10-17\",\"Statement\":[]}" + } + managed_policy_arns = { + scale_up = "arn:aws:iam::123456789012:policy/microvm-scale-up" + pool = "arn:aws:iam::123456789012:policy/microvm-pool" + } + } + } + } + + assert { + condition = ( + output.provider.environment_variables.scale_up["MICROVM_EXECUTION_ROLE_ARN"] == "arn:aws:iam::123456789012:role/external-microvm-execution" + && jsondecode(output.provider.environment_variables.scale_up["MICROVM_RUN_CONFIG"]).logging.disabled == {} + && data.aws_iam_policy_document.scale_up.statement[0].actions == toset(["lambdamicrovms:RunMicrovm"]) + && data.aws_iam_policy_document.scale_up.statement[0].resources == toset(["arn:aws:lambdamicrovms:eu-west-1:123456789012:microvm/*"]) + && data.aws_iam_policy_document.scale_up.statement[1].resources == toset(["arn:aws:iam::123456789012:role/external-microvm-execution"]) + && data.aws_iam_policy_document.scale_down.statement[0].actions == toset(["lambdamicrovms:TerminateMicrovm"]) + ) + error_message = "External execution role and action/resource overrides must reach the MicroVM provider contract." + } + + assert { + condition = ( + output.provider.policies.scale_up.additional_iam_policy_json == "{\"Version\":\"2012-10-17\",\"Statement\":[]}" + && output.provider.policies.scale_up.managed_policy_enabled + && output.provider.policies.scale_up.managed_policy_arn == "arn:aws:iam::123456789012:policy/microvm-scale-up" + && output.provider.policies.pool.managed_policy_enabled + && output.provider.policies.pool.managed_policy_arn == "arn:aws:iam::123456789012:policy/microvm-pool" + ) + error_message = "Optional MicroVM policy attachments must stay controlled by object presence." + } +} + +run "rejects_empty_image_identifier" { + command = plan + + variables { + config = { + image_identifier = " " + } + } + + expect_failures = [terraform_data.validate_config] +} diff --git a/modules/compute-providers/microvm/trust-policy/README.md b/modules/compute-providers/microvm/trust-policy/README.md new file mode 100644 index 0000000000..533f8b657d --- /dev/null +++ b/modules/compute-providers/microvm/trust-policy/README.md @@ -0,0 +1,41 @@ +# MicroVM runner trust policy + +This internal submodule builds the MicroVM runner-role trust policy independently from runtime resources that consume the runner role. It preserves the default Lambda service trust and optionally merges an additional IAM trust policy document supplied by the common runner stack. + + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 1.4.0 | +| [aws](#requirement\_aws) | >= 6.0 | + +## Providers + +| Name | Version | +|------|---------| +| [aws](#provider\_aws) | >= 6.0 | + +## Modules + +No modules. + +## Resources + +| Name | Type | +|------|------| +| [aws_iam_policy_document.assume_role](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source | +| [aws_iam_policy_document.default](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [additional\_trust\_policy\_json](#input\_additional\_trust\_policy\_json) | Optional IAM policy document merged with the MicroVM provider's default runner-role trust policy. | `string` | `null` | no | + +## Outputs + +| Name | Description | +|------|-------------| +| [assume\_role\_policy](#output\_assume\_role\_policy) | MicroVM runner-role trust policy including any additional trust statements. | + diff --git a/modules/compute-providers/microvm/trust-policy/assume-role.tf b/modules/compute-providers/microvm/trust-policy/assume-role.tf new file mode 100644 index 0000000000..3654bce8bf --- /dev/null +++ b/modules/compute-providers/microvm/trust-policy/assume-role.tf @@ -0,0 +1,21 @@ +data "aws_iam_policy_document" "default" { + statement { + effect = "Allow" + actions = [ + "sts:AssumeRole", + "sts:TagSession", + ] + + principals { + type = "Service" + identifiers = ["lambda.amazonaws.com"] + } + } +} + +data "aws_iam_policy_document" "assume_role" { + source_policy_documents = compact([ + data.aws_iam_policy_document.default.json, + var.additional_trust_policy_json, + ]) +} diff --git a/modules/compute-providers/microvm/trust-policy/outputs.tf b/modules/compute-providers/microvm/trust-policy/outputs.tf new file mode 100644 index 0000000000..8564675873 --- /dev/null +++ b/modules/compute-providers/microvm/trust-policy/outputs.tf @@ -0,0 +1,4 @@ +output "assume_role_policy" { + description = "MicroVM runner-role trust policy including any additional trust statements." + value = data.aws_iam_policy_document.assume_role.json +} diff --git a/modules/compute-providers/microvm/trust-policy/tests/trust-policy.tftest.hcl b/modules/compute-providers/microvm/trust-policy/tests/trust-policy.tftest.hcl new file mode 100644 index 0000000000..5f0173ebc1 --- /dev/null +++ b/modules/compute-providers/microvm/trust-policy/tests/trust-policy.tftest.hcl @@ -0,0 +1,59 @@ +mock_provider "aws" { + mock_data "aws_iam_policy_document" { + defaults = { + json = "{}" + } + } +} + +run "returns_default_microvm_trust_policy" { + command = plan + + assert { + condition = toset(data.aws_iam_policy_document.default.statement[0].actions) == toset(["sts:AssumeRole", "sts:TagSession"]) + error_message = "The MicroVM runner role must allow assume-role and tagged sessions." + } + + assert { + condition = anytrue([ + for principal in data.aws_iam_policy_document.default.statement[0].principals : + principal.type == "Service" && toset(principal.identifiers) == toset(["lambda.amazonaws.com"]) + ]) + error_message = "The MicroVM runner role must trust the Lambda service principal required by the provider." + } + + assert { + condition = ( + length(data.aws_iam_policy_document.assume_role.source_policy_documents) == 1 + && output.assume_role_policy == data.aws_iam_policy_document.assume_role.json + ) + error_message = "The MicroVM trust-policy module must return the default trust document as assume_role_policy." + } +} + +run "merges_additional_trust_policy" { + command = plan + + variables { + additional_trust_policy_json = "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Sid\":\"TrustDeploymentRole\",\"Effect\":\"Allow\",\"Action\":\"sts:AssumeRole\",\"Principal\":{\"AWS\":\"arn:aws:iam::123456789012:role/deployer\"}}]}" + } + + assert { + condition = ( + length(data.aws_iam_policy_document.assume_role.source_policy_documents) == 2 + && contains(data.aws_iam_policy_document.assume_role.source_policy_documents, var.additional_trust_policy_json) + && output.assume_role_policy == data.aws_iam_policy_document.assume_role.json + ) + error_message = "The MicroVM trust-policy module must merge and return the additional trust policy document." + } +} + +run "rejects_invalid_additional_trust_policy" { + command = plan + + variables { + additional_trust_policy_json = "{" + } + + expect_failures = [var.additional_trust_policy_json] +} diff --git a/modules/compute-providers/microvm/trust-policy/variables.tf b/modules/compute-providers/microvm/trust-policy/variables.tf new file mode 100644 index 0000000000..1af67309d2 --- /dev/null +++ b/modules/compute-providers/microvm/trust-policy/variables.tf @@ -0,0 +1,10 @@ +variable "additional_trust_policy_json" { + description = "Optional IAM policy document merged with the MicroVM provider's default runner-role trust policy." + type = string + default = null + + validation { + condition = var.additional_trust_policy_json == null ? true : can(jsondecode(var.additional_trust_policy_json)) + error_message = "additional_trust_policy_json must be valid JSON when set." + } +} diff --git a/modules/compute-providers/microvm/trust-policy/versions.tf b/modules/compute-providers/microvm/trust-policy/versions.tf new file mode 100644 index 0000000000..18ab313d76 --- /dev/null +++ b/modules/compute-providers/microvm/trust-policy/versions.tf @@ -0,0 +1,10 @@ +terraform { + required_version = ">= 1.4.0" + + required_providers { + aws = { + source = "hashicorp/aws" + version = ">= 6.0" + } + } +} diff --git a/modules/compute-providers/microvm/validations.tf b/modules/compute-providers/microvm/validations.tf new file mode 100644 index 0000000000..4acbaecfa7 --- /dev/null +++ b/modules/compute-providers/microvm/validations.tf @@ -0,0 +1,48 @@ +resource "terraform_data" "validate_config" { + lifecycle { + precondition { + condition = trimspace(var.config.image_identifier) != "" + error_message = "compute_provider.microvm.image_identifier must not be empty." + } + + precondition { + condition = var.config.maximum_duration_in_seconds == null ? true : ( + var.config.maximum_duration_in_seconds >= 1 && + var.config.maximum_duration_in_seconds <= 28800 + ) + error_message = "compute_provider.microvm.maximum_duration_in_seconds must be null or between 1 and 28800." + } + + precondition { + condition = var.config.run_hook_payload == null ? true : length(var.config.run_hook_payload) <= 16384 + error_message = "compute_provider.microvm.run_hook_payload must be 16384 characters or less." + } + + precondition { + condition = var.config.logging == null ? true : ( + (var.config.logging.cloud_watch == null ? 0 : 1) + + (var.config.logging.disabled ? 1 : 0) == 1 + ) + error_message = "compute_provider.microvm.logging must set exactly one of cloud_watch or disabled." + } + + precondition { + condition = try(var.config.iam.additional_policy_json.scale_up, null) == null ? true : can(jsondecode(var.config.iam.additional_policy_json.scale_up)) + error_message = "compute_provider.microvm.iam.additional_policy_json.scale_up must be valid JSON when set." + } + } +} + +resource "terraform_data" "validate_runner" { + lifecycle { + precondition { + condition = contains(["linux", "osx", "windows"], var.runner.os) + error_message = "runner.os must be linux, osx, or windows." + } + + precondition { + condition = length(var.runner.name_prefix) <= 45 + error_message = "runner.name_prefix must be at most 45 characters." + } + } +} diff --git a/modules/compute-providers/microvm/variables.tf b/modules/compute-providers/microvm/variables.tf new file mode 100644 index 0000000000..8e0aed08f6 --- /dev/null +++ b/modules/compute-providers/microvm/variables.tf @@ -0,0 +1,200 @@ +variable "aws_partition" { + description = "AWS partition used to construct IAM ARNs." + type = string + default = "aws" +} + +variable "aws_region" { + description = "AWS region used by compute-provider resources and policy documents." + type = string +} + +variable "prefix" { + description = "Prefix used to identify resources created for the runner stack." + type = string + default = "github-actions" +} + +variable "tags" { + description = "Base tags available to taggable compute-provider resources. Provider-specific tags override this map within their documented scopes." + type = map(string) + default = {} +} + +variable "config" { + description = <<-EOT + Lambda MicroVM compute-provider configuration. Paths match `compute_provider.microvm` in the runner stack. + + - `image_identifier`: ARN or ID of the MicroVM image used to run GitHub runners. + - `image_version`: Optional MicroVM image version. + - `execution_role`: Optional externally managed execution role assumed by MicroVMs. Null uses the common runner role. + - `execution_role.arn`: ARN of the externally managed MicroVM execution role. + - `egress_network_connectors`: Egress network connectors passed to RunMicrovm. + - `idle_policy`: Optional auto-suspend and auto-resume configuration passed to RunMicrovm. + - `idle_policy.max_idle_duration_seconds`: Maximum idle time before MicroVM auto-suspend. + - `idle_policy.suspended_duration_seconds`: Maximum suspended time before MicroVM termination. + - `idle_policy.auto_resume_enabled`: Enables automatic resume on inbound traffic while suspended. + - `logging`: Optional RunMicrovm logging union. Exactly one of `cloud_watch` or `disabled` must be selected when set. + - `logging.cloud_watch.log_group`: Optional CloudWatch Logs log group used by MicroVM runtime logs. + - `logging.cloud_watch.log_stream`: Optional CloudWatch Logs log stream used by MicroVM runtime logs. + - `logging.disabled`: Disables MicroVM runtime logging when true. + - `run_hook_payload`: Optional payload delivered to the MicroVM `/run` hook. Maximum 16,384 characters. + - `maximum_duration_in_seconds`: Optional maximum MicroVM lifetime. Valid range is 1 through 28,800 seconds. + - `environment_variables`: Additional provider-specific Lambda environment variables merged into scale-up, scale-down, and pool. + - `tags`: Tags encoded into the MicroVM runner configuration. + - `iam.resource_arns`: Resource ARNs used by the generated MicroVM control-plane policies. The service is new and some actions may require `*`. + - `iam.actions.scale_up`: MicroVM IAM actions used by scale-up and pool. + - `iam.actions.scale_down`: MicroVM IAM actions used by scale-down. + - `iam.additional_policy_json.scale_up`: Optional additional provider policy attached separately to the scale-up Lambda role. + - `iam.managed_policy_arns.scale_up`: Optional managed policy attached to the scale-up Lambda role. + - `iam.managed_policy_arns.pool`: Optional managed policy attached to the pool Lambda role. + EOT + + type = object({ + image_identifier = string + image_version = optional(string, null) + execution_role = optional(object({ + arn = string + }), null) + egress_network_connectors = optional(list(string), []) + idle_policy = optional(object({ + max_idle_duration_seconds = number + suspended_duration_seconds = number + auto_resume_enabled = bool + }), null) + logging = optional(object({ + cloud_watch = optional(object({ + log_group = optional(string, null) + log_stream = optional(string, null) + }), null) + disabled = optional(bool, false) + }), null) + run_hook_payload = optional(string, null) + maximum_duration_in_seconds = optional(number, null) + environment_variables = optional(map(string), {}) + tags = optional(map(string), {}) + iam = optional(object({ + resource_arns = optional(list(string), ["*"]) + actions = optional(object({ + scale_up = optional(list(string), null) + scale_down = optional(list(string), null) + }), {}) + additional_policy_json = optional(object({ + scale_up = optional(string, null) + }), {}) + managed_policy_arns = optional(object({ + scale_up = optional(string, null) + pool = optional(string, null) + }), {}) + }), {}) + }) + + nullable = false +} + +variable "runner" { + description = <<-EOT + Provider-neutral runner settings consumed by compute providers. + + - `os`: Runner operating system. Supported values are `linux`, `osx`, and `windows`. + - `architecture`: Runner distribution architecture. + - `boot_time_in_minutes`: Expected boot and registration duration used by scale-down and pool. + - `name_prefix`: Prefix added to registered runner names. + - `run_as_root`: Runs the runner service as root. + - `run_as`: Operating-system user used when `run_as_root` is false. + - `hooks.job_started`: Script installed as the runner job-started hook. + - `hooks.job_completed`: Script installed as the runner job-completed hook. + - `iam.role.arn`: Resolved runner-role ARN referenced by provider policies and resources. + - `iam.role.name`: Resolved runner-role name used by provider resources. + - `iam.role.managed`: Whether runner-stack manages the resolved runner role. + - `iam.managed_policy_arns`: Common managed-policy ARNs returned with the provider-specific runner policies for attachment by runner-stack. + - `iam.path`: IAM path available to provider-managed IAM resources. Null derives the path from `prefix`. + EOT + type = object({ + os = optional(string, "linux") + architecture = optional(string, "x64") + boot_time_in_minutes = optional(number, 5) + name_prefix = optional(string, "") + run_as_root = optional(bool, false) + run_as = optional(string, "ec2-user") + hooks = optional(object({ + job_started = optional(string, "") + job_completed = optional(string, "") + }), {}) + iam = object({ + role = object({ + arn = string + name = string + managed = optional(bool, true) + }) + managed_policy_arns = optional(map(string), {}) + path = optional(string, null) + }) + }) + + nullable = false +} + +# tflint-ignore: terraform_unused_declarations +variable "github" { + description = <<-EOT + GitHub Enterprise Server settings available to compute-provider bootstrap data. + + - `enterprise_server.url`: Optional GitHub Enterprise Server base URL. Null selects GitHub.com. + - `enterprise_server.ssl_verify`: Enables TLS certificate verification for GitHub Enterprise Server. + EOT + type = object({ + enterprise_server = optional(object({ + url = optional(string, null) + ssl_verify = optional(bool, true) + }), {}) + }) + default = {} + nullable = false +} + +# tflint-ignore: terraform_unused_declarations +variable "ssm" { + description = <<-EOT + Parameter Store paths and tag scopes available to compute-provider bootstrap resources. + + - `paths.root`: Root Parameter Store path for the runner stack. + - `paths.tokens`: Path segment used for registration tokens and just-in-time configuration. + - `paths.config`: Path segment used for persistent runner and provider configuration. + - `tags`: Shared SSM tags that override module-level `tags`. + - `parameters.tags`: Parameter-specific tags that override module-level and shared SSM tags. + EOT + type = object({ + paths = object({ + root = string + tokens = string + config = string + }) + tags = optional(map(string), {}) + parameters = optional(object({ + tags = optional(map(string), {}) + }), {}) + }) + + nullable = false +} + +# tflint-ignore: terraform_unused_declarations +variable "observability" { + description = <<-EOT + CloudWatch Logs settings available to compute-provider runner log groups. + + - `logs.retention_in_days`: Retention period for provider-owned runner log groups. + - `logs.kms_key_id`: Optional KMS key ID or ARN used to encrypt runner log groups. + - `logs.tags`: Shared log-group tags that override module-level `tags`. + EOT + type = object({ + logs = optional(object({ + retention_in_days = optional(number, 180) + kms_key_id = optional(string, null) + tags = optional(map(string), {}) + }), {}) + }) + default = {} + nullable = false +} diff --git a/modules/compute-providers/microvm/versions.tf b/modules/compute-providers/microvm/versions.tf new file mode 100644 index 0000000000..3ef011ea0a --- /dev/null +++ b/modules/compute-providers/microvm/versions.tf @@ -0,0 +1,10 @@ +terraform { + required_version = ">= 1.4.0" + + required_providers { + aws = { + source = "hashicorp/aws" + version = ">= 6.33" + } + } +} diff --git a/modules/multi-runner/README.md b/modules/multi-runner/README.md index 612df47652..5f3b56377c 100644 --- a/modules/multi-runner/README.md +++ b/modules/multi-runner/README.md @@ -6,6 +6,50 @@ This module creates many runners with a single GitHub app. The module utilizes t The module takes a configuration as input containing a matcher for the labels. The [webhook](https://github-aws-runners.github.io/terraform-aws-github-runner/modules/internal/webhook/) lambda is using the configuration to delegate events based on the labels in the workflow job and sent them to a dedicated queue based on the configuration. Events on each queue are processed by a dedicated lambda per configuration to scale runners. +## Provider boundary + +See [Experimental compute-provider refactor](https://github-aws-runners.github.io/terraform-aws-github-runner/modules/internal/compute-provider-refactor/) for the motivation, ownership contract, opt-in flow, state guarantees, and migration phases. + +The multi-runner module owns provider-neutral runner-configuration normalization, queues, shared runner-binary discovery, and webhook routing. Stable `multi_runner_config` entries continue to use the existing `modules/runners` module at their historical `module.runners["configuration"]` addresses. + +To opt into v2, leave `multi_runner_config` empty and populate `experimental.multi_runner_config_v2`. The whole module instance then uses `modules/runner-stack` at `module.runner_stacks["configuration"]`. That stack coordinates internal provider-neutral modules for scale-up and scale-down, pool, job retry, and SSM housekeeping, and owns the common runner role and attachments. It selects the provider from the single populated typed block under `compute_provider`; the EC2 provider owns EC2 policy requirements, the instance profile, launch template, bootstrap resources, and provider-specific Lambda fragments. The MicroVM provider owns the Lambda MicroVM runtime configuration, execution-role policy, and provider-specific Lambda fragments while the runtime Lambdas create and terminate MicroVMs. The selected block must be known during planning because it determines the module graph. The runner-stack child modules are implementation details rather than standalone public entry points. CodeBuild and other provider modules are future work. + +In v2, common runner-role configuration belongs under `runner.iam`; EC2's optional external instance-profile selection belongs under `compute_provider.ec2.instance_profile`. Provider policy documents are generated internally and attached by the common stack when it creates the role. An external role remains unmanaged and must already contain the required policies. + +Phase 1 supports both input contracts, but callers must populate only one runner configuration map per module instance. When `experimental.multi_runner_config_v2` is empty, `multi_runner_config` follows the unchanged legacy path. To use v2, `multi_runner_config` must be empty and `experimental.multi_runner_config_v2` becomes the complete runner map. The maps are not merged, and populating both is unsupported. + +### V2 tagging + +For v2 runner configurations, top-level module `tags` are merged with configuration `tags`. Shared `lambda.tags`, `queue.tags`, and `observability.logs.tags` are then merged with component tags such as `runner.tags`, `scale_up.tags`, `scale_down.tags`, `pool.tags`, `job_retry.tags`, and the nested SSM tag scopes. Narrower scopes win repeated keys. Queue tags also apply to the configuration build queue and dead-letter queue owned by multi-runner. Stable v1 configurations keep their existing tag behavior unchanged. + +The output contracts are separated so a single map never contains two incompatible entry schemas. In v1 mode, entries remain exclusively in `runners_map` with their existing flat shape and `runners_map_v2` is empty. In v2 mode, entries are exposed exclusively through `runners_map_v2`, grouped under `runner`, `scale_up`, `scale_down`, `pool`, and `provider`, while `runners_map` is empty. Use `runners_map_v2["configuration"].runner.role` for the common runner role and `runners_map_v2["configuration"].scale_up.lambda`, `.log_group`, and `.role` for the scale-up resources. The same resource shape is used for `scale_down` and an enabled `pool`; `pool` is null when it is disabled. Provider-specific resources are grouped under the selected provider key, which also identifies the compute provider. For EC2, launch-template and runner-log resources remain under `runners_map_v2["configuration"].provider.ec2`. For MicroVM, the selected image and execution-role reference are exposed under `runners_map_v2["configuration"].provider.microvm`. + +### Multi-runner v2 migration roadmap + +Here, v1 and v2 refer to `multi_runner_config` and `experimental.multi_runner_config_v2`, not module release versions. The migration is intentionally split across releases so configuration migration, state migration, and interface removal do not happen at the same time. + +#### Phase 1 — Add v2 as a module-level opt-in (current) + +Both input contracts are available in the same module release, but callers must populate only one in a module instance. An empty `experimental.multi_runner_config_v2` keeps every `multi_runner_config` entry on the unchanged `modules/runners` implementation at `module.runners["configuration"]`, retaining its input contract, flat `runners_map` output, and Terraform addresses. To select `module.runner_stacks["configuration"]` and the nested `runners_map_v2` output shape, leave `multi_runner_config` empty and populate the v2 map. Populating both maps is unsupported. + +Compatibility guarantee: upgrading while leaving `experimental.multi_runner_config_v2` empty requires no state migration and must not move or replace legacy runner resources. Phase 1 does not support mixing implementations or migrating an existing v1 deployment by enabling v2; existing deployments should wait for phase 2 state mapping. The v2 opt-in is for new or explicitly experimental deployments. + +#### Phase 2 — Translate v1 and migrate state + +`multi_runner_config` remains accepted but is deprecated and translated to the v2 contract before dispatching through `runner-stack`. Existing users can therefore migrate the implementation and state without first combining v1 and v2 inputs. This phase will include tested `moved` blocks wherever Terraform can express the mapping and exact state-migration instructions for remaining addresses. The legacy flat output shape remains available as a compatibility adapter while users update configuration and output references. + +Compatibility guarantee: users can migrate implementation state before rewriting their configuration. With equivalent inputs, the documented migration must produce a plan without unintended runner-resource destruction or replacement. + +#### Phase 3 — Remove v1 from multi-runner + +After the announced migration window, a breaking release removes `multi_runner_config`, its translation, and the legacy flat output adapter from the multi-runner module. Only the v2 provider-oriented contract remains. Phase 3 will not introduce another state-address migration. + +Compatibility guarantee: phase 3 will not be released together with phase 2. Users will have at least one released migration version in which v1 is still accepted before its removal. + +#### Future — Retire the legacy runners module + +Removing `modules/runners` is a separate future change. It requires its own compatibility analysis, migration instructions, and deprecation window for direct and top-level consumers; it is not part of this provider-boundary refactor. + For each configuration: - When enabled, the [distribution syncer](https://github-aws-runners.github.io/terraform-aws-github-runner/modules/internal/runner-binaries-syncer/) is deployed for each unique combination of OS and architecture. @@ -96,6 +140,7 @@ module "multi-runner" { | [ami\_housekeeper](#module\_ami\_housekeeper) | ../ami-housekeeper | n/a | | [instance\_termination\_watcher](#module\_instance\_termination\_watcher) | ../termination-watcher | n/a | | [runner\_binaries](#module\_runner\_binaries) | ../runner-binaries-syncer | n/a | +| [runner\_stacks](#module\_runner\_stacks) | ../runner-stack | n/a | | [runners](#module\_runners) | ../runners | n/a | | [ssm](#module\_ssm) | ../ssm | n/a | | [webhook](#module\_webhook) | ../webhook | n/a | @@ -129,6 +174,7 @@ module "multi-runner" { | [enable\_ami\_housekeeper](#input\_enable\_ami\_housekeeper) | Option to disable the lambda to clean up old AMIs. | `bool` | `false` | no | | [enable\_managed\_runner\_security\_group](#input\_enable\_managed\_runner\_security\_group) | Enabling the default managed security group creation. Unmanaged security groups can be specified via `runner_additional_security_group_ids`. | `bool` | `true` | no | | [eventbridge](#input\_eventbridge) | Enable the use of EventBridge by the module. By enabling this feature events will be put on the EventBridge by the webhook instead of directly dispatching to queues for scaling. | 
object({
    enable        = optional(bool, true)
    accept_events = optional(list(string), [])
  })
| `{}` | no | +| [experimental](#input\_experimental) | Opt-in experimental features. Omit this object to retain only the stable `multi_runner_config` behavior. Experimental schemas can change before they become stable.

- `multi_runner_config_v2`: Provider-oriented runner configurations keyed by configuration name. To opt into v2, leave `multi_runner_config` empty and populate this map. When this map is empty, stable `multi_runner_config` entries continue to use the unchanged `runners` module. Populating both maps in the same module instance is unsupported.

Each `multi_runner_config_v2` entry supports the following nested fields:

- `tags`: Configuration-wide tags. These override module-level `tags`; narrower component and compute-provider tag maps take precedence for their resources.
- `runner.os`: Runner operating system.
- `runner.architecture`: Runner distribution architecture.
- `runner.boot_time_in_minutes`: Expected boot duration used before a runner is considered stale.
- `runner.disable_default_labels`: Prevents GitHub default labels from being registered.
- `runner.extra_labels`: Additional labels combined with `matcherConfig.labelMatchers`. Default self-hosted, operating-system, and architecture labels are also included unless `runner.disable_default_labels` is true.
- `runner.group_name`: GitHub runner group used during registration.
- `runner.name_prefix`: Prefix added to registered runner names.
- `runner.run_as_root`: Runs the runner service as root when supported by the compute provider.
- `runner.run_as`: Operating-system user used when `run_as_root` is false.
- `runner.maximum_count`: Maximum number of runners for this configuration.
- `runner.ephemeral`: Registers runners in ephemeral mode.
- `runner.jit_config_enabled`: Explicitly enables or disables just-in-time configuration. Null follows `ephemeral`.
- `runner.auto_update_disabled`: Disables the GitHub runner application's built-in updater.
- `runner.tags`: Tags for common runner resources, currently the managed runner IAM role. These override entry-level `tags`.
- `runner.hooks.job_started`: Script content installed as the runner job-started hook.
- `runner.hooks.job_completed`: Script content installed as the runner job-completed hook.
- `runner.iam.role.arn`: ARN of an externally managed runner role. When set, `runner-stack` does not create or modify that role.
- `runner.iam.managed_policy_arns`: Named managed-policy ARNs attached to the module-managed runner role.
- `runner.iam.additional_trust_policy_json`: Optional IAM policy document merged with the selected compute provider's default runner-role trust policy.
- `runner.iam.path`: IAM path for the module-managed runner role.
- `runner.iam.permissions_boundary`: Permissions-boundary ARN for the module-managed runner role.
- `github.organization_runners`: Registers runners at organization scope when true; otherwise repository-scoped registration is used.
- `lambda.tags`: Shared tags for control-plane Lambda functions. Component tags override this map.
- `queue.delay_webhook_event`: Delay in seconds applied to webhook job messages.
- `queue.job_queue_retention_in_seconds`: Build-queue message retention period in seconds.
- `queue.event_source_mapping.batch_size`: Maximum build-queue records delivered to one scale-up Lambda invocation. Null uses the module-level setting.
- `queue.event_source_mapping.maximum_batching_window_in_seconds`: Maximum batching window for build-queue records. Null uses the module-level setting.
- `queue.redrive_build_queue.enabled`: Creates and attaches a dead-letter queue for the build queue.
- `queue.redrive_build_queue.maxReceiveCount`: Number of receives before a build message moves to the dead-letter queue.
- `queue.tags`: Tags for configuration-owned queue resources. These override entry-level `tags`; component tags override this map.
- `scale_up.reserved_concurrent_executions`: Reserved concurrency for the scale-up Lambda. Use `-1` for unreserved concurrency.
- `scale_up.job_queued_check_enabled`: Enables the queued-job verification before scaling. Null follows the runner mode default.
- `scale_up.tags`: Tags for scale-up resources. These override entry-level and shared Lambda, queue, and log-group tags within their resource scopes.
- `scale_down.schedule_expression`: EventBridge schedule expression that invokes scale-down.
- `scale_down.minimum_running_time_in_minutes`: Minimum runner age before scale-down may terminate it. Null selects the operating-system default.
- `scale_down.tags`: Tags for scale-down resources. These override entry-level and shared Lambda and log-group tags within their resource scopes.
- `scale_down.idle_config`: Time-based desired idle-runner configurations.
- `scale_down.idle_config[].cron`: Cron expression identifying when the idle configuration applies.
- `scale_down.idle_config[].timeZone`: IANA time zone used to evaluate `cron`.
- `scale_down.idle_config[].idleCount`: Number of idle runners to retain during the matching period.
- `scale_down.idle_config[].evictionStrategy`: Selection strategy used when excess idle runners are removed.
- `pool.config`: Scheduled target pool sizes. An empty list disables the pool component.
- `pool.config[].schedule_expression`: Scheduler expression that activates the target size.
- `pool.config[].schedule_expression_timezone`: Optional IANA time zone used to evaluate the schedule.
- `pool.config[].size`: Desired number of runners for the schedule.
- `pool.runner_owner`: Optional GitHub organization or repository owner used when creating pooled runners.
- `pool.tags`: Tags for pool resources. These override entry-level and shared Lambda and log-group tags within their resource scopes.
- `job_retry.enabled`: Creates the retry queue, Lambda function, event-source mapping, and related IAM resources.
- `job_retry.delay_in_seconds`: Initial delay before a queued-job retry check.
- `job_retry.delay_backoff`: Multiplier applied to the delay after each unsuccessful check.
- `job_retry.max_attempts`: Maximum retry-check attempts before the message is no longer republished.
- `job_retry.tags`: Tags for job-retry resources. These override entry-level and shared Lambda, queue, and log-group tags within their resource scopes.
- `job_retry.lambda.memory_size`: Memory allocated to the job-retry Lambda in MB.
- `job_retry.lambda.reserved_concurrent_executions`: Reserved concurrency for the job-retry Lambda. Use `-1` for unreserved concurrency.
- `job_retry.lambda.timeout`: Job-retry Lambda timeout in seconds and visibility timeout for its retry queue.
- `ssm.tags`: Shared tags for SSM-related resources. These override entry-level `tags`.
- `ssm.kms_key`: Optional customer-managed KMS key used for temporary registration parameters. The wrapper's presence selects the KMS policy at plan time.
- `ssm.kms_key.arn`: ARN of the customer-managed KMS key. The ARN may be unknown until apply.
- `ssm.parameters.tags`: Tags for Terraform-managed and runtime-created runner configuration parameters. These override `ssm.tags`.
- `ssm.housekeeper.tags`: Tags for SSM housekeeper resources. These override entry-level, shared Lambda, shared log, and `ssm.tags` values.
- `observability.logs.tags`: Shared tags for CloudWatch log groups. Component tags override this map.
- `compute_provider`: Typed compute-provider blocks. Exactly one block must be non-null, and the populated block selects the provider. Its presence must be known during planning; values inside it may remain unknown until apply.
- `compute_provider.ec2`: EC2-specific configuration.
- `compute_provider.ec2.ami.filter`: EC2 AMI filters combined with the default AMI-name filter.
- `compute_provider.ec2.ami.owners`: AWS account IDs or aliases allowed to own the selected AMI.
- `compute_provider.ec2.ami.id_ssm_parameter`: Optional externally managed SSM parameter containing the AMI ID. The wrapper's presence selects external ownership at plan time.
- `compute_provider.ec2.ami.id_ssm_parameter.arn`: ARN of the externally managed SSM parameter. The ARN may be unknown until apply.
- `compute_provider.ec2.ami.kms_key`: Optional KMS key required to launch encrypted AMIs or snapshots. The wrapper's presence selects the KMS policy at plan time.
- `compute_provider.ec2.ami.kms_key.arn`: ARN of the KMS key. The ARN may be unknown until apply.
- `compute_provider.ec2.block_device_mappings`: EBS mappings added to the runner launch template.
- `compute_provider.ec2.block_device_mappings[].delete_on_termination`: Deletes the volume when its runner instance terminates.
- `compute_provider.ec2.block_device_mappings[].device_name`: Device name exposed to the runner instance.
- `compute_provider.ec2.block_device_mappings[].encrypted`: Enables EBS encryption.
- `compute_provider.ec2.block_device_mappings[].iops`: Provisioned IOPS for supported volume types.
- `compute_provider.ec2.block_device_mappings[].kms_key_id`: KMS key ID or ARN used to encrypt the volume.
- `compute_provider.ec2.block_device_mappings[].snapshot_id`: Snapshot used to initialize the volume.
- `compute_provider.ec2.block_device_mappings[].throughput`: Provisioned throughput for supported volume types.
- `compute_provider.ec2.block_device_mappings[].volume_initialization_rate`: Fixed initialization rate in MiB/s for supported snapshot-backed volumes.
- `compute_provider.ec2.block_device_mappings[].volume_size`: Volume size in GiB.
- `compute_provider.ec2.block_device_mappings[].volume_type`: EBS volume type.
- `compute_provider.ec2.create_service_linked_role_spot`: Allows scale-up to create the EC2 Spot service-linked role.
- `compute_provider.ec2.credit_specification`: CPU credit mode for burstable instance types.
- `compute_provider.ec2.ebs_optimized`: Requests EBS-optimized runner instances.
- `compute_provider.ec2.cloudwatch_agent.enabled`: Installs and configures the CloudWatch agent through the default bootstrap flow.
- `compute_provider.ec2.cloudwatch_agent.config`: Optional complete CloudWatch agent configuration.
- `compute_provider.ec2.binaries_syncer.enabled`: Enables use of the module-level synchronized runner distribution from S3.
- `compute_provider.ec2.detailed_monitoring_enabled`: Enables detailed EC2 monitoring for runner instances.
- `compute_provider.ec2.ssm_enabled`: Attaches runner permissions and policies required for AWS Systems Manager access.
- `compute_provider.ec2.user_data.enabled`: Enables launch-template user data.
- `compute_provider.ec2.user_data.template`: Optional path to a custom user-data template.
- `compute_provider.ec2.user_data.content`: Optional complete user-data content used instead of rendering a template.
- `compute_provider.ec2.user_data.pre_install`: Script content inserted before runner installation in the default template.
- `compute_provider.ec2.user_data.post_install`: Script content inserted after runner installation in the default template.
- `compute_provider.ec2.user_data.debug_logging_enabled`: Enables verbose user-data tracing, which can expose secrets in logs.
- `compute_provider.ec2.instance_allocation_strategy`: EC2 Fleet allocation strategy used to select capacity.
- `compute_provider.ec2.instance_max_spot_price`: Optional maximum hourly Spot price.
- `compute_provider.ec2.instance_target_capacity_type`: Primary capacity type, either `spot` or `on-demand`.
- `compute_provider.ec2.instance_type_priorities`: Optional numeric priorities keyed by instance type.
- `compute_provider.ec2.instance_types`: EC2 instance types available to the scale-up and pool functions.
- `compute_provider.ec2.additional_security_group_ids`: Existing security groups attached to runner instances.
- `compute_provider.ec2.instance_profile.name`: Name of an externally managed instance profile. Setting it also requires `runner.iam.role`.
- `compute_provider.ec2.enable_on_demand_failover_for_errors`: EC2 error codes that trigger an on-demand fallback after a Spot launch failure.
- `compute_provider.ec2.scale_errors`: EC2 error codes treated as retryable scale-up failures.
- `compute_provider.ec2.subnet_ids`: Subnets from which scale-up may launch runners. Null uses the module-level value.
- `compute_provider.ec2.vpc_id`: VPC in which runner networking resources are created. Null uses the module-level value.
- `compute_provider.ec2.cpu_options.core_count`: Number of CPU cores exposed to the runner instance.
- `compute_provider.ec2.cpu_options.threads_per_core`: Number of hardware threads exposed per CPU core.
- `compute_provider.ec2.cpu_options.amd_sev_snp`: Enables or disables AMD SEV-SNP on supported instance types.
- `compute_provider.ec2.cpu_options.nested_virtualization`: Enables or disables nested virtualization on supported instance types.
- `compute_provider.ec2.placement.affinity`: Host affinity setting.
- `compute_provider.ec2.placement.availability_zone`: Availability Zone in which the instance is placed.
- `compute_provider.ec2.placement.group_id`: Placement-group ID.
- `compute_provider.ec2.placement.group_name`: Placement-group name.
- `compute_provider.ec2.placement.host_id`: Dedicated Host ID.
- `compute_provider.ec2.placement.host_resource_group_arn`: ARN of the host resource group used for placement.
- `compute_provider.ec2.placement.spread_domain`: Spread-domain placement value.
- `compute_provider.ec2.placement.tenancy`: Instance tenancy.
- `compute_provider.ec2.placement.partition_number`: Placement-group partition number.
- `compute_provider.ec2.license_specifications[].license_configuration_arn`: ARN of a License Manager license configuration.
- `compute_provider.ec2.use_dedicated_host`: Enables the dedicated-host launch path required for macOS runners.
- `compute_provider.ec2.log_files`: Optional log files collected by the CloudWatch agent.
- `compute_provider.ec2.log_files[].log_group_name`: CloudWatch log-group name before optional prefixing.
- `compute_provider.ec2.log_files[].prefix_log_group`: Prefixes the log-group name with the runner stack path when true.
- `compute_provider.ec2.log_files[].file_path`: File or glob read by the CloudWatch agent.
- `compute_provider.ec2.log_files[].log_stream_name`: CloudWatch log-stream name template.
- `compute_provider.ec2.log_files[].log_class`: CloudWatch log-group class for the collected file.
- `compute_provider.ec2.tags`: Tags for runtime EC2 instances, volumes, network interfaces, and eligible Spot requests. These override entry-level tags and the generated runner `Name`; provider-required bootstrap tags take final precedence.
- `compute_provider.ec2.metadata_options.instance_metadata_tags`: Exposes instance tags through Instance Metadata Service when enabled.
- `compute_provider.ec2.metadata_options.http_endpoint`: Enables or disables the Instance Metadata Service endpoint.
- `compute_provider.ec2.metadata_options.http_tokens`: Controls whether IMDSv2 session tokens are optional or required.
- `compute_provider.ec2.metadata_options.http_put_response_hop_limit`: Network hop limit for Instance Metadata Service token responses.
- `compute_provider.microvm`: Lambda MicroVM-specific configuration.
- `compute_provider.microvm.image_identifier`: ARN or ID of the MicroVM image used to run GitHub runners.
- `compute_provider.microvm.image_version`: Optional MicroVM image version.
- `compute_provider.microvm.execution_role.arn`: Optional externally managed execution role assumed by MicroVMs. Null uses the common runner role.
- `compute_provider.microvm.egress_network_connectors`: Egress network connectors passed to RunMicrovm.
- `compute_provider.microvm.idle_policy`: Optional auto-suspend and auto-resume configuration passed to RunMicrovm.
- `compute_provider.microvm.logging`: Optional RunMicrovm logging union. Exactly one of `cloud_watch` or `disabled` must be selected when set.
- `compute_provider.microvm.run_hook_payload`: Optional payload delivered to the MicroVM `/run` hook. Maximum 16,384 characters.
- `compute_provider.microvm.maximum_duration_in_seconds`: Optional maximum MicroVM lifetime. Valid range is 1 through 28,800 seconds.
- `compute_provider.microvm.environment_variables`: Additional provider-specific Lambda environment variables merged into scale-up, scale-down, and pool.
- `compute_provider.microvm.tags`: Tags encoded into the MicroVM runner configuration.
- `compute_provider.microvm.iam`: Optional MicroVM control-plane IAM overrides and managed policy attachments.
- `matcherConfig.labelMatchers`: Groups of labels used to match webhook jobs to this configuration.
- `matcherConfig.exactMatch`: Requires the job labels to exactly match a configured label group.
- `matcherConfig.bidirectionalLabelMatch`: Requires labels to match in both directions instead of allowing configured subsets.
- `matcherConfig.priority`: Ordering used when multiple configurations match the same job.
- `matcherConfig.enableDynamicLabels`: Enables runtime interpretation of supported dynamic AWS labels.
- `matcherConfig.awsDynamicLabelsPolicy`: Optional policy restricting values accepted from dynamic AWS labels. | 
object({
    multi_runner_config_v2 = optional(map(object({
      tags = optional(map(string), {})

      runner = object({
        os                     = string
        architecture           = string
        boot_time_in_minutes   = optional(number, 5)
        disable_default_labels = optional(bool, false)
        extra_labels           = optional(list(string), [])
        group_name             = optional(string, "Default")
        name_prefix            = optional(string, "")
        run_as_root            = optional(bool, false)
        run_as                 = optional(string, "ec2-user")
        maximum_count          = number
        ephemeral              = optional(bool, false)
        jit_config_enabled     = optional(bool, null)
        auto_update_disabled   = optional(bool, false)
        tags                   = optional(map(string), {})
        hooks = optional(object({
          job_started   = optional(string, "")
          job_completed = optional(string, "")
        }), {})
        iam = optional(object({
          role = optional(object({
            arn = string
          }), null)
          managed_policy_arns          = optional(map(string), {})
          additional_trust_policy_json = optional(string, null)
          path                         = optional(string, null)
          permissions_boundary         = optional(string, null)
        }), {})
      })

      github = optional(object({
        organization_runners = optional(bool, false)
      }), {})

      lambda = optional(object({
        tags = optional(map(string), {})
      }), {})

      queue = optional(object({
        delay_webhook_event            = optional(number, 30)
        job_queue_retention_in_seconds = optional(number, 86400)
        event_source_mapping = optional(object({
          batch_size                         = optional(number, null)
          maximum_batching_window_in_seconds = optional(number, null)
        }), {})
        redrive_build_queue = optional(object({
          enabled         = bool
          maxReceiveCount = number
          }), {
          enabled         = false
          maxReceiveCount = null
        })
        tags = optional(map(string), {})
      }), {})

      scale_up = optional(object({
        reserved_concurrent_executions = optional(number, 1)
        job_queued_check_enabled       = optional(bool, null)
        tags                           = optional(map(string), {})
      }), {})

      scale_down = optional(object({
        schedule_expression             = optional(string, "cron(*/5 * * * ? *)")
        minimum_running_time_in_minutes = optional(number, null)
        tags                            = optional(map(string), {})
        idle_config = optional(list(object({
          cron             = string
          timeZone         = string
          idleCount        = number
          evictionStrategy = optional(string, "oldest_first")
        })), [])
      }), {})

      pool = optional(object({
        config = optional(list(object({
          schedule_expression          = string
          schedule_expression_timezone = optional(string)
          size                         = number
        })), [])
        runner_owner = optional(string, null)
        tags         = optional(map(string), {})
      }), {})

      job_retry = optional(object({
        enabled          = optional(bool, false)
        delay_in_seconds = optional(number, 300)
        delay_backoff    = optional(number, 2)
        max_attempts     = optional(number, 1)
        tags             = optional(map(string), {})
        lambda = optional(object({
          memory_size                    = optional(number, 256)
          reserved_concurrent_executions = optional(number, 1)
          timeout                        = optional(number, 30)
        }), {})
      }), {})

      ssm = optional(object({
        tags = optional(map(string), {})
        kms_key = optional(object({
          arn = string
        }), null)
        parameters = optional(object({
          tags = optional(map(string), {})
        }), {})
        housekeeper = optional(object({
          tags = optional(map(string), {})
        }), {})
      }), {})

      observability = optional(object({
        logs = optional(object({
          tags = optional(map(string), {})
        }), {})
      }), {})

      compute_provider = object({
        ec2 = optional(object({
          metadata_options = optional(object({
            instance_metadata_tags      = optional(string, "enabled")
            http_endpoint               = optional(string, "enabled")
            http_tokens                 = optional(string, "required")
            http_put_response_hop_limit = optional(number, 1)
          }), {})
          ami = optional(object({
            filter = optional(map(list(string)), { state = ["available"] })
            owners = optional(list(string), ["amazon"])
            id_ssm_parameter = optional(object({
              arn = string
            }), null)
            kms_key = optional(object({
              arn = string
            }), null)
          }), null)
          block_device_mappings = optional(list(object({
            delete_on_termination      = optional(bool, true)
            device_name                = optional(string, "/dev/xvda")
            encrypted                  = optional(bool, true)
            iops                       = optional(number)
            kms_key_id                 = optional(string)
            snapshot_id                = optional(string)
            throughput                 = optional(number)
            volume_initialization_rate = optional(number)
            volume_size                = number
            volume_type                = optional(string, "gp3")
            })), [{
            volume_size = 30
          }])
          create_service_linked_role_spot = optional(bool, false)
          credit_specification            = optional(string, null)
          ebs_optimized                   = optional(bool, false)
          cloudwatch_agent = optional(object({
            enabled = optional(bool, true)
            config  = optional(string, null)
          }), {})
          binaries_syncer = optional(object({
            enabled = optional(bool, true)
          }), {})
          detailed_monitoring_enabled = optional(bool, false)
          ssm_enabled                 = optional(bool, false)
          user_data = optional(object({
            enabled               = optional(bool, true)
            template              = optional(string, null)
            content               = optional(string, null)
            pre_install           = optional(string, "")
            post_install          = optional(string, "")
            debug_logging_enabled = optional(bool, false)
          }), {})
          instance_allocation_strategy  = optional(string, "lowest-price")
          instance_max_spot_price       = optional(string, null)
          instance_target_capacity_type = optional(string, "spot")
          instance_type_priorities      = optional(map(number), null)
          instance_types                = list(string)
          additional_security_group_ids = optional(list(string), [])
          instance_profile = optional(object({
            name = string
          }), null)
          enable_on_demand_failover_for_errors = optional(list(string), [])
          scale_errors = optional(list(string), [
            "UnfulfillableCapacity",
            "MaxSpotInstanceCountExceeded",
            "TargetCapacityLimitExceededException",
            "RequestLimitExceeded",
            "ResourceLimitExceeded",
            "MaxSpotInstanceCountExceeded",
            "MaxSpotFleetRequestCountExceeded",
            "InsufficientInstanceCapacity",
            "InsufficientCapacityOnHost",
          ])
          subnet_ids = optional(list(string), null)
          vpc_id     = optional(string, null)
          cpu_options = optional(object({
            core_count            = optional(number)
            threads_per_core      = optional(number)
            amd_sev_snp           = optional(string)
            nested_virtualization = optional(string)
          }), null)
          placement = optional(object({
            affinity                = optional(string)
            availability_zone       = optional(string)
            group_id                = optional(string)
            group_name              = optional(string)
            host_id                 = optional(string)
            host_resource_group_arn = optional(string)
            spread_domain           = optional(string)
            tenancy                 = optional(string)
            partition_number        = optional(number)
          }), null)
          license_specifications = optional(list(object({
            license_configuration_arn = string
          })), [])
          use_dedicated_host = optional(bool, false)
          log_files = optional(list(object({
            log_group_name   = string
            prefix_log_group = bool
            file_path        = string
            log_stream_name  = string
            log_class        = optional(string, "STANDARD")
          })), null)
          tags = optional(map(string), {})
        }), null)

        microvm = optional(object({
          image_identifier = string
          image_version    = optional(string, null)
          execution_role = optional(object({
            arn = string
          }), null)
          egress_network_connectors = optional(list(string), [])
          idle_policy = optional(object({
            max_idle_duration_seconds  = number
            suspended_duration_seconds = number
            auto_resume_enabled        = bool
          }), null)
          logging = optional(object({
            cloud_watch = optional(object({
              log_group  = optional(string, null)
              log_stream = optional(string, null)
            }), null)
            disabled = optional(bool, false)
          }), null)
          run_hook_payload            = optional(string, null)
          maximum_duration_in_seconds = optional(number, null)
          environment_variables       = optional(map(string), {})
          tags                        = optional(map(string), {})
          iam = optional(object({
            resource_arns = optional(list(string), ["*"])
            actions = optional(object({
              scale_up   = optional(list(string), null)
              scale_down = optional(list(string), null)
            }), {})
            additional_policy_json = optional(object({
              scale_up = optional(string, null)
            }), {})
            managed_policy_arns = optional(object({
              scale_up = optional(string, null)
              pool     = optional(string, null)
            }), {})
          }), {})
        }), null)
      })

      matcherConfig = object({
        labelMatchers           = list(list(string))
        exactMatch              = optional(bool, false)
        bidirectionalLabelMatch = optional(bool, false)
        priority                = optional(number, 999)
        enableDynamicLabels     = optional(bool, false)
        awsDynamicLabelsPolicy = optional(object({
          blocked_keys = optional(list(string), [])
          restricted_keys = optional(map(object({
            allowed = optional(list(string), [])
            denied  = optional(list(string), [])
            max     = optional(string, null)
          })), {})
        }), null)
      })
    })), {})
  })
| `{}` | no | | [ghes\_ssl\_verify](#input\_ghes\_ssl\_verify) | GitHub Enterprise SSL verification. Set to 'false' when custom certificate (chains) is used for GitHub Enterprise Server (insecure). | `bool` | `true` | no | | [ghes\_url](#input\_ghes\_url) | GitHub Enterprise Server URL. Example: https://github.internal.co - DO NOT SET IF USING PUBLIC GITHUB. .However if you are using GitHub Enterprise Cloud with data-residency (ghe.com), set the endpoint here. Example - https://companyname.ghe.com\| | `string` | `null` | no | | [github\_app](#input\_github\_app) | GitHub app parameters, see your github app.
You can optionally create the SSM parameters yourself and provide the ARN and name here, through the `*_ssm` attributes.
If you chose to provide the configuration values directly here,
please ensure the key is the base64-encoded `.pem` file (the output of `base64 app.private-key.pem`, not the content of `private-key.pem`).
Note: the provided SSM parameters arn and name have a precedence over the actual value (i.e `key_base64_ssm` has a precedence over `key_base64` etc). | 
object({
    key_base64 = optional(string)
    key_base64_ssm = optional(object({
      arn  = string
      name = string
    }))
    id = optional(string)
    id_ssm = optional(object({
      arn  = string
      name = string
    }))
    webhook_secret = optional(string)
    webhook_secret_ssm = optional(object({
      arn  = string
      name = string
    }))
  })
| n/a | yes | @@ -152,7 +198,7 @@ module "multi-runner" { | [logging\_retention\_in\_days](#input\_logging\_retention\_in\_days) | Specifies the number of days you want to retain log events for the lambda log group. Possible values are: 0, 1, 3, 5, 7, 14, 30, 60, 90, 120, 150, 180, 365, 400, 545, 731, 1827, and 3653. | `number` | `180` | no | | [matcher\_config\_parameter\_store\_tier](#input\_matcher\_config\_parameter\_store\_tier) | The tier of the parameter store for the matcher configuration. Valid values are `Standard`, and `Advanced`. | `string` | `"Standard"` | no | | [metrics](#input\_metrics) | Configuration for metrics created by the module, by default metrics are disabled to avoid additional costs. When metrics are enable all metrics are created unless explicit configured otherwise. | 
object({
    enable    = optional(bool, false)
    namespace = optional(string, "GitHub Runners")
    metric = optional(object({
      enable_github_app_rate_limit    = optional(bool, true)
      enable_job_retry                = optional(bool, true)
      enable_spot_termination_warning = optional(bool, true)
    }), {})
  })
| `{}` | no | -| [multi\_runner\_config](#input\_multi\_runner\_config) | multi\_runner\_config = {
runner\_config: {
runner\_os: "The EC2 Operating System type to use for action runner instances (linux, osx, windows)."
runner\_architecture: "The platform architecture of the runner instance\_type."
runner\_metadata\_options: "(Optional) Metadata options for the ec2 runner instances."
ami: "(Optional) AMI configuration for the action runner instances. This object allows you to specify all AMI-related settings in one place."
create\_service\_linked\_role\_spot: (Optional) create the serviced linked role for spot instances that is required by the scale-up lambda.
credit\_specification: "(Optional) The credit specification of the runner instance\_type. Can be unset, `standard` or `unlimited`.
delay\_webhook\_event: "The number of seconds the event accepted by the webhook is invisible on the queue before the scale up lambda will receive the event."
disable\_runner\_autoupdate: "Disable the auto update of the github runner agent. Be aware there is a grace period of 30 days, see also the [GitHub article](https://github.blog/changelog/2022-02-01-github-actions-self-hosted-runners-can-now-disable-automatic-updates/)"
ebs\_optimized: "The EC2 EBS optimized configuration."
enable\_ephemeral\_runners: "Enable ephemeral runners, runners will only be used once."
enable\_job\_queued\_check: Enables JIT configuration for creating runners instead of registration token based registraton. JIT configuration will only be applied for ephemeral runners. By default JIT configuration is enabled for ephemeral runners an can be disabled via this override. When running on GHES without support for JIT configuration this variable should be set to true for ephemeral runners."
enable\_on\_demand\_failover\_for\_errors: "Enable on-demand failover. For example to fall back to on demand when no spot capacity is available the variable can be set to `InsufficientInstanceCapacity`. When not defined the default behavior is to retry later."
scale\_errors: "List of AWS error codes that should trigger retry during scale up. This list replaces the module default scale-up retry errors"
enable\_organization\_runners: "Register runners to organization, instead of repo level"
enable\_runner\_binaries\_syncer: "Option to disable the lambda to sync GitHub runner distribution, useful when using a pre-build AMI."
enable\_ssm\_on\_runners: "Enable to allow access the runner instances for debugging purposes via SSM. Note that this adds additional permissions to the runner instances."
enable\_userdata: "Should the userdata script be enabled for the runner. Set this to false if you are using your own prebuilt AMI."
instance\_allocation\_strategy: "The allocation strategy for creating instances. For spot, AWS recommends `price-capacity-optimized`; for on-demand, use `lowest-price` or `prioritized`. The AWS default is `lowest-price`."
instance\_type\_priorities: "A map of instance type to priority for the `prioritized` and `capacity-optimized-prioritized` allocation strategies. Lower numbers mean higher priority. If not provided, priorities are assigned based on the order of `instance_types`."
instance\_max\_spot\_price: "Max price price for spot instances per hour. This variable will be passed to the create fleet as max spot price for the fleet."
instance\_target\_capacity\_type: "Default lifecycle used for runner instances, can be either `spot` or `on-demand`."
instance\_types: "List of instance types for the action runner. Defaults are based on runner\_os (al2023 for linux, macOS Sequoia for osx, Windows Server Core for win)."
job\_queue\_retention\_in\_seconds: "The number of seconds the job is held in the queue before it is purged"
minimum\_running\_time\_in\_minutes: "The time an ec2 action runner should be running at minimum before terminated if not busy."
pool\_runner\_owner: "The pool will deploy runners to the GitHub org ID, set this value to the org to which you want the runners deployed. Repo level is not supported."
runner\_additional\_security\_group\_ids: "List of additional security groups IDs to apply to the runner. If added outside the multi\_runner\_config block, the additional security group(s) will be applied to all runner configs. If added inside the multi\_runner\_config, the additional security group(s) will be applied to the individual runner."
runner\_as\_root: "Run the action runner under the root user. Variable `runner_run_as` will be ignored."
runner\_boot\_time\_in\_minutes: "The minimum time for an EC2 runner to boot and register as a runner."
runner\_disable\_default\_labels: "Disable default labels for the runners (os, architecture and `self-hosted`). If enabled, the runner will only have the extra labels provided in `runner_extra_labels`. In case you on own start script is used, this configuration parameter needs to be parsed via SSM."
runner\_extra\_labels: "Extra (custom) labels for the runners (GitHub). Separate each label by a comma. Labels checks on the webhook can be enforced by setting `multi_runner_config.matcherConfig.exactMatch`. GitHub read-only labels should not be provided."
runner\_group\_name: "Name of the runner group."
runner\_name\_prefix: "Prefix for the GitHub runner name."
runner\_run\_as: "Run the GitHub actions agent as user."
runners\_maximum\_count: "The maximum number of runners that will be created. Setting the variable to `-1` disables the maximum check."
scale\_down\_schedule\_expression: "Scheduler expression to check every x for scale down."
scale\_up\_reserved\_concurrent\_executions: "Amount of reserved concurrent executions for the scale-up lambda function. A value of 0 disables lambda from being triggered and -1 removes any concurrency limitations."
lambda\_event\_source\_mapping\_batch\_size: "(Optional) Maximum number of records per Lambda invocation for this runner flavor. Overrides the module-level `lambda_event_source_mapping_batch_size` when set."
lambda\_event\_source\_mapping\_maximum\_batching\_window\_in\_seconds: "(Optional) Maximum seconds to gather records before invoking Lambda for this runner flavor. Overrides the module-level `lambda_event_source_mapping_maximum_batching_window_in_seconds` when set."
userdata\_template: "Alternative user-data template, replacing the default template. By providing your own user\_data you have to take care of installing all required software, including the action runner. Variables userdata\_pre/post\_install are ignored."
enable\_jit\_config: "Overwrite the default behavior for JIT configuration. By default JIT configuration is enabled for ephemeral runners and disabled for non-ephemeral runners. In case of GHES check first if the JIT config API is available. In case you are upgrading from 3.x to 4.x you can set `enable_jit_config` to `false` to avoid a breaking change when having your own AMI."
enable\_runner\_detailed\_monitoring: "Should detailed monitoring be enabled for the runner. Set this to true if you want to use detailed monitoring. See https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/using-cloudwatch-new.html for details."
enable\_cloudwatch\_agent: "Enabling the cloudwatch agent on the ec2 runner instances, the runner contains default config. Configuration can be overridden via `cloudwatch_config`."
cloudwatch\_config: "(optional) Replaces the module default cloudwatch log config. See https://docs.aws.amazon.com/AmazonCloudWatch/latest/monitoring/CloudWatch-Agent-Configuration-File-Details.html for details."
userdata\_pre\_install: "Script to be ran before the GitHub Actions runner is installed on the EC2 instances"
userdata\_post\_install: "Script to be ran after the GitHub Actions runner is installed on the EC2 instances"
runner\_hook\_job\_started: "Script to be ran in the runner environment at the beginning of every job"
runner\_hook\_job\_completed: "Script to be ran in the runner environment at the end of every job"
runner\_ec2\_tags: "Map of tags that will be added to the launch template instance tag specifications."
runner\_iam\_role\_managed\_policy\_arns: "Attach AWS or customer-managed IAM policies (by ARN) to the runner IAM role"
vpc\_id: "The VPC for security groups of the action runners. If not set uses the value of `var.vpc_id`."
subnet\_ids: "List of subnets in which the action runners will be launched, the subnets needs to be subnets in the `vpc_id`. If not set, uses the value of `var.subnet_ids`."
idle\_config: "List of time period that can be defined as cron expression to keep a minimum amount of runners active instead of scaling down to 0. By defining this list you can ensure that in time periods that match the cron expression within 5 seconds a runner is kept idle."
license\_specifications: "Optional EC2 License Manager license configuration ARNs for the runner launch template. Required for macOS dedicated-host runners when the host resource group uses a Mac dedicated host license configuration."
use\_dedicated\_host: "Experimental! Can be removed / changed without trigger a major release. Whether to use EC2 dedicated hosts for the runners. Needed for macos runners Note that using dedicated hosts can increase cost significantly."
runner\_log\_files: "(optional) Replaces the module default cloudwatch log config. See https://docs.aws.amazon.com/AmazonCloudWatch/latest/monitoring/CloudWatch-Agent-Configuration-File-Details.html for details."
block\_device\_mappings: "The EC2 instance block device configuration. Takes the following keys: `device_name`, `delete_on_termination`, `volume_type`, `volume_size`, `encrypted`, `iops`, `throughput`, `kms_key_id`, `snapshot_id`, `volume_initialization_rate`."
job\_retry: "Experimental! Can be removed / changed without trigger a major release. Configure job retries. The configuration enables job retries (for ephemeral runners). After creating the instances a message will be published to a job retry queue. The job retry check lambda is checking after a delay if the job is queued. If not the message will be published again on the scale-up (build queue). Using this feature can impact the rate limit of the GitHub app."
pool\_config: "The configuration for updating the pool. The `pool_size` to adjust to by the events triggered by the `schedule_expression`. For example you can configure a cron expression for week days to adjust the pool to 10 and another expression for the weekend to adjust the pool to 1. Use `schedule_expression_timezone` to override the schedule time zone (defaults to UTC)."
iam\_overrides: "Allows to (optionally) override the instance profile and runner role created by the module. Set `override_instance_profile` to true and provide the `instance_profile_name` to use an existing instance profile. Set `override_runner_role` to true and provide the `runner_role_arn` to use an existing role for the runner instances."
}
matcherConfig: {
labelMatchers: "The list of list of labels supported by the runner configuration. `[[self-hosted, linux, x64, example]]`"
exactMatch: "DEPRECATED: Use `bidirectionalLabelMatch` instead. If set to true all labels in the workflow job must match the GitHub labels (os, architecture and `self-hosted`). When false if __any__ workflow label matches it will trigger the webhook. Note: this only checks that workflow labels are a subset of runner labels, not the reverse."
bidirectionalLabelMatch: "If set to true, the runner labels and workflow job labels must be an exact two-way match (same set, any order, no extras or missing labels). This is stricter than `exactMatch` which only checks that workflow labels are a subset of runner labels. When false, if __any__ workflow label matches it will trigger the webhook."
priority: "If set it defines the priority of the matcher, the matcher with the lowest priority will be evaluated first. Default is 999, allowed values 0-999."
enableDynamicLabels: "Experimental! When true the dispatcher allows `ghr-*` dynamic labels for jobs routed to this runner. Default false."
awsDynamicLabelsPolicy: "Optional AWS dynamic label policy evaluated by the dispatcher. Only effective when `enableDynamicLabels = true`. Jobs whose provider dynamic labels violate every matching runner's policy are rejected with a 202 (a warning is logged). Evaluation: keys in `blocked_keys` are always rejected; keys in `restricted_keys` are allowed only when their value passes the rule; unlisted keys are allowed. Schema: `{ blocked_keys = [], restricted_keys = { = { allowed = [globs], denied = [globs], max = number|string } } }`. Keys use the dynamic label suffix, e.g. `instance-type` for `ghr-ec2-instance-type`."
}
redrive\_build\_queue: "Set options to attach (optional) a dead letter queue to the build queue, the queue between the webhook and the scale up lambda. You have the following options. 1. Disable by setting `enabled` to false. 2. Enable by setting `enabled` to `true`, `maxReceiveCount` to a number of max retries."
} | 
map(object({
    runner_config = object({
      runner_os           = string
      runner_architecture = string
      runner_metadata_options = optional(map(any), {
        instance_metadata_tags      = "enabled"
        http_endpoint               = "enabled"
        http_tokens                 = "required"
        http_put_response_hop_limit = 1
      })
      ami = optional(object({
        filter               = optional(map(list(string)), { state = ["available"] })
        owners               = optional(list(string), ["amazon"])
        id_ssm_parameter_arn = optional(string, null)
        kms_key_arn          = optional(string, null)
      }), null)
      create_service_linked_role_spot      = optional(bool, false)
      credit_specification                 = optional(string, null)
      delay_webhook_event                  = optional(number, 30)
      disable_runner_autoupdate            = optional(bool, false)
      ebs_optimized                        = optional(bool, false)
      enable_ephemeral_runners             = optional(bool, false)
      enable_job_queued_check              = optional(bool, null)
      enable_on_demand_failover_for_errors = optional(list(string), [])
      scale_errors = optional(list(string), [
        "UnfulfillableCapacity",
        "MaxSpotInstanceCountExceeded",
        "TargetCapacityLimitExceededException",
        "RequestLimitExceeded",
        "ResourceLimitExceeded",
        "MaxSpotInstanceCountExceeded",
        "MaxSpotFleetRequestCountExceeded",
        "InsufficientInstanceCapacity",
        "InsufficientCapacityOnHost",
      ])
      enable_organization_runners                                    = optional(bool, false)
      enable_runner_binaries_syncer                                  = optional(bool, true)
      enable_ssm_on_runners                                          = optional(bool, false)
      enable_userdata                                                = optional(bool, true)
      instance_allocation_strategy                                   = optional(string, "lowest-price")
      instance_type_priorities                                       = optional(map(number), null)
      instance_max_spot_price                                        = optional(string, null)
      instance_target_capacity_type                                  = optional(string, "spot")
      instance_types                                                 = list(string)
      job_queue_retention_in_seconds                                 = optional(number, 86400)
      minimum_running_time_in_minutes                                = optional(number, null)
      pool_runner_owner                                              = optional(string, null)
      runner_as_root                                                 = optional(bool, false)
      runner_boot_time_in_minutes                                    = optional(number, 5)
      runner_disable_default_labels                                  = optional(bool, false)
      runner_extra_labels                                            = optional(list(string), [])
      runner_group_name                                              = optional(string, "Default")
      runner_name_prefix                                             = optional(string, "")
      runner_run_as                                                  = optional(string, "ec2-user")
      runners_maximum_count                                          = number
      runner_additional_security_group_ids                           = optional(list(string), [])
      scale_down_schedule_expression                                 = optional(string, "cron(*/5 * * * ? *)")
      scale_up_reserved_concurrent_executions                        = optional(number, 1)
      lambda_event_source_mapping_batch_size                         = optional(number, null)
      lambda_event_source_mapping_maximum_batching_window_in_seconds = optional(number, null)
      userdata_template                                              = optional(string, null)
      userdata_content                                               = optional(string, null)
      enable_jit_config                                              = optional(bool, null)
      enable_runner_detailed_monitoring                              = optional(bool, false)
      enable_cloudwatch_agent                                        = optional(bool, true)
      cloudwatch_config                                              = optional(string, null)
      userdata_pre_install                                           = optional(string, "")
      userdata_post_install                                          = optional(string, "")
      runner_hook_job_started                                        = optional(string, "")
      runner_hook_job_completed                                      = optional(string, "")
      runner_ec2_tags                                                = optional(map(string), {})
      runner_iam_role_managed_policy_arns                            = optional(list(string), [])
      vpc_id                                                         = optional(string, null)
      subnet_ids                                                     = optional(list(string), null)
      idle_config = optional(list(object({
        cron             = string
        timeZone         = string
        idleCount        = number
        evictionStrategy = optional(string, "oldest_first")
      })), [])
      cpu_options = optional(object({
        core_count            = optional(number)
        threads_per_core      = optional(number)
        amd_sev_snp           = optional(string)
        nested_virtualization = optional(string)
      }), null)
      placement = optional(object({
        affinity                = optional(string)
        availability_zone       = optional(string)
        group_id                = optional(string)
        group_name              = optional(string)
        host_id                 = optional(string)
        host_resource_group_arn = optional(string)
        spread_domain           = optional(string)
        tenancy                 = optional(string)
        partition_number        = optional(number)
      }), null)
      license_specifications = optional(list(object({
        license_configuration_arn = string
      })), [])
      use_dedicated_host = optional(bool, false)
      runner_log_files = optional(list(object({
        log_group_name   = string
        prefix_log_group = bool
        file_path        = string
        log_stream_name  = string
        log_class        = optional(string, "STANDARD")
      })), null)
      block_device_mappings = optional(list(object({
        delete_on_termination      = optional(bool, true)
        device_name                = optional(string, "/dev/xvda")
        encrypted                  = optional(bool, true)
        iops                       = optional(number)
        kms_key_id                 = optional(string)
        snapshot_id                = optional(string)
        throughput                 = optional(number)
        volume_initialization_rate = optional(number)
        volume_size                = number
        volume_type                = optional(string, "gp3")
        })), [{
        volume_size = 30
      }])
      pool_config = optional(list(object({
        schedule_expression          = string
        schedule_expression_timezone = optional(string)
        size                         = number
      })), [])
      job_retry = optional(object({
        enable             = optional(bool, false)
        delay_in_seconds   = optional(number, 300)
        delay_backoff      = optional(number, 2)
        lambda_memory_size = optional(number, 256)
        lambda_timeout     = optional(number, 30)
        max_attempts       = optional(number, 1)
      }), {})
      iam_overrides = optional(object({
        override_instance_profile = optional(bool, null)
        instance_profile_name     = optional(string, null)
        override_runner_role      = optional(bool, null)
        runner_role_arn           = optional(string, null)
        }), {
        override_instance_profile = false
        instance_profile_name     = null
        override_runner_role      = false
        runner_role_arn           = null
      })
    })
    matcherConfig = object({
      labelMatchers           = list(list(string))
      exactMatch              = optional(bool, false)
      bidirectionalLabelMatch = optional(bool, false)
      priority                = optional(number, 999)
      enableDynamicLabels     = optional(bool, false)
      awsDynamicLabelsPolicy  = optional(any, null)
    })
    redrive_build_queue = optional(object({
      enabled         = bool
      maxReceiveCount = number
      }), {
      enabled         = false
      maxReceiveCount = null
    })
  }))
| n/a | yes | +| [multi\_runner\_config](#input\_multi\_runner\_config) | multi\_runner\_config = {
runner\_config: {
runner\_os: "The EC2 Operating System type to use for action runner instances (linux, osx, windows)."
runner\_architecture: "The platform architecture of the runner instance\_type."
runner\_metadata\_options: "(Optional) Metadata options for the ec2 runner instances."
ami: "(Optional) AMI configuration for the action runner instances. This object allows you to specify all AMI-related settings in one place."
create\_service\_linked\_role\_spot: (Optional) create the serviced linked role for spot instances that is required by the scale-up lambda.
credit\_specification: "(Optional) The credit specification of the runner instance\_type. Can be unset, `standard` or `unlimited`.
delay\_webhook\_event: "The number of seconds the event accepted by the webhook is invisible on the queue before the scale up lambda will receive the event."
disable\_runner\_autoupdate: "Disable the auto update of the github runner agent. Be aware there is a grace period of 30 days, see also the [GitHub article](https://github.blog/changelog/2022-02-01-github-actions-self-hosted-runners-can-now-disable-automatic-updates/)"
ebs\_optimized: "The EC2 EBS optimized configuration."
enable\_ephemeral\_runners: "Enable ephemeral runners, runners will only be used once."
enable\_job\_queued\_check: Enables JIT configuration for creating runners instead of registration token based registraton. JIT configuration will only be applied for ephemeral runners. By default JIT configuration is enabled for ephemeral runners an can be disabled via this override. When running on GHES without support for JIT configuration this variable should be set to true for ephemeral runners."
enable\_on\_demand\_failover\_for\_errors: "Enable on-demand failover. For example to fall back to on demand when no spot capacity is available the variable can be set to `InsufficientInstanceCapacity`. When not defined the default behavior is to retry later."
scale\_errors: "List of AWS error codes that should trigger retry during scale up. This list replaces the module default scale-up retry errors"
enable\_organization\_runners: "Register runners to organization, instead of repo level"
enable\_runner\_binaries\_syncer: "Option to disable the lambda to sync GitHub runner distribution, useful when using a pre-build AMI."
enable\_ssm\_on\_runners: "Enable to allow access the runner instances for debugging purposes via SSM. Note that this adds additional permissions to the runner instances."
enable\_userdata: "Should the userdata script be enabled for the runner. Set this to false if you are using your own prebuilt AMI."
instance\_allocation\_strategy: "The allocation strategy for creating instances. For spot, AWS recommends `price-capacity-optimized`; for on-demand, use `lowest-price` or `prioritized`. The AWS default is `lowest-price`."
instance\_type\_priorities: "A map of instance type to priority for the `prioritized` and `capacity-optimized-prioritized` allocation strategies. Lower numbers mean higher priority. If not provided, priorities are assigned based on the order of `instance_types`."
instance\_max\_spot\_price: "Max price price for spot instances per hour. This variable will be passed to the create fleet as max spot price for the fleet."
instance\_target\_capacity\_type: "Default lifecycle used for runner instances, can be either `spot` or `on-demand`."
instance\_types: "List of instance types for the action runner. Defaults are based on runner\_os (al2023 for linux, macOS Sequoia for osx, Windows Server Core for win)."
job\_queue\_retention\_in\_seconds: "The number of seconds the job is held in the queue before it is purged"
minimum\_running\_time\_in\_minutes: "The time an ec2 action runner should be running at minimum before terminated if not busy."
pool\_runner\_owner: "The pool will deploy runners to the GitHub org ID, set this value to the org to which you want the runners deployed. Repo level is not supported."
runner\_additional\_security\_group\_ids: "List of additional security groups IDs to apply to the runner. If added outside the multi\_runner\_config block, the additional security group(s) will be applied to all runner configs. If added inside the multi\_runner\_config, the additional security group(s) will be applied to the individual runner."
runner\_as\_root: "Run the action runner under the root user. Variable `runner_run_as` will be ignored."
runner\_boot\_time\_in\_minutes: "The minimum time for an EC2 runner to boot and register as a runner."
runner\_disable\_default\_labels: "Disable default labels for the runners (os, architecture and `self-hosted`). If enabled, the runner will only have the extra labels provided in `runner_extra_labels`. In case you on own start script is used, this configuration parameter needs to be parsed via SSM."
runner\_extra\_labels: "Extra (custom) labels for the runners (GitHub). Separate each label by a comma. Labels checks on the webhook can be enforced by setting `multi_runner_config.matcherConfig.exactMatch`. GitHub read-only labels should not be provided."
runner\_group\_name: "Name of the runner group."
runner\_name\_prefix: "Prefix for the GitHub runner name."
runner\_run\_as: "Run the GitHub actions agent as user."
runners\_maximum\_count: "The maximum number of runners that will be created. Setting the variable to `-1` disables the maximum check."
scale\_down\_schedule\_expression: "Scheduler expression to check every x for scale down."
scale\_up\_reserved\_concurrent\_executions: "Amount of reserved concurrent executions for the scale-up lambda function. A value of 0 disables lambda from being triggered and -1 removes any concurrency limitations."
lambda\_event\_source\_mapping\_batch\_size: "(Optional) Maximum number of records per Lambda invocation for this runner flavor. Overrides the module-level `lambda_event_source_mapping_batch_size` when set."
lambda\_event\_source\_mapping\_maximum\_batching\_window\_in\_seconds: "(Optional) Maximum seconds to gather records before invoking Lambda for this runner flavor. Overrides the module-level `lambda_event_source_mapping_maximum_batching_window_in_seconds` when set."
userdata\_template: "Alternative user-data template, replacing the default template. By providing your own user\_data you have to take care of installing all required software, including the action runner. Variables userdata\_pre/post\_install are ignored."
enable\_jit\_config: "Overwrite the default behavior for JIT configuration. By default JIT configuration is enabled for ephemeral runners and disabled for non-ephemeral runners. In case of GHES check first if the JIT config API is available. In case you are upgrading from 3.x to 4.x you can set `enable_jit_config` to `false` to avoid a breaking change when having your own AMI."
enable\_runner\_detailed\_monitoring: "Should detailed monitoring be enabled for the runner. Set this to true if you want to use detailed monitoring. See https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/using-cloudwatch-new.html for details."
enable\_cloudwatch\_agent: "Enabling the cloudwatch agent on the ec2 runner instances, the runner contains default config. Configuration can be overridden via `cloudwatch_config`."
cloudwatch\_config: "(optional) Replaces the module default cloudwatch log config. See https://docs.aws.amazon.com/AmazonCloudWatch/latest/monitoring/CloudWatch-Agent-Configuration-File-Details.html for details."
userdata\_pre\_install: "Script to be ran before the GitHub Actions runner is installed on the EC2 instances"
userdata\_post\_install: "Script to be ran after the GitHub Actions runner is installed on the EC2 instances"
runner\_hook\_job\_started: "Script to be ran in the runner environment at the beginning of every job"
runner\_hook\_job\_completed: "Script to be ran in the runner environment at the end of every job"
runner\_ec2\_tags: "Map of tags that will be added to the launch template instance tag specifications."
runner\_iam\_role\_managed\_policy\_arns: "Attach AWS or customer-managed IAM policies (by ARN) to the runner IAM role"
vpc\_id: "The VPC for security groups of the action runners. If not set uses the value of `var.vpc_id`."
subnet\_ids: "List of subnets in which the action runners will be launched, the subnets needs to be subnets in the `vpc_id`. If not set, uses the value of `var.subnet_ids`."
idle\_config: "List of time period that can be defined as cron expression to keep a minimum amount of runners active instead of scaling down to 0. By defining this list you can ensure that in time periods that match the cron expression within 5 seconds a runner is kept idle."
license\_specifications: "Optional EC2 License Manager license configuration ARNs for the runner launch template. Required for macOS dedicated-host runners when the host resource group uses a Mac dedicated host license configuration."
use\_dedicated\_host: "Experimental! Can be removed / changed without trigger a major release. Whether to use EC2 dedicated hosts for the runners. Needed for macos runners Note that using dedicated hosts can increase cost significantly."
runner\_log\_files: "(optional) Replaces the module default cloudwatch log config. See https://docs.aws.amazon.com/AmazonCloudWatch/latest/monitoring/CloudWatch-Agent-Configuration-File-Details.html for details."
block\_device\_mappings: "The EC2 instance block device configuration. Takes the following keys: `device_name`, `delete_on_termination`, `volume_type`, `volume_size`, `encrypted`, `iops`, `throughput`, `kms_key_id`, `snapshot_id`, `volume_initialization_rate`."
job\_retry: "Experimental! Can be removed / changed without trigger a major release. Configure job retries. The configuration enables job retries (for ephemeral runners). After creating the instances a message will be published to a job retry queue. The job retry check lambda is checking after a delay if the job is queued. If not the message will be published again on the scale-up (build queue). Using this feature can impact the rate limit of the GitHub app."
pool\_config: "The configuration for updating the pool. The `pool_size` to adjust to by the events triggered by the `schedule_expression`. For example you can configure a cron expression for week days to adjust the pool to 10 and another expression for the weekend to adjust the pool to 1. Use `schedule_expression_timezone` to override the schedule time zone (defaults to UTC)."
iam\_overrides: "Allows to (optionally) override the instance profile and runner role created by the module. Set `override_instance_profile` to true and provide the `instance_profile_name` to use an existing instance profile. Set `override_runner_role` to true and provide the `runner_role_arn` to use an existing role for the runner instances."
}
matcherConfig: {
labelMatchers: "The list of list of labels supported by the runner configuration. `[[self-hosted, linux, x64, example]]`"
exactMatch: "DEPRECATED: Use `bidirectionalLabelMatch` instead. If set to true all labels in the workflow job must match the GitHub labels (os, architecture and `self-hosted`). When false if __any__ workflow label matches it will trigger the webhook. Note: this only checks that workflow labels are a subset of runner labels, not the reverse."
bidirectionalLabelMatch: "If set to true, the runner labels and workflow job labels must be an exact two-way match (same set, any order, no extras or missing labels). This is stricter than `exactMatch` which only checks that workflow labels are a subset of runner labels. When false, if __any__ workflow label matches it will trigger the webhook."
priority: "If set it defines the priority of the matcher, the matcher with the lowest priority will be evaluated first. Default is 999, allowed values 0-999."
enableDynamicLabels: "Experimental! When true the dispatcher allows `ghr-*` dynamic labels for jobs routed to this runner. Default false."
awsDynamicLabelsPolicy: "Optional AWS dynamic label policy evaluated by the dispatcher. Only effective when `enableDynamicLabels = true`. Jobs whose provider dynamic labels violate every matching runner's policy are rejected with a 202 (a warning is logged). Evaluation: keys in `blocked_keys` are always rejected; keys in `restricted_keys` are allowed only when their value passes the rule; unlisted keys are allowed. Schema: `{ blocked_keys = [], restricted_keys = { = { allowed = [globs], denied = [globs], max = number|string } } }`. Keys use the dynamic label suffix, e.g. `instance-type` for `ghr-ec2-instance-type`."
}
redrive\_build\_queue: "Set options to attach (optional) a dead letter queue to the build queue, the queue between the webhook and the scale up lambda. You have the following options. 1. Disable by setting `enabled` to false. 2. Enable by setting `enabled` to `true`, `maxReceiveCount` to a number of max retries."
} | 
map(object({
    runner_config = object({
      runner_os           = string
      runner_architecture = string
      runner_metadata_options = optional(map(any), {
        instance_metadata_tags      = "enabled"
        http_endpoint               = "enabled"
        http_tokens                 = "required"
        http_put_response_hop_limit = 1
      })
      ami = optional(object({
        filter               = optional(map(list(string)), { state = ["available"] })
        owners               = optional(list(string), ["amazon"])
        id_ssm_parameter_arn = optional(string, null)
        kms_key_arn          = optional(string, null)
      }), null)
      create_service_linked_role_spot      = optional(bool, false)
      credit_specification                 = optional(string, null)
      delay_webhook_event                  = optional(number, 30)
      disable_runner_autoupdate            = optional(bool, false)
      ebs_optimized                        = optional(bool, false)
      enable_ephemeral_runners             = optional(bool, false)
      enable_job_queued_check              = optional(bool, null)
      enable_on_demand_failover_for_errors = optional(list(string), [])
      scale_errors = optional(list(string), [
        "UnfulfillableCapacity",
        "MaxSpotInstanceCountExceeded",
        "TargetCapacityLimitExceededException",
        "RequestLimitExceeded",
        "ResourceLimitExceeded",
        "MaxSpotInstanceCountExceeded",
        "MaxSpotFleetRequestCountExceeded",
        "InsufficientInstanceCapacity",
        "InsufficientCapacityOnHost",
      ])
      enable_organization_runners                                    = optional(bool, false)
      enable_runner_binaries_syncer                                  = optional(bool, true)
      enable_ssm_on_runners                                          = optional(bool, false)
      enable_userdata                                                = optional(bool, true)
      instance_allocation_strategy                                   = optional(string, "lowest-price")
      instance_type_priorities                                       = optional(map(number), null)
      instance_max_spot_price                                        = optional(string, null)
      instance_target_capacity_type                                  = optional(string, "spot")
      instance_types                                                 = list(string)
      job_queue_retention_in_seconds                                 = optional(number, 86400)
      minimum_running_time_in_minutes                                = optional(number, null)
      pool_runner_owner                                              = optional(string, null)
      runner_as_root                                                 = optional(bool, false)
      runner_boot_time_in_minutes                                    = optional(number, 5)
      runner_disable_default_labels                                  = optional(bool, false)
      runner_extra_labels                                            = optional(list(string), [])
      runner_group_name                                              = optional(string, "Default")
      runner_name_prefix                                             = optional(string, "")
      runner_run_as                                                  = optional(string, "ec2-user")
      runners_maximum_count                                          = number
      runner_additional_security_group_ids                           = optional(list(string), [])
      scale_down_schedule_expression                                 = optional(string, "cron(*/5 * * * ? *)")
      scale_up_reserved_concurrent_executions                        = optional(number, 1)
      lambda_event_source_mapping_batch_size                         = optional(number, null)
      lambda_event_source_mapping_maximum_batching_window_in_seconds = optional(number, null)
      userdata_template                                              = optional(string, null)
      userdata_content                                               = optional(string, null)
      enable_jit_config                                              = optional(bool, null)
      enable_runner_detailed_monitoring                              = optional(bool, false)
      enable_cloudwatch_agent                                        = optional(bool, true)
      cloudwatch_config                                              = optional(string, null)
      userdata_pre_install                                           = optional(string, "")
      userdata_post_install                                          = optional(string, "")
      runner_hook_job_started                                        = optional(string, "")
      runner_hook_job_completed                                      = optional(string, "")
      runner_ec2_tags                                                = optional(map(string), {})
      runner_iam_role_managed_policy_arns                            = optional(list(string), [])
      vpc_id                                                         = optional(string, null)
      subnet_ids                                                     = optional(list(string), null)
      idle_config = optional(list(object({
        cron             = string
        timeZone         = string
        idleCount        = number
        evictionStrategy = optional(string, "oldest_first")
      })), [])
      cpu_options = optional(object({
        core_count            = optional(number)
        threads_per_core      = optional(number)
        amd_sev_snp           = optional(string)
        nested_virtualization = optional(string)
      }), null)
      placement = optional(object({
        affinity                = optional(string)
        availability_zone       = optional(string)
        group_id                = optional(string)
        group_name              = optional(string)
        host_id                 = optional(string)
        host_resource_group_arn = optional(string)
        spread_domain           = optional(string)
        tenancy                 = optional(string)
        partition_number        = optional(number)
      }), null)
      license_specifications = optional(list(object({
        license_configuration_arn = string
      })), [])
      use_dedicated_host = optional(bool, false)
      runner_log_files = optional(list(object({
        log_group_name   = string
        prefix_log_group = bool
        file_path        = string
        log_stream_name  = string
        log_class        = optional(string, "STANDARD")
      })), null)
      block_device_mappings = optional(list(object({
        delete_on_termination      = optional(bool, true)
        device_name                = optional(string, "/dev/xvda")
        encrypted                  = optional(bool, true)
        iops                       = optional(number)
        kms_key_id                 = optional(string)
        snapshot_id                = optional(string)
        throughput                 = optional(number)
        volume_initialization_rate = optional(number)
        volume_size                = number
        volume_type                = optional(string, "gp3")
        })), [{
        volume_size = 30
      }])
      pool_config = optional(list(object({
        schedule_expression          = string
        schedule_expression_timezone = optional(string)
        size                         = number
      })), [])
      job_retry = optional(object({
        enable             = optional(bool, false)
        delay_in_seconds   = optional(number, 300)
        delay_backoff      = optional(number, 2)
        lambda_memory_size = optional(number, 256)
        lambda_timeout     = optional(number, 30)
        max_attempts       = optional(number, 1)
      }), {})
      iam_overrides = optional(object({
        override_instance_profile = optional(bool, null)
        instance_profile_name     = optional(string, null)
        override_runner_role      = optional(bool, null)
        runner_role_arn           = optional(string, null)
        }), {
        override_instance_profile = false
        instance_profile_name     = null
        override_runner_role      = false
        runner_role_arn           = null
      })
    })
    matcherConfig = object({
      labelMatchers           = list(list(string))
      exactMatch              = optional(bool, false)
      bidirectionalLabelMatch = optional(bool, false)
      priority                = optional(number, 999)
      enableDynamicLabels     = optional(bool, false)
      awsDynamicLabelsPolicy = optional(object({
        blocked_keys = optional(list(string), [])
        restricted_keys = optional(map(object({
          allowed = optional(list(string), [])
          denied  = optional(list(string), [])
          max     = optional(string, null)
        })), {})
      }), null)
    })
    redrive_build_queue = optional(object({
      enabled         = bool
      maxReceiveCount = number
      }), {
      enabled         = false
      maxReceiveCount = null
    })
  }))
| `{}` | no | | [parameter\_store\_tags](#input\_parameter\_store\_tags) | Map of tags that will be added to all the SSM Parameter Store parameters created by the Lambda function. | `map(string)` | `{}` | no | | [pool\_lambda\_reserved\_concurrent\_executions](#input\_pool\_lambda\_reserved\_concurrent\_executions) | Amount of reserved concurrent executions for the scale-up lambda function. A value of 0 disables lambda from being triggered and -1 removes any concurrency limitations. | `number` | `1` | no | | [pool\_lambda\_timeout](#input\_pool\_lambda\_timeout) | Time out for the pool lambda in seconds. | `number` | `60` | no | @@ -201,7 +247,8 @@ module "multi-runner" { | [binaries\_syncer\_map](#output\_binaries\_syncer\_map) | n/a | | [instance\_termination\_handler](#output\_instance\_termination\_handler) | n/a | | [instance\_termination\_watcher](#output\_instance\_termination\_watcher) | n/a | -| [runners\_map](#output\_runners\_map) | n/a | +| [runners\_map](#output\_runners\_map) | Stable v1 runner resources keyed by runner configuration. Entries retain the historical flat output shape. | +| [runners\_map\_v2](#output\_runners\_map\_v2) | Experimental v2 runner resources keyed by runner configuration and grouped by common or compute-provider ownership. | | [ssm\_parameters](#output\_ssm\_parameters) | n/a | | [webhook](#output\_webhook) | n/a | diff --git a/modules/multi-runner/compute-provider.tf b/modules/multi-runner/compute-provider.tf new file mode 100644 index 0000000000..83793f34f2 --- /dev/null +++ b/modules/multi-runner/compute-provider.tf @@ -0,0 +1,16 @@ +locals { + compute_provider_types = { + for runner_key, runner_config in local.multi_runner_config : runner_key => one([ + for provider_type, provider_config in runner_config.compute_provider : provider_type + if provider_config != null + ]) + } + + runner_config_by_provider = { + for provider_type in toset(values(local.compute_provider_types)) : + provider_type => { + for runner_key, runner_config in local.multi_runner_config : runner_key => runner_config + if local.compute_provider_types[runner_key] == provider_type + } + } +} diff --git a/modules/multi-runner/config.experimental.tf b/modules/multi-runner/config.experimental.tf new file mode 100644 index 0000000000..38720b9c51 --- /dev/null +++ b/modules/multi-runner/config.experimental.tf @@ -0,0 +1,178 @@ +locals { + use_multi_runner_config_v2 = length(var.experimental.multi_runner_config_v2) > 0 + selected_multi_runner_config_v1 = local.use_multi_runner_config_v2 ? {} : var.multi_runner_config + selected_multi_runner_config_v2 = local.use_multi_runner_config_v2 ? var.experimental.multi_runner_config_v2 : {} + + # Stable v1 remains an external flat contract. Normalize it once so common + # multi-runner consumers can use the same ownership model as experimental v2. + multi_runner_config_v1_as_v2 = { + for k, v in local.selected_multi_runner_config_v1 : k => { + tags = {} + + runner = { + os = v.runner_config.runner_os + architecture = v.runner_config.runner_architecture + boot_time_in_minutes = v.runner_config.runner_boot_time_in_minutes + disable_default_labels = v.runner_config.runner_disable_default_labels + extra_labels = v.runner_config.runner_extra_labels + group_name = v.runner_config.runner_group_name + name_prefix = v.runner_config.runner_name_prefix + run_as_root = v.runner_config.runner_as_root + run_as = v.runner_config.runner_run_as + maximum_count = v.runner_config.runners_maximum_count + ephemeral = v.runner_config.enable_ephemeral_runners + jit_config_enabled = v.runner_config.enable_jit_config + auto_update_disabled = v.runner_config.disable_runner_autoupdate + tags = {} + hooks = { + job_started = v.runner_config.runner_hook_job_started + job_completed = v.runner_config.runner_hook_job_completed + } + iam = { + role = v.runner_config.iam_overrides.override_runner_role == true ? { + arn = v.runner_config.iam_overrides.runner_role_arn + } : null + managed_policy_arns = { + for policy_index, policy_arn in v.runner_config.runner_iam_role_managed_policy_arns : + "legacy-${policy_index}" => policy_arn + } + path = var.role_path + permissions_boundary = var.role_permissions_boundary + } + } + + github = { + organization_runners = v.runner_config.enable_organization_runners + } + + lambda = { + tags = {} + } + + queue = { + delay_webhook_event = v.runner_config.delay_webhook_event + job_queue_retention_in_seconds = v.runner_config.job_queue_retention_in_seconds + event_source_mapping = { + batch_size = v.runner_config.lambda_event_source_mapping_batch_size + maximum_batching_window_in_seconds = v.runner_config.lambda_event_source_mapping_maximum_batching_window_in_seconds + } + redrive_build_queue = v.redrive_build_queue + tags = {} + } + + scale_up = { + reserved_concurrent_executions = v.runner_config.scale_up_reserved_concurrent_executions + job_queued_check_enabled = v.runner_config.enable_job_queued_check + tags = {} + } + + scale_down = { + schedule_expression = v.runner_config.scale_down_schedule_expression + minimum_running_time_in_minutes = v.runner_config.minimum_running_time_in_minutes + idle_config = v.runner_config.idle_config + tags = {} + } + + pool = { + config = v.runner_config.pool_config + runner_owner = v.runner_config.pool_runner_owner + tags = {} + } + + job_retry = { + enabled = v.runner_config.job_retry.enable + delay_in_seconds = v.runner_config.job_retry.delay_in_seconds + delay_backoff = v.runner_config.job_retry.delay_backoff + max_attempts = v.runner_config.job_retry.max_attempts + tags = {} + lambda = { + memory_size = v.runner_config.job_retry.lambda_memory_size + timeout = v.runner_config.job_retry.lambda_timeout + reserved_concurrent_executions = 1 + } + } + + ssm = { + tags = {} + kms_key = null + parameters = { + tags = {} + } + housekeeper = { + tags = {} + } + } + + observability = { + logs = { + tags = {} + } + } + + compute_provider = { + ec2 = { + metadata_options = v.runner_config.runner_metadata_options + # Stable v1 keeps its nullable `id_ssm_parameter_arn` leaf. Translate + # it once into v2's caller-known ownership wrapper without changing + # the input passed to the legacy runners module. + ami = v.runner_config.ami == null ? null : { + filter = v.runner_config.ami.filter + owners = v.runner_config.ami.owners + id_ssm_parameter = v.runner_config.ami.id_ssm_parameter_arn == null ? null : { + arn = v.runner_config.ami.id_ssm_parameter_arn + } + kms_key = v.runner_config.ami.kms_key_arn == null ? null : { + arn = v.runner_config.ami.kms_key_arn + } + } + block_device_mappings = v.runner_config.block_device_mappings + create_service_linked_role_spot = v.runner_config.create_service_linked_role_spot + credit_specification = v.runner_config.credit_specification + ebs_optimized = v.runner_config.ebs_optimized + cloudwatch_agent = { + enabled = v.runner_config.enable_cloudwatch_agent + config = v.runner_config.cloudwatch_config + } + binaries_syncer = { + enabled = v.runner_config.enable_runner_binaries_syncer + } + detailed_monitoring_enabled = v.runner_config.enable_runner_detailed_monitoring + ssm_enabled = v.runner_config.enable_ssm_on_runners + user_data = { + enabled = v.runner_config.enable_userdata + template = v.runner_config.userdata_template + content = v.runner_config.userdata_content + pre_install = v.runner_config.userdata_pre_install + post_install = v.runner_config.userdata_post_install + debug_logging_enabled = false + } + instance_allocation_strategy = v.runner_config.instance_allocation_strategy + instance_max_spot_price = v.runner_config.instance_max_spot_price + instance_target_capacity_type = v.runner_config.instance_target_capacity_type + instance_type_priorities = v.runner_config.instance_type_priorities + instance_types = v.runner_config.instance_types + additional_security_group_ids = v.runner_config.runner_additional_security_group_ids + instance_profile = v.runner_config.iam_overrides.override_instance_profile == true ? { + name = v.runner_config.iam_overrides.instance_profile_name + } : null + enable_on_demand_failover_for_errors = v.runner_config.enable_on_demand_failover_for_errors + scale_errors = v.runner_config.scale_errors + subnet_ids = v.runner_config.subnet_ids + vpc_id = v.runner_config.vpc_id + cpu_options = v.runner_config.cpu_options + placement = v.runner_config.placement + license_specifications = v.runner_config.license_specifications + use_dedicated_host = v.runner_config.use_dedicated_host + log_files = v.runner_config.runner_log_files + tags = v.runner_config.runner_ec2_tags + } + } + + matcherConfig = v.matcherConfig + } + } + + # A non-empty v2 map is a module-level opt-in. Never combine v1 and v2 in one + # deployment: this keeps module addresses and output contracts unambiguous. + multi_runner_config = local.use_multi_runner_config_v2 ? local.selected_multi_runner_config_v2 : local.multi_runner_config_v1_as_v2 +} diff --git a/modules/multi-runner/main.tf b/modules/multi-runner/main.tf index caef8fdbcc..64b119d18a 100644 --- a/modules/multi-runner/main.tf +++ b/modules/multi-runner/main.tf @@ -20,8 +20,14 @@ locals { merge(v, { runner_config = merge(v.runner_config, { runner_extra_labels = local.runner_extra_labels[k] }) }), ) } - tmp_distinct_list_unique_os_and_arch = distinct([for i, config in local.runner_config : { "os_type" : config.runner_config.runner_os, "architecture" : config.runner_config.runner_architecture } if config.runner_config.enable_runner_binaries_syncer]) - unique_os_and_arch = { for i, v in local.tmp_distinct_list_unique_os_and_arch : "${v.os_type}_${v.architecture}" => v } + tmp_distinct_list_unique_os_and_arch = distinct([ + for _, config in try(local.runner_config_by_provider.ec2, {}) : { + "os_type" : config.runner.os, + "architecture" : config.runner.architecture + } + if config.compute_provider.ec2.binaries_syncer.enabled + ]) + unique_os_and_arch = { for _, v in local.tmp_distinct_list_unique_os_and_arch : "${v.os_type}_${v.architecture}" => v } ssm_root_path = "/${var.ssm_paths.root}/${var.prefix}" } diff --git a/modules/multi-runner/outputs.tf b/modules/multi-runner/outputs.tf index 7ce7171faf..bae66faecf 100644 --- a/modules/multi-runner/outputs.tf +++ b/modules/multi-runner/outputs.tf @@ -1,5 +1,6 @@ output "runners_map" { + description = "Stable v1 runner resources keyed by runner configuration. Entries retain the historical flat output shape." value = { for runner_key, runner in module.runners : runner_key => { launch_template_name = runner.launch_template.name launch_template_id = runner.launch_template.id @@ -21,6 +22,18 @@ output "runners_map" { } } +output "runners_map_v2" { + description = "Experimental v2 runner resources keyed by runner configuration and grouped by common or compute-provider ownership." + value = { for runner_key, runner in module.runner_stacks : runner_key => { + runner = runner.runner + scale_up = runner.scale_up + scale_down = runner.scale_down + pool = runner.pool + provider = runner.provider + } + } +} + output "binaries_syncer_map" { value = { for runner_binary_key, runner_binary in module.runner_binaries : runner_binary_key => { lambda = runner_binary.lambda diff --git a/modules/multi-runner/queues.tf b/modules/multi-runner/queues.tf index bcc75f99cc..615cd1187d 100644 --- a/modules/multi-runner/queues.tf +++ b/modules/multi-runner/queues.tf @@ -1,3 +1,12 @@ +locals { + sqs_tags = { + for k, v in local.multi_runner_config : k => merge( + var.tags, + v.tags, + v.queue.tags, + ) + } +} data "aws_iam_policy_document" "deny_insecure_transport" { statement { @@ -27,42 +36,42 @@ data "aws_iam_policy_document" "deny_insecure_transport" { } resource "aws_sqs_queue" "queued_builds" { - for_each = var.multi_runner_config + for_each = local.multi_runner_config name = "${var.prefix}-${each.key}-queued-builds" - delay_seconds = each.value.runner_config.delay_webhook_event + delay_seconds = each.value.queue.delay_webhook_event visibility_timeout_seconds = var.runners_scale_up_lambda_timeout - message_retention_seconds = each.value.runner_config.job_queue_retention_in_seconds + message_retention_seconds = each.value.queue.job_queue_retention_in_seconds receive_wait_time_seconds = 0 - redrive_policy = each.value.redrive_build_queue.enabled ? jsonencode({ + redrive_policy = each.value.queue.redrive_build_queue.enabled ? jsonencode({ deadLetterTargetArn = aws_sqs_queue.queued_builds_dlq[each.key].arn, - maxReceiveCount = each.value.redrive_build_queue.maxReceiveCount + maxReceiveCount = each.value.queue.redrive_build_queue.maxReceiveCount }) : null sqs_managed_sse_enabled = var.queue_encryption.sqs_managed_sse_enabled kms_master_key_id = var.queue_encryption.kms_master_key_id kms_data_key_reuse_period_seconds = var.queue_encryption.kms_data_key_reuse_period_seconds - tags = var.tags + tags = local.sqs_tags[each.key] } resource "aws_sqs_queue_policy" "build_queue_policy" { - for_each = var.multi_runner_config + for_each = local.multi_runner_config queue_url = aws_sqs_queue.queued_builds[each.key].id policy = data.aws_iam_policy_document.deny_insecure_transport.json } resource "aws_sqs_queue" "queued_builds_dlq" { - for_each = { for config, values in var.multi_runner_config : config => values if values.redrive_build_queue.enabled } + for_each = { for config, values in local.multi_runner_config : config => values if values.queue.redrive_build_queue.enabled } name = "${var.prefix}-${each.key}-queued-builds_dead_letter" sqs_managed_sse_enabled = var.queue_encryption.sqs_managed_sse_enabled kms_master_key_id = var.queue_encryption.kms_master_key_id kms_data_key_reuse_period_seconds = var.queue_encryption.kms_data_key_reuse_period_seconds - tags = var.tags + tags = local.sqs_tags[each.key] } resource "aws_sqs_queue_policy" "build_queue_dlq_policy" { - for_each = { for config, values in var.multi_runner_config : config => values if values.redrive_build_queue.enabled } + for_each = { for config, values in local.multi_runner_config : config => values if values.queue.redrive_build_queue.enabled } queue_url = aws_sqs_queue.queued_builds_dlq[each.key].id policy = data.aws_iam_policy_document.deny_insecure_transport.json } diff --git a/modules/multi-runner/runners.experimental.tf b/modules/multi-runner/runners.experimental.tf new file mode 100644 index 0000000000..22fd453f1b --- /dev/null +++ b/modules/multi-runner/runners.experimental.tf @@ -0,0 +1,206 @@ +locals { + runner_config_v2 = { + for k, v in local.selected_multi_runner_config_v2 : k => merge(v, { + id = aws_sqs_queue.queued_builds[k].id + arn = aws_sqs_queue.queued_builds[k].arn + url = aws_sqs_queue.queued_builds[k].url + runner = merge(v.runner, { + extra_labels = sort(setunion(flatten(v.matcherConfig.labelMatchers), compact(v.runner.extra_labels))) + }) + }) + } + + runner_config_v2_compute_provider = { + for k, v in local.runner_config_v2 : k => merge( + local.compute_provider_types[k] == "ec2" ? { + (local.compute_provider_types[k]) = { + ami = v.compute_provider.ec2.ami + vpc_id = coalesce(v.compute_provider.ec2.vpc_id, var.vpc_id) + subnet_ids = coalesce(v.compute_provider.ec2.subnet_ids, var.subnet_ids) + instance_types = v.compute_provider.ec2.instance_types + instance_target_capacity_type = v.compute_provider.ec2.instance_target_capacity_type + instance_allocation_strategy = v.compute_provider.ec2.instance_allocation_strategy + instance_type_priorities = v.compute_provider.ec2.instance_type_priorities + instance_max_spot_price = v.compute_provider.ec2.instance_max_spot_price + block_device_mappings = v.compute_provider.ec2.block_device_mappings + ebs_optimized = v.compute_provider.ec2.ebs_optimized + instance_profile = v.compute_provider.ec2.instance_profile + instance_profile_path = var.instance_profile_path + enable_on_demand_failover_for_errors = v.compute_provider.ec2.enable_on_demand_failover_for_errors + scale_errors = v.compute_provider.ec2.scale_errors + managed_security_group_enabled = var.enable_managed_runner_security_group + detailed_monitoring_enabled = v.compute_provider.ec2.detailed_monitoring_enabled + ssm_enabled = v.compute_provider.ec2.ssm_enabled + egress_rules = var.runner_egress_rules + additional_security_group_ids = try(coalescelist(v.compute_provider.ec2.additional_security_group_ids, var.runner_additional_security_group_ids), []) + metadata_options = v.compute_provider.ec2.metadata_options + credit_specification = v.compute_provider.ec2.credit_specification + cpu_options = v.compute_provider.ec2.cpu_options + placement = v.compute_provider.ec2.placement + license_specifications = v.compute_provider.ec2.license_specifications + use_dedicated_host = v.compute_provider.ec2.use_dedicated_host + binaries_syncer = { + enabled = v.compute_provider.ec2.binaries_syncer.enabled + s3 = v.compute_provider.ec2.binaries_syncer.enabled ? local.runner_binaries_by_os_and_arch_map["${v.runner.os}_${v.runner.architecture}"] : null + } + cloudwatch_agent = { + enabled = v.compute_provider.ec2.cloudwatch_agent.enabled + config = try(coalesce(v.compute_provider.ec2.cloudwatch_agent.config, var.cloudwatch_config), null) + } + log_files = v.compute_provider.ec2.log_files + user_data = v.compute_provider.ec2.user_data + key_name = var.key_name + tags = v.compute_provider.ec2.tags + + create_service_linked_role_spot = v.compute_provider.ec2.create_service_linked_role_spot + associate_public_ipv4_address = var.associate_public_ipv4_address + } + } : {}, + local.compute_provider_types[k] != "ec2" ? { + (local.compute_provider_types[k]) = v.compute_provider[local.compute_provider_types[k]] + } : {}, + ) + } +} + +module "runner_stacks" { + source = "../runner-stack" + for_each = local.runner_config_v2 + + aws_region = var.aws_region + aws_partition = var.aws_partition + prefix = "${var.prefix}-${each.key}" + tags = merge(var.tags, each.value.tags) + + runner = { + os = each.value.runner.os + architecture = each.value.runner.architecture + boot_time_in_minutes = each.value.runner.boot_time_in_minutes + disable_default_labels = each.value.runner.disable_default_labels + labels = each.value.runner.disable_default_labels ? sort(distinct(each.value.runner.extra_labels)) : sort(distinct(concat(["self-hosted", each.value.runner.os, each.value.runner.architecture], each.value.runner.extra_labels))) + group_name = each.value.runner.group_name + name_prefix = each.value.runner.name_prefix + run_as_root = each.value.runner.run_as_root + run_as = each.value.runner.run_as + maximum_count = each.value.runner.maximum_count + ephemeral = each.value.runner.ephemeral + jit_config_enabled = each.value.runner.jit_config_enabled + auto_update_disabled = each.value.runner.auto_update_disabled + tags = each.value.runner.tags + hooks = each.value.runner.hooks + iam = { + role = each.value.runner.iam.role + managed_policy_arns = each.value.runner.iam.managed_policy_arns + additional_trust_policy_json = each.value.runner.iam.additional_trust_policy_json + path = each.value.runner.iam.path != null ? each.value.runner.iam.path : var.role_path + permissions_boundary = each.value.runner.iam.permissions_boundary != null ? each.value.runner.iam.permissions_boundary : var.role_permissions_boundary + } + } + + github = { + app_parameters = local.github_app_parameters + organization_runners = each.value.github.organization_runners + enterprise_server = { + url = var.ghes_url + ssl_verify = var.ghes_ssl_verify + } + user_agent = var.user_agent + } + + queue = { + build = { + arn = each.value.arn + url = each.value.url + } + event_source_mapping = { + batch_size = coalesce(each.value.queue.event_source_mapping.batch_size, var.lambda_event_source_mapping_batch_size) + maximum_batching_window_in_seconds = coalesce(each.value.queue.event_source_mapping.maximum_batching_window_in_seconds, var.lambda_event_source_mapping_maximum_batching_window_in_seconds) + } + tags = each.value.queue.tags + } + + lambda = { + zip = var.runners_lambda_zip + s3 = { + bucket = var.lambda_s3_bucket + key = var.runners_lambda_s3_key + object_version = var.runners_lambda_s3_object_version + } + runtime = var.lambda_runtime + architecture = var.lambda_architecture + subnet_ids = var.lambda_subnet_ids + security_group_ids = var.lambda_security_group_ids + tags = merge(var.lambda_tags, each.value.lambda.tags) + role = { + path = var.role_path + permissions_boundary = var.role_permissions_boundary + } + } + + scale_up = { + memory_size = var.scale_up_lambda_memory_size + timeout = var.runners_scale_up_lambda_timeout + reserved_concurrent_executions = each.value.scale_up.reserved_concurrent_executions + job_queued_check_enabled = each.value.scale_up.job_queued_check_enabled + tags = each.value.scale_up.tags + } + + scale_down = { + memory_size = var.scale_down_lambda_memory_size + timeout = var.runners_scale_down_lambda_timeout + schedule_expression = each.value.scale_down.schedule_expression + minimum_running_time_in_minutes = each.value.scale_down.minimum_running_time_in_minutes + idle_config = each.value.scale_down.idle_config + tags = each.value.scale_down.tags + } + + pool = { + config = each.value.pool.config + include_busy_runners = false + runner_owner = each.value.pool.runner_owner + tags = each.value.pool.tags + lambda = { + timeout = var.pool_lambda_timeout + reserved_concurrent_executions = var.pool_lambda_reserved_concurrent_executions + } + } + + job_retry = each.value.job_retry + + ssm = { + paths = { + root = "${local.ssm_root_path}/${each.key}" + tokens = "${var.ssm_paths.runners}/tokens" + config = "${var.ssm_paths.runners}/config" + } + kms_key = each.value.ssm.kms_key + tags = each.value.ssm.tags + parameters = { + tags = merge(var.parameter_store_tags, each.value.ssm.parameters.tags) + } + housekeeper = { + schedule_expression = var.runners_ssm_housekeeper.schedule_expression + state = var.runners_ssm_housekeeper.enabled ? "ENABLED" : "DISABLED" + tags = each.value.ssm.housekeeper.tags + lambda = { + memory_size = var.runners_ssm_housekeeper.lambda_memory_size + timeout = var.runners_ssm_housekeeper.lambda_timeout + } + config = var.runners_ssm_housekeeper.config + } + } + + observability = { + logs = { + level = var.log_level + retention_in_days = var.logging_retention_in_days + kms_key_id = var.logging_kms_key_id + class = var.log_class + tags = each.value.observability.logs.tags + } + tracing = var.tracing_config + metrics = var.metrics + } + + compute_provider = local.runner_config_v2_compute_provider[each.key] +} diff --git a/modules/multi-runner/tests/provider-routing.tftest.hcl b/modules/multi-runner/tests/provider-routing.tftest.hcl new file mode 100644 index 0000000000..c00e41f4b9 --- /dev/null +++ b/modules/multi-runner/tests/provider-routing.tftest.hcl @@ -0,0 +1,610 @@ +mock_provider "aws" { + mock_data "aws_caller_identity" { + defaults = { + account_id = "123456789012" + } + } + + mock_data "aws_iam_policy_document" { + defaults = { + json = "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Effect\":\"Allow\",\"Principal\":{\"Service\":\"lambda.amazonaws.com\"},\"Action\":\"sts:AssumeRole\"}]}" + } + } +} + +mock_provider "random" {} +mock_provider "null" {} + +variables { + aws_region = "eu-west-1" + vpc_id = "vpc-12345678" + subnet_ids = ["subnet-12345678"] + + github_app = { + id = "123456" + key_base64 = "dGVzdA==" + webhook_secret = "test-secret" + } + + lambda_s3_bucket = "lambda-artifacts" + webhook_lambda_s3_key = "webhook.zip" + runners_lambda_s3_key = "runners.zip" + syncer_lambda_s3_key = "runner-binaries-syncer.zip" +} + +run "empty_runner_configurations_return_empty_output_maps" { + command = plan + + assert { + condition = length(output.runners_map) == 0 && length(output.runners_map_v2) == 0 + error_message = "Stable and experimental runner outputs must both be empty when no runner configurations are supplied." + } +} + +run "stable_v1_keeps_legacy_runner_module" { + command = plan + + variables { + tags = { + StableGlobal = "global" + Precedence = "global" + } + + multi_runner_config = { + linux = { + runner_config = { + runner_os = "linux" + runner_architecture = "x64" + instance_types = ["m5.large"] + runners_maximum_count = 2 + enable_runner_binaries_syncer = false + enable_organization_runners = true + } + matcherConfig = { + labelMatchers = [["self-hosted", "linux", "x64"]] + } + redrive_build_queue = { + enabled = true + maxReceiveCount = 3 + } + } + } + } + + assert { + condition = keys(local.runner_config_by_provider.ec2) == ["linux"] + error_message = "Stable multi_runner_config entries must route to the EC2 provider." + } + + assert { + condition = keys(local.runner_config) == ["linux"] && length(local.runner_config_v2) == 0 + error_message = "Stable multi_runner_config entries must keep the original runner configuration and remain isolated from v2." + } + + assert { + condition = ( + contains(keys(local.runner_config["linux"]), "runner_config") + && !contains(keys(local.runner_config["linux"]), "compute_provider") + && local.runner_config["linux"].runner_config.enable_organization_runners + ) + error_message = "Stable module inputs must retain the original local.runner_config shape instead of using the v1-to-v2 translation." + } + + assert { + condition = keys(module.runners) == ["linux"] && length(module.runner_stacks) == 0 + error_message = "Stable multi_runner_config entries must retain the historical module.runners address." + } + + assert { + condition = keys(aws_sqs_queue.queued_builds) == ["linux"] + error_message = "Common queue ownership must preserve the stable runner configuration key." + } + + assert { + condition = ( + aws_sqs_queue.queued_builds["linux"].tags == var.tags + && aws_sqs_queue.queued_builds_dlq["linux"].tags == var.tags + ) + error_message = "Stable multi_runner_config queues must continue to receive exactly the module-level tags." + } + + assert { + condition = keys(output.runners_map) == ["linux"] + error_message = "Stable multi_runner_config must preserve the public runner map key." + } + + assert { + condition = length(output.runners_map_v2) == 0 + error_message = "Stable multi_runner_config must not add entries to the experimental runners_map_v2 output." + } + + assert { + condition = toset(keys(output.runners_map["linux"])) == toset( + [ + "launch_template_name", + "launch_template_id", + "launch_template_version", + "launch_template_ami_id", + "lambda_up", + "lambda_up_log_group", + "lambda_down", + "lambda_down_log_group", + "lambda_pool", + "lambda_pool_log_group", + "role_runner", + "role_scale_up", + "role_scale_down", + "role_pool", + "runners_log_groups", + "logfiles", + ] + ) + error_message = "Stable multi_runner_config must retain its existing flat runners_map entry shape." + } +} + +run "experimental_v2_routes_through_provider_stack" { + command = plan + + variables { + experimental = { + multi_runner_config_v2 = { + linux = { + runner = { + os = "linux" + architecture = "x64" + maximum_count = 2 + hooks = { + job_started = "/opt/actions/job-started.sh" + } + iam = { + managed_policy_arns = { + readonly = "arn:aws:iam::aws:policy/ReadOnlyAccess" + } + } + } + github = { + organization_runners = true + } + scale_down = { + idle_config = [{ + cron = "* * * * *" + timeZone = "UTC" + idleCount = 1 + }] + } + pool = { + config = [{ + schedule_expression = "cron(0 8 * * ? *)" + size = 1 + }] + } + compute_provider = { + ec2 = { + instance_types = ["m5.large"] + binaries_syncer = { + enabled = false + } + } + } + matcherConfig = { + labelMatchers = [["self-hosted", "linux", "x64"]] + enableDynamicLabels = true + awsDynamicLabelsPolicy = { + blocked_keys = ["image-id"] + restricted_keys = { + "instance-type" = { + allowed = ["m5.*", "c5.*"] + denied = ["*.metal"] + } + "ebs-volume-size" = { + max = 200 + } + } + } + } + } + } + } + } + + assert { + condition = keys(local.runner_config_by_provider.ec2) == ["linux"] + error_message = "Experimental multi_runner_config_v2 entries must route to the EC2 provider." + } + + assert { + condition = ( + local.compute_provider_types["linux"] == "ec2" + && local.runner_matcher_config["linux"].computeProvider == "ec2" + ) + error_message = "Compute-provider selection must supply the webhook routing contract." + } + + assert { + condition = length(local.runner_config) == 0 && keys(local.runner_config_v2) == ["linux"] + error_message = "Experimental multi_runner_config_v2 entries must remain isolated in the v2 configuration map." + } + + assert { + condition = toset(local.runner_config_v2["linux"].runner.extra_labels) == toset(["self-hosted", "linux", "x64"]) + error_message = "Experimental runner labels must include labels declared by its matcher configuration." + } + + assert { + condition = ( + local.runner_matcher_config["linux"].matcherConfig.awsDynamicLabelsPolicy.blocked_keys == tolist(["image-id"]) + && local.runner_matcher_config["linux"].matcherConfig.awsDynamicLabelsPolicy.restricted_keys["instance-type"].allowed == tolist(["m5.*", "c5.*"]) + && local.runner_matcher_config["linux"].matcherConfig.awsDynamicLabelsPolicy.restricted_keys["instance-type"].denied == tolist(["*.metal"]) + && local.runner_matcher_config["linux"].matcherConfig.awsDynamicLabelsPolicy.restricted_keys["ebs-volume-size"].max == "200" + ) + error_message = "Experimental matcher config must preserve the typed AWS dynamic-label policy contract." + } + + assert { + condition = length(module.runners) == 0 && keys(module.runner_stacks) == ["linux"] + error_message = "Experimental multi_runner_config_v2 entries must dispatch through module.runner_stacks." + } + + assert { + condition = keys(aws_sqs_queue.queued_builds) == ["linux"] + error_message = "Common queue ownership must preserve the experimental runner configuration key." + } + + assert { + condition = length(output.runners_map) == 0 + error_message = "Experimental multi_runner_config_v2 must not add nested entries to the stable runners_map output." + } + + assert { + condition = keys(output.runners_map_v2) == ["linux"] + error_message = "Experimental multi_runner_config_v2 must expose its runner configuration key through runners_map_v2." + } + + assert { + condition = toset(keys(output.runners_map_v2["linux"])) == toset( + [ + "provider", + "runner", + "scale_up", + "scale_down", + "pool", + ] + ) + error_message = "Experimental v2 runners_map_v2 entries must group common and provider resources by owner." + } + + assert { + condition = ( + toset(keys(output.runners_map_v2["linux"].runner)) == toset(["role"]) + && toset(keys(output.runners_map_v2["linux"].scale_up)) == toset(["lambda", "log_group", "role"]) + && toset(keys(output.runners_map_v2["linux"].scale_down)) == toset(["lambda", "log_group", "role"]) + && toset(keys(output.runners_map_v2["linux"].pool)) == toset(["lambda", "log_group", "role"]) + ) + error_message = "Experimental v2 common resources must use the nested runner, scale-up, scale-down, and pool contracts." + } + + assert { + condition = ( + toset(keys(output.runners_map_v2["linux"].provider)) == toset(["ec2"]) + && toset(keys(output.runners_map_v2["linux"].provider.ec2)) == toset([ + "launch_template", + "runners_log_groups", + "logfiles", + ]) + ) + error_message = "Experimental v2 must expose only EC2-owned resources under runners_map_v2..provider.ec2." + } + + assert { + condition = ( + !contains(keys(output.runners_map_v2["linux"]), "launch_template_name") + && output.runners_map_v2["linux"].runner.role != null + && !contains(keys(output.runners_map_v2["linux"].provider.ec2), "role_runner") + && !contains(keys(output.runners_map_v2["linux"]), "runners_log_groups") + && !contains(keys(output.runners_map_v2["linux"]), "logfiles") + ) + error_message = "Experimental v2 must expose only its nested schema through runners_map_v2 without legacy flat fields." + } + + assert { + condition = local.runner_config_by_provider.ec2["linux"].scale_down.idle_config[0].idleCount == 1 + error_message = "Provider-neutral idle configuration must remain in the common runner contract." + } + + assert { + condition = local.runner_config_by_provider.ec2["linux"].runner.iam.managed_policy_arns.readonly == "arn:aws:iam::aws:policy/ReadOnlyAccess" + error_message = "Runner-role policies must remain in the common runner contract." + } + + assert { + condition = ( + local.runner_config_by_provider.ec2["linux"].runner.hooks.job_started == "/opt/actions/job-started.sh" + && !contains(keys(local.runner_config_by_provider.ec2["linux"].compute_provider.ec2), "hooks") + ) + error_message = "Runner lifecycle hooks must remain in the common runner contract." + } +} + +run "experimental_v2_routes_microvm_through_provider_stack" { + command = plan + + variables { + experimental = { + multi_runner_config_v2 = { + micro = { + runner = { + os = "linux" + architecture = "arm64" + maximum_count = 4 + name_prefix = "microvm-" + } + pool = { + config = [{ + schedule_expression = "cron(0 8 * * ? *)" + size = 2 + }] + } + compute_provider = { + microvm = { + image_identifier = "arn:aws:lambdamicrovms:eu-west-1:123456789012:image/runner" + image_version = "1" + environment_variables = { + MICROVM_CLUSTER = "runner-cluster" + } + } + } + matcherConfig = { + labelMatchers = [["self-hosted", "linux", "arm64", "microvm"]] + } + } + } + } + } + + assert { + condition = ( + keys(try(local.runner_config_by_provider.ec2, {})) == [] + && keys(local.runner_config_by_provider.microvm) == ["micro"] + && local.compute_provider_types["micro"] == "microvm" + && local.runner_matcher_config["micro"].computeProvider == "microvm" + ) + error_message = "Experimental multi_runner_config_v2 entries must route MicroVM lanes to the MicroVM provider." + } + + assert { + condition = ( + length(module.runners) == 0 + && keys(module.runner_stacks) == ["micro"] + && length(module.runner_binaries) == 0 + ) + error_message = "MicroVM v2 lanes must dispatch through runner_stack without creating EC2 runner binaries." + } + + assert { + condition = ( + keys(output.runners_map_v2) == ["micro"] + && toset(keys(output.runners_map_v2["micro"].provider)) == toset(["microvm"]) + && output.runners_map_v2["micro"].provider.microvm.image_identifier == "arn:aws:lambdamicrovms:eu-west-1:123456789012:image/runner" + && output.runners_map_v2["micro"].provider.microvm.image_version == "1" + ) + error_message = "MicroVM v2 lanes must expose MicroVM-owned resources under runners_map_v2..provider.microvm." + } + + assert { + condition = ( + module.runner_stacks["micro"].scale_up.lambda.environment[0].variables["COMPUTE_PROVIDER_TYPE"] == "microvm" + && module.runner_stacks["micro"].scale_up.lambda.environment[0].variables["MICROVM_IMAGE_IDENTIFIER"] == "arn:aws:lambdamicrovms:eu-west-1:123456789012:image/runner" + && module.runner_stacks["micro"].scale_up.lambda.environment[0].variables["MICROVM_CLUSTER"] == "runner-cluster" + && module.runner_stacks["micro"].pool.lambda.environment[0].variables["MICROVM_IMAGE_IDENTIFIER"] == "arn:aws:lambdamicrovms:eu-west-1:123456789012:image/runner" + && !contains(keys(module.runner_stacks["micro"].scale_up.lambda.environment[0].variables), "INSTANCE_TYPES") + ) + error_message = "MicroVM v2 lanes must pass MicroVM provider fragments to scale-up and pool without EC2 environment variables." + } +} + +run "experimental_v2_layers_shared_and_component_tags" { + command = plan + + variables { + tags = { + GlobalOnly = "global" + Precedence = "global" + } + + lambda_tags = { + SharedLambdaOnly = "shared-lambda" + Precedence = "shared-lambda" + } + + experimental = { + multi_runner_config_v2 = { + tagged = { + tags = { + RunnerConfigOnly = "runner-config" + Precedence = "runner-config" + } + + runner = { + os = "linux" + architecture = "x64" + maximum_count = 2 + tags = { + RunnerOnly = "runner" + Precedence = "runner" + } + } + + lambda = { + tags = { + ConfigLambdaOnly = "config-lambda" + Precedence = "config-lambda" + } + } + + queue = { + redrive_build_queue = { + enabled = true + maxReceiveCount = 3 + } + tags = { + SharedQueueOnly = "shared-queue" + Precedence = "shared-queue" + } + } + + scale_up = { + tags = { + ScaleUpOnly = "scale-up" + Precedence = "scale-up" + } + } + + scale_down = { + tags = { + ScaleDownOnly = "scale-down" + Precedence = "scale-down" + } + } + + observability = { + logs = { + tags = { + SharedLogOnly = "shared-log" + Precedence = "shared-log" + } + } + } + + compute_provider = { + ec2 = { + instance_types = ["m5.large"] + binaries_syncer = { + enabled = false + } + } + } + + matcherConfig = { + labelMatchers = [["self-hosted", "linux", "x64", "tagged"]] + } + } + } + } + } + + assert { + condition = aws_sqs_queue.queued_builds["tagged"].tags == tomap({ + GlobalOnly = "global" + RunnerConfigOnly = "runner-config" + SharedQueueOnly = "shared-queue" + Precedence = "shared-queue" + }) + error_message = "Experimental v2 build queue tags must merge global, runner-configuration, and queue tags in that precedence order." + } + + assert { + condition = aws_sqs_queue.queued_builds_dlq["tagged"].tags == tomap({ + GlobalOnly = "global" + RunnerConfigOnly = "runner-config" + SharedQueueOnly = "shared-queue" + Precedence = "shared-queue" + }) + error_message = "Experimental v2 dead-letter queue tags must use the same layered precedence as the build queue." + } + + assert { + condition = module.runner_stacks["tagged"].scale_up.lambda.tags == tomap({ + GlobalOnly = "global" + RunnerConfigOnly = "runner-config" + SharedLambdaOnly = "shared-lambda" + ConfigLambdaOnly = "config-lambda" + ScaleUpOnly = "scale-up" + Precedence = "scale-up" + }) + error_message = "Scale-up Lambda tags must merge global, runner-configuration, shared Lambda, configuration Lambda, and component tags in that precedence order." + } + + assert { + condition = module.runner_stacks["tagged"].scale_up.log_group.tags == tomap({ + GlobalOnly = "global" + RunnerConfigOnly = "runner-config" + SharedLogOnly = "shared-log" + ScaleUpOnly = "scale-up" + Precedence = "scale-up" + }) + error_message = "Scale-up log-group tags must merge global, runner-configuration, shared log, and component tags in that precedence order." + } + + assert { + condition = module.runner_stacks["tagged"].scale_up.role.tags == tomap({ + GlobalOnly = "global" + RunnerConfigOnly = "runner-config" + ScaleUpOnly = "scale-up" + Precedence = "scale-up" + }) + error_message = "Scale-up role tags must merge global, runner-configuration, and component tags without Lambda- or log-only tags." + } + + assert { + condition = module.runner_stacks["tagged"].runner.role.tags == tomap({ + GlobalOnly = "global" + RunnerConfigOnly = "runner-config" + RunnerOnly = "runner" + Precedence = "runner" + }) + error_message = "Runner role tags must merge global, runner-configuration, and runner-component tags in that precedence order." + } + + assert { + condition = module.runner_stacks["tagged"].scale_down.lambda.tags == tomap({ + GlobalOnly = "global" + RunnerConfigOnly = "runner-config" + SharedLambdaOnly = "shared-lambda" + ConfigLambdaOnly = "config-lambda" + ScaleDownOnly = "scale-down" + Precedence = "scale-down" + }) + error_message = "Scale-down Lambda tags must preserve shared layers before applying scale-down component tags." + } + + assert { + condition = module.runner_stacks["tagged"].scale_down.log_group.tags == tomap({ + GlobalOnly = "global" + RunnerConfigOnly = "runner-config" + SharedLogOnly = "shared-log" + ScaleDownOnly = "scale-down" + Precedence = "scale-down" + }) + error_message = "Scale-down log-group tags must preserve shared log tags before applying scale-down component tags." + } + + assert { + condition = output.runners_map_v2["tagged"].pool == null + error_message = "Experimental v2 must expose a null pool object when no pool configuration is supplied." + } +} + +run "experimental_v2_rejects_empty_compute_provider" { + command = plan + + variables { + experimental = { + multi_runner_config_v2 = { + microvm = { + runner = { + os = "linux" + architecture = "x64" + maximum_count = 2 + } + compute_provider = {} + matcherConfig = { + labelMatchers = [["self-hosted", "linux", "x64"]] + } + } + } + } + } + + expect_failures = [var.experimental] +} diff --git a/modules/multi-runner/variables.experimental.tf b/modules/multi-runner/variables.experimental.tf new file mode 100644 index 0000000000..4a07c5021f --- /dev/null +++ b/modules/multi-runner/variables.experimental.tf @@ -0,0 +1,463 @@ +variable "experimental" { + description = <<-EOT + Opt-in experimental features. Omit this object to retain only the stable `multi_runner_config` behavior. Experimental schemas can change before they become stable. + + - `multi_runner_config_v2`: Provider-oriented runner configurations keyed by configuration name. To opt into v2, leave `multi_runner_config` empty and populate this map. When this map is empty, stable `multi_runner_config` entries continue to use the unchanged `runners` module. Populating both maps in the same module instance is unsupported. + + Each `multi_runner_config_v2` entry supports the following nested fields: + + - `tags`: Configuration-wide tags. These override module-level `tags`; narrower component and compute-provider tag maps take precedence for their resources. + - `runner.os`: Runner operating system. + - `runner.architecture`: Runner distribution architecture. + - `runner.boot_time_in_minutes`: Expected boot duration used before a runner is considered stale. + - `runner.disable_default_labels`: Prevents GitHub default labels from being registered. + - `runner.extra_labels`: Additional labels combined with `matcherConfig.labelMatchers`. Default self-hosted, operating-system, and architecture labels are also included unless `runner.disable_default_labels` is true. + - `runner.group_name`: GitHub runner group used during registration. + - `runner.name_prefix`: Prefix added to registered runner names. + - `runner.run_as_root`: Runs the runner service as root when supported by the compute provider. + - `runner.run_as`: Operating-system user used when `run_as_root` is false. + - `runner.maximum_count`: Maximum number of runners for this configuration. + - `runner.ephemeral`: Registers runners in ephemeral mode. + - `runner.jit_config_enabled`: Explicitly enables or disables just-in-time configuration. Null follows `ephemeral`. + - `runner.auto_update_disabled`: Disables the GitHub runner application's built-in updater. + - `runner.tags`: Tags for common runner resources, currently the managed runner IAM role. These override entry-level `tags`. + - `runner.hooks.job_started`: Script content installed as the runner job-started hook. + - `runner.hooks.job_completed`: Script content installed as the runner job-completed hook. + - `runner.iam.role.arn`: ARN of an externally managed runner role. When set, `runner-stack` does not create or modify that role. + - `runner.iam.managed_policy_arns`: Named managed-policy ARNs attached to the module-managed runner role. + - `runner.iam.additional_trust_policy_json`: Optional IAM policy document merged with the selected compute provider's default runner-role trust policy. + - `runner.iam.path`: IAM path for the module-managed runner role. + - `runner.iam.permissions_boundary`: Permissions-boundary ARN for the module-managed runner role. + - `github.organization_runners`: Registers runners at organization scope when true; otherwise repository-scoped registration is used. + - `lambda.tags`: Shared tags for control-plane Lambda functions. Component tags override this map. + - `queue.delay_webhook_event`: Delay in seconds applied to webhook job messages. + - `queue.job_queue_retention_in_seconds`: Build-queue message retention period in seconds. + - `queue.event_source_mapping.batch_size`: Maximum build-queue records delivered to one scale-up Lambda invocation. Null uses the module-level setting. + - `queue.event_source_mapping.maximum_batching_window_in_seconds`: Maximum batching window for build-queue records. Null uses the module-level setting. + - `queue.redrive_build_queue.enabled`: Creates and attaches a dead-letter queue for the build queue. + - `queue.redrive_build_queue.maxReceiveCount`: Number of receives before a build message moves to the dead-letter queue. + - `queue.tags`: Tags for configuration-owned queue resources. These override entry-level `tags`; component tags override this map. + - `scale_up.reserved_concurrent_executions`: Reserved concurrency for the scale-up Lambda. Use `-1` for unreserved concurrency. + - `scale_up.job_queued_check_enabled`: Enables the queued-job verification before scaling. Null follows the runner mode default. + - `scale_up.tags`: Tags for scale-up resources. These override entry-level and shared Lambda, queue, and log-group tags within their resource scopes. + - `scale_down.schedule_expression`: EventBridge schedule expression that invokes scale-down. + - `scale_down.minimum_running_time_in_minutes`: Minimum runner age before scale-down may terminate it. Null selects the operating-system default. + - `scale_down.tags`: Tags for scale-down resources. These override entry-level and shared Lambda and log-group tags within their resource scopes. + - `scale_down.idle_config`: Time-based desired idle-runner configurations. + - `scale_down.idle_config[].cron`: Cron expression identifying when the idle configuration applies. + - `scale_down.idle_config[].timeZone`: IANA time zone used to evaluate `cron`. + - `scale_down.idle_config[].idleCount`: Number of idle runners to retain during the matching period. + - `scale_down.idle_config[].evictionStrategy`: Selection strategy used when excess idle runners are removed. + - `pool.config`: Scheduled target pool sizes. An empty list disables the pool component. + - `pool.config[].schedule_expression`: Scheduler expression that activates the target size. + - `pool.config[].schedule_expression_timezone`: Optional IANA time zone used to evaluate the schedule. + - `pool.config[].size`: Desired number of runners for the schedule. + - `pool.runner_owner`: Optional GitHub organization or repository owner used when creating pooled runners. + - `pool.tags`: Tags for pool resources. These override entry-level and shared Lambda and log-group tags within their resource scopes. + - `job_retry.enabled`: Creates the retry queue, Lambda function, event-source mapping, and related IAM resources. + - `job_retry.delay_in_seconds`: Initial delay before a queued-job retry check. + - `job_retry.delay_backoff`: Multiplier applied to the delay after each unsuccessful check. + - `job_retry.max_attempts`: Maximum retry-check attempts before the message is no longer republished. + - `job_retry.tags`: Tags for job-retry resources. These override entry-level and shared Lambda, queue, and log-group tags within their resource scopes. + - `job_retry.lambda.memory_size`: Memory allocated to the job-retry Lambda in MB. + - `job_retry.lambda.reserved_concurrent_executions`: Reserved concurrency for the job-retry Lambda. Use `-1` for unreserved concurrency. + - `job_retry.lambda.timeout`: Job-retry Lambda timeout in seconds and visibility timeout for its retry queue. + - `ssm.tags`: Shared tags for SSM-related resources. These override entry-level `tags`. + - `ssm.kms_key`: Optional customer-managed KMS key used for temporary registration parameters. The wrapper's presence selects the KMS policy at plan time. + - `ssm.kms_key.arn`: ARN of the customer-managed KMS key. The ARN may be unknown until apply. + - `ssm.parameters.tags`: Tags for Terraform-managed and runtime-created runner configuration parameters. These override `ssm.tags`. + - `ssm.housekeeper.tags`: Tags for SSM housekeeper resources. These override entry-level, shared Lambda, shared log, and `ssm.tags` values. + - `observability.logs.tags`: Shared tags for CloudWatch log groups. Component tags override this map. + - `compute_provider`: Typed compute-provider blocks. Exactly one block must be non-null, and the populated block selects the provider. Its presence must be known during planning; values inside it may remain unknown until apply. + - `compute_provider.ec2`: EC2-specific configuration. + - `compute_provider.ec2.ami.filter`: EC2 AMI filters combined with the default AMI-name filter. + - `compute_provider.ec2.ami.owners`: AWS account IDs or aliases allowed to own the selected AMI. + - `compute_provider.ec2.ami.id_ssm_parameter`: Optional externally managed SSM parameter containing the AMI ID. The wrapper's presence selects external ownership at plan time. + - `compute_provider.ec2.ami.id_ssm_parameter.arn`: ARN of the externally managed SSM parameter. The ARN may be unknown until apply. + - `compute_provider.ec2.ami.kms_key`: Optional KMS key required to launch encrypted AMIs or snapshots. The wrapper's presence selects the KMS policy at plan time. + - `compute_provider.ec2.ami.kms_key.arn`: ARN of the KMS key. The ARN may be unknown until apply. + - `compute_provider.ec2.block_device_mappings`: EBS mappings added to the runner launch template. + - `compute_provider.ec2.block_device_mappings[].delete_on_termination`: Deletes the volume when its runner instance terminates. + - `compute_provider.ec2.block_device_mappings[].device_name`: Device name exposed to the runner instance. + - `compute_provider.ec2.block_device_mappings[].encrypted`: Enables EBS encryption. + - `compute_provider.ec2.block_device_mappings[].iops`: Provisioned IOPS for supported volume types. + - `compute_provider.ec2.block_device_mappings[].kms_key_id`: KMS key ID or ARN used to encrypt the volume. + - `compute_provider.ec2.block_device_mappings[].snapshot_id`: Snapshot used to initialize the volume. + - `compute_provider.ec2.block_device_mappings[].throughput`: Provisioned throughput for supported volume types. + - `compute_provider.ec2.block_device_mappings[].volume_initialization_rate`: Fixed initialization rate in MiB/s for supported snapshot-backed volumes. + - `compute_provider.ec2.block_device_mappings[].volume_size`: Volume size in GiB. + - `compute_provider.ec2.block_device_mappings[].volume_type`: EBS volume type. + - `compute_provider.ec2.create_service_linked_role_spot`: Allows scale-up to create the EC2 Spot service-linked role. + - `compute_provider.ec2.credit_specification`: CPU credit mode for burstable instance types. + - `compute_provider.ec2.ebs_optimized`: Requests EBS-optimized runner instances. + - `compute_provider.ec2.cloudwatch_agent.enabled`: Installs and configures the CloudWatch agent through the default bootstrap flow. + - `compute_provider.ec2.cloudwatch_agent.config`: Optional complete CloudWatch agent configuration. + - `compute_provider.ec2.binaries_syncer.enabled`: Enables use of the module-level synchronized runner distribution from S3. + - `compute_provider.ec2.detailed_monitoring_enabled`: Enables detailed EC2 monitoring for runner instances. + - `compute_provider.ec2.ssm_enabled`: Attaches runner permissions and policies required for AWS Systems Manager access. + - `compute_provider.ec2.user_data.enabled`: Enables launch-template user data. + - `compute_provider.ec2.user_data.template`: Optional path to a custom user-data template. + - `compute_provider.ec2.user_data.content`: Optional complete user-data content used instead of rendering a template. + - `compute_provider.ec2.user_data.pre_install`: Script content inserted before runner installation in the default template. + - `compute_provider.ec2.user_data.post_install`: Script content inserted after runner installation in the default template. + - `compute_provider.ec2.user_data.debug_logging_enabled`: Enables verbose user-data tracing, which can expose secrets in logs. + - `compute_provider.ec2.instance_allocation_strategy`: EC2 Fleet allocation strategy used to select capacity. + - `compute_provider.ec2.instance_max_spot_price`: Optional maximum hourly Spot price. + - `compute_provider.ec2.instance_target_capacity_type`: Primary capacity type, either `spot` or `on-demand`. + - `compute_provider.ec2.instance_type_priorities`: Optional numeric priorities keyed by instance type. + - `compute_provider.ec2.instance_types`: EC2 instance types available to the scale-up and pool functions. + - `compute_provider.ec2.additional_security_group_ids`: Existing security groups attached to runner instances. + - `compute_provider.ec2.instance_profile.name`: Name of an externally managed instance profile. Setting it also requires `runner.iam.role`. + - `compute_provider.ec2.enable_on_demand_failover_for_errors`: EC2 error codes that trigger an on-demand fallback after a Spot launch failure. + - `compute_provider.ec2.scale_errors`: EC2 error codes treated as retryable scale-up failures. + - `compute_provider.ec2.subnet_ids`: Subnets from which scale-up may launch runners. Null uses the module-level value. + - `compute_provider.ec2.vpc_id`: VPC in which runner networking resources are created. Null uses the module-level value. + - `compute_provider.ec2.cpu_options.core_count`: Number of CPU cores exposed to the runner instance. + - `compute_provider.ec2.cpu_options.threads_per_core`: Number of hardware threads exposed per CPU core. + - `compute_provider.ec2.cpu_options.amd_sev_snp`: Enables or disables AMD SEV-SNP on supported instance types. + - `compute_provider.ec2.cpu_options.nested_virtualization`: Enables or disables nested virtualization on supported instance types. + - `compute_provider.ec2.placement.affinity`: Host affinity setting. + - `compute_provider.ec2.placement.availability_zone`: Availability Zone in which the instance is placed. + - `compute_provider.ec2.placement.group_id`: Placement-group ID. + - `compute_provider.ec2.placement.group_name`: Placement-group name. + - `compute_provider.ec2.placement.host_id`: Dedicated Host ID. + - `compute_provider.ec2.placement.host_resource_group_arn`: ARN of the host resource group used for placement. + - `compute_provider.ec2.placement.spread_domain`: Spread-domain placement value. + - `compute_provider.ec2.placement.tenancy`: Instance tenancy. + - `compute_provider.ec2.placement.partition_number`: Placement-group partition number. + - `compute_provider.ec2.license_specifications[].license_configuration_arn`: ARN of a License Manager license configuration. + - `compute_provider.ec2.use_dedicated_host`: Enables the dedicated-host launch path required for macOS runners. + - `compute_provider.ec2.log_files`: Optional log files collected by the CloudWatch agent. + - `compute_provider.ec2.log_files[].log_group_name`: CloudWatch log-group name before optional prefixing. + - `compute_provider.ec2.log_files[].prefix_log_group`: Prefixes the log-group name with the runner stack path when true. + - `compute_provider.ec2.log_files[].file_path`: File or glob read by the CloudWatch agent. + - `compute_provider.ec2.log_files[].log_stream_name`: CloudWatch log-stream name template. + - `compute_provider.ec2.log_files[].log_class`: CloudWatch log-group class for the collected file. + - `compute_provider.ec2.tags`: Tags for runtime EC2 instances, volumes, network interfaces, and eligible Spot requests. These override entry-level tags and the generated runner `Name`; provider-required bootstrap tags take final precedence. + - `compute_provider.ec2.metadata_options.instance_metadata_tags`: Exposes instance tags through Instance Metadata Service when enabled. + - `compute_provider.ec2.metadata_options.http_endpoint`: Enables or disables the Instance Metadata Service endpoint. + - `compute_provider.ec2.metadata_options.http_tokens`: Controls whether IMDSv2 session tokens are optional or required. + - `compute_provider.ec2.metadata_options.http_put_response_hop_limit`: Network hop limit for Instance Metadata Service token responses. + - `compute_provider.microvm`: Lambda MicroVM-specific configuration. + - `compute_provider.microvm.image_identifier`: ARN or ID of the MicroVM image used to run GitHub runners. + - `compute_provider.microvm.image_version`: Optional MicroVM image version. + - `compute_provider.microvm.execution_role.arn`: Optional externally managed execution role assumed by MicroVMs. Null uses the common runner role. + - `compute_provider.microvm.egress_network_connectors`: Egress network connectors passed to RunMicrovm. + - `compute_provider.microvm.idle_policy`: Optional auto-suspend and auto-resume configuration passed to RunMicrovm. + - `compute_provider.microvm.logging`: Optional RunMicrovm logging union. Exactly one of `cloud_watch` or `disabled` must be selected when set. + - `compute_provider.microvm.run_hook_payload`: Optional payload delivered to the MicroVM `/run` hook. Maximum 16,384 characters. + - `compute_provider.microvm.maximum_duration_in_seconds`: Optional maximum MicroVM lifetime. Valid range is 1 through 28,800 seconds. + - `compute_provider.microvm.environment_variables`: Additional provider-specific Lambda environment variables merged into scale-up, scale-down, and pool. + - `compute_provider.microvm.tags`: Tags encoded into the MicroVM runner configuration. + - `compute_provider.microvm.iam`: Optional MicroVM control-plane IAM overrides and managed policy attachments. + - `matcherConfig.labelMatchers`: Groups of labels used to match webhook jobs to this configuration. + - `matcherConfig.exactMatch`: Requires the job labels to exactly match a configured label group. + - `matcherConfig.bidirectionalLabelMatch`: Requires labels to match in both directions instead of allowing configured subsets. + - `matcherConfig.priority`: Ordering used when multiple configurations match the same job. + - `matcherConfig.enableDynamicLabels`: Enables runtime interpretation of supported dynamic AWS labels. + - `matcherConfig.awsDynamicLabelsPolicy`: Optional policy restricting values accepted from dynamic AWS labels. + EOT + + type = object({ + multi_runner_config_v2 = optional(map(object({ + tags = optional(map(string), {}) + + runner = object({ + os = string + architecture = string + boot_time_in_minutes = optional(number, 5) + disable_default_labels = optional(bool, false) + extra_labels = optional(list(string), []) + group_name = optional(string, "Default") + name_prefix = optional(string, "") + run_as_root = optional(bool, false) + run_as = optional(string, "ec2-user") + maximum_count = number + ephemeral = optional(bool, false) + jit_config_enabled = optional(bool, null) + auto_update_disabled = optional(bool, false) + tags = optional(map(string), {}) + hooks = optional(object({ + job_started = optional(string, "") + job_completed = optional(string, "") + }), {}) + iam = optional(object({ + role = optional(object({ + arn = string + }), null) + managed_policy_arns = optional(map(string), {}) + additional_trust_policy_json = optional(string, null) + path = optional(string, null) + permissions_boundary = optional(string, null) + }), {}) + }) + + github = optional(object({ + organization_runners = optional(bool, false) + }), {}) + + lambda = optional(object({ + tags = optional(map(string), {}) + }), {}) + + queue = optional(object({ + delay_webhook_event = optional(number, 30) + job_queue_retention_in_seconds = optional(number, 86400) + event_source_mapping = optional(object({ + batch_size = optional(number, null) + maximum_batching_window_in_seconds = optional(number, null) + }), {}) + redrive_build_queue = optional(object({ + enabled = bool + maxReceiveCount = number + }), { + enabled = false + maxReceiveCount = null + }) + tags = optional(map(string), {}) + }), {}) + + scale_up = optional(object({ + reserved_concurrent_executions = optional(number, 1) + job_queued_check_enabled = optional(bool, null) + tags = optional(map(string), {}) + }), {}) + + scale_down = optional(object({ + schedule_expression = optional(string, "cron(*/5 * * * ? *)") + minimum_running_time_in_minutes = optional(number, null) + tags = optional(map(string), {}) + idle_config = optional(list(object({ + cron = string + timeZone = string + idleCount = number + evictionStrategy = optional(string, "oldest_first") + })), []) + }), {}) + + pool = optional(object({ + config = optional(list(object({ + schedule_expression = string + schedule_expression_timezone = optional(string) + size = number + })), []) + runner_owner = optional(string, null) + tags = optional(map(string), {}) + }), {}) + + job_retry = optional(object({ + enabled = optional(bool, false) + delay_in_seconds = optional(number, 300) + delay_backoff = optional(number, 2) + max_attempts = optional(number, 1) + tags = optional(map(string), {}) + lambda = optional(object({ + memory_size = optional(number, 256) + reserved_concurrent_executions = optional(number, 1) + timeout = optional(number, 30) + }), {}) + }), {}) + + ssm = optional(object({ + tags = optional(map(string), {}) + kms_key = optional(object({ + arn = string + }), null) + parameters = optional(object({ + tags = optional(map(string), {}) + }), {}) + housekeeper = optional(object({ + tags = optional(map(string), {}) + }), {}) + }), {}) + + observability = optional(object({ + logs = optional(object({ + tags = optional(map(string), {}) + }), {}) + }), {}) + + compute_provider = object({ + ec2 = optional(object({ + metadata_options = optional(object({ + instance_metadata_tags = optional(string, "enabled") + http_endpoint = optional(string, "enabled") + http_tokens = optional(string, "required") + http_put_response_hop_limit = optional(number, 1) + }), {}) + ami = optional(object({ + filter = optional(map(list(string)), { state = ["available"] }) + owners = optional(list(string), ["amazon"]) + id_ssm_parameter = optional(object({ + arn = string + }), null) + kms_key = optional(object({ + arn = string + }), null) + }), null) + block_device_mappings = optional(list(object({ + delete_on_termination = optional(bool, true) + device_name = optional(string, "/dev/xvda") + encrypted = optional(bool, true) + iops = optional(number) + kms_key_id = optional(string) + snapshot_id = optional(string) + throughput = optional(number) + volume_initialization_rate = optional(number) + volume_size = number + volume_type = optional(string, "gp3") + })), [{ + volume_size = 30 + }]) + create_service_linked_role_spot = optional(bool, false) + credit_specification = optional(string, null) + ebs_optimized = optional(bool, false) + cloudwatch_agent = optional(object({ + enabled = optional(bool, true) + config = optional(string, null) + }), {}) + binaries_syncer = optional(object({ + enabled = optional(bool, true) + }), {}) + detailed_monitoring_enabled = optional(bool, false) + ssm_enabled = optional(bool, false) + user_data = optional(object({ + enabled = optional(bool, true) + template = optional(string, null) + content = optional(string, null) + pre_install = optional(string, "") + post_install = optional(string, "") + debug_logging_enabled = optional(bool, false) + }), {}) + instance_allocation_strategy = optional(string, "lowest-price") + instance_max_spot_price = optional(string, null) + instance_target_capacity_type = optional(string, "spot") + instance_type_priorities = optional(map(number), null) + instance_types = list(string) + additional_security_group_ids = optional(list(string), []) + instance_profile = optional(object({ + name = string + }), null) + enable_on_demand_failover_for_errors = optional(list(string), []) + scale_errors = optional(list(string), [ + "UnfulfillableCapacity", + "MaxSpotInstanceCountExceeded", + "TargetCapacityLimitExceededException", + "RequestLimitExceeded", + "ResourceLimitExceeded", + "MaxSpotInstanceCountExceeded", + "MaxSpotFleetRequestCountExceeded", + "InsufficientInstanceCapacity", + "InsufficientCapacityOnHost", + ]) + subnet_ids = optional(list(string), null) + vpc_id = optional(string, null) + cpu_options = optional(object({ + core_count = optional(number) + threads_per_core = optional(number) + amd_sev_snp = optional(string) + nested_virtualization = optional(string) + }), null) + placement = optional(object({ + affinity = optional(string) + availability_zone = optional(string) + group_id = optional(string) + group_name = optional(string) + host_id = optional(string) + host_resource_group_arn = optional(string) + spread_domain = optional(string) + tenancy = optional(string) + partition_number = optional(number) + }), null) + license_specifications = optional(list(object({ + license_configuration_arn = string + })), []) + use_dedicated_host = optional(bool, false) + log_files = optional(list(object({ + log_group_name = string + prefix_log_group = bool + file_path = string + log_stream_name = string + log_class = optional(string, "STANDARD") + })), null) + tags = optional(map(string), {}) + }), null) + + microvm = optional(object({ + image_identifier = string + image_version = optional(string, null) + execution_role = optional(object({ + arn = string + }), null) + egress_network_connectors = optional(list(string), []) + idle_policy = optional(object({ + max_idle_duration_seconds = number + suspended_duration_seconds = number + auto_resume_enabled = bool + }), null) + logging = optional(object({ + cloud_watch = optional(object({ + log_group = optional(string, null) + log_stream = optional(string, null) + }), null) + disabled = optional(bool, false) + }), null) + run_hook_payload = optional(string, null) + maximum_duration_in_seconds = optional(number, null) + environment_variables = optional(map(string), {}) + tags = optional(map(string), {}) + iam = optional(object({ + resource_arns = optional(list(string), ["*"]) + actions = optional(object({ + scale_up = optional(list(string), null) + scale_down = optional(list(string), null) + }), {}) + additional_policy_json = optional(object({ + scale_up = optional(string, null) + }), {}) + managed_policy_arns = optional(object({ + scale_up = optional(string, null) + pool = optional(string, null) + }), {}) + }), {}) + }), null) + }) + + matcherConfig = object({ + labelMatchers = list(list(string)) + exactMatch = optional(bool, false) + bidirectionalLabelMatch = optional(bool, false) + priority = optional(number, 999) + enableDynamicLabels = optional(bool, false) + awsDynamicLabelsPolicy = optional(object({ + blocked_keys = optional(list(string), []) + restricted_keys = optional(map(object({ + allowed = optional(list(string), []) + denied = optional(list(string), []) + max = optional(string, null) + })), {}) + }), null) + }) + })), {}) + }) + default = {} + + validation { + condition = alltrue([ + for runner_config in values(var.experimental.multi_runner_config_v2) : + length([ + for provider_type, provider_config in runner_config.compute_provider : provider_type + if provider_config != null + ]) == 1 + ]) + error_message = "Each experimental runner configuration must set exactly one compute-provider block. Supported compute-provider blocks: ec2, microvm." + } + + validation { + condition = alltrue([ + for runner_config in values(var.experimental.multi_runner_config_v2) : + runner_config.runner.iam.role == null || length(runner_config.runner.iam.managed_policy_arns) == 0 + ]) + error_message = "runner.iam.managed_policy_arns cannot be set with an external runner.iam.role because external roles are not managed by this module." + } +} diff --git a/modules/multi-runner/variables.tf b/modules/multi-runner/variables.tf index df6fb77473..d243f355ea 100644 --- a/modules/multi-runner/variables.tf +++ b/modules/multi-runner/variables.tf @@ -208,7 +208,14 @@ variable "multi_runner_config" { bidirectionalLabelMatch = optional(bool, false) priority = optional(number, 999) enableDynamicLabels = optional(bool, false) - awsDynamicLabelsPolicy = optional(any, null) + awsDynamicLabelsPolicy = optional(object({ + blocked_keys = optional(list(string), []) + restricted_keys = optional(map(object({ + allowed = optional(list(string), []) + denied = optional(list(string), []) + max = optional(string, null) + })), {}) + }), null) }) redrive_build_queue = optional(object({ enabled = bool @@ -292,8 +299,8 @@ variable "multi_runner_config" { redrive_build_queue: "Set options to attach (optional) a dead letter queue to the build queue, the queue between the webhook and the scale up lambda. You have the following options. 1. Disable by setting `enabled` to false. 2. Enable by setting `enabled` to `true`, `maxReceiveCount` to a number of max retries." } EOT + default = {} } - variable "scale_up_lambda_memory_size" { description = "Memory size limit in MB for scale_up lambda." type = number @@ -800,6 +807,8 @@ variable "user_agent" { default = "github-aws-runners" } +# TODO: Remove this standalone multi-runner input in a future breaking cleanup; per-configuration runner_config.iam_overrides is the value used by EC2 runner modules. +# tflint-ignore: terraform_unused_declarations variable "iam_overrides" { description = "This map provides the possibility to override some IAM defaults. The following attributes are supported: `instance_profile_name` overrides the instance profile name used in the launch template. `runner_role_arn` overrides the IAM role ARN used for the runner instances." type = object({ diff --git a/modules/multi-runner/webhook.tf b/modules/multi-runner/webhook.tf index 6ee9b4b2ec..96850ba485 100644 --- a/modules/multi-runner/webhook.tf +++ b/modules/multi-runner/webhook.tf @@ -1,10 +1,21 @@ +locals { + runner_matcher_config = { + for k, v in local.multi_runner_config : k => { + id = aws_sqs_queue.queued_builds[k].id + arn = aws_sqs_queue.queued_builds[k].arn + computeProvider = local.compute_provider_types[k] + matcherConfig = v.matcherConfig + } + } +} + module "webhook" { source = "../webhook" prefix = var.prefix tags = local.tags kms_key_arn = var.kms_key_arn eventbridge = var.eventbridge - runner_matcher_config = local.runner_config + runner_matcher_config = local.runner_matcher_config matcher_config_parameter_store_tier = var.matcher_config_parameter_store_tier ssm_paths = { diff --git a/modules/runner-stack/README.md b/modules/runner-stack/README.md new file mode 100644 index 0000000000..65a8e34bc0 --- /dev/null +++ b/modules/runner-stack/README.md @@ -0,0 +1,131 @@ +# Runner stack module + +> This module is treated as an internal module; breaking changes do not trigger a major release bump. + +This internal module implements the experimental provider-neutral runner control plane selected by `experimental.multi_runner_config_v2`. It is composed by `multi-runner` and is not intended as a standalone public entry point. Its direct contract may change while v2 remains experimental. + +The stack coordinates internal modules for [`scale-runners`](./scale-runners), [`pool`](./pool), [`job-retry`](./job-retry), and [`ssm-housekeeper`](./ssm-housekeeper). These modules own their provider-neutral Lambda, scheduler, logging, and IAM resources. The stack also creates or selects the runner IAM role, manages shared runner configuration in SSM, and dispatches the selected compute provider. + +Provider-owned settings are typed and nested under the selected provider block; for example, AMI, VPC, instance-profile, capacity, userdata, and runner-host logging settings live under `compute_provider.ec2`, while Lambda MicroVM image and runtime settings live under `compute_provider.microvm`. Exactly one provider block must be populated, and its presence must be known during planning; there is no separate input discriminator. Before creating the common runner role, the stack calls the selected provider's isolated `trust-policy` module to combine its default trust with `runner.iam.additional_trust_policy_json`. The resulting assume-role policy does not depend on the full compute-provider module, which receives the resolved runner role only after it is created. EC2 owns the instance profile, launch template, EC2 bootstrap parameters, runner log groups, and its provider policies and Lambda environment variables. MicroVM owns runtime configuration, execution-role policy, and its provider Lambda environment variables; the runtime Lambdas create and terminate MicroVMs. The common stack attaches each returned policy group to its runner, scale-up, scale-down, or pool role. The runner-stack output groups provider-specific resources under the matching dynamic provider key, which also identifies the selected provider. + +## Tagging + +`tags` supplies module-wide defaults. Shared resource tags are set with `lambda.tags`, `queue.tags`, and `observability.logs.tags`. Component tags under `runner`, `scale_up`, `scale_down`, `pool`, `job_retry`, and `ssm` apply to the taggable resources owned by that component. `ssm.parameters.tags` and `ssm.housekeeper.tags` provide narrower SSM scopes. + +Tags are merged from broadest to narrowest: module tags, shared resource tags, component tags, and then subcomponent tags. The narrowest value wins when a key is repeated. For example, a scale-up Lambda receives `tags`, `lambda.tags`, and `scale_up.tags`, while its log group receives `tags`, `observability.logs.tags`, and `scale_up.tags`. + +Provider-specific runner tags remain inside the provider boundary. `compute_provider.ec2.tags` applies to runtime EC2 instance, volume, network-interface, and spot-request tag specifications. The EC2 provider applies the bootstrap tags `ghr:environment`, `ghr:ssm_config_path`, and `ghr:runner_name_prefix` last so they cannot be overridden; those tags are not added to common Lambda, IAM, queue, log-group, or SSM resources. + +## Overview + +### Action runners on EC2 + +The action runners are created via a launch template; in the launch template only the subnet needs to be provided. During launch the installation is handled via a user data script. The configuration is fetched from SSM parameter store. + +### Lambda scale up + +The scale up lambda is triggered by events on a SQS queue. Events on this queue are delayed, which will give the workflow some time to start running on available runners. For each event the lambda will check if the workflow is still queued and no other limits are reached. In that case the lambda will create a new EC2 instance. The lambda only needs to know which launch template to use and which subnets are available. From the available subnets a random one will be chosen. Once the instance is created the event is assumed as handled, and we assume the workflow wil start at some moment once the created instance is ready. + +### Lambda scale down + +The scale down lambda is triggered via a CloudWatch event. The event is triggered by a cron expression defined in `scale_down.schedule_expression` (https://docs.aws.amazon.com/AmazonCloudWatch/latest/events/ScheduledEvents.html). For scaling down GitHub does not provide a good API yet, therefore we run the scaling down based on this event every x minutes. Each time the lambda is triggered it tries to remove all runners older than x minutes (configurable) managed in this deployment. In case the runner can be removed from GitHub, which means it is not executing a workflow, the lambda will terminate the EC2 instance. + +--8<-- "modules/runner-stack/scale-down-state-diagram.md:mkdocs_scale_down_state_diagram" + +## Lambda Function + +The Lambda function is written in [TypeScript](https://www.typescriptlang.org/) and requires Node 12.x and yarn. Sources are located in [./lambdas/runners]. Two lambda functions share the same sources, there is one entry point for `scaleDown` and another one for `scaleUp`. + +### Install + +```bash +cd lambdas/runners +yarn install +``` + +### Test + +Test are implemented with [vitest][https://vitest.dev/]), calls to AWS and GitHub are mocked. + +```bash +yarn run test +``` + +### Package + +To compile all TypeScript/JavaScript sources in a single file [ncc](https://github.com/zeit/ncc) is used. + +```bash +yarn run dist +``` + + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 1.3.0 | +| [aws](#requirement\_aws) | >= 6.33 | + +## Providers + +| Name | Version | +|------|---------| +| [aws](#provider\_aws) | >= 6.33 | + +## Modules + +| Name | Source | Version | +|------|--------|---------| +| [ec2](#module\_ec2) | ../compute-providers/ec2 | n/a | +| [ec2\_trust\_policy](#module\_ec2\_trust\_policy) | ../compute-providers/ec2/trust-policy | n/a | +| [job\_retry](#module\_job\_retry) | ./job-retry | n/a | +| [microvm](#module\_microvm) | ../compute-providers/microvm | n/a | +| [microvm\_trust\_policy](#module\_microvm\_trust\_policy) | ../compute-providers/microvm/trust-policy | n/a | +| [pool](#module\_pool) | ./pool | n/a | +| [scale\_runners](#module\_scale\_runners) | ./scale-runners | n/a | +| [ssm\_housekeeper](#module\_ssm\_housekeeper) | ./ssm-housekeeper | n/a | + +## Resources + +| Name | Type | +|------|------| +| [aws_iam_role.runner](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role) | resource | +| [aws_iam_role_policy.runner_provider](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy) | resource | +| [aws_iam_role_policy_attachment.runner](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy_attachment) | resource | +| [aws_ssm_parameter.disable_default_labels](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/ssm_parameter) | resource | +| [aws_ssm_parameter.jit_config_enabled](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/ssm_parameter) | resource | +| [aws_ssm_parameter.runner_agent_mode](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/ssm_parameter) | resource | +| [aws_ssm_parameter.token_path](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/ssm_parameter) | resource | +| [aws_caller_identity.current](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/caller_identity) | data source | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [aws\_partition](#input\_aws\_partition) | AWS partition used to construct ARNs. | `string` | `"aws"` | no | +| [aws\_region](#input\_aws\_region) | AWS region. | `string` | n/a | yes | +| [compute\_provider](#input\_compute\_provider) | Typed compute-provider configuration. Provider-owned settings remain inside the selected compute-provider block.

Exactly one compute-provider block must be non-null. The populated block selects the provider, and its presence must be known during planning. Values inside the selected block may remain unknown until apply.

- `ec2`: EC2 compute-provider configuration.
- `ec2.ami`: Optional AMI discovery or external AMI-parameter configuration. Null uses the operating-system and architecture defaults.
- `ec2.ami.filter`: EC2 AMI filters combined with the provider's default AMI-name filter.
- `ec2.ami.owners`: AWS account IDs or aliases allowed to own the selected AMI.
- `ec2.ami.id_ssm_parameter`: Optional externally managed SSM parameter containing the AMI ID. Null creates a provider-managed AMI-ID parameter. The wrapper's presence is the plan-time ownership discriminator, so keep the object literal even when its ARN comes from another resource.
- `ec2.ami.id_ssm_parameter.arn`: ARN of the externally managed SSM parameter. The ARN may be unknown until apply.
- `ec2.ami.kms_key`: Optional KMS key required to launch encrypted AMIs or snapshots. The wrapper's presence is the plan-time policy discriminator.
- `ec2.ami.kms_key.arn`: ARN of the KMS key. The ARN may be unknown until apply.
- `ec2.vpc_id`: VPC in which runner networking resources are created.
- `ec2.subnet_ids`: Subnets from which scale-up may launch runner instances.
- `ec2.overrides`: Optional resource-name overrides.
- `ec2.overrides.name_runner`: Name tag used for runner compute resources. An empty value uses the generated provider name.
- `ec2.overrides.name_sg`: Name tag used for the managed runner security group. An empty value uses the generated provider name.
- `ec2.instance_profile`: Optional externally managed instance profile used by the launch template.
- `ec2.instance_profile.name`: Name of the externally managed instance profile.
- `ec2.instance_profile_path`: IAM path for the provider-managed instance profile. Null uses a path derived from the stack prefix.
- `ec2.binaries_syncer`: Runner-distribution synchronization configuration.
- `ec2.binaries_syncer.enabled`: Enables use of a synchronized runner distribution from S3.
- `ec2.binaries_syncer.s3`: S3 object containing the synchronized runner distribution. Required when synchronization is enabled.
- `ec2.binaries_syncer.s3.arn`: ARN of the runner-distribution bucket, used by IAM policies.
- `ec2.binaries_syncer.s3.id`: Bucket name used to construct the runner-distribution S3 URI.
- `ec2.binaries_syncer.s3.key`: Object key of the runner distribution.
- `ec2.block_device_mappings`: EBS mappings added to the runner launch template.
- `ec2.block_device_mappings[].delete_on_termination`: Deletes the volume when its runner instance terminates.
- `ec2.block_device_mappings[].device_name`: Device name exposed to the runner instance.
- `ec2.block_device_mappings[].encrypted`: Enables EBS encryption.
- `ec2.block_device_mappings[].iops`: Provisioned IOPS for volume types that support it.
- `ec2.block_device_mappings[].kms_key_id`: KMS key ID or ARN used to encrypt the volume.
- `ec2.block_device_mappings[].snapshot_id`: Snapshot used to initialize the volume.
- `ec2.block_device_mappings[].throughput`: Provisioned throughput for volume types that support it.
- `ec2.block_device_mappings[].volume_initialization_rate`: Fixed initialization rate in MiB/s for supported snapshot-backed volumes.
- `ec2.block_device_mappings[].volume_size`: Volume size in GiB.
- `ec2.block_device_mappings[].volume_type`: EBS volume type.
- `ec2.ebs_optimized`: Requests EBS-optimized runner instances.
- `ec2.instance_target_capacity_type`: Primary capacity type, either `spot` or `on-demand`.
- `ec2.instance_allocation_strategy`: EC2 Fleet allocation strategy used to select instance capacity.
- `ec2.instance_type_priorities`: Optional numeric priorities keyed by instance type.
- `ec2.instance_max_spot_price`: Optional maximum hourly Spot price.
- `ec2.instance_types`: EC2 instance types available to the scale-up and pool functions.
- `ec2.user_data`: Runner bootstrap user-data configuration.
- `ec2.user_data.enabled`: Enables launch-template user data.
- `ec2.user_data.template`: Optional path to a custom user-data template.
- `ec2.user_data.content`: Optional complete user-data content. When set, it is used instead of rendering a template.
- `ec2.user_data.pre_install`: Script content inserted before runner installation in the default template.
- `ec2.user_data.post_install`: Script content inserted after runner installation in the default template.
- `ec2.user_data.debug_logging_enabled`: Enables verbose user-data tracing, which can expose secrets in logs.
- `ec2.ssm_enabled`: Attaches runner permissions and policies required for AWS Systems Manager access.
- `ec2.create_service_linked_role_spot`: Allows scale-up to create the EC2 Spot service-linked role.
- `ec2.cloudwatch_agent`: CloudWatch agent configuration for runner instances.
- `ec2.cloudwatch_agent.enabled`: Installs and configures the CloudWatch agent through the default bootstrap flow.
- `ec2.cloudwatch_agent.config`: Optional complete CloudWatch agent configuration. Null renders the provider default from `log_files`.
- `ec2.managed_security_group_enabled`: Creates and attaches the provider-managed runner security group.
- `ec2.log_files`: Optional log files collected by the CloudWatch agent. Null uses the provider defaults.
- `ec2.log_files[].log_group_name`: CloudWatch log-group name, before optional prefixing.
- `ec2.log_files[].prefix_log_group`: Prefixes the log-group name with the runner stack path when true.
- `ec2.log_files[].file_path`: File or glob read by the CloudWatch agent.
- `ec2.log_files[].log_stream_name`: CloudWatch log-stream name template.
- `ec2.log_files[].log_class`: CloudWatch log-group class for the collected file.
- `ec2.key_name`: Optional EC2 key-pair name added to the launch template.
- `ec2.additional_security_group_ids`: Existing security groups attached in addition to the managed security group.
- `ec2.detailed_monitoring_enabled`: Enables detailed EC2 monitoring for runner instances.
- `ec2.egress_rules`: Egress rules created on the managed runner security group.
- `ec2.egress_rules[].cidr_blocks`: IPv4 CIDR destinations.
- `ec2.egress_rules[].ipv6_cidr_blocks`: IPv6 CIDR destinations.
- `ec2.egress_rules[].prefix_list_ids`: AWS prefix-list destinations.
- `ec2.egress_rules[].from_port`: First destination port in the permitted range.
- `ec2.egress_rules[].protocol`: IP protocol name or number. Use `-1` for all protocols.
- `ec2.egress_rules[].security_groups`: Destination security-group IDs.
- `ec2.egress_rules[].self`: Allows traffic to the managed security group itself when true.
- `ec2.egress_rules[].to_port`: Last destination port in the permitted range.
- `ec2.egress_rules[].description`: Optional rule description.
- `ec2.tags`: Additional tags for runner instances, EBS volumes, network interfaces, and eligible Spot instance requests created from the launch template. They override module-level tags and the generated runner `Name`; the provider-managed `ghr:environment`, `ghr:ssm_config_path`, and `ghr:runner_name_prefix` bootstrap tags take final precedence. These tags do not apply to static provider resources such as the launch template, security group, IAM resources, SSM parameters, or log groups.
- `ec2.metadata_options`: Instance Metadata Service configuration in the launch template.
- `ec2.metadata_options.instance_metadata_tags`: Exposes instance tags through Instance Metadata Service when `enabled`.
- `ec2.metadata_options.http_endpoint`: Enables or disables the Instance Metadata Service endpoint.
- `ec2.metadata_options.http_tokens`: Controls whether IMDSv2 session tokens are optional or required.
- `ec2.metadata_options.http_put_response_hop_limit`: Network hop limit for Instance Metadata Service token responses.
- `ec2.credit_specification`: CPU credit mode for burstable instance types, either `standard` or `unlimited`.
- `ec2.cpu_options`: CPU topology and processor-feature configuration.
- `ec2.cpu_options.core_count`: Number of CPU cores exposed to the runner instance.
- `ec2.cpu_options.threads_per_core`: Number of hardware threads exposed per CPU core.
- `ec2.cpu_options.amd_sev_snp`: Enables or disables AMD SEV-SNP on supported instance types.
- `ec2.cpu_options.nested_virtualization`: Enables or disables nested virtualization on supported instance types.
- `ec2.placement`: EC2 placement configuration for runner instances.
- `ec2.placement.affinity`: Host affinity setting.
- `ec2.placement.availability_zone`: Availability Zone in which the instance is placed.
- `ec2.placement.group_id`: Placement-group ID.
- `ec2.placement.group_name`: Placement-group name.
- `ec2.placement.host_id`: Dedicated Host ID.
- `ec2.placement.host_resource_group_arn`: ARN of the host resource group used for placement.
- `ec2.placement.spread_domain`: Spread-domain placement value.
- `ec2.placement.tenancy`: Instance tenancy, such as `default`, `dedicated`, or `host`.
- `ec2.placement.partition_number`: Placement-group partition number.
- `ec2.license_specifications`: License Manager configurations added to the launch template.
- `ec2.license_specifications[].license_configuration_arn`: ARN of a License Manager license configuration.
- `ec2.associate_public_ipv4_address`: Associates a public IPv4 address with runner network interfaces.
- `ec2.enable_on_demand_failover_for_errors`: EC2 error codes that trigger an on-demand fallback after a Spot launch failure.
- `ec2.scale_errors`: EC2 error codes treated as retryable scale-up failures.
- `ec2.use_dedicated_host`: Enables the dedicated-host launch path, required for macOS runners.
- `microvm`: Lambda MicroVM compute-provider configuration.
- `microvm.image_identifier`: ARN or ID of the MicroVM image used to run GitHub runners.
- `microvm.image_version`: Optional MicroVM image version.
- `microvm.execution_role.arn`: Optional externally managed execution role assumed by MicroVMs. Null uses the common runner role.
- `microvm.egress_network_connectors`: Egress network connectors passed to RunMicrovm.
- `microvm.idle_policy`: Optional auto-suspend and auto-resume configuration passed to RunMicrovm.
- `microvm.logging`: Optional RunMicrovm logging union. Exactly one of `cloud_watch` or `disabled` must be selected when set.
- `microvm.run_hook_payload`: Optional payload delivered to the MicroVM `/run` hook. Maximum 16,384 characters.
- `microvm.maximum_duration_in_seconds`: Optional maximum MicroVM lifetime. Valid range is 1 through 28,800 seconds.
- `microvm.environment_variables`: Additional provider-specific Lambda environment variables merged into scale-up, scale-down, and pool.
- `microvm.tags`: Tags encoded into the MicroVM runner configuration.
- `microvm.iam`: Optional MicroVM control-plane IAM overrides and managed policy attachments. | 
object({
    ec2 = optional(object({
      ami = optional(object({
        filter = optional(map(list(string)), { state = ["available"] })
        owners = optional(list(string), ["amazon"])
        id_ssm_parameter = optional(object({
          arn = string
        }), null)
        kms_key = optional(object({
          arn = string
        }), null)
      }), null)
      vpc_id     = string
      subnet_ids = list(string)
      overrides = optional(object({
        name_runner = optional(string, "")
        name_sg     = optional(string, "")
      }), {})
      instance_profile = optional(object({
        name = string
      }), null)
      instance_profile_path = optional(string, null)
      binaries_syncer = optional(object({
        enabled = optional(bool, true)
        s3 = optional(object({
          arn = string
          id  = string
          key = string
        }), null)
      }), {})
      block_device_mappings = optional(list(object({
        delete_on_termination      = optional(bool, true)
        device_name                = optional(string, "/dev/xvda")
        encrypted                  = optional(bool, true)
        iops                       = optional(number)
        kms_key_id                 = optional(string)
        snapshot_id                = optional(string)
        throughput                 = optional(number)
        volume_initialization_rate = optional(number)
        volume_size                = number
        volume_type                = optional(string, "gp3")
      })), [{ volume_size = 30 }])
      ebs_optimized                 = optional(bool, false)
      instance_target_capacity_type = optional(string, "spot")
      instance_allocation_strategy  = optional(string, "lowest-price")
      instance_type_priorities      = optional(map(number), null)
      instance_max_spot_price       = optional(string, null)
      instance_types                = list(string)
      user_data = optional(object({
        enabled               = optional(bool, true)
        template              = optional(string, null)
        content               = optional(string, null)
        pre_install           = optional(string, "")
        post_install          = optional(string, "")
        debug_logging_enabled = optional(bool, false)
      }), {})
      ssm_enabled                     = optional(bool, false)
      create_service_linked_role_spot = optional(bool, false)
      cloudwatch_agent = optional(object({
        enabled = optional(bool, true)
        config  = optional(string, null)
      }), {})
      managed_security_group_enabled = optional(bool, true)
      log_files = optional(list(object({
        log_group_name   = string
        prefix_log_group = bool
        file_path        = string
        log_stream_name  = string
        log_class        = optional(string, "STANDARD")
      })), null)
      key_name                      = optional(string, null)
      additional_security_group_ids = optional(list(string), [])
      detailed_monitoring_enabled   = optional(bool, false)
      egress_rules = optional(list(object({
        cidr_blocks      = list(string)
        ipv6_cidr_blocks = list(string)
        prefix_list_ids  = list(string)
        from_port        = number
        protocol         = string
        security_groups  = list(string)
        self             = bool
        to_port          = number
        description      = string
        })), [{
        cidr_blocks      = ["0.0.0.0/0"]
        ipv6_cidr_blocks = ["::/0"]
        prefix_list_ids  = null
        from_port        = 0
        protocol         = "-1"
        security_groups  = null
        self             = null
        to_port          = 0
        description      = null
      }])
      tags = optional(map(string), {})
      metadata_options = optional(object({
        instance_metadata_tags      = optional(string, "enabled")
        http_endpoint               = optional(string, "enabled")
        http_tokens                 = optional(string, "required")
        http_put_response_hop_limit = optional(number, 1)
      }), {})
      credit_specification = optional(string, null)
      cpu_options = optional(object({
        core_count            = optional(number)
        threads_per_core      = optional(number)
        amd_sev_snp           = optional(string)
        nested_virtualization = optional(string)
      }), null)
      placement = optional(object({
        affinity                = optional(string)
        availability_zone       = optional(string)
        group_id                = optional(string)
        group_name              = optional(string)
        host_id                 = optional(string)
        host_resource_group_arn = optional(string)
        spread_domain           = optional(string)
        tenancy                 = optional(string)
        partition_number        = optional(number)
      }), null)
      license_specifications = optional(list(object({
        license_configuration_arn = string
      })), [])
      associate_public_ipv4_address        = optional(bool, false)
      enable_on_demand_failover_for_errors = optional(list(string), [])
      scale_errors = optional(list(string), [
        "UnfulfillableCapacity",
        "MaxSpotInstanceCountExceeded",
        "TargetCapacityLimitExceededException",
        "RequestLimitExceeded",
        "ResourceLimitExceeded",
        "MaxSpotInstanceCountExceeded",
        "MaxSpotFleetRequestCountExceeded",
        "InsufficientInstanceCapacity",
        "InsufficientCapacityOnHost",
      ])
      use_dedicated_host = optional(bool, false)
    }), null)

    microvm = optional(object({
      image_identifier = string
      image_version    = optional(string, null)
      execution_role = optional(object({
        arn = string
      }), null)
      egress_network_connectors = optional(list(string), [])
      idle_policy = optional(object({
        max_idle_duration_seconds  = number
        suspended_duration_seconds = number
        auto_resume_enabled        = bool
      }), null)
      logging = optional(object({
        cloud_watch = optional(object({
          log_group  = optional(string, null)
          log_stream = optional(string, null)
        }), null)
        disabled = optional(bool, false)
      }), null)
      run_hook_payload            = optional(string, null)
      maximum_duration_in_seconds = optional(number, null)
      environment_variables       = optional(map(string), {})
      tags                        = optional(map(string), {})
      iam = optional(object({
        resource_arns = optional(list(string), ["*"])
        actions = optional(object({
          scale_up   = optional(list(string), null)
          scale_down = optional(list(string), null)
        }), {})
        additional_policy_json = optional(object({
          scale_up = optional(string, null)
        }), {})
        managed_policy_arns = optional(object({
          scale_up = optional(string, null)
          pool     = optional(string, null)
        }), {})
      }), {})
    }), null)
  })
| n/a | yes | +| [github](#input\_github) | GitHub API and runner-registration configuration.

- `app_parameters.key_base64`: Parameter Store reference for the GitHub App private key.
- `app_parameters.key_base64.name`: Name of the private-key parameter supplied to the control-plane functions.
- `app_parameters.key_base64.arn`: ARN of the private-key parameter used by IAM policies.
- `app_parameters.id`: Parameter Store reference for the GitHub App ID.
- `app_parameters.id.name`: Name of the App-ID parameter supplied to the control-plane functions.
- `app_parameters.id.arn`: ARN of the App-ID parameter used by IAM policies.
- `organization_runners`: Registers runners at organization scope when true; otherwise repository-scoped registration is used.
- `enterprise_server.url`: Optional GitHub Enterprise Server base URL. Null selects GitHub.com.
- `enterprise_server.ssl_verify`: Enables TLS certificate verification for GitHub Enterprise Server requests.
- `user_agent`: Optional User-Agent value added to GitHub API requests. | 
object({
    app_parameters = object({
      key_base64 = map(string)
      id         = map(string)
    })
    organization_runners = bool
    enterprise_server = optional(object({
      url        = optional(string, null)
      ssl_verify = optional(bool, true)
    }), {})
    user_agent = optional(string, null)
  })
| n/a | yes | +| [job\_retry](#input\_job\_retry) | Job-retry queue and Lambda configuration.

- `enabled`: Creates the retry queue, Lambda function, event-source mapping, and related IAM resources.
- `delay_in_seconds`: Initial delay before a queued-job retry check. AWS SQS limits this value to 900 seconds.
- `delay_backoff`: Multiplier applied to the delay after each unsuccessful check.
- `max_attempts`: Maximum retry-check attempts before the message is no longer republished.
- `tags`: Tags for job-retry resources, including the Lambda function, log group, IAM role, retry queue, and event-source mapping. These override module-level tags and the shared `lambda.tags`, `queue.tags`, and `observability.logs.tags` maps when keys conflict.
- `lambda.memory_size`: Memory allocated to the job-retry Lambda in MB.
- `lambda.reserved_concurrent_executions`: Reserved concurrency for the job-retry Lambda. Use `-1` for unreserved concurrency.
- `lambda.timeout`: Job-retry Lambda timeout in seconds and visibility timeout for its retry queue. | 
object({
    enabled          = optional(bool, false)
    delay_in_seconds = optional(number, 300)
    delay_backoff    = optional(number, 2)
    max_attempts     = optional(number, 1)
    tags             = optional(map(string), {})
    lambda = optional(object({
      memory_size                    = optional(number, 256)
      reserved_concurrent_executions = optional(number, 1)
      timeout                        = optional(number, 30)
    }), {})
  })
| `{}` | no | +| [lambda](#input\_lambda) | Configuration shared by the control-plane Lambda functions.

- `zip`: Local control-plane archive. When null, the module's packaged runner archive is used.
- `s3.bucket`: Optional S3 bucket containing the Lambda archive. Setting this selects S3 instead of a local archive.
- `s3.key`: Object key of the Lambda archive in `s3.bucket`.
- `s3.object_version`: Optional version of the Lambda archive object.
- `runtime`: Runtime used by all control-plane Lambda functions.
- `architecture`: Instruction-set architecture used by all control-plane Lambda functions. Supported values are `arm64` and `x86_64`.
- `subnet_ids`: Subnets used for Lambda VPC configuration.
- `security_group_ids`: Security groups used for Lambda VPC configuration.
- `tags`: Shared tags applied to Lambda function resources only. These override module-level `tags`; component `tags` override this map when keys conflict.
- `role.path`: IAM path for module-managed Lambda execution roles. Defaults to a path derived from `prefix`.
- `role.permissions_boundary`: Permissions-boundary ARN applied to module-managed Lambda execution roles. | 
object({
    zip = optional(string, null)
    s3 = optional(object({
      bucket         = optional(string, null)
      key            = optional(string, null)
      object_version = optional(string, null)
    }), {})
    runtime            = optional(string, "nodejs24.x")
    architecture       = optional(string, "arm64")
    subnet_ids         = optional(list(string), [])
    security_group_ids = optional(list(string), [])
    tags               = optional(map(string), {})
    role = optional(object({
      path                 = optional(string, null)
      permissions_boundary = optional(string, null)
    }), {})
  })
| `{}` | no | +| [observability](#input\_observability) | Logging, tracing, and metrics configuration for control-plane and provider resources.

- `logs.level`: Application log level supplied to the control-plane functions.
- `logs.retention_in_days`: CloudWatch Logs retention period.
- `logs.kms_key_id`: Optional KMS key ID or ARN used to encrypt CloudWatch log groups.
- `logs.class`: CloudWatch log-group class. Supported values are `STANDARD` and `INFREQUENT_ACCESS`.
- `logs.tags`: Shared tags for CloudWatch log groups. These override module-level `tags`; component `tags` override this map when keys conflict.
- `tracing.mode`: Optional Lambda active-tracing mode. Null disables X-Ray tracing configuration.
- `tracing.capture_http_requests`: Enables HTTP request capture in the tracing helper.
- `tracing.capture_error`: Enables error capture in the tracing helper.
- `metrics.enable`: Enables module-emitted metrics.
- `metrics.namespace`: CloudWatch namespace used for emitted metrics.
- `metrics.metric.enable_github_app_rate_limit`: Emits GitHub App rate-limit metrics.
- `metrics.metric.enable_job_retry`: Emits job-retry metrics.
- `metrics.metric.enable_spot_termination_warning`: Emits spot-termination warning metrics where supported. | 
object({
    logs = optional(object({
      level             = optional(string, "info")
      retention_in_days = optional(number, 180)
      kms_key_id        = optional(string, null)
      class             = optional(string, "STANDARD")
      tags              = optional(map(string), {})
    }), {})
    tracing = optional(object({
      mode                  = optional(string, null)
      capture_http_requests = optional(bool, false)
      capture_error         = optional(bool, false)
    }), {})
    metrics = optional(object({
      enable    = optional(bool, false)
      namespace = optional(string, "GitHub Runners")
      metric = optional(object({
        enable_github_app_rate_limit    = optional(bool, true)
        enable_job_retry                = optional(bool, true)
        enable_spot_termination_warning = optional(bool, true)
      }), {})
    }), {})
  })
| `{}` | no | +| [pool](#input\_pool) | Scheduled runner-pool configuration. The pool component is created only when `config` is non-empty.

- `config`: Scheduled target pool sizes.
- `config[].schedule_expression`: Scheduler expression that activates the target size.
- `config[].schedule_expression_timezone`: Optional IANA time zone used to evaluate the schedule.
- `config[].size`: Desired number of runners for the schedule.
- `include_busy_runners`: Includes busy runners when calculating the current pool size.
- `runner_owner`: Optional GitHub organization or repository owner used when creating pooled runners.
- `tags`: Tags for pool resources, including the Lambda function, log group, IAM roles, and scheduler group. These override module-level tags and the shared `lambda.tags` and `observability.logs.tags` maps when keys conflict.
- `lambda.memory_size`: Memory allocated to the pool Lambda in MB.
- `lambda.timeout`: Pool Lambda timeout in seconds.
- `lambda.reserved_concurrent_executions`: Reserved concurrency for the pool Lambda. Use `-1` for unreserved concurrency. | 
object({
    config = optional(list(object({
      schedule_expression          = string
      schedule_expression_timezone = optional(string)
      size                         = number
    })), [])
    include_busy_runners = optional(bool, false)
    runner_owner         = optional(string, null)
    tags                 = optional(map(string), {})
    lambda = optional(object({
      memory_size                    = optional(number, 512)
      timeout                        = optional(number, 60)
      reserved_concurrent_executions = optional(number, 1)
    }), {})
  })
| `{}` | no | +| [prefix](#input\_prefix) | The prefix used for naming resources. | `string` | `"github-actions"` | no | +| [queue](#input\_queue) | Build queue reference and queue-integrated Lambda configuration.

- `build.arn`: ARN of the externally managed build queue consumed by scale-up.
- `build.url`: URL of the externally managed build queue used when messages are published.
- `event_source_mapping.batch_size`: Maximum records delivered to a Lambda invocation.
- `event_source_mapping.maximum_batching_window_in_seconds`: Maximum time Lambda may buffer records before invocation.
- `tags`: Shared tags for queue-related resources created by this stack, including event-source mappings and the optional job-retry queue. These override module-level `tags`; component `tags` override this map when keys conflict. The referenced build queue is not managed or tagged by this module. | 
object({
    build = object({
      arn = string
      url = string
    })
    event_source_mapping = optional(object({
      batch_size                         = optional(number, 10)
      maximum_batching_window_in_seconds = optional(number, 0)
    }), {})
    tags = optional(map(string), {})
  })
| n/a | yes | +| [runner](#input\_runner) | Provider-neutral GitHub runner configuration.

- `os`: Runner operating system. Supported values are `linux`, `osx`, and `windows`.
- `architecture`: Runner distribution architecture, such as `x64` or `arm64`.
- `boot_time_in_minutes`: Expected instance boot duration used before a runner is considered stale.
- `disable_default_labels`: Prevents GitHub's default self-hosted, operating-system, and architecture labels from being registered.
- `labels`: Complete set of labels supplied to the control-plane functions.
- `group_name`: GitHub runner group used during registration.
- `name_prefix`: Prefix added to registered runner names.
- `run_as_root`: Runs the runner service as root when supported by the compute provider.
- `run_as`: Operating-system user used when `run_as_root` is false.
- `maximum_count`: Maximum number of runners that may exist for this stack.
- `ephemeral`: Registers runners in ephemeral mode.
- `jit_config_enabled`: Explicitly enables or disables just-in-time configuration. When null, runtime behavior follows `ephemeral`.
- `auto_update_disabled`: Disables the GitHub runner application's built-in updater.
- `tags`: Additional tags for common runner resources, currently the managed runner IAM role. These override module-level `tags` with the same key.
- `hooks.job_started`: Script content installed as the runner job-started hook.
- `hooks.job_completed`: Script content installed as the runner job-completed hook.
- `iam.role.arn`: ARN of an externally managed runner role. When set, this module does not create or modify that role.
- `iam.managed_policy_arns`: Named managed-policy ARNs attached to the module-managed runner role.
- `iam.additional_trust_policy_json`: Optional IAM policy document merged with the selected compute provider's default runner-role trust policy.
- `iam.path`: IAM path for the module-managed runner role. Defaults to a path derived from `prefix`.
- `iam.permissions_boundary`: Permissions-boundary ARN for the module-managed runner role. | 
object({
    os                     = optional(string, "linux")
    architecture           = optional(string, "x64")
    boot_time_in_minutes   = optional(number, 5)
    disable_default_labels = optional(bool, false)
    labels                 = list(string)
    group_name             = optional(string, "Default")
    name_prefix            = optional(string, "")
    run_as_root            = optional(bool, false)
    run_as                 = optional(string, "ec2-user")
    maximum_count          = optional(number, 3)
    ephemeral              = optional(bool, false)
    jit_config_enabled     = optional(bool, null)
    auto_update_disabled   = optional(bool, false)
    tags                   = optional(map(string), {})
    hooks = optional(object({
      job_started   = optional(string, "")
      job_completed = optional(string, "")
    }), {})
    iam = optional(object({
      role = optional(object({
        arn = string
      }), null)
      managed_policy_arns          = optional(map(string), {})
      additional_trust_policy_json = optional(string, null)
      path                         = optional(string, null)
      permissions_boundary         = optional(string, null)
    }), {})
  })
| n/a | yes | +| [scale\_down](#input\_scale\_down) | Scale-down Lambda, schedule, and idle-runner configuration.

- `memory_size`: Memory allocated to the scale-down Lambda in MB.
- `timeout`: Scale-down Lambda timeout in seconds.
- `schedule_expression`: EventBridge schedule expression that invokes scale-down.
- `minimum_running_time_in_minutes`: Minimum runner age before scale-down may terminate it. Null selects the operating-system default.
- `tags`: Tags for scale-down resources, including the Lambda function, log group, EventBridge rule, and IAM role. These override module-level tags and the shared `lambda.tags` and `observability.logs.tags` maps when keys conflict.
- `idle_config`: Time-based desired idle-runner configurations.
- `idle_config[].cron`: Cron expression identifying when the configuration applies.
- `idle_config[].timeZone`: IANA time zone used to evaluate `cron`.
- `idle_config[].idleCount`: Number of idle runners to retain during the matching period.
- `idle_config[].evictionStrategy`: Selection strategy used when excess idle runners are removed. | 
object({
    memory_size                     = optional(number, 512)
    timeout                         = optional(number, 60)
    schedule_expression             = optional(string, "cron(*/5 * * * ? *)")
    minimum_running_time_in_minutes = optional(number, null)
    tags                            = optional(map(string), {})
    idle_config = optional(list(object({
      cron             = string
      timeZone         = string
      idleCount        = number
      evictionStrategy = optional(string, "oldest_first")
    })), [])
  })
| `{}` | no | +| [scale\_up](#input\_scale\_up) | Scale-up component configuration.

- `memory_size`: Memory allocated to the scale-up Lambda in MB.
- `timeout`: Scale-up Lambda timeout in seconds.
- `reserved_concurrent_executions`: Reserved concurrency for the scale-up Lambda. Use `-1` for unreserved concurrency.
- `job_queued_check_enabled`: Enables the queued-job verification before scaling. When null, the default is enabled for persistent runners and disabled for ephemeral runners.
- `tags`: Tags for scale-up resources, including the Lambda function, log group, event-source mapping, and IAM role. These override module-level tags and the shared `lambda.tags`, `queue.tags`, and `observability.logs.tags` maps when keys conflict. | 
object({
    memory_size                    = optional(number, 512)
    timeout                        = optional(number, 60)
    reserved_concurrent_executions = optional(number, 1)
    job_queued_check_enabled       = optional(bool, null)
    tags                           = optional(map(string), {})
  })
| `{}` | no | +| [ssm](#input\_ssm) | Parameter Store paths, encryption, tag scopes, and housekeeper configuration.

- `paths.root`: Root Parameter Store path for this runner stack.
- `paths.tokens`: Path segment under `paths.root` used for registration tokens and just-in-time configuration.
- `paths.config`: Path segment under `paths.root` used for persistent runner configuration.
- `kms_key`: Optional customer-managed KMS key used to encrypt temporary registration parameters. The wrapper's presence is the plan-time policy discriminator.
- `kms_key.arn`: ARN of the customer-managed KMS key. The ARN may be unknown until apply.
- `tags`: Shared tags for SSM-related resources. These override module-level `tags` and are inherited by parameter and housekeeper resources.
- `parameters.tags`: Tags for Terraform-managed runner configuration parameters and temporary parameters created by the scale-up and pool Lambdas. These override module-level and `ssm.tags` values with the same key.
- `housekeeper.schedule_expression`: EventBridge schedule expression that invokes the SSM housekeeper.
- `housekeeper.state`: EventBridge rule state, such as `ENABLED` or `DISABLED`.
- `housekeeper.tags`: Tags for housekeeper resources, including the Lambda function, log group, EventBridge rule, and IAM role. These override module-level, `ssm.tags`, shared Lambda, and shared log tags when keys conflict.
- `housekeeper.lambda.memory_size`: Memory allocated to the SSM housekeeper Lambda in MB.
- `housekeeper.lambda.timeout`: SSM housekeeper Lambda timeout in seconds.
- `housekeeper.config.tokenPath`: Parameter Store token path cleaned by the housekeeper. When omitted, the configured runner token path is used.
- `housekeeper.config.minimumDaysOld`: Minimum parameter age in days before deletion is allowed.
- `housekeeper.config.dryRun`: Reports eligible parameters without deleting them when true. | 
object({
    paths = object({
      root   = string
      tokens = string
      config = string
    })
    kms_key = optional(object({
      arn = string
    }), null)
    tags = optional(map(string), {})
    parameters = optional(object({
      tags = optional(map(string), {})
    }), {})
    housekeeper = optional(object({
      schedule_expression = optional(string, "rate(1 day)")
      state               = optional(string, "ENABLED")
      tags                = optional(map(string), {})
      lambda = optional(object({
        memory_size = optional(number, 512)
        timeout     = optional(number, 60)
      }), {})
      config = optional(object({
        tokenPath      = optional(string)
        minimumDaysOld = optional(number, 1)
        dryRun         = optional(bool, false)
      }), {})
    }), {})
  })
| n/a | yes | +| [tags](#input\_tags) | Base tags added to taggable resources created by this stack. Shared, component, and compute-provider tag maps override matching keys within their documented resource scopes. | `map(string)` | `{}` | no | + +## Outputs + +| Name | Description | +|------|-------------| +| [pool](#output\_pool) | Scheduled pool resources. Null when no pool configuration is supplied. | +| [provider](#output\_provider) | Provider-specific resources grouped under the selected provider key. | +| [runner](#output\_runner) | Common runner resources. The role is null when an external runner role is used. | +| [scale\_down](#output\_scale\_down) | Scale-down control-plane resources. | +| [scale\_up](#output\_scale\_up) | Scale-up control-plane resources. | + diff --git a/modules/runner-stack/common-config.tf b/modules/runner-stack/common-config.tf new file mode 100644 index 0000000000..bbbe12b65f --- /dev/null +++ b/modules/runner-stack/common-config.tf @@ -0,0 +1,49 @@ +# Shared control-plane configuration: naming, paths, tags, and normalized values. +locals { + common_tags = var.tags + runner_tags = merge(local.common_tags, var.runner.tags) + lambda_tags = merge(local.common_tags, var.lambda.tags) + queue_tags = merge(local.common_tags, var.queue.tags) + observability_log_tags = merge(local.common_tags, var.observability.logs.tags) + + scale_up_tags = merge(local.common_tags, var.scale_up.tags) + scale_up_lambda_tags = merge(local.lambda_tags, var.scale_up.tags) + scale_up_log_tags = merge(local.observability_log_tags, var.scale_up.tags) + scale_up_queue_tags = merge(local.queue_tags, var.scale_up.tags) + + scale_down_tags = merge(local.common_tags, var.scale_down.tags) + scale_down_lambda_tags = merge(local.lambda_tags, var.scale_down.tags) + scale_down_log_tags = merge(local.observability_log_tags, var.scale_down.tags) + + pool_tags = merge(local.common_tags, var.pool.tags) + pool_lambda_tags = merge(local.lambda_tags, var.pool.tags) + pool_log_tags = merge(local.observability_log_tags, var.pool.tags) + + job_retry_tags = merge(local.common_tags, var.job_retry.tags) + job_retry_lambda_tags = merge(local.lambda_tags, var.job_retry.tags) + job_retry_log_tags = merge(local.observability_log_tags, var.job_retry.tags) + job_retry_queue_tags = merge(local.queue_tags, var.job_retry.tags) + + ssm_tags = merge(local.common_tags, var.ssm.tags) + ssm_parameter_tags = merge(local.ssm_tags, var.ssm.parameters.tags) + ssm_housekeeper_tags = merge(local.ssm_tags, var.ssm.housekeeper.tags) + ssm_housekeeper_lambda_tags = merge(local.lambda_tags, var.ssm.tags, var.ssm.housekeeper.tags) + ssm_housekeeper_log_tags = merge(local.observability_log_tags, var.ssm.tags, var.ssm.housekeeper.tags) + + lambda_role_path = var.lambda.role.path == null ? "/${var.prefix}/" : var.lambda.role.path + runner_role_path = var.runner.iam.path == null ? "/${var.prefix}/" : var.runner.iam.path + lambda_zip = var.lambda.zip == null ? "${path.module}/../../lambdas/functions/control-plane/runners.zip" : var.lambda.zip + kms_key = var.ssm.kms_key + enable_job_queued_check = var.scale_up.job_queued_check_enabled == null ? !var.runner.ephemeral : var.scale_up.job_queued_check_enabled + token_path = "${var.ssm.paths.root}/${var.ssm.paths.tokens}" + arn_ssm_parameters_path_config = "arn:${var.aws_partition}:ssm:${var.aws_region}:${data.aws_caller_identity.current.account_id}:parameter${var.ssm.paths.root}/${var.ssm.paths.config}" + + parameter_store_tags = jsonencode([ + for key, value in local.ssm_parameter_tags : { + Key = key + Value = value + } + ]) +} + +data "aws_caller_identity" "current" {} diff --git a/modules/runner-stack/compute-provider.tf b/modules/runner-stack/compute-provider.tf new file mode 100644 index 0000000000..af53e415e3 --- /dev/null +++ b/modules/runner-stack/compute-provider.tf @@ -0,0 +1,20 @@ +locals { + provider_type = one([ + for provider_type, provider_config in var.compute_provider : provider_type + if provider_config != null + ]) + + provider_assume_role_policies = { + ec2 = try(module.ec2_trust_policy[0].assume_role_policy, null) + microvm = try(module.microvm_trust_policy[0].assume_role_policy, null) + } + + provider_assume_role_policy = local.provider_assume_role_policies[local.provider_type] + + provider_contracts = { + ec2 = one(module.ec2[*].provider) + microvm = one(module.microvm[*].provider) + } + + provider_contract = local.provider_contracts[local.provider_type] +} diff --git a/modules/runner-stack/ec2.tf b/modules/runner-stack/ec2.tf new file mode 100644 index 0000000000..071174f859 --- /dev/null +++ b/modules/runner-stack/ec2.tf @@ -0,0 +1,27 @@ +module "ec2_trust_policy" { + count = local.provider_type == "ec2" ? 1 : 0 + source = "../compute-providers/ec2/trust-policy" + + additional_trust_policy_json = var.runner.iam.additional_trust_policy_json +} + +module "ec2" { + count = local.provider_type == "ec2" ? 1 : 0 + source = "../compute-providers/ec2" + + aws_partition = var.aws_partition + aws_region = var.aws_region + prefix = var.prefix + tags = var.tags + + config = var.compute_provider.ec2 + runner = merge(var.runner, { + iam = merge(var.runner.iam, { + role = local.runner_role + managed_policy_arns = local.common_runner_managed_policy_arns + }) + }) + github = var.github + ssm = var.ssm + observability = var.observability +} diff --git a/modules/runner-stack/job-retry.tf b/modules/runner-stack/job-retry.tf new file mode 100644 index 0000000000..2631f1c279 --- /dev/null +++ b/modules/runner-stack/job-retry.tf @@ -0,0 +1,62 @@ + +locals { + job_retry_enabled = var.job_retry.enabled +} + +module "job_retry" { + source = "./job-retry" + count = local.job_retry_enabled ? 1 : 0 + + config = { + prefix = var.prefix + aws_partition = var.aws_partition + lambda = { + artifact = { + zip = local.lambda_zip + s3 = var.lambda.s3 + } + runtime = var.lambda.runtime + architecture = var.lambda.architecture + memory_size = var.job_retry.lambda.memory_size + timeout = var.job_retry.lambda.timeout + reserved_concurrent_executions = var.job_retry.lambda.reserved_concurrent_executions + environment_variables = {} + vpc = { + subnet_ids = var.lambda.subnet_ids + security_group_ids = var.lambda.security_group_ids + } + role = { + path = local.lambda_role_path + permissions_boundary = var.lambda.role.permissions_boundary + principals = [] + } + } + runner = { + name_prefix = var.runner.name_prefix + } + github = var.github + queue = { + build = var.queue.build + event_source_mapping = { + batch_size = var.queue.event_source_mapping.batch_size + maximum_batching_window_in_seconds = var.queue.event_source_mapping.maximum_batching_window_in_seconds + } + encryption = { + sqs_managed_sse_enabled = true + kms_master_key_id = null + kms_data_key_reuse_period_seconds = null + } + } + ssm = { + kms_key = local.kms_key + } + observability = var.observability + tags = { + resources = local.job_retry_tags + lambda = local.job_retry_lambda_tags + log_group = local.job_retry_log_tags + queue = local.job_retry_queue_tags + event_source_mapping = local.job_retry_queue_tags + } + } +} diff --git a/modules/runner-stack/job-retry/README.md b/modules/runner-stack/job-retry/README.md new file mode 100644 index 0000000000..ffba2f9636 --- /dev/null +++ b/modules/runner-stack/job-retry/README.md @@ -0,0 +1,61 @@ +# Module - Job Retry + +This module is listening to a SQS queue where the scale-up lambda publishes messages for jobs that needs to trigger a retry if still queued. The job retry module lambda function is handling the messages, checking if the job is queued. Next for queued jobs a message is published to the build queue for the scale-up lambda. The scale-up lambda will handle the message as any other workflow job event. + +## Usages + +The module is an inner module used by the runner stack when the opt-in feature for job retry is enabled. The module is not intended to be used standalone. + + + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 1.3.0 | +| [aws](#requirement\_aws) | >= 6.21 | + +## Providers + +| Name | Version | +|------|---------| +| [aws](#provider\_aws) | >= 6.21 | + +## Modules + +No modules. + +## Resources + +| Name | Type | +|------|------| +| [aws_cloudwatch_log_group.job_retry](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/cloudwatch_log_group) | resource | +| [aws_iam_role.job_retry](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role) | resource | +| [aws_iam_role_policy.job_retry](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy) | resource | +| [aws_iam_role_policy.job_retry_logging](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy) | resource | +| [aws_iam_role_policy.job_retry_xray](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy) | resource | +| [aws_iam_role_policy_attachment.job_retry_vpc_execution_role](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy_attachment) | resource | +| [aws_lambda_event_source_mapping.job_retry](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/lambda_event_source_mapping) | resource | +| [aws_lambda_function.job_retry](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/lambda_function) | resource | +| [aws_lambda_permission.job_retry](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/lambda_permission) | resource | +| [aws_sqs_queue.job_retry_check_queue](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/sqs_queue) | resource | +| [aws_sqs_queue_policy.job_retry_check_queue_policy](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/sqs_queue_policy) | resource | +| [aws_iam_policy_document.deny_insecure_transport](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source | +| [aws_iam_policy_document.job_retry](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source | +| [aws_iam_policy_document.job_retry_logging](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source | +| [aws_iam_policy_document.lambda_assume_role](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source | +| [aws_iam_policy_document.lambda_xray](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [config](#input\_config) | Provider-neutral job-retry configuration assembled by runner-stack.

- `prefix`: Prefix used to name job-retry resources.
- `aws_partition`: AWS partition used to construct the Lambda VPC managed-policy ARN.
- `lambda.artifact.zip`: Resolved local control-plane archive.
- `lambda.artifact.s3.bucket`: Optional S3 bucket containing the Lambda archive.
- `lambda.artifact.s3.key`: Object key of the Lambda archive.
- `lambda.artifact.s3.object_version`: Optional object version of the Lambda archive.
- `lambda.runtime`: Runtime used by the job-retry Lambda.
- `lambda.architecture`: Instruction-set architecture used by the job-retry Lambda.
- `lambda.memory_size`: Memory allocated to the job-retry Lambda.
- `lambda.timeout`: Lambda timeout and retry-queue visibility timeout in seconds.
- `lambda.reserved_concurrent_executions`: Reserved concurrency for the Lambda. Use `-1` for unreserved concurrency.
- `lambda.environment_variables`: Additional Lambda environment variables. Required job-retry variables override matching keys.
- `lambda.vpc.subnet_ids`: Subnets used for Lambda VPC configuration.
- `lambda.vpc.security_group_ids`: Security groups used for Lambda VPC configuration.
- `lambda.role.path`: IAM path used for the job-retry Lambda role.
- `lambda.role.permissions_boundary`: Optional permissions boundary for the Lambda role.
- `lambda.role.principals`: Extra principals allowed to assume the Lambda role, for example during local testing.
- `runner.name_prefix`: Prefix used to identify runners belonging to this runner configuration.
- `github.organization_runners`: Enables organization runners.
- `github.enterprise_server.url`: Optional GitHub Enterprise Server URL.
- `github.user_agent`: Optional User-Agent sent to GitHub.
- `github.app_parameters.key_base64`: Name and ARN of the GitHub App private-key parameter.
- `github.app_parameters.id`: Name and ARN of the GitHub App ID parameter.
- `queue.build`: URL and ARN of the build queue to which retry messages are published.
- `queue.event_source_mapping.batch_size`: Maximum records delivered per job-retry invocation.
- `queue.event_source_mapping.maximum_batching_window_in_seconds`: Maximum event batching window.
- `queue.encryption`: Server-side encryption configuration for the retry queue.
- `ssm.kms_key`: Optional KMS key used by the job-retry IAM policy.
- `observability.logs`: Logging level, retention, encryption, and log-class configuration.
- `observability.tracing`: Lambda X-Ray and tracing-helper configuration.
- `observability.metrics`: Metrics enablement, namespace, and job-retry metric configuration.
- `tags.resources`: Tags for the job-retry Lambda role and component resources.
- `tags.lambda`: Tags for the job-retry Lambda function.
- `tags.log_group`: Tags for the job-retry log group.
- `tags.queue`: Tags for the retry queue.
- `tags.event_source_mapping`: Tags for the retry-queue event-source mapping. | 
object({
    prefix        = string
    aws_partition = string
    lambda = object({
      artifact = object({
        zip = string
        s3 = object({
          bucket         = optional(string, null)
          key            = optional(string, null)
          object_version = optional(string, null)
        })
      })
      runtime                        = string
      architecture                   = string
      memory_size                    = number
      timeout                        = number
      reserved_concurrent_executions = number
      environment_variables          = map(string)
      vpc = object({
        subnet_ids         = list(string)
        security_group_ids = list(string)
      })
      role = object({
        path                 = string
        permissions_boundary = optional(string, null)
        principals = list(object({
          type        = string
          identifiers = list(string)
        }))
      })
    })
    runner = object({
      name_prefix = string
    })
    github = object({
      organization_runners = bool
      enterprise_server = object({
        url = optional(string, null)
      })
      user_agent = optional(string, null)
      app_parameters = object({
        key_base64 = object({
          name = string
          arn  = string
        })
        id = object({
          name = string
          arn  = string
        })
      })
    })
    queue = object({
      build = object({
        url = string
        arn = string
      })
      event_source_mapping = object({
        batch_size                         = number
        maximum_batching_window_in_seconds = number
      })
      encryption = object({
        sqs_managed_sse_enabled           = bool
        kms_master_key_id                 = optional(string, null)
        kms_data_key_reuse_period_seconds = optional(number, null)
      })
    })
    ssm = object({
      kms_key = optional(object({
        arn = string
      }), null)
    })
    observability = object({
      logs = object({
        level             = string
        retention_in_days = number
        kms_key_id        = optional(string, null)
        class             = string
      })
      tracing = object({
        mode                  = optional(string, null)
        capture_http_requests = bool
        capture_error         = bool
      })
      metrics = object({
        enable    = bool
        namespace = string
        metric = object({
          enable_github_app_rate_limit = bool
          enable_job_retry             = bool
        })
      })
    })
    tags = object({
      resources            = map(string)
      lambda               = map(string)
      log_group            = map(string)
      queue                = map(string)
      event_source_mapping = map(string)
    })
  })
| n/a | yes | + +## Outputs + +| Name | Description | +|------|-------------| +| [job\_retry\_check\_queue](#output\_job\_retry\_check\_queue) | Queue consumed by the job-retry Lambda. | +| [lambda](#output\_lambda) | Job-retry Lambda resources. | + diff --git a/modules/runner-stack/job-retry/iam-policies.tf b/modules/runner-stack/job-retry/iam-policies.tf new file mode 100644 index 0000000000..6f0a3b215e --- /dev/null +++ b/modules/runner-stack/job-retry/iam-policies.tf @@ -0,0 +1,104 @@ +# IAM policies attached to the job-retry Lambda role. +data "aws_iam_policy_document" "lambda_assume_role" { + statement { + actions = ["sts:AssumeRole"] + + principals { + type = "Service" + identifiers = ["lambda.amazonaws.com"] + } + + dynamic "principals" { + for_each = var.config.lambda.role.principals + + content { + type = principals.value.type + identifiers = principals.value.identifiers + } + } + } +} + +data "aws_iam_policy_document" "job_retry_logging" { + statement { + effect = "Allow" + + actions = [ + "logs:CreateLogStream", + "logs:PutLogEvents", + ] + + resources = ["${aws_cloudwatch_log_group.job_retry.arn}*"] + } +} + +data "aws_iam_policy_document" "lambda_xray" { + count = var.config.observability.tracing.mode != null ? 1 : 0 + + statement { + sid = "AllowXRay" + effect = "Allow" + actions = [ + "xray:BatchGetTraces", + "xray:GetTraceSummaries", + "xray:PutTelemetryRecords", + "xray:PutTraceSegments", + ] + resources = ["*"] + } +} + +data "aws_iam_policy_document" "job_retry" { + statement { + effect = "Allow" + + actions = [ + "ssm:GetParameter", + "ssm:GetParameters", + ] + + resources = [ + var.config.github.app_parameters.key_base64.arn, + var.config.github.app_parameters.id.arn, + ] + } + + statement { + effect = "Allow" + + actions = [ + "sqs:ReceiveMessage", + "sqs:GetQueueAttributes", + "sqs:DeleteMessage", + ] + + resources = [aws_sqs_queue.job_retry_check_queue.arn] + } + + statement { + effect = "Allow" + + actions = [ + "sqs:SendMessage", + "sqs:GetQueueAttributes", + ] + + resources = [var.config.queue.build.arn] + } + + dynamic "statement" { + for_each = var.config.ssm.kms_key == null ? [] : [var.config.ssm.kms_key] + + content { + effect = "Allow" + + actions = [ + "kms:Encrypt", + "kms:Decrypt", + "kms:GenerateDataKey", + ] + + resources = [statement.value.arn] + } + } +} diff --git a/modules/runner-stack/job-retry/job-retry.tf b/modules/runner-stack/job-retry/job-retry.tf new file mode 100644 index 0000000000..a8e873ed3c --- /dev/null +++ b/modules/runner-stack/job-retry/job-retry.tf @@ -0,0 +1,177 @@ +# Provider-neutral job-retry queue and Lambda resources. +locals { + name = "job-retry" + vpc_enabled = ( + length(var.config.lambda.vpc.subnet_ids) > 0 && + length(var.config.lambda.vpc.security_group_ids) > 0 + ) + + lambda_environment_variables = { + ENVIRONMENT = var.config.prefix + LOG_LEVEL = var.config.observability.logs.level + PREFIX = var.config.prefix + POWERTOOLS_LOGGER_LOG_EVENT = var.config.observability.logs.level == "debug" ? "true" : "false" + POWERTOOLS_SERVICE_NAME = local.name + POWERTOOLS_TRACE_ENABLED = var.config.observability.tracing.mode != null + POWERTOOLS_TRACER_CAPTURE_HTTPS_REQUESTS = var.config.observability.tracing.capture_http_requests + POWERTOOLS_TRACER_CAPTURE_ERROR = var.config.observability.tracing.capture_error + POWERTOOLS_METRICS_NAMESPACE = var.config.observability.metrics.namespace + } + + job_retry_environment_variables = { + ENABLE_ORGANIZATION_RUNNERS = var.config.github.organization_runners + ENABLE_METRIC_JOB_RETRY = var.config.observability.metrics.enable && var.config.observability.metrics.metric.enable_job_retry + ENABLE_METRIC_GITHUB_APP_RATE_LIMIT = var.config.observability.metrics.enable && var.config.observability.metrics.metric.enable_github_app_rate_limit + GHES_URL = var.config.github.enterprise_server.url + USER_AGENT = var.config.github.user_agent + JOB_QUEUE_SCALE_UP_URL = var.config.queue.build.url + PARAMETER_GITHUB_APP_ID_NAME = var.config.github.app_parameters.id.name + PARAMETER_GITHUB_APP_KEY_BASE64_NAME = var.config.github.app_parameters.key_base64.name + RUNNER_NAME_PREFIX = var.config.runner.name_prefix + } + + environment_variables = merge( + local.lambda_environment_variables, + var.config.lambda.environment_variables, + local.job_retry_environment_variables, + ) +} + +resource "aws_sqs_queue_policy" "job_retry_check_queue_policy" { + queue_url = aws_sqs_queue.job_retry_check_queue.id + policy = data.aws_iam_policy_document.deny_insecure_transport.json +} + +resource "aws_sqs_queue" "job_retry_check_queue" { + name = "${var.config.prefix}-job-retry" + visibility_timeout_seconds = var.config.lambda.timeout + + sqs_managed_sse_enabled = var.config.queue.encryption.sqs_managed_sse_enabled + kms_master_key_id = var.config.queue.encryption.kms_master_key_id + kms_data_key_reuse_period_seconds = var.config.queue.encryption.kms_data_key_reuse_period_seconds + + tags = var.config.tags.queue +} + +resource "aws_lambda_function" "job_retry" { + s3_bucket = var.config.lambda.artifact.s3.bucket + s3_key = var.config.lambda.artifact.s3.key + s3_object_version = var.config.lambda.artifact.s3.object_version + filename = var.config.lambda.artifact.s3.bucket == null ? var.config.lambda.artifact.zip : null + source_code_hash = var.config.lambda.artifact.s3.bucket == null ? filebase64sha256(var.config.lambda.artifact.zip) : null + function_name = "${var.config.prefix}-${local.name}" + role = aws_iam_role.job_retry.arn + handler = "index.jobRetryCheck" + runtime = var.config.lambda.runtime + timeout = var.config.lambda.timeout + memory_size = var.config.lambda.memory_size + reserved_concurrent_executions = var.config.lambda.reserved_concurrent_executions + architectures = [var.config.lambda.architecture] + + environment { + variables = local.environment_variables + } + + dynamic "vpc_config" { + for_each = local.vpc_enabled ? [true] : [] + + content { + security_group_ids = var.config.lambda.vpc.security_group_ids + subnet_ids = var.config.lambda.vpc.subnet_ids + } + } + + dynamic "tracing_config" { + for_each = var.config.observability.tracing.mode != null ? [true] : [] + + content { + mode = var.config.observability.tracing.mode + } + } + + tags = var.config.tags.lambda +} + +resource "aws_cloudwatch_log_group" "job_retry" { + name = "/aws/lambda/${aws_lambda_function.job_retry.function_name}" + retention_in_days = var.config.observability.logs.retention_in_days + kms_key_id = var.config.observability.logs.kms_key_id + log_group_class = var.config.observability.logs.class + tags = var.config.tags.log_group +} + +resource "aws_iam_role" "job_retry" { + name = "${substr("${var.config.prefix}-${local.name}", 0, 54)}-${substr(md5("${var.config.prefix}-${local.name}"), 0, 8)}" + assume_role_policy = data.aws_iam_policy_document.lambda_assume_role.json + path = var.config.lambda.role.path + permissions_boundary = var.config.lambda.role.permissions_boundary + tags = var.config.tags.resources +} + +resource "aws_iam_role_policy" "job_retry_logging" { + name = "logging-policy" + role = aws_iam_role.job_retry.name + policy = data.aws_iam_policy_document.job_retry_logging.json +} + +resource "aws_iam_role_policy_attachment" "job_retry_vpc_execution_role" { + count = local.vpc_enabled ? 1 : 0 + role = aws_iam_role.job_retry.name + policy_arn = "arn:${var.config.aws_partition}:iam::aws:policy/service-role/AWSLambdaVPCAccessExecutionRole" +} + +resource "aws_iam_role_policy" "job_retry_xray" { + count = var.config.observability.tracing.mode != null ? 1 : 0 + name = "xray-policy" + policy = data.aws_iam_policy_document.lambda_xray[0].json + role = aws_iam_role.job_retry.name +} + +resource "aws_lambda_event_source_mapping" "job_retry" { + event_source_arn = aws_sqs_queue.job_retry_check_queue.arn + function_name = aws_lambda_function.job_retry.arn + batch_size = var.config.queue.event_source_mapping.batch_size + maximum_batching_window_in_seconds = var.config.queue.event_source_mapping.maximum_batching_window_in_seconds + tags = var.config.tags.event_source_mapping +} + +resource "aws_lambda_permission" "job_retry" { + statement_id = "AllowExecutionFromSQS" + action = "lambda:InvokeFunction" + function_name = aws_lambda_function.job_retry.function_name + principal = "sqs.amazonaws.com" + source_arn = aws_sqs_queue.job_retry_check_queue.arn +} + +resource "aws_iam_role_policy" "job_retry" { + name = "job_retry-policy" + role = aws_iam_role.job_retry.name + policy = data.aws_iam_policy_document.job_retry.json +} + +data "aws_iam_policy_document" "deny_insecure_transport" { + statement { + sid = "DenyInsecureTransport" + + effect = "Deny" + + principals { + type = "AWS" + identifiers = ["*"] + } + + actions = [ + "sqs:*" + ] + + resources = [ + "*" + ] + + condition { + test = "Bool" + variable = "aws:SecureTransport" + values = ["false"] + } + } +} diff --git a/modules/runner-stack/job-retry/outputs.tf b/modules/runner-stack/job-retry/outputs.tf new file mode 100644 index 0000000000..4f08cc4498 --- /dev/null +++ b/modules/runner-stack/job-retry/outputs.tf @@ -0,0 +1,13 @@ +output "lambda" { + description = "Job-retry Lambda resources." + value = { + function = aws_lambda_function.job_retry + log_group = aws_cloudwatch_log_group.job_retry + role = aws_iam_role.job_retry + } +} + +output "job_retry_check_queue" { + description = "Queue consumed by the job-retry Lambda." + value = aws_sqs_queue.job_retry_check_queue +} diff --git a/modules/runner-stack/job-retry/tests/job-retry.tftest.hcl b/modules/runner-stack/job-retry/tests/job-retry.tftest.hcl new file mode 100644 index 0000000000..40cd279e30 --- /dev/null +++ b/modules/runner-stack/job-retry/tests/job-retry.tftest.hcl @@ -0,0 +1,260 @@ +mock_provider "aws" { + mock_data "aws_iam_policy_document" { + defaults = { + json = "{\"Version\":\"2012-10-17\",\"Statement\":[]}" + } + } + + mock_resource "aws_iam_role" { + defaults = { + arn = "arn:aws:iam::123456789012:role/job-retry-test" + } + } +} + +variables { + config = { + prefix = "job-retry-test" + aws_partition = "aws" + lambda = { + artifact = { + zip = "unused.zip" + s3 = { + bucket = "lambda-artifacts" + key = "job-retry.zip" + } + } + architecture = "arm64" + runtime = "nodejs24.x" + memory_size = 256 + timeout = 30 + reserved_concurrent_executions = 1 + environment_variables = { + CUSTOM_ENV = "preserved" + RUNNER_NAME_PREFIX = "caller-prefix-" + } + vpc = { + security_group_ids = ["sg-12345678"] + subnet_ids = ["subnet-12345678"] + } + role = { + path = "/job-retry-test/" + principals = [{ + type = "AWS" + identifiers = ["arn:aws:iam::123456789012:root"] + }] + } + } + runner = { + name_prefix = "required-prefix-" + } + github = { + organization_runners = false + enterprise_server = { + url = "" + } + user_agent = "job-retry-test" + app_parameters = { + key_base64 = { + name = "/github-runner/key-base64" + arn = "arn:aws:ssm:eu-west-1:123456789012:parameter/github-runner/key-base64" + } + id = { + name = "/github-runner/app-id" + arn = "arn:aws:ssm:eu-west-1:123456789012:parameter/github-runner/app-id" + } + } + } + queue = { + build = { + url = "https://sqs.eu-west-1.amazonaws.com/123456789012/build-queue" + arn = "arn:aws:sqs:eu-west-1:123456789012:build-queue" + } + event_source_mapping = { + batch_size = 10 + maximum_batching_window_in_seconds = 0 + } + encryption = { + sqs_managed_sse_enabled = true + } + } + ssm = { + kms_key = { + arn = "arn:aws:kms:eu-west-1:123456789012:key/job-retry-test" + } + } + observability = { + logs = { + level = "trace" + class = "INFREQUENT_ACCESS" + retention_in_days = 180 + } + tracing = { + mode = "Active" + capture_http_requests = false + capture_error = false + } + metrics = { + enable = false + namespace = "JobRetryTest" + metric = { + enable_github_app_rate_limit = true + enable_job_retry = true + } + } + } + tags = { + resources = { scope = "resources" } + lambda = { scope = "lambda" } + log_group = { scope = "log-group" } + queue = { scope = "queue" } + event_source_mapping = { scope = "event-source-mapping" } + } + } +} + +run "preserves_nested_job_retry_configuration" { + command = plan + + assert { + condition = output.lambda.function.environment[0].variables["CUSTOM_ENV"] == "preserved" + error_message = "Caller-provided job-retry environment variables must be preserved." + } + + assert { + condition = output.lambda.function.environment[0].variables["RUNNER_NAME_PREFIX"] == "required-prefix-" + error_message = "Required job-retry environment variables must override caller-provided values." + } + + assert { + condition = ( + toset(keys(output.lambda)) == toset(["function", "log_group", "role"]) + && output.lambda.function.s3_bucket == "lambda-artifacts" + && output.lambda.function.s3_key == "job-retry.zip" + && output.lambda.function.reserved_concurrent_executions == 1 + ) + error_message = "The nested Lambda configuration and direct resource output contract must be preserved." + } + + assert { + condition = ( + output.lambda.function.tags == tomap({ scope = "lambda" }) + && output.lambda.log_group.tags == tomap({ scope = "log-group" }) + && output.lambda.role.tags == tomap({ scope = "resources" }) + && output.job_retry_check_queue.tags == tomap({ scope = "queue" }) + && aws_lambda_event_source_mapping.job_retry.tags == tomap({ scope = "event-source-mapping" }) + ) + error_message = "Resolved nested tag maps must be applied to their owned resources." + } + + assert { + condition = ( + output.lambda.log_group.log_group_class == "INFREQUENT_ACCESS" + && length(data.aws_iam_policy_document.job_retry.statement) == 4 + && length(aws_lambda_function.job_retry.vpc_config) == 1 + && length(aws_iam_role_policy_attachment.job_retry_vpc_execution_role) == 1 + && length(aws_iam_role_policy.job_retry_xray) == 1 + && length(data.aws_iam_policy_document.lambda_assume_role.statement[0].principals) == 2 + ) + error_message = "Logging, KMS, complete VPC, tracing, and extra role-principal configuration must be preserved." + } +} + +run "does_not_enable_partial_vpc_configuration" { + command = plan + + variables { + config = { + prefix = "job-retry-test" + aws_partition = "aws" + lambda = { + artifact = { + zip = "unused.zip" + s3 = { + bucket = "lambda-artifacts" + key = "job-retry.zip" + } + } + architecture = "arm64" + runtime = "nodejs24.x" + memory_size = 256 + timeout = 30 + reserved_concurrent_executions = 1 + environment_variables = {} + vpc = { + security_group_ids = [] + subnet_ids = ["subnet-12345678"] + } + role = { + path = "/job-retry-test/" + principals = [] + } + } + runner = { + name_prefix = "" + } + github = { + organization_runners = false + enterprise_server = {} + app_parameters = { + key_base64 = { + name = "/github-runner/key-base64" + arn = "arn:aws:ssm:eu-west-1:123456789012:parameter/github-runner/key-base64" + } + id = { + name = "/github-runner/app-id" + arn = "arn:aws:ssm:eu-west-1:123456789012:parameter/github-runner/app-id" + } + } + } + queue = { + build = { + url = "https://sqs.eu-west-1.amazonaws.com/123456789012/build-queue" + arn = "arn:aws:sqs:eu-west-1:123456789012:build-queue" + } + event_source_mapping = { + batch_size = 10 + maximum_batching_window_in_seconds = 0 + } + encryption = { + sqs_managed_sse_enabled = true + } + } + ssm = {} + observability = { + logs = { + level = "info" + class = "STANDARD" + retention_in_days = 180 + } + tracing = { + capture_http_requests = false + capture_error = false + } + metrics = { + enable = false + namespace = "GitHub Runners" + metric = { + enable_github_app_rate_limit = true + enable_job_retry = true + } + } + } + tags = { + resources = {} + lambda = {} + log_group = {} + queue = {} + event_source_mapping = {} + } + } + } + + assert { + condition = ( + length(aws_lambda_function.job_retry.vpc_config) == 0 + && length(aws_iam_role_policy_attachment.job_retry_vpc_execution_role) == 0 + ) + error_message = "The VPC block and managed policy must both remain disabled until subnet and security-group lists are complete." + } +} diff --git a/modules/runner-stack/job-retry/variables.tf b/modules/runner-stack/job-retry/variables.tf new file mode 100644 index 0000000000..950bd6eb8b --- /dev/null +++ b/modules/runner-stack/job-retry/variables.tf @@ -0,0 +1,168 @@ +variable "config" { + description = <<-EOT + Provider-neutral job-retry configuration assembled by runner-stack. + + - `prefix`: Prefix used to name job-retry resources. + - `aws_partition`: AWS partition used to construct the Lambda VPC managed-policy ARN. + - `lambda.artifact.zip`: Resolved local control-plane archive. + - `lambda.artifact.s3.bucket`: Optional S3 bucket containing the Lambda archive. + - `lambda.artifact.s3.key`: Object key of the Lambda archive. + - `lambda.artifact.s3.object_version`: Optional object version of the Lambda archive. + - `lambda.runtime`: Runtime used by the job-retry Lambda. + - `lambda.architecture`: Instruction-set architecture used by the job-retry Lambda. + - `lambda.memory_size`: Memory allocated to the job-retry Lambda. + - `lambda.timeout`: Lambda timeout and retry-queue visibility timeout in seconds. + - `lambda.reserved_concurrent_executions`: Reserved concurrency for the Lambda. Use `-1` for unreserved concurrency. + - `lambda.environment_variables`: Additional Lambda environment variables. Required job-retry variables override matching keys. + - `lambda.vpc.subnet_ids`: Subnets used for Lambda VPC configuration. + - `lambda.vpc.security_group_ids`: Security groups used for Lambda VPC configuration. + - `lambda.role.path`: IAM path used for the job-retry Lambda role. + - `lambda.role.permissions_boundary`: Optional permissions boundary for the Lambda role. + - `lambda.role.principals`: Extra principals allowed to assume the Lambda role, for example during local testing. + - `runner.name_prefix`: Prefix used to identify runners belonging to this runner configuration. + - `github.organization_runners`: Enables organization runners. + - `github.enterprise_server.url`: Optional GitHub Enterprise Server URL. + - `github.user_agent`: Optional User-Agent sent to GitHub. + - `github.app_parameters.key_base64`: Name and ARN of the GitHub App private-key parameter. + - `github.app_parameters.id`: Name and ARN of the GitHub App ID parameter. + - `queue.build`: URL and ARN of the build queue to which retry messages are published. + - `queue.event_source_mapping.batch_size`: Maximum records delivered per job-retry invocation. + - `queue.event_source_mapping.maximum_batching_window_in_seconds`: Maximum event batching window. + - `queue.encryption`: Server-side encryption configuration for the retry queue. + - `ssm.kms_key`: Optional KMS key used by the job-retry IAM policy. + - `observability.logs`: Logging level, retention, encryption, and log-class configuration. + - `observability.tracing`: Lambda X-Ray and tracing-helper configuration. + - `observability.metrics`: Metrics enablement, namespace, and job-retry metric configuration. + - `tags.resources`: Tags for the job-retry Lambda role and component resources. + - `tags.lambda`: Tags for the job-retry Lambda function. + - `tags.log_group`: Tags for the job-retry log group. + - `tags.queue`: Tags for the retry queue. + - `tags.event_source_mapping`: Tags for the retry-queue event-source mapping. + EOT + + type = object({ + prefix = string + aws_partition = string + lambda = object({ + artifact = object({ + zip = string + s3 = object({ + bucket = optional(string, null) + key = optional(string, null) + object_version = optional(string, null) + }) + }) + runtime = string + architecture = string + memory_size = number + timeout = number + reserved_concurrent_executions = number + environment_variables = map(string) + vpc = object({ + subnet_ids = list(string) + security_group_ids = list(string) + }) + role = object({ + path = string + permissions_boundary = optional(string, null) + principals = list(object({ + type = string + identifiers = list(string) + })) + }) + }) + runner = object({ + name_prefix = string + }) + github = object({ + organization_runners = bool + enterprise_server = object({ + url = optional(string, null) + }) + user_agent = optional(string, null) + app_parameters = object({ + key_base64 = object({ + name = string + arn = string + }) + id = object({ + name = string + arn = string + }) + }) + }) + queue = object({ + build = object({ + url = string + arn = string + }) + event_source_mapping = object({ + batch_size = number + maximum_batching_window_in_seconds = number + }) + encryption = object({ + sqs_managed_sse_enabled = bool + kms_master_key_id = optional(string, null) + kms_data_key_reuse_period_seconds = optional(number, null) + }) + }) + ssm = object({ + kms_key = optional(object({ + arn = string + }), null) + }) + observability = object({ + logs = object({ + level = string + retention_in_days = number + kms_key_id = optional(string, null) + class = string + }) + tracing = object({ + mode = optional(string, null) + capture_http_requests = bool + capture_error = bool + }) + metrics = object({ + enable = bool + namespace = string + metric = object({ + enable_github_app_rate_limit = bool + enable_job_retry = bool + }) + }) + }) + tags = object({ + resources = map(string) + lambda = map(string) + log_group = map(string) + queue = map(string) + event_source_mapping = map(string) + }) + }) + + nullable = false + + validation { + condition = contains(["arm64", "x86_64"], var.config.lambda.architecture) + error_message = "config.lambda.architecture must be arm64 or x86_64." + } + + validation { + condition = contains([ + "silly", + "trace", + "debug", + "info", + "warn", + "error", + "fatal", + ], var.config.observability.logs.level) + error_message = "config.observability.logs.level must be one of silly, trace, debug, info, warn, error, or fatal." + } + + validation { + condition = length(var.config.prefix) + length("job-retry") <= 63 + error_message = "The length of config.prefix plus job-retry must be less than or equal to 63." + } +} diff --git a/modules/runner-stack/job-retry/versions.tf b/modules/runner-stack/job-retry/versions.tf new file mode 100644 index 0000000000..42a40b33fd --- /dev/null +++ b/modules/runner-stack/job-retry/versions.tf @@ -0,0 +1,10 @@ +terraform { + required_version = ">= 1.3.0" + + required_providers { + aws = { + source = "hashicorp/aws" + version = ">= 6.21" + } + } +} diff --git a/modules/runner-stack/microvm.tf b/modules/runner-stack/microvm.tf new file mode 100644 index 0000000000..f768d5092c --- /dev/null +++ b/modules/runner-stack/microvm.tf @@ -0,0 +1,27 @@ +module "microvm_trust_policy" { + count = local.provider_type == "microvm" ? 1 : 0 + source = "../compute-providers/microvm/trust-policy" + + additional_trust_policy_json = var.runner.iam.additional_trust_policy_json +} + +module "microvm" { + count = local.provider_type == "microvm" ? 1 : 0 + source = "../compute-providers/microvm" + + aws_partition = var.aws_partition + aws_region = var.aws_region + prefix = var.prefix + tags = var.tags + + config = var.compute_provider.microvm + runner = merge(var.runner, { + iam = merge(var.runner.iam, { + role = local.runner_role + managed_policy_arns = local.common_runner_managed_policy_arns + }) + }) + github = var.github + ssm = var.ssm + observability = var.observability +} diff --git a/modules/runner-stack/outputs.tf b/modules/runner-stack/outputs.tf new file mode 100644 index 0000000000..b7b03822cb --- /dev/null +++ b/modules/runner-stack/outputs.tf @@ -0,0 +1,28 @@ +output "runner" { + description = "Common runner resources. The role is null when an external runner role is used." + value = { + role = one(aws_iam_role.runner[*]) + } +} + +output "scale_up" { + description = "Scale-up control-plane resources." + value = module.scale_runners.scale_up +} + +output "scale_down" { + description = "Scale-down control-plane resources." + value = module.scale_runners.scale_down +} + +output "pool" { + description = "Scheduled pool resources. Null when no pool configuration is supplied." + value = one(module.pool[*].pool) +} + +output "provider" { + description = "Provider-specific resources grouped under the selected provider key." + value = { + (local.provider_type) = local.provider_contract.resources + } +} diff --git a/modules/runner-stack/pool.tf b/modules/runner-stack/pool.tf new file mode 100644 index 0000000000..0e075b1378 --- /dev/null +++ b/modules/runner-stack/pool.tf @@ -0,0 +1,64 @@ +module "pool" { + count = length(var.pool.config) == 0 ? 0 : 1 + + source = "./pool" + + config = { + prefix = var.prefix + ghes = { + ssl_verify = var.github.enterprise_server.ssl_verify + url = var.github.enterprise_server.url + } + user_agent = var.github.user_agent + github_app_parameters = var.github.app_parameters + runners_maximum_count = var.runner.maximum_count + kms_key = local.kms_key + lambda = { + log_level = var.observability.logs.level + logging_retention_in_days = var.observability.logs.retention_in_days + logging_kms_key_id = var.observability.logs.kms_key_id + log_class = var.observability.logs.class + reserved_concurrent_executions = var.pool.lambda.reserved_concurrent_executions + s3_bucket = var.lambda.s3.bucket + s3_key = var.lambda.s3.key + s3_object_version = var.lambda.s3.object_version + security_group_ids = var.lambda.security_group_ids + subnet_ids = var.lambda.subnet_ids + architecture = var.lambda.architecture + memory_size = var.pool.lambda.memory_size + runtime = var.lambda.runtime + timeout = var.pool.lambda.timeout + zip = local.lambda_zip + parameter_store_tags = local.parameter_store_tags + } + pool = var.pool.config + include_busy_runners = var.pool.include_busy_runners + role_path = local.lambda_role_path + role_permissions_boundary = var.lambda.role.permissions_boundary + runner = { + disable_runner_autoupdate = var.runner.auto_update_disabled + ephemeral = var.runner.ephemeral + enable_jit_config = var.runner.jit_config_enabled + labels = var.runner.labels + group_name = var.runner.group_name + name_prefix = var.runner.name_prefix + pool_owner = var.pool.runner_owner + } + ssm_token_path = "${var.ssm.paths.root}/${var.ssm.paths.tokens}" + ssm_config_path = "${var.ssm.paths.root}/${var.ssm.paths.config}" + tags = local.pool_tags + lambda_tags = local.pool_lambda_tags + log_group_tags = local.pool_log_tags + arn_ssm_parameters_path_config = local.arn_ssm_parameters_path_config + } + + aws_partition = var.aws_partition + tracing_config = var.observability.tracing + runner_provider = { + type = local.provider_type + environment_variables = local.provider_contract.environment_variables.pool + iam_policy_json = local.provider_contract.policies.pool.iam_policy_json + managed_policy_enabled = local.provider_contract.policies.pool.managed_policy_enabled + managed_policy_arn = local.provider_contract.policies.pool.managed_policy_arn + } +} diff --git a/modules/runner-stack/pool/README.md b/modules/runner-stack/pool/README.md new file mode 100644 index 0000000000..64553579ff --- /dev/null +++ b/modules/runner-stack/pool/README.md @@ -0,0 +1,64 @@ +# Pool module + +This module creates the AWS resources required to maintain a pool of runners. However terraform modules are always exposed and theoretically can be used anywhere. This module is seen as a strict inner module. + +## Why a submodule for the pool + +The pool is an opt-in feature. To be able to use the count on a module level to avoid counts per resources a module is created. All inputs of the module are already defined on a higher level. See the mapping of the variables in [`pool.tf`](../pool.tf) + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 1.3.0 | +| [aws](#requirement\_aws) | >= 6.21 | + +## Providers + +| Name | Version | +|------|---------| +| [aws](#provider\_aws) | >= 6.21 | + +## Modules + +No modules. + +## Resources + +| Name | Type | +|------|------| +| [aws_cloudwatch_log_group.pool](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/cloudwatch_log_group) | resource | +| [aws_iam_role.pool](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role) | resource | +| [aws_iam_role.scheduler](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role) | resource | +| [aws_iam_role_policy.pool](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy) | resource | +| [aws_iam_role_policy.pool_logging](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy) | resource | +| [aws_iam_role_policy.pool_xray](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy) | resource | +| [aws_iam_role_policy.scheduler](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy) | resource | +| [aws_iam_role_policy_attachment.pool_vpc_execution_role](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy_attachment) | resource | +| [aws_iam_role_policy_attachment.provider](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy_attachment) | resource | +| [aws_lambda_function.pool](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/lambda_function) | resource | +| [aws_scheduler_schedule.pool](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/scheduler_schedule) | resource | +| [aws_scheduler_schedule_group.pool](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/scheduler_schedule_group) | resource | +| [aws_iam_policy_document.lambda_assume_role_policy](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source | +| [aws_iam_policy_document.lambda_xray](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source | +| [aws_iam_policy_document.pool](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source | +| [aws_iam_policy_document.pool_common](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source | +| [aws_iam_policy_document.pool_logging](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source | +| [aws_iam_policy_document.scheduler](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source | +| [aws_iam_policy_document.scheduler_assume](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [aws\_partition](#input\_aws\_partition) | (optional) partition for the arn if not 'aws' | `string` | `"aws"` | no | +| [config](#input\_config) | Configuration passed from the runner stack to the pool Lambda and scheduler.

- `lambda`: Pool Lambda runtime and deployment configuration.
- `lambda.log_level`: Logging level used by the pool Lambda.
- `lambda.logging_retention_in_days`: Number of days to retain events in the pool Lambda log group.
- `lambda.logging_kms_key_id`: KMS key ID used to encrypt the pool Lambda log group.
- `lambda.log_class`: CloudWatch Logs class for the pool Lambda log group.
- `lambda.reserved_concurrent_executions`: Reserved concurrency for the pool Lambda. Use -1 for no reservation.
- `lambda.s3_bucket`: S3 bucket containing the pool Lambda deployment package.
- `lambda.s3_key`: S3 key of the pool Lambda deployment package.
- `lambda.s3_object_version`: S3 object version of the pool Lambda deployment package.
- `lambda.security_group_ids`: Security group IDs associated with the pool Lambda.
- `lambda.runtime`: AWS Lambda runtime used by the pool Lambda.
- `lambda.architecture`: AWS Lambda architecture used by the pool Lambda.
- `lambda.memory_size`: Memory allocated to the pool Lambda in MB.
- `lambda.timeout`: Pool Lambda timeout in seconds.
- `lambda.zip`: Local path to the pool Lambda deployment package when S3 is not used.
- `lambda.subnet_ids`: Subnet IDs in which the pool Lambda runs.
- `lambda.parameter_store_tags`: JSON-encoded tags supplied to the pool Lambda for SSM parameters it creates.
- `tags`: Common tags added to pool resources.
- `ghes`: GitHub Enterprise Server connection configuration.
- `ghes.url`: GitHub Enterprise Server URL; null when using public GitHub.
- `ghes.ssl_verify`: Whether the pool Lambda verifies the GitHub Enterprise Server TLS certificate.
- `github_app_parameters`: SSM parameter metadata for GitHub App credentials.
- `github_app_parameters.key_base64`: Metadata for the SSM parameter containing the base64-encoded GitHub App private key.
- `github_app_parameters.key_base64.name`: Name of the private-key parameter supplied to the pool Lambda.
- `github_app_parameters.key_base64.arn`: ARN of the private-key parameter used by the pool IAM policy.
- `github_app_parameters.id`: Metadata for the SSM parameter containing the GitHub App ID.
- `github_app_parameters.id.name`: Name of the App-ID parameter supplied to the pool Lambda.
- `github_app_parameters.id.arn`: ARN of the App-ID parameter used by the pool IAM policy.
- `runner`: Runner registration configuration used by the pool Lambda.
- `runner.disable_runner_autoupdate`: Whether GitHub runner automatic updates are disabled.
- `runner.ephemeral`: Whether runners register as ephemeral runners.
- `runner.enable_jit_config`: Whether runners use just-in-time registration configuration.
- `runner.labels`: Labels assigned to runners created by the pool Lambda.
- `runner.group_name`: GitHub runner group assigned to runners created by the pool Lambda.
- `runner.name_prefix`: Prefix used for runner names.
- `runner.pool_owner`: GitHub organization or repository that owns the runner pool.
- `runners_maximum_count`: Maximum number of runners that the pool Lambda may create.
- `prefix`: Prefix used to name pool resources.
- `pool`: Scheduled pool targets.
- `pool[*].schedule_expression`: EventBridge Scheduler expression for a pool target.
- `pool[*].schedule_expression_timezone`: Time zone used to evaluate the schedule expression.
- `pool[*].size`: Desired runner count for the scheduled pool target.
- `include_busy_runners`: Whether busy runners count toward the desired pool size.
- `role_permissions_boundary`: Permissions boundary applied to IAM roles created for the pool.
- `kms_key`: Optional customer-managed KMS key that the pool Lambda may use to decrypt encrypted parameters. Object presence controls whether the KMS statement exists.
- `kms_key.arn`: ARN of the customer-managed KMS key. The ARN may be unknown until apply.
- `role_path`: IAM path applied to roles created for the pool.
- `ssm_token_path`: SSM path under which runner registration tokens are stored.
- `ssm_config_path`: SSM path under which runner configuration is stored.
- `arn_ssm_parameters_path_config`: ARN matching the runner configuration SSM path.
- `lambda_tags`: Tags added specifically to the pool Lambda function, overriding common tags with the same key.
- `log_group_tags`: Tags added specifically to the pool Lambda log group, overriding common tags with the same key.
- `user_agent`: User-Agent header used for GitHub API requests. | 
object({
    lambda = object({
      log_level                      = string
      logging_retention_in_days      = number
      logging_kms_key_id             = string
      log_class                      = string
      reserved_concurrent_executions = number
      s3_bucket                      = string
      s3_key                         = string
      s3_object_version              = string
      security_group_ids             = list(string)
      runtime                        = string
      architecture                   = string
      memory_size                    = number
      timeout                        = number
      zip                            = string
      subnet_ids                     = list(string)
      parameter_store_tags           = string
    })
    tags = map(string)
    ghes = object({
      url        = string
      ssl_verify = string
    })
    github_app_parameters = object({
      key_base64 = map(string)
      id         = map(string)
    })
    runner = object({
      disable_runner_autoupdate = bool
      ephemeral                 = bool
      enable_jit_config         = bool
      labels                    = list(string)
      group_name                = string
      name_prefix               = string
      pool_owner                = string
    })
    runners_maximum_count = number
    prefix                = string
    pool = list(object({
      schedule_expression          = string
      schedule_expression_timezone = string
      size                         = number
    }))
    include_busy_runners      = bool
    role_permissions_boundary = string
    kms_key = optional(object({
      arn = string
    }), null)
    role_path                      = string
    ssm_token_path                 = string
    ssm_config_path                = string
    arn_ssm_parameters_path_config = string
    lambda_tags                    = map(string)
    log_group_tags                 = optional(map(string), {})
    user_agent                     = string
  })
| n/a | yes | +| [runner\_provider](#input\_runner\_provider) | Compute provider integration used by the pool Lambda.

- `type`: Compute provider type passed to scheduled pool invocations.
- `environment_variables`: Provider-specific environment variables added to the pool Lambda.
- `iam_policy_json`: Provider-specific IAM policy document merged into the pool Lambda policy.
- `managed_policy_enabled`: Whether to attach a provider-specific managed IAM policy to the pool Lambda role.
- `managed_policy_arn`: ARN of the provider-specific managed IAM policy to attach when enabled. | 
object({
    type                   = string
    environment_variables  = map(string)
    iam_policy_json        = string
    managed_policy_enabled = bool
    managed_policy_arn     = optional(string, null)
  })
| n/a | yes | +| [tracing\_config](#input\_tracing\_config) | Tracing configuration for the pool Lambda.

- `mode`: AWS X-Ray tracing mode. A null value disables tracing.
- `capture_http_requests`: Whether Powertools tracing captures outgoing HTTP requests.
- `capture_error`: Whether Powertools tracing captures errors as tracing metadata. | 
object({
    mode                  = optional(string, null)
    capture_http_requests = optional(bool, false)
    capture_error         = optional(bool, false)
  })
| `{}` | no | + +## Outputs + +| Name | Description | +|------|-------------| +| [pool](#output\_pool) | Scheduled pool Lambda resources. | + diff --git a/modules/runner-stack/pool/iam-policies.tf b/modules/runner-stack/pool/iam-policies.tf new file mode 100644 index 0000000000..13690cce09 --- /dev/null +++ b/modules/runner-stack/pool/iam-policies.tf @@ -0,0 +1,66 @@ +# IAM policies attached to the pool Lambda role. +data "aws_iam_policy_document" "pool_common" { + statement { + effect = "Allow" + + actions = [ + "ssm:AddTagsToResource", + "ssm:PutParameter", + ] + + resources = ["*"] + } + + statement { + effect = "Allow" + + actions = [ + "ssm:GetParameter", + "ssm:GetParameters", + "ssm:GetParametersByPath", + ] + + resources = [ + var.config.arn_ssm_parameters_path_config, + "${var.config.arn_ssm_parameters_path_config}/*", + ] + } + + statement { + effect = "Allow" + + actions = [ + "ssm:GetParameter", + "ssm:GetParameters", + ] + + resources = [ + var.config.github_app_parameters.key_base64.arn, + var.config.github_app_parameters.id.arn, + ] + } + + dynamic "statement" { + for_each = var.config.kms_key == null ? [] : [var.config.kms_key] + + content { + effect = "Allow" + + actions = ["kms:Decrypt"] + resources = [statement.value.arn] + } + } +} + +data "aws_iam_policy_document" "pool_logging" { + statement { + effect = "Allow" + + actions = [ + "logs:CreateLogStream", + "logs:PutLogEvents", + ] + + resources = ["${aws_cloudwatch_log_group.pool.arn}*"] + } +} diff --git a/modules/runner-stack/pool/outputs.tf b/modules/runner-stack/pool/outputs.tf new file mode 100644 index 0000000000..cfc429ecce --- /dev/null +++ b/modules/runner-stack/pool/outputs.tf @@ -0,0 +1,8 @@ +output "pool" { + description = "Scheduled pool Lambda resources." + value = { + lambda = aws_lambda_function.pool + log_group = aws_cloudwatch_log_group.pool + role = aws_iam_role.pool + } +} diff --git a/modules/runner-stack/pool/pool.tf b/modules/runner-stack/pool/pool.tf new file mode 100644 index 0000000000..5e95c897ca --- /dev/null +++ b/modules/runner-stack/pool/pool.tf @@ -0,0 +1,225 @@ +# Provider-neutral pool Lambda and scheduler wiring. +locals { + pool_name_prefix = ( + length("${var.config.prefix}-pool") <= 38 + ? "${var.config.prefix}-pool" + : "${substr("${var.config.prefix}-pool", 0, 29)}-${substr(md5("${var.config.prefix}-pool"), 0, 8)}" + ) + + common_environment_variables = { + DISABLE_RUNNER_AUTOUPDATE = var.config.runner.disable_runner_autoupdate + ENABLE_EPHEMERAL_RUNNERS = var.config.runner.ephemeral + ENABLE_JIT_CONFIG = var.config.runner.enable_jit_config + ENVIRONMENT = var.config.prefix + GHES_URL = var.config.ghes.url + USER_AGENT = var.config.user_agent + LOG_LEVEL = upper(var.config.lambda.log_level) + NODE_TLS_REJECT_UNAUTHORIZED = var.config.ghes.url != null && !var.config.ghes.ssl_verify ? 0 : 1 + PARAMETER_GITHUB_APP_ID_NAME = var.config.github_app_parameters.id.name + PARAMETER_GITHUB_APP_KEY_BASE64_NAME = var.config.github_app_parameters.key_base64.name + POWERTOOLS_LOGGER_LOG_EVENT = var.config.lambda.log_level == "debug" ? "true" : "false" + RUNNER_LABELS = lower(join(",", var.config.runner.labels)) + RUNNER_GROUP_NAME = var.config.runner.group_name + RUNNER_NAME_PREFIX = var.config.runner.name_prefix + RUNNER_OWNER = var.config.runner.pool_owner + RUNNERS_MAXIMUM_COUNT = var.config.runners_maximum_count + SSM_TOKEN_PATH = var.config.ssm_token_path + SSM_CONFIG_PATH = var.config.ssm_config_path + POWERTOOLS_SERVICE_NAME = "${var.config.prefix}-pool" + POWERTOOLS_TRACE_ENABLED = var.tracing_config.mode != null ? true : false + POWERTOOLS_TRACER_CAPTURE_HTTPS_REQUESTS = var.tracing_config.capture_http_requests + POWERTOOLS_TRACER_CAPTURE_ERROR = var.tracing_config.capture_error + SSM_PARAMETER_STORE_TAGS = var.config.lambda.parameter_store_tags + INCLUDE_BUSY_RUNNERS = var.config.include_busy_runners + } +} + +resource "aws_lambda_function" "pool" { + + s3_bucket = var.config.lambda.s3_bucket != null ? var.config.lambda.s3_bucket : null + s3_key = var.config.lambda.s3_key != null ? var.config.lambda.s3_key : null + s3_object_version = var.config.lambda.s3_object_version != null ? var.config.lambda.s3_object_version : null + filename = var.config.lambda.s3_bucket == null ? var.config.lambda.zip : null + source_code_hash = var.config.lambda.s3_bucket == null ? filebase64sha256(var.config.lambda.zip) : null + function_name = "${var.config.prefix}-pool" + role = aws_iam_role.pool.arn + handler = "index.adjustPool" + architectures = [var.config.lambda.architecture] + runtime = var.config.lambda.runtime + timeout = var.config.lambda.timeout + reserved_concurrent_executions = var.config.lambda.reserved_concurrent_executions + memory_size = var.config.lambda.memory_size + tags = merge(var.config.tags, var.config.lambda_tags) + + environment { + variables = merge(var.runner_provider.environment_variables, local.common_environment_variables) + } + + dynamic "vpc_config" { + for_each = var.config.lambda.subnet_ids != null && var.config.lambda.security_group_ids != null ? [true] : [] + content { + security_group_ids = var.config.lambda.security_group_ids + subnet_ids = var.config.lambda.subnet_ids + } + } + + dynamic "tracing_config" { + for_each = var.tracing_config.mode != null ? [true] : [] + content { + mode = var.tracing_config.mode + } + } +} + +resource "aws_cloudwatch_log_group" "pool" { + name = "/aws/lambda/${aws_lambda_function.pool.function_name}" + retention_in_days = var.config.lambda.logging_retention_in_days + kms_key_id = var.config.lambda.logging_kms_key_id + log_group_class = var.config.lambda.log_class + tags = merge(var.config.tags, var.config.log_group_tags) +} + +resource "aws_iam_role" "pool" { + name = "${substr("${var.config.prefix}-pool-lambda", 0, 54)}-${substr(md5("${var.config.prefix}-pool-lambda"), 0, 8)}" + assume_role_policy = data.aws_iam_policy_document.lambda_assume_role_policy.json + path = var.config.role_path + permissions_boundary = var.config.role_permissions_boundary + tags = var.config.tags +} + +resource "aws_iam_role_policy" "pool" { + name = "pool-policy" + role = aws_iam_role.pool.name + policy = data.aws_iam_policy_document.pool.json +} + +data "aws_iam_policy_document" "pool" { + source_policy_documents = [ + data.aws_iam_policy_document.pool_common.json, + var.runner_provider.iam_policy_json, + ] +} + +resource "aws_iam_role_policy" "pool_logging" { + name = "logging-policy" + role = aws_iam_role.pool.name + policy = data.aws_iam_policy_document.pool_logging.json +} + +resource "aws_iam_role_policy_attachment" "pool_vpc_execution_role" { + count = length(var.config.lambda.subnet_ids) > 0 ? 1 : 0 + role = aws_iam_role.pool.name + policy_arn = "arn:${var.aws_partition}:iam::aws:policy/service-role/AWSLambdaVPCAccessExecutionRole" +} + +data "aws_iam_policy_document" "lambda_assume_role_policy" { + statement { + actions = ["sts:AssumeRole"] + + principals { + type = "Service" + identifiers = ["lambda.amazonaws.com"] + } + } +} + +resource "aws_iam_role_policy_attachment" "provider" { + count = var.runner_provider.managed_policy_enabled ? 1 : 0 + role = aws_iam_role.pool.name + policy_arn = var.runner_provider.managed_policy_arn +} + +# lambda xray policy +data "aws_iam_policy_document" "lambda_xray" { + count = var.tracing_config.mode != null ? 1 : 0 + statement { + actions = [ + "xray:BatchGetTraces", + "xray:GetTraceSummaries", + "xray:PutTelemetryRecords", + "xray:PutTraceSegments" + ] + effect = "Allow" + resources = [ + "*" + ] + sid = "AllowXRay" + } +} + +resource "aws_iam_role_policy" "pool_xray" { + count = var.tracing_config.mode != null ? 1 : 0 + name = "xray-policy" + policy = data.aws_iam_policy_document.lambda_xray[0].json + role = aws_iam_role.pool.name +} + +resource "aws_scheduler_schedule_group" "pool" { + name_prefix = local.pool_name_prefix + + tags = var.config.tags +} + +data "aws_iam_policy_document" "scheduler_assume" { + statement { + sid = "ScheduleGroupAssumeRole" + actions = ["sts:AssumeRole"] + principals { + type = "Service" + identifiers = ["scheduler.amazonaws.com"] + } + + condition { + test = "StringEquals" + variable = "aws:SourceArn" + values = [aws_scheduler_schedule_group.pool.arn] + } + } +} + +data "aws_iam_policy_document" "scheduler" { + statement { + sid = "InvokePoolLambda" + actions = ["lambda:InvokeFunction"] + resources = [aws_lambda_function.pool.arn] + } +} + +resource "aws_iam_role" "scheduler" { + name_prefix = local.pool_name_prefix + + path = var.config.role_path + permissions_boundary = var.config.role_permissions_boundary + + assume_role_policy = data.aws_iam_policy_document.scheduler_assume.json + tags = var.config.tags +} + +resource "aws_iam_role_policy" "scheduler" { + name = "terraform" + role = aws_iam_role.scheduler.name + policy = data.aws_iam_policy_document.scheduler.json +} + +resource "aws_scheduler_schedule" "pool" { + for_each = { for i, v in var.config.pool : i => v } + + name = "${var.config.prefix}-pool-${each.key}-rule" + group_name = aws_scheduler_schedule_group.pool.name + + flexible_time_window { + mode = "OFF" + } + + schedule_expression = each.value.schedule_expression + schedule_expression_timezone = each.value.schedule_expression_timezone + + target { + arn = aws_lambda_function.pool.arn + role_arn = aws_iam_role.scheduler.arn + input = jsonencode({ + poolSize = each.value.size + type = var.runner_provider.type + }) + } +} diff --git a/modules/runner-stack/pool/tests/provider.tftest.hcl b/modules/runner-stack/pool/tests/provider.tftest.hcl new file mode 100644 index 0000000000..b352a03c26 --- /dev/null +++ b/modules/runner-stack/pool/tests/provider.tftest.hcl @@ -0,0 +1,135 @@ +mock_provider "aws" { + mock_data "aws_iam_policy_document" { + defaults = { + json = "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Effect\":\"Allow\",\"Action\":\"logs:CreateLogStream\",\"Resource\":\"*\"}]}" + } + } +} + +variables { + config = { + lambda = { + log_level = "info" + logging_retention_in_days = 14 + logging_kms_key_id = null + log_class = "STANDARD" + reserved_concurrent_executions = 1 + s3_bucket = "lambda-artifacts" + s3_key = "runners.zip" + s3_object_version = null + security_group_ids = [] + runtime = "nodejs24.x" + architecture = "arm64" + memory_size = 256 + timeout = 60 + zip = "runners.zip" + subnet_ids = [] + parameter_store_tags = "{}" + } + tags = { + Environment = "pool-test" + } + ghes = { + url = null + ssl_verify = true + } + github_app_parameters = { + key_base64 = { + name = "/github-runner/key-base64" + arn = "arn:aws:ssm:eu-west-1:123456789012:parameter/github-runner/key-base64" + } + id = { + name = "/github-runner/app-id" + arn = "arn:aws:ssm:eu-west-1:123456789012:parameter/github-runner/app-id" + } + } + runner = { + disable_runner_autoupdate = false + ephemeral = true + enable_jit_config = true + labels = ["self-hosted", "microvm"] + group_name = "default" + name_prefix = "microvm" + pool_owner = "example" + } + runners_maximum_count = 10 + prefix = "pool-test" + pool = [{ + schedule_expression = "cron(0 8 * * ? *)" + schedule_expression_timezone = "UTC" + size = 2 + }] + include_busy_runners = false + role_permissions_boundary = null + kms_key = { + arn = "arn:aws:kms:eu-west-1:123456789012:key/pool-test" + } + role_path = "/" + ssm_token_path = "/github-runner/tokens" + ssm_config_path = "/github-runner/config" + arn_ssm_parameters_path_config = "arn:aws:ssm:eu-west-1:123456789012:parameter/github-runner/config" + lambda_tags = {} + user_agent = "terraform-aws-github-runner" + } + + runner_provider = { + type = "microvm" + environment_variables = { + MICROVM_CLUSTER = "runner-cluster" + } + iam_policy_json = jsonencode({ + Version = "2012-10-17" + Statement = [{ + Effect = "Allow" + Action = ["microvm:CreateRunner"] + Resource = ["*"] + }] + }) + managed_policy_enabled = true + managed_policy_arn = "arn:aws:iam::123456789012:policy/microvm-pool" + } +} + +run "provider_supplies_only_compute_specific_pool_configuration" { + command = plan + + assert { + condition = toset(keys(output.pool)) == toset(["lambda", "log_group", "role"]) + error_message = "The pool module must expose its resources through one nested output." + } + + assert { + condition = aws_lambda_function.pool.environment[0].variables["RUNNER_OWNER"] == "example" + error_message = "The pool module must continue to assemble common runner environment variables." + } + + assert { + condition = aws_lambda_function.pool.environment[0].variables["MICROVM_CLUSTER"] == "runner-cluster" + error_message = "The pool module must merge compute-provider environment variables into the Lambda environment." + } + + assert { + condition = !contains(keys(aws_lambda_function.pool.environment[0].variables), "AMI_ID_SSM_PARAMETER_NAME") + error_message = "The common pool module must not add EC2-specific environment variables." + } + + assert { + condition = jsondecode(aws_scheduler_schedule.pool["0"].target[0].input).type == "microvm" + error_message = "The pool scheduler payload must select the configured compute provider." + } + + assert { + condition = length(data.aws_iam_policy_document.pool.source_policy_documents) == 2 + error_message = "The pool role policy must merge the common and compute-provider policy documents." + } + + assert { + condition = length(data.aws_iam_policy_document.pool_common.statement) == 4 + error_message = "A present KMS key object must add the pool KMS policy statement." + } + + assert { + condition = length(aws_iam_role_policy_attachment.provider) == 1 + error_message = "The optional compute-provider managed policy must be attached to the pool role." + } +} diff --git a/modules/runner-stack/pool/variables.tf b/modules/runner-stack/pool/variables.tf new file mode 100644 index 0000000000..63cbf90e30 --- /dev/null +++ b/modules/runner-stack/pool/variables.tf @@ -0,0 +1,172 @@ +variable "config" { + description = <<-EOF + Configuration passed from the runner stack to the pool Lambda and scheduler. + + - `lambda`: Pool Lambda runtime and deployment configuration. + - `lambda.log_level`: Logging level used by the pool Lambda. + - `lambda.logging_retention_in_days`: Number of days to retain events in the pool Lambda log group. + - `lambda.logging_kms_key_id`: KMS key ID used to encrypt the pool Lambda log group. + - `lambda.log_class`: CloudWatch Logs class for the pool Lambda log group. + - `lambda.reserved_concurrent_executions`: Reserved concurrency for the pool Lambda. Use -1 for no reservation. + - `lambda.s3_bucket`: S3 bucket containing the pool Lambda deployment package. + - `lambda.s3_key`: S3 key of the pool Lambda deployment package. + - `lambda.s3_object_version`: S3 object version of the pool Lambda deployment package. + - `lambda.security_group_ids`: Security group IDs associated with the pool Lambda. + - `lambda.runtime`: AWS Lambda runtime used by the pool Lambda. + - `lambda.architecture`: AWS Lambda architecture used by the pool Lambda. + - `lambda.memory_size`: Memory allocated to the pool Lambda in MB. + - `lambda.timeout`: Pool Lambda timeout in seconds. + - `lambda.zip`: Local path to the pool Lambda deployment package when S3 is not used. + - `lambda.subnet_ids`: Subnet IDs in which the pool Lambda runs. + - `lambda.parameter_store_tags`: JSON-encoded tags supplied to the pool Lambda for SSM parameters it creates. + - `tags`: Common tags added to pool resources. + - `ghes`: GitHub Enterprise Server connection configuration. + - `ghes.url`: GitHub Enterprise Server URL; null when using public GitHub. + - `ghes.ssl_verify`: Whether the pool Lambda verifies the GitHub Enterprise Server TLS certificate. + - `github_app_parameters`: SSM parameter metadata for GitHub App credentials. + - `github_app_parameters.key_base64`: Metadata for the SSM parameter containing the base64-encoded GitHub App private key. + - `github_app_parameters.key_base64.name`: Name of the private-key parameter supplied to the pool Lambda. + - `github_app_parameters.key_base64.arn`: ARN of the private-key parameter used by the pool IAM policy. + - `github_app_parameters.id`: Metadata for the SSM parameter containing the GitHub App ID. + - `github_app_parameters.id.name`: Name of the App-ID parameter supplied to the pool Lambda. + - `github_app_parameters.id.arn`: ARN of the App-ID parameter used by the pool IAM policy. + - `runner`: Runner registration configuration used by the pool Lambda. + - `runner.disable_runner_autoupdate`: Whether GitHub runner automatic updates are disabled. + - `runner.ephemeral`: Whether runners register as ephemeral runners. + - `runner.enable_jit_config`: Whether runners use just-in-time registration configuration. + - `runner.labels`: Labels assigned to runners created by the pool Lambda. + - `runner.group_name`: GitHub runner group assigned to runners created by the pool Lambda. + - `runner.name_prefix`: Prefix used for runner names. + - `runner.pool_owner`: GitHub organization or repository that owns the runner pool. + - `runners_maximum_count`: Maximum number of runners that the pool Lambda may create. + - `prefix`: Prefix used to name pool resources. + - `pool`: Scheduled pool targets. + - `pool[*].schedule_expression`: EventBridge Scheduler expression for a pool target. + - `pool[*].schedule_expression_timezone`: Time zone used to evaluate the schedule expression. + - `pool[*].size`: Desired runner count for the scheduled pool target. + - `include_busy_runners`: Whether busy runners count toward the desired pool size. + - `role_permissions_boundary`: Permissions boundary applied to IAM roles created for the pool. + - `kms_key`: Optional customer-managed KMS key that the pool Lambda may use to decrypt encrypted parameters. Object presence controls whether the KMS statement exists. + - `kms_key.arn`: ARN of the customer-managed KMS key. The ARN may be unknown until apply. + - `role_path`: IAM path applied to roles created for the pool. + - `ssm_token_path`: SSM path under which runner registration tokens are stored. + - `ssm_config_path`: SSM path under which runner configuration is stored. + - `arn_ssm_parameters_path_config`: ARN matching the runner configuration SSM path. + - `lambda_tags`: Tags added specifically to the pool Lambda function, overriding common tags with the same key. + - `log_group_tags`: Tags added specifically to the pool Lambda log group, overriding common tags with the same key. + - `user_agent`: User-Agent header used for GitHub API requests. + EOF + type = object({ + lambda = object({ + log_level = string + logging_retention_in_days = number + logging_kms_key_id = string + log_class = string + reserved_concurrent_executions = number + s3_bucket = string + s3_key = string + s3_object_version = string + security_group_ids = list(string) + runtime = string + architecture = string + memory_size = number + timeout = number + zip = string + subnet_ids = list(string) + parameter_store_tags = string + }) + tags = map(string) + ghes = object({ + url = string + ssl_verify = string + }) + github_app_parameters = object({ + key_base64 = map(string) + id = map(string) + }) + runner = object({ + disable_runner_autoupdate = bool + ephemeral = bool + enable_jit_config = bool + labels = list(string) + group_name = string + name_prefix = string + pool_owner = string + }) + runners_maximum_count = number + prefix = string + pool = list(object({ + schedule_expression = string + schedule_expression_timezone = string + size = number + })) + include_busy_runners = bool + role_permissions_boundary = string + kms_key = optional(object({ + arn = string + }), null) + role_path = string + ssm_token_path = string + ssm_config_path = string + arn_ssm_parameters_path_config = string + lambda_tags = map(string) + log_group_tags = optional(map(string), {}) + user_agent = string + }) +} + +variable "runner_provider" { + description = <<-EOF + Compute provider integration used by the pool Lambda. + + - `type`: Compute provider type passed to scheduled pool invocations. + - `environment_variables`: Provider-specific environment variables added to the pool Lambda. + - `iam_policy_json`: Provider-specific IAM policy document merged into the pool Lambda policy. + - `managed_policy_enabled`: Whether to attach a provider-specific managed IAM policy to the pool Lambda role. + - `managed_policy_arn`: ARN of the provider-specific managed IAM policy to attach when enabled. + EOF + type = object({ + type = string + environment_variables = map(string) + iam_policy_json = string + managed_policy_enabled = bool + managed_policy_arn = optional(string, null) + }) + + validation { + condition = trimspace(var.runner_provider.type) != "" + error_message = "The compute provider type must not be empty." + } + + validation { + condition = can(jsondecode(var.runner_provider.iam_policy_json)) + error_message = "The compute provider IAM policy must be valid JSON." + } + + validation { + condition = !var.runner_provider.managed_policy_enabled || var.runner_provider.managed_policy_arn != null + error_message = "The compute provider managed policy ARN must be set when its attachment is enabled." + } +} + +variable "aws_partition" { + description = "(optional) partition for the arn if not 'aws'" + type = string + default = "aws" +} + +variable "tracing_config" { + description = <<-EOF + Tracing configuration for the pool Lambda. + + - `mode`: AWS X-Ray tracing mode. A null value disables tracing. + - `capture_http_requests`: Whether Powertools tracing captures outgoing HTTP requests. + - `capture_error`: Whether Powertools tracing captures errors as tracing metadata. + EOF + type = object({ + mode = optional(string, null) + capture_http_requests = optional(bool, false) + capture_error = optional(bool, false) + }) + default = {} +} diff --git a/modules/runner-stack/pool/versions.tf b/modules/runner-stack/pool/versions.tf new file mode 100644 index 0000000000..42a40b33fd --- /dev/null +++ b/modules/runner-stack/pool/versions.tf @@ -0,0 +1,10 @@ +terraform { + required_version = ">= 1.3.0" + + required_providers { + aws = { + source = "hashicorp/aws" + version = ">= 6.21" + } + } +} diff --git a/modules/runner-stack/runner-role.tf b/modules/runner-stack/runner-role.tf new file mode 100644 index 0000000000..0422487d7a --- /dev/null +++ b/modules/runner-stack/runner-role.tf @@ -0,0 +1,48 @@ +locals { + # Role ownership belongs to the common stack. The selected trust-policy + # submodule supplies the assume-role document, while the full compute provider + # supplies permissions after the role has been resolved. + create_runner_role = var.runner.iam.role == null + + runner_role = { + arn = local.create_runner_role ? one(aws_iam_role.runner[*].arn) : var.runner.iam.role.arn + name = local.create_runner_role ? one(aws_iam_role.runner[*].name) : basename(var.runner.iam.role.arn) + managed = local.create_runner_role + } + + common_runner_managed_policy_arns = merge( + { + for policy_name, policy_arn in var.runner.iam.managed_policy_arns : + "user-${policy_name}" => policy_arn + }, + var.observability.tracing.mode != null ? { + xray = "arn:${var.aws_partition}:iam::aws:policy/AWSXRayDaemonWriteAccess" + } : {}, + ) + + provider_runner_policies = local.provider_contract.policies.runner +} + +resource "aws_iam_role" "runner" { + count = local.create_runner_role ? 1 : 0 + name = "${substr("${var.prefix}-runner", 0, 54)}-${substr(md5("${var.prefix}-runner"), 0, 8)}" + assume_role_policy = local.provider_assume_role_policy + path = local.runner_role_path + permissions_boundary = var.runner.iam.permissions_boundary + tags = local.runner_tags +} + +resource "aws_iam_role_policy" "runner_provider" { + for_each = local.create_runner_role ? local.provider_runner_policies.inline_policies : {} + + name = each.value.name + role = aws_iam_role.runner[0].name + policy = each.value.policy_json +} + +resource "aws_iam_role_policy_attachment" "runner" { + for_each = local.create_runner_role ? local.provider_runner_policies.managed_policy_arns : {} + + role = aws_iam_role.runner[0].name + policy_arn = each.value +} diff --git a/modules/runner-stack/runner-ssm-parameters.tf b/modules/runner-stack/runner-ssm-parameters.tf new file mode 100644 index 0000000000..43708f1d02 --- /dev/null +++ b/modules/runner-stack/runner-ssm-parameters.tf @@ -0,0 +1,28 @@ +# Shared runner configuration stored in SSM Parameter Store. +resource "aws_ssm_parameter" "runner_agent_mode" { + name = "${var.ssm.paths.root}/${var.ssm.paths.config}/agent_mode" + type = "String" + value = var.runner.ephemeral ? "ephemeral" : "persistent" + tags = local.ssm_parameter_tags +} + +resource "aws_ssm_parameter" "disable_default_labels" { + name = "${var.ssm.paths.root}/${var.ssm.paths.config}/disable_default_labels" + type = "String" + value = var.runner.disable_default_labels + tags = local.ssm_parameter_tags +} + +resource "aws_ssm_parameter" "jit_config_enabled" { + name = "${var.ssm.paths.root}/${var.ssm.paths.config}/enable_jit_config" + type = "String" + value = var.runner.jit_config_enabled == null ? var.runner.ephemeral : var.runner.jit_config_enabled + tags = local.ssm_parameter_tags +} + +resource "aws_ssm_parameter" "token_path" { + name = "${var.ssm.paths.root}/${var.ssm.paths.config}/token_path" + type = "String" + value = "${var.ssm.paths.root}/${var.ssm.paths.tokens}" + tags = local.ssm_parameter_tags +} diff --git a/modules/runner-stack/scale-down-state-diagram.md b/modules/runner-stack/scale-down-state-diagram.md new file mode 100644 index 0000000000..64e32bc141 --- /dev/null +++ b/modules/runner-stack/scale-down-state-diagram.md @@ -0,0 +1,150 @@ +# GitHub Actions Runner Scale-Down State Diagram + + + +The scale-down Lambda function runs on a scheduled basis (every 5 minutes by default) to manage GitHub Actions runner instances. It performs a two-phase cleanup process: first terminating confirmed orphaned instances, then evaluating active runners to maintain the desired idle capacity while removing unnecessary instances. + +```mermaid +stateDiagram-v2 + [*] --> ScheduledExecution : Cron Trigger every 5 min + + ScheduledExecution --> Phase1_OrphanTermination : Start Phase 1 + + state Phase1_OrphanTermination { + [*] --> ListOrphanInstances : Query EC2 for ghr orphan true + + ListOrphanInstances --> CheckOrphanType : For each orphan + + state CheckOrphanType <> + CheckOrphanType --> HasRunnerIdTag : Has ghr github runner id + CheckOrphanType --> TerminateOrphan : No runner ID tag + + HasRunnerIdTag --> LastChanceCheck : Query GitHub API + + state LastChanceCheck <> + LastChanceCheck --> ConfirmedOrphan : Offline and busy + LastChanceCheck --> FalsePositive : Exists and not problematic + + ConfirmedOrphan --> TerminateOrphan + FalsePositive --> RemoveOrphanTag + + TerminateOrphan --> NextOrphan : Continue processing + RemoveOrphanTag --> NextOrphan + + NextOrphan --> CheckOrphanType : More orphans? + NextOrphan --> Phase2_ActiveRunners : All processed + } + + Phase1_OrphanTermination --> Phase2_ActiveRunners : Phase 1 Complete + + state Phase2_ActiveRunners { + [*] --> ListActiveRunners : Query non-orphan EC2 instances + + ListActiveRunners --> GroupByOwner : Sort by owner and repo + + GroupByOwner --> ProcessOwnerGroup : For each owner + + state ProcessOwnerGroup { + [*] --> SortByStrategy : Apply eviction strategy + SortByStrategy --> ProcessRunner : Oldest first or newest first + + ProcessRunner --> QueryGitHub : Get GitHub runners for owner + + QueryGitHub --> MatchRunner : Find runner by instance ID suffix + + state MatchRunner <> + MatchRunner --> FoundInGitHub : Runner exists in GitHub + MatchRunner --> NotFoundInGitHub : Runner not in GitHub + + state FoundInGitHub { + [*] --> CheckMinimumTime : Has minimum runtime passed? + + state CheckMinimumTime <> + CheckMinimumTime --> TooYoung : Runtime less than minimum + CheckMinimumTime --> CheckIdleQuota : Runtime greater than or equal to minimum + + TooYoung --> NextRunner + + state CheckIdleQuota <> + CheckIdleQuota --> KeepIdle : Idle quota available + CheckIdleQuota --> CheckBusyState : Quota full + + KeepIdle --> NextRunner + + state CheckBusyState <> + CheckBusyState --> KeepBusy : Runner busy + CheckBusyState --> TerminateIdle : Runner idle + + KeepBusy --> NextRunner + TerminateIdle --> DeregisterFromGitHub + DeregisterFromGitHub --> TerminateInstance + TerminateInstance --> NextRunner + } + + state NotFoundInGitHub { + [*] --> CheckBootTime : Has boot time exceeded? + + state CheckBootTime <> + CheckBootTime --> StillBooting : Boot time less than threshold + CheckBootTime --> MarkOrphan : Boot time greater than or equal to threshold + + StillBooting --> NextRunner + MarkOrphan --> TagAsOrphan : Set ghr orphan true + TagAsOrphan --> NextRunner + } + + NextRunner --> ProcessRunner : More runners in group? + NextRunner --> NextOwnerGroup : Group complete + } + + NextOwnerGroup --> ProcessOwnerGroup : More owner groups? + NextOwnerGroup --> ExecutionComplete : All groups processed + } + + Phase2_ActiveRunners --> ExecutionComplete : Phase 2 Complete + + ExecutionComplete --> [*] : Wait for next cron trigger + + note right of LastChanceCheck + Uses ghr github runner id tag + for precise GitHub API lookup + end note + + note right of MatchRunner + Matches GitHub runner name + ending with EC2 instance ID + end note + + note right of CheckMinimumTime + Minimum running time in minutes + (Linux: 5min, Windows: 15min, OSX: 20min) + end note + + note right of CheckBootTime + Runner boot time in minutes + Default configuration value + end note +``` + + + +## Key Decision Points + +| State | Condition | Action | +|-------|-----------|--------| +| **Orphan w/ Runner ID** | GitHub: offline + busy | Terminate (confirmed orphan) | +| **Orphan w/ Runner ID** | GitHub: exists + healthy | Remove orphan tag (false positive) | +| **Orphan w/o Runner ID** | Always | Terminate (no way to verify) | +| **Active Runner Found** | Runtime < minimum | Keep (too young) | +| **Active Runner Found** | Idle quota available | Keep as idle | +| **Active Runner Found** | Quota full + idle | Terminate + deregister | +| **Active Runner Found** | Quota full + busy | Keep running | +| **Active Runner Missing** | Boot time exceeded | Mark as orphan | +| **Active Runner Missing** | Still booting | Wait | + +## Configuration Parameters + +- **Cron Schedule**: `cron(*/5 * * * ? *)` (every 5 minutes) +- **Minimum Runtime**: Linux 5min, Windows 15min, OSX 20min +- **Boot Timeout**: Configurable via `runner_boot_time_in_minutes` +- **Idle Config**: Per-environment configuration for desired idle runners diff --git a/modules/runner-stack/scale-runners.tf b/modules/runner-stack/scale-runners.tf new file mode 100644 index 0000000000..ac5ca69845 --- /dev/null +++ b/modules/runner-stack/scale-runners.tf @@ -0,0 +1,86 @@ +module "scale_runners" { + source = "./scale-runners" + + aws_partition = var.aws_partition + + config = { + prefix = var.prefix + lambda = { + artifact = { + zip = local.lambda_zip + s3 = var.lambda.s3 + } + runtime = var.lambda.runtime + architecture = var.lambda.architecture + vpc = { + subnet_ids = var.lambda.subnet_ids + security_group_ids = var.lambda.security_group_ids + } + role = { + path = local.lambda_role_path + permissions_boundary = var.lambda.role.permissions_boundary + } + } + runner = var.runner + github = var.github + queue = { + build = var.queue.build + event_source_mapping = var.queue.event_source_mapping + } + ssm = { + token_path = local.token_path + config_path = "${var.ssm.paths.root}/${var.ssm.paths.config}" + config_path_arn = local.arn_ssm_parameters_path_config + kms_key = local.kms_key + parameter_store_tags = local.parameter_store_tags + } + observability = var.observability + scale_up = { + memory_size = var.scale_up.memory_size + timeout = var.scale_up.timeout + reserved_concurrent_executions = var.scale_up.reserved_concurrent_executions + job_queued_check_enabled = local.enable_job_queued_check + tags = { + resources = local.scale_up_tags + lambda = local.scale_up_lambda_tags + log_group = local.scale_up_log_tags + event_source_mapping = local.scale_up_queue_tags + } + } + scale_down = { + memory_size = var.scale_down.memory_size + timeout = var.scale_down.timeout + schedule_expression = var.scale_down.schedule_expression + minimum_running_time_in_minutes = var.scale_down.minimum_running_time_in_minutes + idle_config = var.scale_down.idle_config + tags = { + resources = local.scale_down_tags + lambda = local.scale_down_lambda_tags + log_group = local.scale_down_log_tags + } + } + job_retry = { + enabled = local.job_retry_enabled + max_attempts = var.job_retry.max_attempts + delay_in_seconds = var.job_retry.delay_in_seconds + delay_backoff = var.job_retry.delay_backoff + queue = one(module.job_retry[*].job_retry_check_queue) + } + } + + runner_provider = { + type = local.provider_type + scale_up = { + environment_variables = local.provider_contract.environment_variables.scale_up + iam_policy_json = local.provider_contract.policies.scale_up.iam_policy_json + additional_iam_policy_json = local.provider_contract.policies.scale_up.additional_iam_policy_json + managed_policy = local.provider_contract.policies.scale_up.managed_policy_enabled ? { + arn = local.provider_contract.policies.scale_up.managed_policy_arn + } : null + } + scale_down = { + environment_variables = local.provider_contract.environment_variables.scale_down + iam_policy_json = local.provider_contract.policies.scale_down.iam_policy_json + } + } +} diff --git a/modules/runner-stack/scale-runners/README.md b/modules/runner-stack/scale-runners/README.md new file mode 100644 index 0000000000..861792c99a --- /dev/null +++ b/modules/runner-stack/scale-runners/README.md @@ -0,0 +1,77 @@ +# Scale runners module + +> This module is treated as an internal module; breaking changes do not trigger a major release bump. + +This provider-neutral child module owns the scale-up and scale-down Lambda functions, their event sources and schedules, and their IAM and logging resources. `runner-stack` supplies common configuration together with the selected compute provider's environment and IAM fragments. + +The module is an implementation detail of the experimental runner stack. It is composed by `runner-stack` and is not intended to be called directly. + + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 1.3.0 | +| [aws](#requirement\_aws) | >= 6.33 | + +## Providers + +| Name | Version | +|------|---------| +| [aws](#provider\_aws) | >= 6.33 | + +## Modules + +No modules. + +## Resources + +| Name | Type | +|------|------| +| [aws_cloudwatch_event_rule.scale_down](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/cloudwatch_event_rule) | resource | +| [aws_cloudwatch_event_target.scale_down](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/cloudwatch_event_target) | resource | +| [aws_cloudwatch_log_group.scale_down](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/cloudwatch_log_group) | resource | +| [aws_cloudwatch_log_group.scale_up](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/cloudwatch_log_group) | resource | +| [aws_iam_role.scale_down](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role) | resource | +| [aws_iam_role.scale_up](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role) | resource | +| [aws_iam_role_policy.job_retry_sqs_publish](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy) | resource | +| [aws_iam_role_policy.scale_down](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy) | resource | +| [aws_iam_role_policy.scale_down_logging](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy) | resource | +| [aws_iam_role_policy.scale_down_xray](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy) | resource | +| [aws_iam_role_policy.scale_up](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy) | resource | +| [aws_iam_role_policy.scale_up_logging](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy) | resource | +| [aws_iam_role_policy.scale_up_xray](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy) | resource | +| [aws_iam_role_policy.service_linked_role](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy) | resource | +| [aws_iam_role_policy_attachment.provider](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy_attachment) | resource | +| [aws_iam_role_policy_attachment.scale_down_vpc_execution_role](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy_attachment) | resource | +| [aws_iam_role_policy_attachment.scale_up_vpc_execution_role](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy_attachment) | resource | +| [aws_lambda_event_source_mapping.scale_up](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/lambda_event_source_mapping) | resource | +| [aws_lambda_function.scale_down](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/lambda_function) | resource | +| [aws_lambda_function.scale_up](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/lambda_function) | resource | +| [aws_lambda_permission.scale_down](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/lambda_permission) | resource | +| [aws_lambda_permission.scale_runners_lambda](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/lambda_permission) | resource | +| [aws_iam_policy_document.lambda_assume_role](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source | +| [aws_iam_policy_document.lambda_xray](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source | +| [aws_iam_policy_document.scale_down](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source | +| [aws_iam_policy_document.scale_down_common](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source | +| [aws_iam_policy_document.scale_down_logging](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source | +| [aws_iam_policy_document.scale_up](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source | +| [aws_iam_policy_document.scale_up_common](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source | +| [aws_iam_policy_document.scale_up_job_retry_publish](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source | +| [aws_iam_policy_document.scale_up_logging](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [aws\_partition](#input\_aws\_partition) | AWS partition used to construct IAM policy ARNs. | `string` | `"aws"` | no | +| [config](#input\_config) | Provider-neutral scale-up and scale-down configuration assembled by runner-stack.

- `prefix`: Prefix used to name scaling resources.
- `lambda.artifact.zip`: Resolved local control-plane archive.
- `lambda.artifact.s3.bucket`: Optional S3 bucket containing the Lambda archive.
- `lambda.artifact.s3.key`: Object key of the Lambda archive.
- `lambda.artifact.s3.object_version`: Optional object version of the Lambda archive.
- `lambda.runtime`: Runtime used by both scaling Lambdas.
- `lambda.architecture`: Instruction-set architecture used by both scaling Lambdas.
- `lambda.vpc.subnet_ids`: Subnets used for Lambda VPC configuration.
- `lambda.vpc.security_group_ids`: Security groups used for Lambda VPC configuration.
- `lambda.role.path`: IAM path used for the scaling Lambda roles.
- `lambda.role.permissions_boundary`: Optional permissions boundary for the scaling Lambda roles.
- `runner.os`: Runner operating system used for the minimum-runtime default.
- `runner.auto_update_disabled`: Disables the GitHub runner application's built-in updater.
- `runner.ephemeral`: Registers runners in ephemeral mode.
- `runner.jit_config_enabled`: Enables or disables just-in-time runner configuration.
- `runner.labels`: Labels supplied when a runner is registered.
- `runner.group_name`: GitHub runner group used during registration.
- `runner.name_prefix`: Prefix added to registered runner names.
- `runner.maximum_count`: Maximum number of runners for this stack.
- `github.organization_runners`: Registers organization runners when true.
- `github.enterprise_server.url`: Optional GitHub Enterprise Server URL.
- `github.enterprise_server.ssl_verify`: Enables TLS verification for GitHub Enterprise Server.
- `github.user_agent`: Optional User-Agent sent to GitHub.
- `github.app_parameters.key_base64`: Name and ARN of the GitHub App private-key parameter.
- `github.app_parameters.id`: Name and ARN of the GitHub App ID parameter.
- `queue.build.arn`: ARN of the build queue consumed by scale-up.
- `queue.event_source_mapping.batch_size`: Maximum records delivered per scale-up invocation.
- `queue.event_source_mapping.maximum_batching_window_in_seconds`: Maximum event batching window.
- `ssm.token_path`: Parameter Store path used for registration tokens.
- `ssm.config_path`: Parameter Store path used for persistent runner configuration.
- `ssm.config_path_arn`: ARN of the persistent runner configuration path.
- `ssm.kms_key`: Optional KMS key used to decrypt shared parameters.
- `ssm.parameter_store_tags`: JSON-encoded tags applied to parameters created at runtime.
- `observability.logs`: Shared logging level, retention, encryption, and log-class configuration.
- `observability.tracing`: Lambda X-Ray and tracing-helper configuration.
- `observability.metrics`: Metrics enablement, namespace, and GitHub rate-limit metric configuration.
- `scale_up`: Scale-up Lambda sizing, concurrency, queued-job behavior, and resolved resource tag maps.
- `scale_up.tags.resources`: Tags for the scale-up IAM role and other component resources.
- `scale_up.tags.lambda`: Tags for the scale-up Lambda function.
- `scale_up.tags.log_group`: Tags for the scale-up log group.
- `scale_up.tags.event_source_mapping`: Tags for the build-queue event-source mapping.
- `scale_down`: Scale-down Lambda sizing, schedule, idle configuration, minimum runtime, and resolved resource tag maps.
- `scale_down.tags.resources`: Tags for the scale-down IAM role and EventBridge rule.
- `scale_down.tags.lambda`: Tags for the scale-down Lambda function.
- `scale_down.tags.log_group`: Tags for the scale-down log group.
- `job_retry.enabled`: Enables publishing retry checks from scale-up.
- `job_retry.queue`: Retry queue ARN and URL. Required when job retry is enabled.
- `job_retry.max_attempts`: Maximum queued-job retry attempts.
- `job_retry.delay_in_seconds`: Initial delay before checking the queued job.
- `job_retry.delay_backoff`: Multiplier applied to subsequent delays. | 
object({
    prefix = string
    lambda = object({
      artifact = object({
        zip = string
        s3 = object({
          bucket         = optional(string, null)
          key            = optional(string, null)
          object_version = optional(string, null)
        })
      })
      runtime      = string
      architecture = string
      vpc = object({
        subnet_ids         = list(string)
        security_group_ids = list(string)
      })
      role = object({
        path                 = string
        permissions_boundary = optional(string, null)
      })
    })
    runner = object({
      os                   = string
      auto_update_disabled = bool
      ephemeral            = bool
      jit_config_enabled   = optional(bool, null)
      labels               = list(string)
      group_name           = string
      name_prefix          = string
      maximum_count        = number
    })
    github = object({
      organization_runners = bool
      enterprise_server = object({
        url        = optional(string, null)
        ssl_verify = bool
      })
      user_agent = optional(string, null)
      app_parameters = object({
        key_base64 = object({
          name = string
          arn  = string
        })
        id = object({
          name = string
          arn  = string
        })
      })
    })
    queue = object({
      build = object({
        arn = string
      })
      event_source_mapping = object({
        batch_size                         = number
        maximum_batching_window_in_seconds = number
      })
    })
    ssm = object({
      token_path           = string
      config_path          = string
      config_path_arn      = string
      parameter_store_tags = string
      kms_key = optional(object({
        arn = string
      }), null)
    })
    observability = object({
      logs = object({
        level             = string
        retention_in_days = number
        kms_key_id        = optional(string, null)
        class             = string
      })
      tracing = object({
        mode                  = optional(string, null)
        capture_http_requests = bool
        capture_error         = bool
      })
      metrics = object({
        enable    = bool
        namespace = string
        metric = object({
          enable_github_app_rate_limit = bool
        })
      })
    })
    scale_up = object({
      memory_size                    = number
      timeout                        = number
      reserved_concurrent_executions = number
      job_queued_check_enabled       = bool
      tags = object({
        resources            = map(string)
        lambda               = map(string)
        log_group            = map(string)
        event_source_mapping = map(string)
      })
    })
    scale_down = object({
      memory_size                     = number
      timeout                         = number
      schedule_expression             = string
      minimum_running_time_in_minutes = optional(number, null)
      idle_config = list(object({
        cron             = string
        timeZone         = string
        idleCount        = number
        evictionStrategy = string
      }))
      tags = object({
        resources = map(string)
        lambda    = map(string)
        log_group = map(string)
      })
    })
    job_retry = object({
      enabled          = bool
      max_attempts     = number
      delay_in_seconds = number
      delay_backoff    = number
      queue = optional(object({
        arn = string
        url = string
      }), null)
    })
  })
| n/a | yes | +| [runner\_provider](#input\_runner\_provider) | Selected compute-provider integration for the scaling control plane.

- `type`: Compute-provider discriminator supplied to both Lambdas.
- `scale_up.environment_variables`: Provider-specific scale-up environment variables.
- `scale_up.iam_policy_json`: Provider-specific IAM policy merged into the common scale-up policy.
- `scale_up.additional_iam_policy_json`: Optional additional provider policy attached separately to the scale-up role.
- `scale_up.managed_policy`: Optional provider-managed policy attachment. Object presence controls attachment creation.
- `scale_up.managed_policy.arn`: ARN of the provider-managed policy. The ARN may remain unknown until apply.
- `scale_down.environment_variables`: Provider-specific scale-down environment variables.
- `scale_down.iam_policy_json`: Provider-specific IAM policy merged into the common scale-down policy. | 
object({
    type = string
    scale_up = object({
      environment_variables      = map(string)
      iam_policy_json            = string
      additional_iam_policy_json = optional(string, null)
      managed_policy = optional(object({
        arn = string
      }), null)
    })
    scale_down = object({
      environment_variables = map(string)
      iam_policy_json       = string
    })
  })
| n/a | yes | + +## Outputs + +| Name | Description | +|------|-------------| +| [scale\_down](#output\_scale\_down) | Scale-down Lambda resources. | +| [scale\_up](#output\_scale\_up) | Scale-up Lambda resources. | + diff --git a/modules/runner-stack/scale-runners/common-config.tf b/modules/runner-stack/scale-runners/common-config.tf new file mode 100644 index 0000000000..7c8a04d095 --- /dev/null +++ b/modules/runner-stack/scale-runners/common-config.tf @@ -0,0 +1,20 @@ +locals { + vpc_enabled = ( + length(var.config.lambda.vpc.subnet_ids) > 0 && + length(var.config.lambda.vpc.security_group_ids) > 0 + ) + + job_retry_config = var.config.job_retry.enabled ? { + enable = true + maxAttempts = var.config.job_retry.max_attempts + delayInSeconds = var.config.job_retry.delay_in_seconds + delayBackoff = var.config.job_retry.delay_backoff + queueUrl = var.config.job_retry.queue.url + } : {} + + min_runtime_defaults = { + windows = 15 + linux = 5 + osx = 20 + } +} diff --git a/modules/runner-stack/scale-runners/lambda-iam-policies.tf b/modules/runner-stack/scale-runners/lambda-iam-policies.tf new file mode 100644 index 0000000000..05922c734e --- /dev/null +++ b/modules/runner-stack/scale-runners/lambda-iam-policies.tf @@ -0,0 +1,26 @@ +data "aws_iam_policy_document" "lambda_assume_role" { + statement { + actions = ["sts:AssumeRole"] + + principals { + type = "Service" + identifiers = ["lambda.amazonaws.com"] + } + } +} + +data "aws_iam_policy_document" "lambda_xray" { + count = var.config.observability.tracing.mode != null ? 1 : 0 + + statement { + sid = "AllowXRay" + effect = "Allow" + actions = [ + "xray:BatchGetTraces", + "xray:GetTraceSummaries", + "xray:PutTelemetryRecords", + "xray:PutTraceSegments", + ] + resources = ["*"] + } +} diff --git a/modules/runner-stack/scale-runners/outputs.tf b/modules/runner-stack/scale-runners/outputs.tf new file mode 100644 index 0000000000..74d54d2101 --- /dev/null +++ b/modules/runner-stack/scale-runners/outputs.tf @@ -0,0 +1,17 @@ +output "scale_up" { + description = "Scale-up Lambda resources." + value = { + lambda = aws_lambda_function.scale_up + log_group = aws_cloudwatch_log_group.scale_up + role = aws_iam_role.scale_up + } +} + +output "scale_down" { + description = "Scale-down Lambda resources." + value = { + lambda = aws_lambda_function.scale_down + log_group = aws_cloudwatch_log_group.scale_down + role = aws_iam_role.scale_down + } +} diff --git a/modules/runner-stack/scale-runners/scale-down-iam-policies.tf b/modules/runner-stack/scale-runners/scale-down-iam-policies.tf new file mode 100644 index 0000000000..c61e8dc68b --- /dev/null +++ b/modules/runner-stack/scale-runners/scale-down-iam-policies.tf @@ -0,0 +1,41 @@ +data "aws_iam_policy_document" "scale_down_common" { + statement { + effect = "Allow" + actions = [ + "ssm:GetParameter", + "ssm:GetParameters", + ] + resources = [ + var.config.github.app_parameters.key_base64.arn, + var.config.github.app_parameters.id.arn, + ] + } + + dynamic "statement" { + for_each = var.config.ssm.kms_key == null ? [] : [var.config.ssm.kms_key] + + content { + effect = "Allow" + actions = ["kms:Decrypt"] + resources = [statement.value.arn] + } + } +} + +data "aws_iam_policy_document" "scale_down" { + source_policy_documents = [ + data.aws_iam_policy_document.scale_down_common.json, + var.runner_provider.scale_down.iam_policy_json, + ] +} + +data "aws_iam_policy_document" "scale_down_logging" { + statement { + effect = "Allow" + actions = [ + "logs:CreateLogStream", + "logs:PutLogEvents", + ] + resources = ["${aws_cloudwatch_log_group.scale_down.arn}*"] + } +} diff --git a/modules/runner-stack/scale-runners/scale-down.tf b/modules/runner-stack/scale-runners/scale-down.tf new file mode 100644 index 0000000000..5e8253728b --- /dev/null +++ b/modules/runner-stack/scale-runners/scale-down.tf @@ -0,0 +1,114 @@ +resource "aws_lambda_function" "scale_down" { + s3_bucket = var.config.lambda.artifact.s3.bucket + s3_key = var.config.lambda.artifact.s3.key + s3_object_version = var.config.lambda.artifact.s3.object_version + filename = var.config.lambda.artifact.s3.bucket == null ? var.config.lambda.artifact.zip : null + source_code_hash = var.config.lambda.artifact.s3.bucket == null ? filebase64sha256(var.config.lambda.artifact.zip) : null + function_name = "${var.config.prefix}-scale-down" + role = aws_iam_role.scale_down.arn + handler = "index.scaleDownHandler" + runtime = var.config.lambda.runtime + timeout = var.config.scale_down.timeout + tags = var.config.scale_down.tags.lambda + memory_size = var.config.scale_down.memory_size + architectures = [var.config.lambda.architecture] + + environment { + variables = merge(var.runner_provider.scale_down.environment_variables, { + ENVIRONMENT = var.config.prefix + ENABLE_METRIC_GITHUB_APP_RATE_LIMIT = var.config.observability.metrics.enable && var.config.observability.metrics.metric.enable_github_app_rate_limit + GHES_URL = var.config.github.enterprise_server.url + USER_AGENT = var.config.github.user_agent + LOG_LEVEL = upper(var.config.observability.logs.level) + MINIMUM_RUNNING_TIME_IN_MINUTES = coalesce(var.config.scale_down.minimum_running_time_in_minutes, local.min_runtime_defaults[var.config.runner.os]) + NODE_TLS_REJECT_UNAUTHORIZED = var.config.github.enterprise_server.url != null && !var.config.github.enterprise_server.ssl_verify ? 0 : 1 + PARAMETER_GITHUB_APP_ID_NAME = var.config.github.app_parameters.id.name + PARAMETER_GITHUB_APP_KEY_BASE64_NAME = var.config.github.app_parameters.key_base64.name + POWERTOOLS_LOGGER_LOG_EVENT = var.config.observability.logs.level == "debug" ? "true" : "false" + SCALE_DOWN_CONFIG = jsonencode(var.config.scale_down.idle_config) + POWERTOOLS_SERVICE_NAME = "${var.config.prefix}-scale-down" + POWERTOOLS_METRICS_NAMESPACE = var.config.observability.metrics.namespace + POWERTOOLS_TRACE_ENABLED = var.config.observability.tracing.mode != null + POWERTOOLS_TRACER_CAPTURE_HTTPS_REQUESTS = var.config.observability.tracing.capture_http_requests + POWERTOOLS_TRACER_CAPTURE_ERROR = var.config.observability.tracing.capture_error + COMPUTE_PROVIDER_TYPE = var.runner_provider.type + }) + } + + dynamic "vpc_config" { + for_each = local.vpc_enabled ? [true] : [] + + content { + security_group_ids = var.config.lambda.vpc.security_group_ids + subnet_ids = var.config.lambda.vpc.subnet_ids + } + } + + dynamic "tracing_config" { + for_each = var.config.observability.tracing.mode != null ? [true] : [] + + content { + mode = var.config.observability.tracing.mode + } + } +} + +resource "aws_cloudwatch_log_group" "scale_down" { + name = "/aws/lambda/${aws_lambda_function.scale_down.function_name}" + retention_in_days = var.config.observability.logs.retention_in_days + kms_key_id = var.config.observability.logs.kms_key_id + log_group_class = var.config.observability.logs.class + tags = var.config.scale_down.tags.log_group +} + +resource "aws_cloudwatch_event_rule" "scale_down" { + name = "${var.config.prefix}-scale-down-rule" + schedule_expression = var.config.scale_down.schedule_expression + tags = var.config.scale_down.tags.resources +} + +resource "aws_cloudwatch_event_target" "scale_down" { + rule = aws_cloudwatch_event_rule.scale_down.name + arn = aws_lambda_function.scale_down.arn +} + +resource "aws_lambda_permission" "scale_down" { + statement_id = "AllowExecutionFromCloudWatch" + action = "lambda:InvokeFunction" + function_name = aws_lambda_function.scale_down.function_name + principal = "events.amazonaws.com" + source_arn = aws_cloudwatch_event_rule.scale_down.arn +} + +resource "aws_iam_role" "scale_down" { + name = "${substr("${var.config.prefix}-scale-down-lambda", 0, 54)}-${substr(md5("${var.config.prefix}-scale-down-lambda"), 0, 8)}" + assume_role_policy = data.aws_iam_policy_document.lambda_assume_role.json + path = var.config.lambda.role.path + permissions_boundary = var.config.lambda.role.permissions_boundary + tags = var.config.scale_down.tags.resources +} + +resource "aws_iam_role_policy" "scale_down" { + name = "scale-down-policy" + role = aws_iam_role.scale_down.name + policy = data.aws_iam_policy_document.scale_down.json +} + +resource "aws_iam_role_policy" "scale_down_logging" { + name = "logging-policy" + role = aws_iam_role.scale_down.name + policy = data.aws_iam_policy_document.scale_down_logging.json +} + +resource "aws_iam_role_policy_attachment" "scale_down_vpc_execution_role" { + count = local.vpc_enabled ? 1 : 0 + role = aws_iam_role.scale_down.name + policy_arn = "arn:${var.aws_partition}:iam::aws:policy/service-role/AWSLambdaVPCAccessExecutionRole" +} + +resource "aws_iam_role_policy" "scale_down_xray" { + count = var.config.observability.tracing.mode != null ? 1 : 0 + name = "xray-policy" + policy = data.aws_iam_policy_document.lambda_xray[0].json + role = aws_iam_role.scale_down.name +} diff --git a/modules/runner-stack/scale-runners/scale-up-iam-policies.tf b/modules/runner-stack/scale-runners/scale-up-iam-policies.tf new file mode 100644 index 0000000000..2e64d54876 --- /dev/null +++ b/modules/runner-stack/scale-runners/scale-up-iam-policies.tf @@ -0,0 +1,74 @@ +data "aws_iam_policy_document" "scale_up_common" { + statement { + effect = "Allow" + actions = [ + "ssm:PutParameter", + "ssm:AddTagsToResource", + ] + resources = ["*"] + } + + statement { + effect = "Allow" + actions = [ + "ssm:GetParameter", + "ssm:GetParameters", + ] + resources = [ + var.config.github.app_parameters.key_base64.arn, + var.config.github.app_parameters.id.arn, + "${var.config.ssm.config_path_arn}/*", + ] + } + + statement { + effect = "Allow" + actions = [ + "sqs:ReceiveMessage", + "sqs:GetQueueAttributes", + "sqs:DeleteMessage", + ] + resources = [var.config.queue.build.arn] + } + + dynamic "statement" { + for_each = var.config.ssm.kms_key == null ? [] : [var.config.ssm.kms_key] + + content { + effect = "Allow" + actions = ["kms:Decrypt"] + resources = [statement.value.arn] + } + } +} + +data "aws_iam_policy_document" "scale_up" { + source_policy_documents = [ + data.aws_iam_policy_document.scale_up_common.json, + var.runner_provider.scale_up.iam_policy_json, + ] +} + +data "aws_iam_policy_document" "scale_up_logging" { + statement { + effect = "Allow" + actions = [ + "logs:CreateLogStream", + "logs:PutLogEvents", + ] + resources = ["${aws_cloudwatch_log_group.scale_up.arn}*"] + } +} + +data "aws_iam_policy_document" "scale_up_job_retry_publish" { + count = var.config.job_retry.enabled ? 1 : 0 + + statement { + effect = "Allow" + actions = [ + "sqs:SendMessage", + "sqs:GetQueueAttributes", + ] + resources = [var.config.job_retry.queue.arn] + } +} diff --git a/modules/runner-stack/scale-runners/scale-up.tf b/modules/runner-stack/scale-runners/scale-up.tf new file mode 100644 index 0000000000..e87ab76c44 --- /dev/null +++ b/modules/runner-stack/scale-runners/scale-up.tf @@ -0,0 +1,145 @@ +resource "aws_lambda_function" "scale_up" { + s3_bucket = var.config.lambda.artifact.s3.bucket + s3_key = var.config.lambda.artifact.s3.key + s3_object_version = var.config.lambda.artifact.s3.object_version + filename = var.config.lambda.artifact.s3.bucket == null ? var.config.lambda.artifact.zip : null + source_code_hash = var.config.lambda.artifact.s3.bucket == null ? filebase64sha256(var.config.lambda.artifact.zip) : null + function_name = "${var.config.prefix}-scale-up" + role = aws_iam_role.scale_up.arn + handler = "index.scaleUpHandler" + runtime = var.config.lambda.runtime + timeout = var.config.scale_up.timeout + reserved_concurrent_executions = var.config.scale_up.reserved_concurrent_executions + memory_size = var.config.scale_up.memory_size + tags = var.config.scale_up.tags.lambda + architectures = [var.config.lambda.architecture] + + environment { + variables = merge(var.runner_provider.scale_up.environment_variables, { + DISABLE_RUNNER_AUTOUPDATE = var.config.runner.auto_update_disabled + ENABLE_EPHEMERAL_RUNNERS = var.config.runner.ephemeral + ENABLE_JIT_CONFIG = var.config.runner.jit_config_enabled + ENABLE_JOB_QUEUED_CHECK = var.config.scale_up.job_queued_check_enabled + ENABLE_METRIC_GITHUB_APP_RATE_LIMIT = var.config.observability.metrics.enable && var.config.observability.metrics.metric.enable_github_app_rate_limit + ENABLE_ORGANIZATION_RUNNERS = var.config.github.organization_runners + ENVIRONMENT = var.config.prefix + GHES_URL = var.config.github.enterprise_server.url + USER_AGENT = var.config.github.user_agent + LOG_LEVEL = upper(var.config.observability.logs.level) + MINIMUM_RUNNING_TIME_IN_MINUTES = coalesce(var.config.scale_down.minimum_running_time_in_minutes, local.min_runtime_defaults[var.config.runner.os]) + NODE_TLS_REJECT_UNAUTHORIZED = var.config.github.enterprise_server.url != null && !var.config.github.enterprise_server.ssl_verify ? 0 : 1 + PARAMETER_GITHUB_APP_ID_NAME = var.config.github.app_parameters.id.name + PARAMETER_GITHUB_APP_KEY_BASE64_NAME = var.config.github.app_parameters.key_base64.name + POWERTOOLS_LOGGER_LOG_EVENT = var.config.observability.logs.level == "debug" ? "true" : "false" + POWERTOOLS_METRICS_NAMESPACE = var.config.observability.metrics.namespace + POWERTOOLS_TRACE_ENABLED = var.config.observability.tracing.mode != null + POWERTOOLS_TRACER_CAPTURE_HTTPS_REQUESTS = var.config.observability.tracing.capture_http_requests + POWERTOOLS_TRACER_CAPTURE_ERROR = var.config.observability.tracing.capture_error + RUNNER_LABELS = lower(join(",", var.config.runner.labels)) + RUNNER_GROUP_NAME = var.config.runner.group_name + RUNNER_NAME_PREFIX = var.config.runner.name_prefix + COMPUTE_PROVIDER_TYPE = var.runner_provider.type + RUNNERS_MAXIMUM_COUNT = var.config.runner.maximum_count + POWERTOOLS_SERVICE_NAME = "${var.config.prefix}-scale-up" + SSM_TOKEN_PATH = var.config.ssm.token_path + SSM_CONFIG_PATH = var.config.ssm.config_path + SSM_PARAMETER_STORE_TAGS = var.config.ssm.parameter_store_tags + JOB_RETRY_CONFIG = jsonencode(local.job_retry_config) + }) + } + + dynamic "vpc_config" { + for_each = local.vpc_enabled ? [true] : [] + + content { + security_group_ids = var.config.lambda.vpc.security_group_ids + subnet_ids = var.config.lambda.vpc.subnet_ids + } + } + + dynamic "tracing_config" { + for_each = var.config.observability.tracing.mode != null ? [true] : [] + + content { + mode = var.config.observability.tracing.mode + } + } +} + +resource "aws_cloudwatch_log_group" "scale_up" { + name = "/aws/lambda/${aws_lambda_function.scale_up.function_name}" + retention_in_days = var.config.observability.logs.retention_in_days + kms_key_id = var.config.observability.logs.kms_key_id + log_group_class = var.config.observability.logs.class + tags = var.config.scale_up.tags.log_group +} + +resource "aws_lambda_event_source_mapping" "scale_up" { + event_source_arn = var.config.queue.build.arn + function_name = aws_lambda_function.scale_up.arn + function_response_types = ["ReportBatchItemFailures"] + batch_size = var.config.queue.event_source_mapping.batch_size + maximum_batching_window_in_seconds = var.config.queue.event_source_mapping.maximum_batching_window_in_seconds + tags = var.config.scale_up.tags.event_source_mapping +} + +resource "aws_lambda_permission" "scale_runners_lambda" { + statement_id = "AllowExecutionFromSQS" + action = "lambda:InvokeFunction" + function_name = aws_lambda_function.scale_up.function_name + principal = "sqs.amazonaws.com" + source_arn = var.config.queue.build.arn +} + +resource "aws_iam_role" "scale_up" { + name = "${substr("${var.config.prefix}-scale-up-lambda", 0, 54)}-${substr(md5("${var.config.prefix}-scale-up-lambda"), 0, 8)}" + assume_role_policy = data.aws_iam_policy_document.lambda_assume_role.json + path = var.config.lambda.role.path + permissions_boundary = var.config.lambda.role.permissions_boundary + tags = var.config.scale_up.tags.resources +} + +resource "aws_iam_role_policy" "scale_up" { + name = "scale-up-policy" + role = aws_iam_role.scale_up.name + policy = data.aws_iam_policy_document.scale_up.json +} + +resource "aws_iam_role_policy" "scale_up_logging" { + name = "logging-policy" + role = aws_iam_role.scale_up.name + policy = data.aws_iam_policy_document.scale_up_logging.json +} + +resource "aws_iam_role_policy" "service_linked_role" { + count = var.runner_provider.scale_up.additional_iam_policy_json != null ? 1 : 0 + name = "service_linked_role" + role = aws_iam_role.scale_up.name + policy = var.runner_provider.scale_up.additional_iam_policy_json +} + +resource "aws_iam_role_policy_attachment" "scale_up_vpc_execution_role" { + count = local.vpc_enabled ? 1 : 0 + role = aws_iam_role.scale_up.name + policy_arn = "arn:${var.aws_partition}:iam::aws:policy/service-role/AWSLambdaVPCAccessExecutionRole" +} + +resource "aws_iam_role_policy_attachment" "provider" { + count = var.runner_provider.scale_up.managed_policy != null ? 1 : 0 + role = aws_iam_role.scale_up.name + policy_arn = var.runner_provider.scale_up.managed_policy.arn +} + +resource "aws_iam_role_policy" "scale_up_xray" { + count = var.config.observability.tracing.mode != null ? 1 : 0 + name = "xray-policy" + policy = data.aws_iam_policy_document.lambda_xray[0].json + role = aws_iam_role.scale_up.name +} + +resource "aws_iam_role_policy" "job_retry_sqs_publish" { + count = var.config.job_retry.enabled ? 1 : 0 + name = "publish-retry-check-sqs-policy" + role = aws_iam_role.scale_up.name + policy = data.aws_iam_policy_document.scale_up_job_retry_publish[0].json +} diff --git a/modules/runner-stack/scale-runners/tests/scale-runners.tftest.hcl b/modules/runner-stack/scale-runners/tests/scale-runners.tftest.hcl new file mode 100644 index 0000000000..d6903a7f74 --- /dev/null +++ b/modules/runner-stack/scale-runners/tests/scale-runners.tftest.hcl @@ -0,0 +1,313 @@ +mock_provider "aws" { + mock_data "aws_iam_policy_document" { + defaults = { + json = "{\"Version\":\"2012-10-17\",\"Statement\":[]}" + } + } + + mock_resource "aws_iam_role" { + defaults = { + arn = "arn:aws:iam::123456789012:role/scale-runners-test" + } + } +} + +variables { + aws_partition = "aws-us-gov" + + config = { + prefix = "scale-runners-test" + lambda = { + artifact = { + zip = "runners.zip" + s3 = { + bucket = "lambda-artifacts" + key = "runners.zip" + object_version = "test-version" + } + } + runtime = "nodejs24.x" + architecture = "arm64" + vpc = { + subnet_ids = ["subnet-12345678"] + security_group_ids = ["sg-12345678"] + } + role = { + path = "/scale-runners-test/" + permissions_boundary = "arn:aws-us-gov:iam::123456789012:policy/permissions-boundary" + } + } + runner = { + os = "windows" + auto_update_disabled = true + ephemeral = true + jit_config_enabled = true + labels = ["Self-Hosted", "MicroVM"] + group_name = "test-group" + name_prefix = "test-runner-" + maximum_count = 7 + } + github = { + organization_runners = true + enterprise_server = { + url = "https://github.example.com" + ssl_verify = false + } + user_agent = "scale-runners-test" + app_parameters = { + key_base64 = { + name = "/github-runner/key-base64" + arn = "arn:aws-us-gov:ssm:us-gov-west-1:123456789012:parameter/github-runner/key-base64" + } + id = { + name = "/github-runner/app-id" + arn = "arn:aws-us-gov:ssm:us-gov-west-1:123456789012:parameter/github-runner/app-id" + } + } + } + queue = { + build = { + arn = "arn:aws-us-gov:sqs:us-gov-west-1:123456789012:build-queue" + } + event_source_mapping = { + batch_size = 25 + maximum_batching_window_in_seconds = 5 + } + } + ssm = { + token_path = "/github-runner/tokens" + config_path = "/github-runner/config" + config_path_arn = "arn:aws-us-gov:ssm:us-gov-west-1:123456789012:parameter/github-runner/config" + parameter_store_tags = jsonencode([{ + Key = "Environment" + Value = "test" + }]) + kms_key = { + arn = "arn:aws-us-gov:kms:us-gov-west-1:123456789012:key/scale-runners-test" + } + } + observability = { + logs = { + level = "debug" + retention_in_days = 14 + kms_key_id = "arn:aws-us-gov:kms:us-gov-west-1:123456789012:key/logs" + class = "INFREQUENT_ACCESS" + } + tracing = { + mode = "Active" + capture_http_requests = true + capture_error = true + } + metrics = { + enable = true + namespace = "ScaleRunnersTest" + metric = { + enable_github_app_rate_limit = true + } + } + } + scale_up = { + memory_size = 768 + timeout = 90 + reserved_concurrent_executions = 2 + job_queued_check_enabled = true + tags = { + resources = { Scope = "scale-up" } + lambda = { Scope = "scale-up-lambda" } + log_group = { Scope = "scale-up-log" } + event_source_mapping = { Scope = "scale-up-queue" } + } + } + scale_down = { + memory_size = 640 + timeout = 75 + schedule_expression = "rate(10 minutes)" + minimum_running_time_in_minutes = null + idle_config = [{ + cron = "* * * * *" + timeZone = "UTC" + idleCount = 2 + evictionStrategy = "oldest_first" + }] + tags = { + resources = { Scope = "scale-down" } + lambda = { Scope = "scale-down-lambda" } + log_group = { Scope = "scale-down-log" } + } + } + job_retry = { + enabled = true + max_attempts = 4 + delay_in_seconds = 120 + delay_backoff = 3 + queue = { + arn = "arn:aws-us-gov:sqs:us-gov-west-1:123456789012:job-retry" + url = "https://sqs.us-gov-west-1.amazonaws.com/123456789012/job-retry" + } + } + } + + runner_provider = { + type = "microvm" + scale_up = { + environment_variables = { + MICROVM_CLUSTER = "runner-cluster" + } + iam_policy_json = jsonencode({ + Version = "2012-10-17" + Statement = [{ + Effect = "Allow" + Action = ["microvm:CreateRunner"] + Resource = ["*"] + }] + }) + additional_iam_policy_json = jsonencode({ + Version = "2012-10-17" + Statement = [{ + Effect = "Allow" + Action = ["iam:CreateServiceLinkedRole"] + Resource = ["*"] + }] + }) + managed_policy = { + arn = "arn:aws-us-gov:iam::123456789012:policy/microvm-scale-up" + } + } + scale_down = { + environment_variables = { + MICROVM_CLUSTER = "runner-cluster" + } + iam_policy_json = jsonencode({ + Version = "2012-10-17" + Statement = [{ + Effect = "Allow" + Action = ["microvm:DeleteRunner"] + Resource = ["*"] + }] + }) + } + } +} + +run "assembles_provider_neutral_scaling_control_plane" { + command = plan + + assert { + condition = ( + toset(keys(output.scale_up)) == toset(["lambda", "log_group", "role"]) + && toset(keys(output.scale_down)) == toset(["lambda", "log_group", "role"]) + ) + error_message = "Scale runners must expose nested scale-up and scale-down Lambda resource contracts." + } + + assert { + condition = ( + aws_lambda_function.scale_up.environment[0].variables["COMPUTE_PROVIDER_TYPE"] == "microvm" + && aws_lambda_function.scale_down.environment[0].variables["COMPUTE_PROVIDER_TYPE"] == "microvm" + && aws_lambda_function.scale_up.environment[0].variables["MICROVM_CLUSTER"] == "runner-cluster" + && aws_lambda_function.scale_down.environment[0].variables["MICROVM_CLUSTER"] == "runner-cluster" + && !contains(keys(aws_lambda_function.scale_up.environment[0].variables), "INSTANCE_TYPES") + ) + error_message = "The common scaling Lambdas must select the provider and merge only its environment fragments." + } + + assert { + condition = ( + aws_lambda_function.scale_up.environment[0].variables["LOG_LEVEL"] == "DEBUG" + && aws_lambda_function.scale_up.environment[0].variables["RUNNER_LABELS"] == "self-hosted,microvm" + && aws_lambda_function.scale_up.environment[0].variables["MINIMUM_RUNNING_TIME_IN_MINUTES"] == "15" + && aws_lambda_function.scale_down.environment[0].variables["MINIMUM_RUNNING_TIME_IN_MINUTES"] == "15" + && aws_lambda_function.scale_up.environment[0].variables["NODE_TLS_REJECT_UNAUTHORIZED"] == "0" + && jsondecode(aws_lambda_function.scale_up.environment[0].variables["SSM_PARAMETER_STORE_TAGS"])[0].Value == "test" + ) + error_message = "Scale runners must assemble shared runner, logging, TLS, lifetime, and Parameter Store environment variables." + } + + assert { + condition = ( + jsondecode(aws_lambda_function.scale_up.environment[0].variables["JOB_RETRY_CONFIG"]).queueUrl == "https://sqs.us-gov-west-1.amazonaws.com/123456789012/job-retry" + && jsondecode(aws_lambda_function.scale_up.environment[0].variables["JOB_RETRY_CONFIG"]).maxAttempts == "4" + && jsondecode(aws_lambda_function.scale_down.environment[0].variables["SCALE_DOWN_CONFIG"])[0].idleCount == 2 + ) + error_message = "Scale runners must preserve job-retry and idle-runner configuration at the Lambda boundary." + } + + assert { + condition = ( + aws_lambda_function.scale_up.memory_size == 768 + && aws_lambda_function.scale_up.timeout == 90 + && aws_lambda_function.scale_up.reserved_concurrent_executions == 2 + && aws_lambda_function.scale_down.memory_size == 640 + && aws_lambda_function.scale_down.timeout == 75 + && aws_cloudwatch_log_group.scale_up.log_group_class == "INFREQUENT_ACCESS" + && aws_cloudwatch_log_group.scale_down.retention_in_days == 14 + ) + error_message = "The child module must preserve Lambda sizing and log-group configuration." + } + + assert { + condition = ( + aws_lambda_event_source_mapping.scale_up.event_source_arn == "arn:aws-us-gov:sqs:us-gov-west-1:123456789012:build-queue" + && aws_lambda_event_source_mapping.scale_up.batch_size == 25 + && aws_lambda_event_source_mapping.scale_up.maximum_batching_window_in_seconds == 5 + && aws_lambda_event_source_mapping.scale_up.tags["Scope"] == "scale-up-queue" + && aws_cloudwatch_event_rule.scale_down.schedule_expression == "rate(10 minutes)" + && aws_cloudwatch_event_rule.scale_down.tags["Scope"] == "scale-down" + ) + error_message = "Scale-up queue and scale-down schedule triggers must remain owned by the child module." + } + + assert { + condition = ( + aws_lambda_function.scale_up.tags["Scope"] == "scale-up-lambda" + && aws_cloudwatch_log_group.scale_up.tags["Scope"] == "scale-up-log" + && aws_iam_role.scale_up.tags["Scope"] == "scale-up" + && aws_lambda_function.scale_down.tags["Scope"] == "scale-down-lambda" + && aws_cloudwatch_log_group.scale_down.tags["Scope"] == "scale-down-log" + && aws_iam_role.scale_down.tags["Scope"] == "scale-down" + ) + error_message = "Resolved component tag maps must reach the resources owned by scale runners." + } + + assert { + condition = ( + length(aws_lambda_function.scale_up.vpc_config) == 1 + && length(aws_lambda_function.scale_down.vpc_config) == 1 + && length(aws_iam_role_policy_attachment.scale_up_vpc_execution_role) == 1 + && length(aws_iam_role_policy_attachment.scale_down_vpc_execution_role) == 1 + && aws_iam_role_policy_attachment.scale_up_vpc_execution_role[0].policy_arn == "arn:aws-us-gov:iam::aws:policy/service-role/AWSLambdaVPCAccessExecutionRole" + ) + error_message = "A complete Lambda VPC configuration must configure both Lambdas and their partition-aware execution policies." + } + + assert { + condition = ( + length(aws_lambda_function.scale_up.tracing_config) == 1 + && length(aws_lambda_function.scale_down.tracing_config) == 1 + && length(aws_iam_role_policy.scale_up_xray) == 1 + && length(aws_iam_role_policy.scale_down_xray) == 1 + ) + error_message = "Active tracing must configure both Lambdas and attach their X-Ray policies." + } + + assert { + condition = ( + length(aws_iam_role_policy.service_linked_role) == 1 + && length(aws_iam_role_policy_attachment.provider) == 1 + && aws_iam_role_policy_attachment.provider[0].policy_arn == "arn:aws-us-gov:iam::123456789012:policy/microvm-scale-up" + && length(aws_iam_role_policy.job_retry_sqs_publish) == 1 + ) + error_message = "Optional compute-provider and job-retry IAM integrations must be attached to the scale-up role." + } + + assert { + condition = ( + length(data.aws_iam_policy_document.scale_up.source_policy_documents) == 2 + && length(data.aws_iam_policy_document.scale_down.source_policy_documents) == 2 + && length(data.aws_iam_policy_document.scale_up_common.statement) == 4 + && length(data.aws_iam_policy_document.scale_down_common.statement) == 2 + && length(data.aws_iam_policy_document.scale_up_job_retry_publish) == 1 + ) + error_message = "Common, provider, KMS, and retry IAM policy fragments must retain their conditional plan shape." + } +} diff --git a/modules/runner-stack/scale-runners/variables.tf b/modules/runner-stack/scale-runners/variables.tf new file mode 100644 index 0000000000..07146601de --- /dev/null +++ b/modules/runner-stack/scale-runners/variables.tf @@ -0,0 +1,231 @@ +variable "aws_partition" { + description = "AWS partition used to construct IAM policy ARNs." + type = string + default = "aws" +} + +variable "config" { + description = <<-EOT + Provider-neutral scale-up and scale-down configuration assembled by runner-stack. + + - `prefix`: Prefix used to name scaling resources. + - `lambda.artifact.zip`: Resolved local control-plane archive. + - `lambda.artifact.s3.bucket`: Optional S3 bucket containing the Lambda archive. + - `lambda.artifact.s3.key`: Object key of the Lambda archive. + - `lambda.artifact.s3.object_version`: Optional object version of the Lambda archive. + - `lambda.runtime`: Runtime used by both scaling Lambdas. + - `lambda.architecture`: Instruction-set architecture used by both scaling Lambdas. + - `lambda.vpc.subnet_ids`: Subnets used for Lambda VPC configuration. + - `lambda.vpc.security_group_ids`: Security groups used for Lambda VPC configuration. + - `lambda.role.path`: IAM path used for the scaling Lambda roles. + - `lambda.role.permissions_boundary`: Optional permissions boundary for the scaling Lambda roles. + - `runner.os`: Runner operating system used for the minimum-runtime default. + - `runner.auto_update_disabled`: Disables the GitHub runner application's built-in updater. + - `runner.ephemeral`: Registers runners in ephemeral mode. + - `runner.jit_config_enabled`: Enables or disables just-in-time runner configuration. + - `runner.labels`: Labels supplied when a runner is registered. + - `runner.group_name`: GitHub runner group used during registration. + - `runner.name_prefix`: Prefix added to registered runner names. + - `runner.maximum_count`: Maximum number of runners for this stack. + - `github.organization_runners`: Registers organization runners when true. + - `github.enterprise_server.url`: Optional GitHub Enterprise Server URL. + - `github.enterprise_server.ssl_verify`: Enables TLS verification for GitHub Enterprise Server. + - `github.user_agent`: Optional User-Agent sent to GitHub. + - `github.app_parameters.key_base64`: Name and ARN of the GitHub App private-key parameter. + - `github.app_parameters.id`: Name and ARN of the GitHub App ID parameter. + - `queue.build.arn`: ARN of the build queue consumed by scale-up. + - `queue.event_source_mapping.batch_size`: Maximum records delivered per scale-up invocation. + - `queue.event_source_mapping.maximum_batching_window_in_seconds`: Maximum event batching window. + - `ssm.token_path`: Parameter Store path used for registration tokens. + - `ssm.config_path`: Parameter Store path used for persistent runner configuration. + - `ssm.config_path_arn`: ARN of the persistent runner configuration path. + - `ssm.kms_key`: Optional KMS key used to decrypt shared parameters. + - `ssm.parameter_store_tags`: JSON-encoded tags applied to parameters created at runtime. + - `observability.logs`: Shared logging level, retention, encryption, and log-class configuration. + - `observability.tracing`: Lambda X-Ray and tracing-helper configuration. + - `observability.metrics`: Metrics enablement, namespace, and GitHub rate-limit metric configuration. + - `scale_up`: Scale-up Lambda sizing, concurrency, queued-job behavior, and resolved resource tag maps. + - `scale_up.tags.resources`: Tags for the scale-up IAM role and other component resources. + - `scale_up.tags.lambda`: Tags for the scale-up Lambda function. + - `scale_up.tags.log_group`: Tags for the scale-up log group. + - `scale_up.tags.event_source_mapping`: Tags for the build-queue event-source mapping. + - `scale_down`: Scale-down Lambda sizing, schedule, idle configuration, minimum runtime, and resolved resource tag maps. + - `scale_down.tags.resources`: Tags for the scale-down IAM role and EventBridge rule. + - `scale_down.tags.lambda`: Tags for the scale-down Lambda function. + - `scale_down.tags.log_group`: Tags for the scale-down log group. + - `job_retry.enabled`: Enables publishing retry checks from scale-up. + - `job_retry.queue`: Retry queue ARN and URL. Required when job retry is enabled. + - `job_retry.max_attempts`: Maximum queued-job retry attempts. + - `job_retry.delay_in_seconds`: Initial delay before checking the queued job. + - `job_retry.delay_backoff`: Multiplier applied to subsequent delays. + EOT + + type = object({ + prefix = string + lambda = object({ + artifact = object({ + zip = string + s3 = object({ + bucket = optional(string, null) + key = optional(string, null) + object_version = optional(string, null) + }) + }) + runtime = string + architecture = string + vpc = object({ + subnet_ids = list(string) + security_group_ids = list(string) + }) + role = object({ + path = string + permissions_boundary = optional(string, null) + }) + }) + runner = object({ + os = string + auto_update_disabled = bool + ephemeral = bool + jit_config_enabled = optional(bool, null) + labels = list(string) + group_name = string + name_prefix = string + maximum_count = number + }) + github = object({ + organization_runners = bool + enterprise_server = object({ + url = optional(string, null) + ssl_verify = bool + }) + user_agent = optional(string, null) + app_parameters = object({ + key_base64 = object({ + name = string + arn = string + }) + id = object({ + name = string + arn = string + }) + }) + }) + queue = object({ + build = object({ + arn = string + }) + event_source_mapping = object({ + batch_size = number + maximum_batching_window_in_seconds = number + }) + }) + ssm = object({ + token_path = string + config_path = string + config_path_arn = string + parameter_store_tags = string + kms_key = optional(object({ + arn = string + }), null) + }) + observability = object({ + logs = object({ + level = string + retention_in_days = number + kms_key_id = optional(string, null) + class = string + }) + tracing = object({ + mode = optional(string, null) + capture_http_requests = bool + capture_error = bool + }) + metrics = object({ + enable = bool + namespace = string + metric = object({ + enable_github_app_rate_limit = bool + }) + }) + }) + scale_up = object({ + memory_size = number + timeout = number + reserved_concurrent_executions = number + job_queued_check_enabled = bool + tags = object({ + resources = map(string) + lambda = map(string) + log_group = map(string) + event_source_mapping = map(string) + }) + }) + scale_down = object({ + memory_size = number + timeout = number + schedule_expression = string + minimum_running_time_in_minutes = optional(number, null) + idle_config = list(object({ + cron = string + timeZone = string + idleCount = number + evictionStrategy = string + })) + tags = object({ + resources = map(string) + lambda = map(string) + log_group = map(string) + }) + }) + job_retry = object({ + enabled = bool + max_attempts = number + delay_in_seconds = number + delay_backoff = number + queue = optional(object({ + arn = string + url = string + }), null) + }) + }) + + nullable = false + + validation { + condition = !var.config.job_retry.enabled || var.config.job_retry.queue != null + error_message = "config.job_retry.queue must be set when config.job_retry.enabled is true." + } +} + +variable "runner_provider" { + description = <<-EOT + Selected compute-provider integration for the scaling control plane. + + - `type`: Compute-provider discriminator supplied to both Lambdas. + - `scale_up.environment_variables`: Provider-specific scale-up environment variables. + - `scale_up.iam_policy_json`: Provider-specific IAM policy merged into the common scale-up policy. + - `scale_up.additional_iam_policy_json`: Optional additional provider policy attached separately to the scale-up role. + - `scale_up.managed_policy`: Optional provider-managed policy attachment. Object presence controls attachment creation. + - `scale_up.managed_policy.arn`: ARN of the provider-managed policy. The ARN may remain unknown until apply. + - `scale_down.environment_variables`: Provider-specific scale-down environment variables. + - `scale_down.iam_policy_json`: Provider-specific IAM policy merged into the common scale-down policy. + EOT + + type = object({ + type = string + scale_up = object({ + environment_variables = map(string) + iam_policy_json = string + additional_iam_policy_json = optional(string, null) + managed_policy = optional(object({ + arn = string + }), null) + }) + scale_down = object({ + environment_variables = map(string) + iam_policy_json = string + }) + }) + + nullable = false +} diff --git a/modules/runner-stack/scale-runners/versions.tf b/modules/runner-stack/scale-runners/versions.tf new file mode 100644 index 0000000000..da9769f550 --- /dev/null +++ b/modules/runner-stack/scale-runners/versions.tf @@ -0,0 +1,10 @@ +terraform { + required_version = ">= 1.3.0" + + required_providers { + aws = { + source = "hashicorp/aws" + version = ">= 6.33" + } + } +} diff --git a/modules/runner-stack/ssm-housekeeper.tf b/modules/runner-stack/ssm-housekeeper.tf new file mode 100644 index 0000000000..18912392d4 --- /dev/null +++ b/modules/runner-stack/ssm-housekeeper.tf @@ -0,0 +1,57 @@ +locals { + ssm_housekeeper_token_path = coalesce(var.ssm.housekeeper.config.tokenPath, local.token_path) + ssm_housekeeper_parameter_path_arn = ( + "arn:${var.aws_partition}:ssm:${var.aws_region}:${data.aws_caller_identity.current.account_id}:parameter${local.ssm_housekeeper_token_path}*" + ) +} + +module "ssm_housekeeper" { + source = "./ssm-housekeeper" + + config = { + prefix = var.prefix + aws_partition = var.aws_partition + schedule = { + expression = var.ssm.housekeeper.schedule_expression + state = var.ssm.housekeeper.state + } + cleanup = { + token_path = local.ssm_housekeeper_token_path + parameter_path_arn = local.ssm_housekeeper_parameter_path_arn + minimum_days_old = var.ssm.housekeeper.config.minimumDaysOld + dry_run = var.ssm.housekeeper.config.dryRun + } + lambda = { + artifact = { + zip = local.lambda_zip + s3 = var.lambda.s3 + } + runtime = var.lambda.runtime + architecture = var.lambda.architecture + memory_size = var.ssm.housekeeper.lambda.memory_size + timeout = var.ssm.housekeeper.lambda.timeout + vpc = { + subnet_ids = var.lambda.subnet_ids + security_group_ids = var.lambda.security_group_ids + } + role = { + path = local.lambda_role_path + permissions_boundary = var.lambda.role.permissions_boundary + } + } + observability = { + logs = { + level = var.observability.logs.level + retention_in_days = var.observability.logs.retention_in_days + kms_key_id = var.observability.logs.kms_key_id + class = var.observability.logs.class + } + tracing = var.observability.tracing + } + tags = { + resources = local.ssm_housekeeper_tags + lambda = local.ssm_housekeeper_lambda_tags + log_group = local.ssm_housekeeper_log_tags + } + } +} diff --git a/modules/runner-stack/ssm-housekeeper/README.md b/modules/runner-stack/ssm-housekeeper/README.md new file mode 100644 index 0000000000..4cc0af9d63 --- /dev/null +++ b/modules/runner-stack/ssm-housekeeper/README.md @@ -0,0 +1,57 @@ +# SSM housekeeper module + +> This module is treated as an internal module; breaking changes do not trigger a major release bump. + +This provider-neutral child module owns the Lambda function, EventBridge schedule, IAM policies, and CloudWatch log group used to remove expired runner registration parameters from Parameter Store. + +The module is an implementation detail of the experimental runner stack. It is composed by `runner-stack` and is not intended to be called directly. + + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 1.3.0 | +| [aws](#requirement\_aws) | >= 6.33 | + +## Providers + +| Name | Version | +|------|---------| +| [aws](#provider\_aws) | >= 6.33 | + +## Modules + +No modules. + +## Resources + +| Name | Type | +|------|------| +| [aws_cloudwatch_event_rule.ssm_housekeeper](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/cloudwatch_event_rule) | resource | +| [aws_cloudwatch_event_target.ssm_housekeeper](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/cloudwatch_event_target) | resource | +| [aws_cloudwatch_log_group.ssm_housekeeper](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/cloudwatch_log_group) | resource | +| [aws_iam_role.ssm_housekeeper](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role) | resource | +| [aws_iam_role_policy.ssm_housekeeper](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy) | resource | +| [aws_iam_role_policy.ssm_housekeeper_logging](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy) | resource | +| [aws_iam_role_policy.ssm_housekeeper_xray](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy) | resource | +| [aws_iam_role_policy_attachment.ssm_housekeeper_vpc_execution_role](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy_attachment) | resource | +| [aws_lambda_function.ssm_housekeeper](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/lambda_function) | resource | +| [aws_lambda_permission.ssm_housekeeper](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/lambda_permission) | resource | +| [aws_iam_policy_document.lambda_assume_role](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source | +| [aws_iam_policy_document.lambda_xray](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source | +| [aws_iam_policy_document.ssm_housekeeper](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source | +| [aws_iam_policy_document.ssm_housekeeper_logging](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [config](#input\_config) | Provider-neutral SSM housekeeper configuration assembled by runner-stack.

- `prefix`: Prefix used to name the housekeeper resources.
- `aws_partition`: AWS partition used to construct IAM policy ARNs.
- `schedule.expression`: EventBridge schedule expression that invokes the housekeeper.
- `schedule.state`: State of the EventBridge rule.
- `cleanup.token_path`: Parameter Store token path supplied to the Lambda.
- `cleanup.parameter_path_arn`: IAM resource ARN matching `cleanup.token_path`.
- `cleanup.minimum_days_old`: Minimum parameter age before deletion.
- `cleanup.dry_run`: Reports eligible parameters without deleting them when true.
- `lambda.artifact.zip`: Resolved local control-plane archive.
- `lambda.artifact.s3.bucket`: Optional S3 bucket containing the Lambda archive.
- `lambda.artifact.s3.key`: Object key of the Lambda archive.
- `lambda.artifact.s3.object_version`: Optional object version of the Lambda archive.
- `lambda.runtime`: Runtime used by the housekeeper Lambda.
- `lambda.architecture`: Instruction-set architecture used by the housekeeper Lambda.
- `lambda.memory_size`: Memory allocated to the housekeeper Lambda.
- `lambda.timeout`: Housekeeper Lambda timeout in seconds.
- `lambda.vpc.subnet_ids`: Subnets used for Lambda VPC configuration.
- `lambda.vpc.security_group_ids`: Security groups used for Lambda VPC configuration.
- `lambda.role.path`: IAM path used for the housekeeper Lambda role.
- `lambda.role.permissions_boundary`: Optional permissions boundary for the housekeeper role.
- `observability.logs`: Logging level, retention, encryption, and log-class configuration.
- `observability.tracing`: Lambda X-Ray and tracing-helper configuration.
- `tags.resources`: Tags for the housekeeper role and EventBridge rule.
- `tags.lambda`: Tags for the housekeeper Lambda function.
- `tags.log_group`: Tags for the housekeeper log group. | 
object({
    prefix        = string
    aws_partition = string
    schedule = object({
      expression = string
      state      = string
    })
    cleanup = object({
      token_path         = string
      parameter_path_arn = string
      minimum_days_old   = number
      dry_run            = bool
    })
    lambda = object({
      artifact = object({
        zip = string
        s3 = object({
          bucket         = optional(string, null)
          key            = optional(string, null)
          object_version = optional(string, null)
        })
      })
      runtime      = string
      architecture = string
      memory_size  = number
      timeout      = number
      vpc = object({
        subnet_ids         = list(string)
        security_group_ids = list(string)
      })
      role = object({
        path                 = string
        permissions_boundary = optional(string, null)
      })
    })
    observability = object({
      logs = object({
        level             = string
        retention_in_days = number
        kms_key_id        = optional(string, null)
        class             = string
      })
      tracing = object({
        mode                  = optional(string, null)
        capture_http_requests = bool
        capture_error         = bool
      })
    })
    tags = object({
      resources = map(string)
      lambda    = map(string)
      log_group = map(string)
    })
  })
| n/a | yes | + +## Outputs + +| Name | Description | +|------|-------------| +| [housekeeper](#output\_housekeeper) | SSM housekeeper Lambda resources. | + diff --git a/modules/runner-stack/ssm-housekeeper/iam-policies.tf b/modules/runner-stack/ssm-housekeeper/iam-policies.tf new file mode 100644 index 0000000000..8599e378f6 --- /dev/null +++ b/modules/runner-stack/ssm-housekeeper/iam-policies.tf @@ -0,0 +1,48 @@ +data "aws_iam_policy_document" "lambda_assume_role" { + statement { + actions = ["sts:AssumeRole"] + + principals { + type = "Service" + identifiers = ["lambda.amazonaws.com"] + } + } +} + +data "aws_iam_policy_document" "lambda_xray" { + count = var.config.observability.tracing.mode != null ? 1 : 0 + + statement { + sid = "AllowXRay" + effect = "Allow" + actions = [ + "xray:BatchGetTraces", + "xray:GetTraceSummaries", + "xray:PutTelemetryRecords", + "xray:PutTraceSegments", + ] + resources = ["*"] + } +} + +data "aws_iam_policy_document" "ssm_housekeeper" { + statement { + effect = "Allow" + actions = [ + "ssm:DeleteParameter", + "ssm:GetParametersByPath", + ] + resources = [var.config.cleanup.parameter_path_arn] + } +} + +data "aws_iam_policy_document" "ssm_housekeeper_logging" { + statement { + effect = "Allow" + actions = [ + "logs:CreateLogStream", + "logs:PutLogEvents", + ] + resources = ["${aws_cloudwatch_log_group.ssm_housekeeper.arn}*"] + } +} diff --git a/modules/runner-stack/ssm-housekeeper/outputs.tf b/modules/runner-stack/ssm-housekeeper/outputs.tf new file mode 100644 index 0000000000..064f5a1ab1 --- /dev/null +++ b/modules/runner-stack/ssm-housekeeper/outputs.tf @@ -0,0 +1,8 @@ +output "housekeeper" { + description = "SSM housekeeper Lambda resources." + value = { + lambda = aws_lambda_function.ssm_housekeeper + log_group = aws_cloudwatch_log_group.ssm_housekeeper + role = aws_iam_role.ssm_housekeeper + } +} diff --git a/modules/runner-stack/ssm-housekeeper/ssm-housekeeper.tf b/modules/runner-stack/ssm-housekeeper/ssm-housekeeper.tf new file mode 100644 index 0000000000..bcafed201a --- /dev/null +++ b/modules/runner-stack/ssm-housekeeper/ssm-housekeeper.tf @@ -0,0 +1,119 @@ +locals { + vpc_enabled = ( + length(var.config.lambda.vpc.subnet_ids) > 0 && + length(var.config.lambda.vpc.security_group_ids) > 0 + ) + + cleanup_config = { + tokenPath = var.config.cleanup.token_path + minimumDaysOld = var.config.cleanup.minimum_days_old + dryRun = var.config.cleanup.dry_run + } +} + +resource "aws_lambda_function" "ssm_housekeeper" { + s3_bucket = var.config.lambda.artifact.s3.bucket + s3_key = var.config.lambda.artifact.s3.key + s3_object_version = var.config.lambda.artifact.s3.object_version + filename = var.config.lambda.artifact.s3.bucket == null ? var.config.lambda.artifact.zip : null + source_code_hash = var.config.lambda.artifact.s3.bucket == null ? filebase64sha256(var.config.lambda.artifact.zip) : null + function_name = "${var.config.prefix}-ssm-housekeeper" + role = aws_iam_role.ssm_housekeeper.arn + handler = "index.ssmHousekeeper" + runtime = var.config.lambda.runtime + timeout = var.config.lambda.timeout + tags = var.config.tags.lambda + memory_size = var.config.lambda.memory_size + architectures = [var.config.lambda.architecture] + + environment { + variables = { + ENVIRONMENT = var.config.prefix + LOG_LEVEL = upper(var.config.observability.logs.level) + SSM_CLEANUP_CONFIG = jsonencode(local.cleanup_config) + POWERTOOLS_SERVICE_NAME = "${var.config.prefix}-ssm-housekeeper" + POWERTOOLS_TRACE_ENABLED = var.config.observability.tracing.mode != null + POWERTOOLS_TRACER_CAPTURE_HTTPS_REQUESTS = var.config.observability.tracing.capture_http_requests + POWERTOOLS_TRACER_CAPTURE_ERROR = var.config.observability.tracing.capture_error + } + } + + dynamic "vpc_config" { + for_each = local.vpc_enabled ? [true] : [] + + content { + security_group_ids = var.config.lambda.vpc.security_group_ids + subnet_ids = var.config.lambda.vpc.subnet_ids + } + } + + dynamic "tracing_config" { + for_each = var.config.observability.tracing.mode != null ? [true] : [] + + content { + mode = var.config.observability.tracing.mode + } + } +} + +resource "aws_cloudwatch_log_group" "ssm_housekeeper" { + name = "/aws/lambda/${aws_lambda_function.ssm_housekeeper.function_name}" + retention_in_days = var.config.observability.logs.retention_in_days + kms_key_id = var.config.observability.logs.kms_key_id + log_group_class = var.config.observability.logs.class + tags = var.config.tags.log_group +} + +resource "aws_cloudwatch_event_rule" "ssm_housekeeper" { + name = "${var.config.prefix}-ssm-housekeeper" + schedule_expression = var.config.schedule.expression + state = var.config.schedule.state + tags = var.config.tags.resources +} + +resource "aws_cloudwatch_event_target" "ssm_housekeeper" { + rule = aws_cloudwatch_event_rule.ssm_housekeeper.name + arn = aws_lambda_function.ssm_housekeeper.arn +} + +resource "aws_lambda_permission" "ssm_housekeeper" { + statement_id = "AllowExecutionFromCloudWatch" + action = "lambda:InvokeFunction" + function_name = aws_lambda_function.ssm_housekeeper.function_name + principal = "events.amazonaws.com" + source_arn = aws_cloudwatch_event_rule.ssm_housekeeper.arn +} + +resource "aws_iam_role" "ssm_housekeeper" { + name = "${substr("${var.config.prefix}-ssm-hk-lambda", 0, 54)}-${substr(md5("${var.config.prefix}-ssm-hk-lambda"), 0, 8)}" + description = "Lambda role for SSM Housekeeper (${var.config.prefix})" + assume_role_policy = data.aws_iam_policy_document.lambda_assume_role.json + path = var.config.lambda.role.path + permissions_boundary = var.config.lambda.role.permissions_boundary + tags = var.config.tags.resources +} + +resource "aws_iam_role_policy" "ssm_housekeeper" { + name = "ssm-policy" + role = aws_iam_role.ssm_housekeeper.name + policy = data.aws_iam_policy_document.ssm_housekeeper.json +} + +resource "aws_iam_role_policy" "ssm_housekeeper_logging" { + name = "logging-policy" + role = aws_iam_role.ssm_housekeeper.name + policy = data.aws_iam_policy_document.ssm_housekeeper_logging.json +} + +resource "aws_iam_role_policy_attachment" "ssm_housekeeper_vpc_execution_role" { + count = local.vpc_enabled ? 1 : 0 + role = aws_iam_role.ssm_housekeeper.name + policy_arn = "arn:${var.config.aws_partition}:iam::aws:policy/service-role/AWSLambdaVPCAccessExecutionRole" +} + +resource "aws_iam_role_policy" "ssm_housekeeper_xray" { + count = var.config.observability.tracing.mode != null ? 1 : 0 + name = "xray-policy" + policy = data.aws_iam_policy_document.lambda_xray[0].json + role = aws_iam_role.ssm_housekeeper.name +} diff --git a/modules/runner-stack/ssm-housekeeper/tests/ssm-housekeeper.tftest.hcl b/modules/runner-stack/ssm-housekeeper/tests/ssm-housekeeper.tftest.hcl new file mode 100644 index 0000000000..38c04e14c5 --- /dev/null +++ b/modules/runner-stack/ssm-housekeeper/tests/ssm-housekeeper.tftest.hcl @@ -0,0 +1,240 @@ +mock_provider "aws" { + mock_data "aws_iam_policy_document" { + defaults = { + json = "{\"Version\":\"2012-10-17\",\"Statement\":[]}" + } + } + + mock_resource "aws_iam_role" { + defaults = { + arn = "arn:aws:iam::123456789012:role/ssm-housekeeper-test" + } + } + + mock_resource "aws_lambda_function" { + defaults = { + arn = "arn:aws:lambda:eu-west-1:123456789012:function:ssm-housekeeper-test" + } + } + + mock_resource "aws_cloudwatch_event_rule" { + defaults = { + arn = "arn:aws:events:eu-west-1:123456789012:rule/ssm-housekeeper-test" + } + } + + mock_resource "aws_cloudwatch_log_group" { + defaults = { + arn = "arn:aws:logs:eu-west-1:123456789012:log-group:/aws/lambda/ssm-housekeeper-test" + } + } +} + +variables { + config = { + prefix = "ssm-housekeeper-test" + aws_partition = "aws-us-gov" + schedule = { + expression = "rate(6 hours)" + state = "DISABLED" + } + cleanup = { + token_path = "/custom/runner/tokens" + parameter_path_arn = "arn:aws-us-gov:ssm:us-gov-west-1:123456789012:parameter/custom/runner/tokens*" + minimum_days_old = 7 + dry_run = true + } + lambda = { + artifact = { + zip = "unused-with-s3.zip" + s3 = { + bucket = "lambda-artifacts" + key = "control-plane/runners.zip" + object_version = "version-1" + } + } + runtime = "nodejs24.x" + architecture = "arm64" + memory_size = 384 + timeout = 45 + vpc = { + subnet_ids = [] + security_group_ids = [] + } + role = { + path = "/runner-stack/" + permissions_boundary = null + } + } + observability = { + logs = { + level = "debug" + retention_in_days = 30 + kms_key_id = null + class = "STANDARD" + } + tracing = { + mode = null + capture_http_requests = false + capture_error = false + } + } + tags = { + resources = { + Scope = "housekeeper" + } + lambda = { + Scope = "housekeeper" + Resource = "lambda" + } + log_group = { + Scope = "housekeeper" + Resource = "logs" + } + } + } +} + +run "configures_schedule_cleanup_and_outputs" { + command = plan + + assert { + condition = ( + aws_cloudwatch_event_rule.ssm_housekeeper.schedule_expression == "rate(6 hours)" && + aws_cloudwatch_event_rule.ssm_housekeeper.state == "DISABLED" + ) + error_message = "The housekeeper EventBridge rule must use the configured schedule and state." + } + + assert { + condition = ( + jsondecode(aws_lambda_function.ssm_housekeeper.environment[0].variables["SSM_CLEANUP_CONFIG"]).tokenPath == "/custom/runner/tokens" && + jsondecode(aws_lambda_function.ssm_housekeeper.environment[0].variables["SSM_CLEANUP_CONFIG"]).minimumDaysOld == 7 && + jsondecode(aws_lambda_function.ssm_housekeeper.environment[0].variables["SSM_CLEANUP_CONFIG"]).dryRun + ) + error_message = "The Lambda cleanup configuration must preserve the configured path override, age, and dry-run setting." + } + + assert { + condition = contains( + data.aws_iam_policy_document.ssm_housekeeper.statement[0].resources, + "arn:aws-us-gov:ssm:us-gov-west-1:123456789012:parameter/custom/runner/tokens*", + ) + error_message = "The housekeeper IAM policy must authorize the same overridden Parameter Store path supplied to the Lambda." + } + + assert { + condition = toset(keys(output.housekeeper)) == toset(["lambda", "log_group", "role"]) + error_message = "The module must expose Lambda, log-group, and role resources through one nested housekeeper output." + } + + assert { + condition = ( + output.housekeeper.lambda.tags == tomap({ + Scope = "housekeeper" + Resource = "lambda" + }) && + output.housekeeper.log_group.tags == tomap({ + Scope = "housekeeper" + Resource = "logs" + }) && + output.housekeeper.role.tags == tomap({ + Scope = "housekeeper" + }) + ) + error_message = "Each nested output resource must retain its resolved component tags." + } + + assert { + condition = ( + length(aws_lambda_function.ssm_housekeeper.vpc_config) == 0 && + length(aws_iam_role_policy_attachment.ssm_housekeeper_vpc_execution_role) == 0 && + length(aws_lambda_function.ssm_housekeeper.tracing_config) == 0 && + length(aws_iam_role_policy.ssm_housekeeper_xray) == 0 + ) + error_message = "Empty VPC configuration and disabled tracing must not create their optional Lambda or IAM configuration." + } +} + +run "enables_vpc_and_xray_together" { + command = plan + + variables { + config = { + prefix = "ssm-housekeeper-vpc-test" + aws_partition = "aws-us-gov" + schedule = { + expression = "rate(1 day)" + state = "ENABLED" + } + cleanup = { + token_path = "/github-runner/tokens" + parameter_path_arn = "arn:aws-us-gov:ssm:us-gov-west-1:123456789012:parameter/github-runner/tokens*" + minimum_days_old = 1 + dry_run = false + } + lambda = { + artifact = { + zip = "unused-with-s3.zip" + s3 = { + bucket = "lambda-artifacts" + key = "control-plane/runners.zip" + } + } + runtime = "nodejs24.x" + architecture = "arm64" + memory_size = 512 + timeout = 60 + vpc = { + subnet_ids = ["subnet-12345678"] + security_group_ids = ["sg-12345678"] + } + role = { + path = "/runner-stack/" + permissions_boundary = null + } + } + observability = { + logs = { + level = "info" + retention_in_days = 14 + kms_key_id = null + class = "STANDARD" + } + tracing = { + mode = "Active" + capture_http_requests = true + capture_error = true + } + } + tags = { + resources = {} + lambda = {} + log_group = {} + } + } + } + + assert { + condition = ( + length(aws_lambda_function.ssm_housekeeper.vpc_config) == 1 && + aws_lambda_function.ssm_housekeeper.vpc_config[0].subnet_ids == toset(["subnet-12345678"]) && + aws_lambda_function.ssm_housekeeper.vpc_config[0].security_group_ids == toset(["sg-12345678"]) && + length(aws_iam_role_policy_attachment.ssm_housekeeper_vpc_execution_role) == 1 && + aws_iam_role_policy_attachment.ssm_housekeeper_vpc_execution_role[0].policy_arn == "arn:aws-us-gov:iam::aws:policy/service-role/AWSLambdaVPCAccessExecutionRole" + ) + error_message = "A complete VPC configuration must configure the Lambda and attach the partition-aware VPC execution policy." + } + + assert { + condition = ( + length(aws_lambda_function.ssm_housekeeper.tracing_config) == 1 && + aws_lambda_function.ssm_housekeeper.tracing_config[0].mode == "Active" && + length(aws_iam_role_policy.ssm_housekeeper_xray) == 1 && + aws_lambda_function.ssm_housekeeper.environment[0].variables["POWERTOOLS_TRACE_ENABLED"] == "true" && + aws_lambda_function.ssm_housekeeper.environment[0].variables["POWERTOOLS_TRACER_CAPTURE_HTTPS_REQUESTS"] == "true" && + aws_lambda_function.ssm_housekeeper.environment[0].variables["POWERTOOLS_TRACER_CAPTURE_ERROR"] == "true" + ) + error_message = "Active tracing must configure Lambda tracing, X-Ray IAM permissions, and tracing-helper environment variables." + } +} diff --git a/modules/runner-stack/ssm-housekeeper/variables.tf b/modules/runner-stack/ssm-housekeeper/variables.tf new file mode 100644 index 0000000000..792b7d75bb --- /dev/null +++ b/modules/runner-stack/ssm-housekeeper/variables.tf @@ -0,0 +1,88 @@ +variable "config" { + description = <<-EOT + Provider-neutral SSM housekeeper configuration assembled by runner-stack. + + - `prefix`: Prefix used to name the housekeeper resources. + - `aws_partition`: AWS partition used to construct IAM policy ARNs. + - `schedule.expression`: EventBridge schedule expression that invokes the housekeeper. + - `schedule.state`: State of the EventBridge rule. + - `cleanup.token_path`: Parameter Store token path supplied to the Lambda. + - `cleanup.parameter_path_arn`: IAM resource ARN matching `cleanup.token_path`. + - `cleanup.minimum_days_old`: Minimum parameter age before deletion. + - `cleanup.dry_run`: Reports eligible parameters without deleting them when true. + - `lambda.artifact.zip`: Resolved local control-plane archive. + - `lambda.artifact.s3.bucket`: Optional S3 bucket containing the Lambda archive. + - `lambda.artifact.s3.key`: Object key of the Lambda archive. + - `lambda.artifact.s3.object_version`: Optional object version of the Lambda archive. + - `lambda.runtime`: Runtime used by the housekeeper Lambda. + - `lambda.architecture`: Instruction-set architecture used by the housekeeper Lambda. + - `lambda.memory_size`: Memory allocated to the housekeeper Lambda. + - `lambda.timeout`: Housekeeper Lambda timeout in seconds. + - `lambda.vpc.subnet_ids`: Subnets used for Lambda VPC configuration. + - `lambda.vpc.security_group_ids`: Security groups used for Lambda VPC configuration. + - `lambda.role.path`: IAM path used for the housekeeper Lambda role. + - `lambda.role.permissions_boundary`: Optional permissions boundary for the housekeeper role. + - `observability.logs`: Logging level, retention, encryption, and log-class configuration. + - `observability.tracing`: Lambda X-Ray and tracing-helper configuration. + - `tags.resources`: Tags for the housekeeper role and EventBridge rule. + - `tags.lambda`: Tags for the housekeeper Lambda function. + - `tags.log_group`: Tags for the housekeeper log group. + EOT + + type = object({ + prefix = string + aws_partition = string + schedule = object({ + expression = string + state = string + }) + cleanup = object({ + token_path = string + parameter_path_arn = string + minimum_days_old = number + dry_run = bool + }) + lambda = object({ + artifact = object({ + zip = string + s3 = object({ + bucket = optional(string, null) + key = optional(string, null) + object_version = optional(string, null) + }) + }) + runtime = string + architecture = string + memory_size = number + timeout = number + vpc = object({ + subnet_ids = list(string) + security_group_ids = list(string) + }) + role = object({ + path = string + permissions_boundary = optional(string, null) + }) + }) + observability = object({ + logs = object({ + level = string + retention_in_days = number + kms_key_id = optional(string, null) + class = string + }) + tracing = object({ + mode = optional(string, null) + capture_http_requests = bool + capture_error = bool + }) + }) + tags = object({ + resources = map(string) + lambda = map(string) + log_group = map(string) + }) + }) + + nullable = false +} diff --git a/modules/runner-stack/ssm-housekeeper/versions.tf b/modules/runner-stack/ssm-housekeeper/versions.tf new file mode 100644 index 0000000000..da9769f550 --- /dev/null +++ b/modules/runner-stack/ssm-housekeeper/versions.tf @@ -0,0 +1,10 @@ +terraform { + required_version = ">= 1.3.0" + + required_providers { + aws = { + source = "hashicorp/aws" + version = ">= 6.33" + } + } +} diff --git a/modules/runner-stack/tests/README.md b/modules/runner-stack/tests/README.md new file mode 100644 index 0000000000..fa55dfecd9 --- /dev/null +++ b/modules/runner-stack/tests/README.md @@ -0,0 +1,72 @@ +# Terraform Tests + +This directory contains [Terraform test files](https://developer.hashicorp.com/terraform/language/tests) (`.tftest.hcl`) for the runners module. + +## Why `terraform test` instead of `terraform validate`? + +`terraform validate` only checks syntax and basic type correctness of the configuration. It **cannot** detect: + +- Conditional expressions with inconsistent result types (e.g., one branch returns an object with 1 attribute, the other returns 16) +- Runtime type mismatches that only surface during `plan` +- Invalid cross-module references that depend on resource attribute shapes + +`terraform test` with `mock_provider` runs a full plan without needing real cloud credentials, catching these classes of bugs in CI. + +## Requirements + +- Terraform >= 1.7 (for `mock_provider` and `mock_data` support) +- No AWS credentials required — all providers are mocked + +## Running locally + +```bash +cd modules/runners +terraform test -test-directory=tests +``` + +Expected output: + +``` +tests/pool.tftest.hcl... in progress + run "plan_with_pool_enabled"... pass +tests/pool.tftest.hcl... pass + +Success! 1 passed, 0 failed. +``` + +## Writing new tests + +1. Create a `.tftest.hcl` file in this directory +2. Use `mock_provider "aws" {}` to avoid needing credentials +3. Use `mock_data` blocks to provide realistic values for data sources that perform validation (e.g., `aws_iam_policy_document` validates JSON) +4. Set all required variables in a `variables {}` block +5. Use `run` blocks with `command = plan` and `assert` conditions + +### Example template + +```hcl +mock_provider "aws" { + mock_data "aws_iam_policy_document" { + defaults = { + json = "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Effect\":\"Allow\",\"Principal\":{\"Service\":\"lambda.amazonaws.com\"},\"Action\":\"sts:AssumeRole\"}]}" + } + } +} + +variables { + # ... required variables ... +} + +run "descriptive_test_name" { + command = plan + + assert { + condition = + error_message = "Explanation of what failed" + } +} +``` + +## CI integration + +These tests run automatically in the `terraform_test` job of `.github/workflows/terraform.yml` on every PR that touches `*.tf` or `*.hcl` files. diff --git a/modules/runner-stack/tests/computed-iam-inputs.tftest.hcl b/modules/runner-stack/tests/computed-iam-inputs.tftest.hcl new file mode 100644 index 0000000000..9e81f63b64 --- /dev/null +++ b/modules/runner-stack/tests/computed-iam-inputs.tftest.hcl @@ -0,0 +1,25 @@ +mock_provider "aws" { + mock_data "aws_iam_policy_document" { + defaults = { + json = "{\"Version\":\"2012-10-17\",\"Statement\":[]}" + } + } +} + +run "computed_external_values_keep_plan_shape_known" { + command = plan + + module { + source = "./tests/fixtures/computed-iam-inputs" + } + + assert { + condition = output.external_role_runner_count == 0 + error_message = "Computed external AMI parameter, KMS key, role, and profile values must not make resource or policy-block counts unknown." + } + + assert { + condition = output.generated_policy_role_runner_count == 1 + error_message = "A computed managed-policy ARN under a caller-known map key must keep attachment planning stable." + } +} diff --git a/modules/runner-stack/tests/fixtures/computed-iam-inputs/README.md b/modules/runner-stack/tests/fixtures/computed-iam-inputs/README.md new file mode 100644 index 0000000000..08c02ba66d --- /dev/null +++ b/modules/runner-stack/tests/fixtures/computed-iam-inputs/README.md @@ -0,0 +1,39 @@ + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 1.3 | +| [random](#requirement\_random) | ~> 3.0 | + +## Providers + +| Name | Version | +|------|---------| +| [random](#provider\_random) | ~> 3.0 | + +## Modules + +| Name | Source | Version | +|------|--------|---------| +| [external\_iam](#module\_external\_iam) | ../../.. | n/a | +| [generated\_policy](#module\_generated\_policy) | ../../.. | n/a | + +## Resources + +| Name | Type | +|------|------| +| [random_id.external](https://registry.terraform.io/providers/hashicorp/random/latest/docs/resources/id) | resource | +| [random_id.generated_policy](https://registry.terraform.io/providers/hashicorp/random/latest/docs/resources/id) | resource | + +## Inputs + +No inputs. + +## Outputs + +| Name | Description | +|------|-------------| +| [external\_role\_runner\_count](#output\_external\_role\_runner\_count) | n/a | +| [generated\_policy\_role\_runner\_count](#output\_generated\_policy\_role\_runner\_count) | n/a | + \ No newline at end of file diff --git a/modules/runner-stack/tests/fixtures/computed-iam-inputs/computed-iam-inputs.tf b/modules/runner-stack/tests/fixtures/computed-iam-inputs/computed-iam-inputs.tf new file mode 100644 index 0000000000..20bcdbef52 --- /dev/null +++ b/modules/runner-stack/tests/fixtures/computed-iam-inputs/computed-iam-inputs.tf @@ -0,0 +1,177 @@ +# A .tftest.hcl variable block supplies plan-known values. This wrapper uses +# random_id results to exercise caller inputs that remain unknown during plan, +# which catches invalid count, for_each, and dynamic-block expressions in the +# IAM boundary. +resource "random_id" "external" { + byte_length = 4 +} + +resource "random_id" "generated_policy" { + byte_length = 4 +} + +module "external_iam" { + source = "../../.." + + aws_region = "eu-west-1" + prefix = "computed-external" + + compute_provider = { + ec2 = { + vpc_id = "vpc-12345678" + subnet_ids = ["subnet-12345678"] + instance_types = ["m5.large"] + ami = { + id_ssm_parameter = { + arn = "arn:aws:ssm:eu-west-1:123456789012:parameter/external-ami-${random_id.external.hex}" + } + kms_key = { + arn = "arn:aws:kms:eu-west-1:123456789012:key/${random_id.external.hex}" + } + } + instance_profile = { + name = "external-runner-${random_id.external.hex}" + } + cloudwatch_agent = { + enabled = false + } + binaries_syncer = { + enabled = false + } + } + } + + runner = { + labels = ["self-hosted", "linux", "x64"] + iam = { + role = { + arn = "arn:aws:iam::123456789012:role/external-runner-${random_id.external.hex}" + } + } + } + + queue = { + build = { + arn = "arn:aws:sqs:eu-west-1:123456789012:computed-external" + url = "https://sqs.eu-west-1.amazonaws.com/123456789012/computed-external" + } + } + + lambda = { + s3 = { + bucket = "lambda-artifacts" + key = "runners.zip" + } + } + + job_retry = { + enabled = true + } + + pool = { + runner_owner = "example" + config = [{ + schedule_expression = "cron(0 8 * * ? *)" + size = 1 + }] + } + + github = { + organization_runners = true + app_parameters = { + key_base64 = { + name = "/github-runner/key-base64" + arn = "arn:aws:ssm:eu-west-1:123456789012:parameter/github-runner/key-base64" + } + id = { + name = "/github-runner/app-id" + arn = "arn:aws:ssm:eu-west-1:123456789012:parameter/github-runner/app-id" + } + } + } + + ssm = { + kms_key = { + arn = "arn:aws:kms:eu-west-1:123456789012:key/${random_id.external.hex}" + } + paths = { + root = "/github-runner/computed-external" + tokens = "tokens" + config = "config" + } + } +} + +module "generated_policy" { + source = "../../.." + + aws_region = "eu-west-1" + prefix = "computed-policy" + + compute_provider = { + ec2 = { + vpc_id = "vpc-12345678" + subnet_ids = ["subnet-12345678"] + instance_types = ["m5.large"] + cloudwatch_agent = { + enabled = false + } + binaries_syncer = { + enabled = false + } + } + } + + runner = { + labels = ["self-hosted", "linux", "x64"] + iam = { + managed_policy_arns = { + generated = "arn:aws:iam::123456789012:policy/generated-runner-${random_id.generated_policy.hex}" + } + } + } + + queue = { + build = { + arn = "arn:aws:sqs:eu-west-1:123456789012:computed-policy" + url = "https://sqs.eu-west-1.amazonaws.com/123456789012/computed-policy" + } + } + + lambda = { + s3 = { + bucket = "lambda-artifacts" + key = "runners.zip" + } + } + + github = { + organization_runners = true + app_parameters = { + key_base64 = { + name = "/github-runner/key-base64" + arn = "arn:aws:ssm:eu-west-1:123456789012:parameter/github-runner/key-base64" + } + id = { + name = "/github-runner/app-id" + arn = "arn:aws:ssm:eu-west-1:123456789012:parameter/github-runner/app-id" + } + } + } + + ssm = { + paths = { + root = "/github-runner/computed-policy" + tokens = "tokens" + config = "config" + } + } +} + +output "external_role_runner_count" { + value = module.external_iam.runner.role == null ? 0 : 1 +} + +output "generated_policy_role_runner_count" { + value = module.generated_policy.runner.role == null ? 0 : 1 +} diff --git a/modules/runner-stack/tests/fixtures/computed-iam-inputs/versions.tf b/modules/runner-stack/tests/fixtures/computed-iam-inputs/versions.tf new file mode 100644 index 0000000000..9fd85fad8f --- /dev/null +++ b/modules/runner-stack/tests/fixtures/computed-iam-inputs/versions.tf @@ -0,0 +1,13 @@ +terraform { + required_version = ">= 1.3" + + required_providers { + aws = { + source = "hashicorp/aws" + } + random = { + source = "hashicorp/random" + version = "~> 3.0" + } + } +} diff --git a/modules/runner-stack/tests/pool.tftest.hcl b/modules/runner-stack/tests/pool.tftest.hcl new file mode 100644 index 0000000000..f93ca2fff6 --- /dev/null +++ b/modules/runner-stack/tests/pool.tftest.hcl @@ -0,0 +1,453 @@ +mock_provider "aws" { + mock_data "aws_iam_policy_document" { + defaults = { + json = "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Effect\":\"Allow\",\"Principal\":{\"Service\":\"lambda.amazonaws.com\"},\"Action\":\"sts:AssumeRole\"}]}" + } + } + + mock_resource "aws_iam_role" { + defaults = { + arn = "arn:aws:iam::123456789012:role/runner-test" + } + } + + mock_resource "aws_ssm_parameter" { + defaults = { + arn = "arn:aws:ssm:eu-west-1:123456789012:parameter/github-runner/ami-id" + } + } +} + +variables { + aws_region = "eu-west-1" + + compute_provider = { + ec2 = { + vpc_id = "vpc-12345678" + subnet_ids = ["subnet-12345678"] + instance_types = ["m5.large"] + ami = { + filter = { state = ["available"] } + owners = ["amazon"] + id_ssm_parameter = { + arn = "arn:aws:ssm:eu-west-1:123456789012:parameter/github-runner/external-ami-id" + } + kms_key = null + } + binaries_syncer = { + s3 = { + arn = "arn:aws:s3:::my-bucket" + id = "my-bucket" + key = "runners/linux/actions-runner.tar.gz" + } + } + ssm_enabled = true + } + } + + runner = { + labels = ["self-hosted", "linux", "x64"] + iam = { + managed_policy_arns = { + readonly = "arn:aws:iam::aws:policy/ReadOnlyAccess" + } + additional_trust_policy_json = jsonencode({ + Version = "2012-10-17" + Statement = [{ + Sid = "AdditionalTrustedAccount" + Effect = "Allow" + Action = "sts:AssumeRole" + Principal = { AWS = "arn:aws:iam::210987654321:root" } + }] + }) + } + } + + queue = { + build = { + arn = "arn:aws:sqs:eu-west-1:123456789012:build-queue" + url = "https://sqs.eu-west-1.amazonaws.com/123456789012/build-queue" + } + } + + # Use S3 bucket to avoid filebase64sha256 needing local zip files + lambda = { + s3 = { + bucket = "my-lambda-bucket" + key = "runners.zip" + } + } + + github = { + organization_runners = true + app_parameters = { + key_base64 = { name = "/github-runner/key-base64", arn = "arn:aws:ssm:eu-west-1:123456789012:parameter/github-runner/key-base64" } + id = { name = "/github-runner/app-id", arn = "arn:aws:ssm:eu-west-1:123456789012:parameter/github-runner/app-id" } + } + } + + ssm = { + paths = { + root = "/github-runner" + tokens = "tokens" + config = "config" + } + } + + # Enable pool to exercise the pool module and its role type + pool = { + config = [{ + schedule_expression = "cron(0 8 * * ? *)" + size = 1 + }] + } +} + +run "plan_with_pool_enabled" { + command = plan + + assert { + condition = length(module.pool) == 1 + error_message = "Pool module should be enabled when pool.config is non-empty" + } + + assert { + condition = toset(keys(output.provider)) == toset(["ec2"]) + error_message = "The runner stack must expose resources only under the selected provider key." + } + + assert { + condition = contains(keys(output.provider.ec2), "launch_template") + error_message = "The runner stack must expose EC2 resources only under provider.ec2." + } + + assert { + condition = length(aws_iam_role.runner) == 1 && output.runner.role != null + error_message = "The common runner stack must create and expose the runner role." + } + + assert { + condition = ( + length(module.ec2_trust_policy) == 1 + && length(module.microvm_trust_policy) == 0 + && aws_iam_role.runner[0].assume_role_policy == module.ec2_trust_policy[0].assume_role_policy + ) + error_message = "The common runner role must use the selected EC2 trust-policy submodule output." + } + + assert { + condition = ( + output.pool != null + && toset(keys(output.pool)) == toset(["lambda", "log_group", "role"]) + ) + error_message = "An enabled pool must expose its Lambda, log group, and role through the nested pool output." + } + + assert { + condition = length(jsondecode(module.scale_runners.scale_up.lambda.environment[0].variables["SSM_PARAMETER_STORE_TAGS"])) == 0 + error_message = "Runtime Parameter Store tags must remain empty when no module or SSM tags are configured; EC2 bootstrap tags must not leak into them." + } + + assert { + condition = !contains(keys(output.provider.ec2), "role_runner") + error_message = "The common runner role must not be duplicated in the EC2 resource output." + } + + assert { + condition = toset(keys(aws_iam_role_policy.runner_provider)) == toset([ + "ssm_parameters", + "describe_tags", + "create_tags", + "terminate_self", + "session_manager", + "distribution_bucket", + "cloudwatch", + ]) + error_message = "The common stack must attach every enabled EC2 runner policy by its stable provider key." + } + + assert { + condition = aws_iam_role_policy_attachment.runner["user-readonly"].policy_arn == "arn:aws:iam::aws:policy/ReadOnlyAccess" + error_message = "The selected EC2 provider contract must return common managed runner policies for one attachment path." + } + + assert { + condition = ( + module.scale_runners.scale_up.lambda.environment[0].variables["COMPUTE_PROVIDER_TYPE"] == "ec2" + && module.scale_runners.scale_down.lambda.environment[0].variables["COMPUTE_PROVIDER_TYPE"] == "ec2" + ) + error_message = "Scaling Lambdas must receive the provider type from the selected provider." + } + + assert { + condition = module.scale_runners.scale_up.lambda.environment[0].variables["INSTANCE_TYPES"] == "m5.large" + error_message = "Scale-up must merge the EC2 environment fragment." + } + + assert { + condition = module.scale_runners.scale_down.lambda.environment[0].variables["RUNNER_BOOT_TIME_IN_MINUTES"] == "5" + error_message = "Scale-down must merge the EC2 environment fragment." + } + + assert { + condition = ( + toset(keys(module.scale_runners.scale_up)) == toset(["lambda", "log_group", "role"]) + && toset(keys(module.scale_runners.scale_down)) == toset(["lambda", "log_group", "role"]) + ) + error_message = "The scale-runners child module must forward the nested scale-up and scale-down resource contracts." + } + +} + +run "plan_with_microvm_provider_enabled" { + command = plan + + variables { + runner = { + labels = ["self-hosted", "linux", "arm64", "microvm"] + name_prefix = "microvm-" + iam = { + managed_policy_arns = { + readonly = "arn:aws:iam::aws:policy/ReadOnlyAccess" + } + } + } + + compute_provider = { + microvm = { + image_identifier = "arn:aws:lambdamicrovms:eu-west-1:123456789012:image/runner" + image_version = "1" + environment_variables = { + MICROVM_CLUSTER = "runner-cluster" + } + tags = { + Lane = "microvm" + } + } + } + } + + assert { + condition = ( + length(module.ec2) == 0 + && length(module.microvm) == 1 + && toset(keys(output.provider)) == toset(["microvm"]) + && toset(keys(output.provider.microvm)) == toset(["image_identifier", "image_version", "execution_role_arn"]) + ) + error_message = "The runner stack must instantiate only the selected MicroVM provider and expose resources under provider.microvm." + } + + assert { + condition = ( + length(module.ec2_trust_policy) == 0 + && length(module.microvm_trust_policy) == 1 + && aws_iam_role.runner[0].assume_role_policy == module.microvm_trust_policy[0].assume_role_policy + ) + error_message = "The common runner role must use the selected MicroVM trust-policy submodule output." + } + + assert { + condition = ( + length(aws_iam_role_policy.runner_provider) == 0 + && aws_iam_role_policy_attachment.runner["user-readonly"].policy_arn == "arn:aws:iam::aws:policy/ReadOnlyAccess" + && output.provider.microvm.image_identifier == "arn:aws:lambdamicrovms:eu-west-1:123456789012:image/runner" + && output.provider.microvm.image_version == "1" + ) + error_message = "MicroVM must return common runner policies without EC2 policies and expose its selected image metadata." + } + + assert { + condition = ( + module.scale_runners.scale_up.lambda.environment[0].variables["COMPUTE_PROVIDER_TYPE"] == "microvm" + && module.scale_runners.scale_up.lambda.environment[0].variables["MICROVM_IMAGE_IDENTIFIER"] == "arn:aws:lambdamicrovms:eu-west-1:123456789012:image/runner" + && module.scale_runners.scale_up.lambda.environment[0].variables["MICROVM_CLUSTER"] == "runner-cluster" + && module.scale_runners.scale_down.lambda.environment[0].variables["RUNNER_BOOT_TIME_IN_MINUTES"] == "5" + && !contains(keys(module.scale_runners.scale_up.lambda.environment[0].variables), "INSTANCE_TYPES") + ) + error_message = "Scale-up and scale-down must receive MicroVM provider fragments without EC2 environment variables." + } + + assert { + condition = ( + output.pool != null + && module.pool[0].pool.lambda.environment[0].variables["MICROVM_IMAGE_IDENTIFIER"] == "arn:aws:lambdamicrovms:eu-west-1:123456789012:image/runner" + && module.pool[0].pool.lambda.environment[0].variables["RUNNER_BOOT_TIME_IN_MINUTES"] == "5" + ) + error_message = "The pool component must receive MicroVM provider fragments when a MicroVM lane has pool config." + } +} + +run "external_runner_role_is_not_managed_by_common" { + command = plan + + variables { + runner = { + labels = ["self-hosted", "linux", "x64"] + iam = { + role = { + arn = "arn:aws:iam::123456789012:role/external/runner-role" + } + } + } + } + + assert { + condition = length(aws_iam_role.runner) == 0 && length(aws_iam_role_policy.runner_provider) == 0 && length(aws_iam_role_policy_attachment.runner) == 0 + error_message = "An external runner role must remain unmanaged by the common stack." + } + + assert { + condition = output.runner.role == null + error_message = "The nested runner role output must be null when an external role is selected." + } + + + assert { + condition = output.provider.ec2.launch_template.iam_instance_profile[0].name == "github-actions-runner-profile" + error_message = "EC2 must create an instance profile around an externally supplied runner role when no profile override is provided." + } +} + +run "external_runner_role_and_profile_remain_external" { + command = plan + + variables { + runner = { + labels = ["self-hosted", "linux", "x64"] + iam = { + role = { + arn = "arn:aws:iam::123456789012:role/external/runner-role" + } + } + } + compute_provider = { + ec2 = { + vpc_id = "vpc-12345678" + subnet_ids = ["subnet-12345678"] + instance_types = ["m5.large"] + instance_profile = { + name = "external-runner-profile" + } + binaries_syncer = { + enabled = false + } + } + } + } + + assert { + condition = length(aws_iam_role.runner) == 0 && length(aws_iam_role_policy.runner_provider) == 0 + error_message = "The common stack must not manage an external role." + } + + assert { + condition = output.provider.ec2.launch_template.iam_instance_profile[0].name == "external-runner-profile" + error_message = "The EC2 launch template must use the external instance profile." + } +} + +run "empty_runner_iam_uses_common_role" { + command = plan + + variables { + runner = { + labels = ["self-hosted", "linux", "x64"] + iam = {} + } + } + + assert { + condition = length(aws_iam_role.runner) == 1 + error_message = "An empty runner.iam object must use common role ownership." + } +} + +run "external_role_rejects_managed_policy_attachments" { + command = plan + + variables { + runner = { + labels = ["self-hosted", "linux", "x64"] + iam = { + role = { + arn = "arn:aws:iam::123456789012:role/external/runner-role" + } + managed_policy_arns = { + readonly = "arn:aws:iam::aws:policy/ReadOnlyAccess" + } + } + } + } + + expect_failures = [var.runner] +} + +run "external_role_rejects_trust_policy_extension" { + command = plan + + variables { + runner = { + labels = ["self-hosted", "linux", "x64"] + iam = { + role = { + arn = "arn:aws:iam::123456789012:role/external/runner-role" + } + additional_trust_policy_json = "{\"Version\":\"2012-10-17\",\"Statement\":[]}" + } + } + } + + expect_failures = [var.runner] +} + +run "rejects_invalid_trust_policy_extension" { + command = plan + + variables { + runner = { + labels = ["self-hosted", "linux", "x64"] + iam = { + additional_trust_policy_json = "{" + } + } + } + + expect_failures = [var.runner] +} + +run "rejects_empty_compute_provider" { + command = plan + + variables { + compute_provider = {} + } + + expect_failures = [var.compute_provider] +} + +run "job_retry_uses_common_runner_configuration_identity" { + command = plan + + variables { + runner = { + labels = ["self-hosted", "linux", "x64"] + name_prefix = "provider-neutral-" + } + job_retry = { + enabled = true + lambda = { + reserved_concurrent_executions = 2 + } + } + } + + assert { + condition = module.job_retry[0].lambda.function.environment[0].variables["RUNNER_NAME_PREFIX"] == "provider-neutral-" + error_message = "Job retry must receive the common runner-configuration name prefix." + } + + assert { + condition = module.job_retry[0].lambda.function.reserved_concurrent_executions == 2 + error_message = "Job retry must apply its configured Lambda reserved concurrency." + } +} diff --git a/modules/runner-stack/tests/tags.tftest.hcl b/modules/runner-stack/tests/tags.tftest.hcl new file mode 100644 index 0000000000..006c57ea97 --- /dev/null +++ b/modules/runner-stack/tests/tags.tftest.hcl @@ -0,0 +1,301 @@ +mock_provider "aws" { + mock_data "aws_iam_policy_document" { + defaults = { + json = "{\"Version\":\"2012-10-17\",\"Statement\":[]}" + } + } + + mock_resource "aws_iam_role" { + defaults = { + arn = "arn:aws:iam::123456789012:role/runner-test" + } + } + + mock_resource "aws_ssm_parameter" { + defaults = { + arn = "arn:aws:ssm:eu-west-1:123456789012:parameter/github-runner/config" + } + } +} + +variables { + aws_region = "eu-west-1" + + tags = { + precedence = "module" + module = "yes" + } + + compute_provider = { + ec2 = { + vpc_id = "vpc-12345678" + subnet_ids = ["subnet-12345678"] + instance_types = ["m5.large"] + ami = { + filter = { state = ["available"] } + owners = ["amazon"] + id_ssm_parameter = { + arn = "arn:aws:ssm:eu-west-1:123456789012:parameter/github-runner/external-ami-id" + } + kms_key = null + } + binaries_syncer = { + s3 = { + arn = "arn:aws:s3:::my-bucket" + id = "my-bucket" + key = "runners/linux/actions-runner.tar.gz" + } + } + } + } + + runner = { + labels = ["self-hosted", "linux", "x64"] + tags = { + precedence = "runner" + runner = "yes" + } + } + + queue = { + build = { + arn = "arn:aws:sqs:eu-west-1:123456789012:build-queue" + url = "https://sqs.eu-west-1.amazonaws.com/123456789012/build-queue" + } + tags = { + precedence = "queue" + queue = "yes" + } + } + + lambda = { + s3 = { + bucket = "my-lambda-bucket" + key = "runners.zip" + } + tags = { + precedence = "lambda" + lambda = "yes" + } + } + + github = { + organization_runners = true + app_parameters = { + key_base64 = { + name = "/github-runner/key-base64" + arn = "arn:aws:ssm:eu-west-1:123456789012:parameter/github-runner/key-base64" + } + id = { + name = "/github-runner/app-id" + arn = "arn:aws:ssm:eu-west-1:123456789012:parameter/github-runner/app-id" + } + } + } + + scale_up = { + tags = { + precedence = "scale-up" + scale_up = "yes" + } + } + + scale_down = { + tags = { + precedence = "scale-down" + scale_down = "yes" + } + } + + pool = { + config = [{ + schedule_expression = "cron(0 8 * * ? *)" + size = 1 + }] + tags = { + precedence = "pool" + pool = "yes" + } + } + + job_retry = { + enabled = true + tags = { + precedence = "job-retry" + job_retry = "yes" + } + } + + ssm = { + paths = { + root = "/github-runner" + tokens = "tokens" + config = "config" + } + tags = { + precedence = "ssm" + ssm = "yes" + } + parameters = { + tags = { + precedence = "ssm-parameter" + parameter = "yes" + } + } + housekeeper = { + tags = { + precedence = "ssm-housekeeper" + housekeeper = "yes" + } + } + } + + observability = { + logs = { + level = "debug" + tags = { + precedence = "log" + log = "yes" + } + } + } +} + +run "layered_component_tags" { + command = plan + + assert { + condition = module.scale_runners.scale_up.lambda.environment[0].variables["LOG_LEVEL"] == "DEBUG" + error_message = "The nested observability.logs.level value must configure the control-plane functions." + } + + assert { + condition = module.scale_runners.scale_up.lambda.tags == tomap({ + precedence = "scale-up" + module = "yes" + lambda = "yes" + scale_up = "yes" + }) && module.scale_runners.scale_up.log_group.tags == tomap({ + precedence = "scale-up" + module = "yes" + log = "yes" + scale_up = "yes" + }) && module.scale_runners.scale_up.role.tags == tomap({ + precedence = "scale-up" + module = "yes" + scale_up = "yes" + }) + error_message = "Scale-up tags must layer module, shared resource, and component tags with the component taking precedence." + } + + assert { + condition = module.scale_runners.scale_down.lambda.tags == tomap({ + precedence = "scale-down" + module = "yes" + lambda = "yes" + scale_down = "yes" + }) && module.scale_runners.scale_down.log_group.tags == tomap({ + precedence = "scale-down" + module = "yes" + log = "yes" + scale_down = "yes" + }) && module.scale_runners.scale_down.role.tags == tomap({ + precedence = "scale-down" + module = "yes" + scale_down = "yes" + }) + error_message = "Scale-down tags must layer module, shared resource, and component tags with the component taking precedence." + } + + assert { + condition = aws_iam_role.runner[0].tags == tomap({ + precedence = "runner" + module = "yes" + runner = "yes" + }) + error_message = "Runner tags must override module tags on the common runner role." + } + + assert { + condition = aws_ssm_parameter.runner_agent_mode.tags == tomap({ + precedence = "ssm-parameter" + module = "yes" + ssm = "yes" + parameter = "yes" + }) && tomap({ + for tag in jsondecode(module.scale_runners.scale_up.lambda.environment[0].variables["SSM_PARAMETER_STORE_TAGS"]) : + tag.Key => tag.Value + }) == tomap({ + precedence = "ssm-parameter" + module = "yes" + ssm = "yes" + parameter = "yes" + }) + error_message = "Terraform-managed and runtime-created SSM parameters must use the same layered parameter tags." + } + + assert { + condition = module.ssm_housekeeper.housekeeper.lambda.tags == tomap({ + precedence = "ssm-housekeeper" + module = "yes" + lambda = "yes" + ssm = "yes" + housekeeper = "yes" + }) && module.ssm_housekeeper.housekeeper.log_group.tags == tomap({ + precedence = "ssm-housekeeper" + module = "yes" + log = "yes" + ssm = "yes" + housekeeper = "yes" + }) && module.ssm_housekeeper.housekeeper.role.tags == tomap({ + precedence = "ssm-housekeeper" + module = "yes" + ssm = "yes" + housekeeper = "yes" + }) + error_message = "SSM housekeeper tags must layer module, SSM, shared resource, and housekeeper tags." + } + + assert { + condition = module.pool[0].pool.lambda.tags == tomap({ + precedence = "pool" + module = "yes" + lambda = "yes" + pool = "yes" + }) && module.pool[0].pool.log_group.tags == tomap({ + precedence = "pool" + module = "yes" + log = "yes" + pool = "yes" + }) && module.pool[0].pool.role.tags == tomap({ + precedence = "pool" + module = "yes" + pool = "yes" + }) + error_message = "Pool tags must layer module, shared resource, and component tags with the component taking precedence." + } + + assert { + condition = module.job_retry[0].lambda.function.tags == tomap({ + precedence = "job-retry" + module = "yes" + lambda = "yes" + job_retry = "yes" + }) && module.job_retry[0].lambda.log_group.tags == tomap({ + precedence = "job-retry" + module = "yes" + log = "yes" + job_retry = "yes" + }) && module.job_retry[0].lambda.role.tags == tomap({ + precedence = "job-retry" + module = "yes" + job_retry = "yes" + }) && module.job_retry[0].job_retry_check_queue.tags == tomap({ + precedence = "job-retry" + module = "yes" + queue = "yes" + job_retry = "yes" + }) + error_message = "Job-retry tags must layer module, shared resource, and component tags with the component taking precedence." + } +} diff --git a/modules/runner-stack/variables.compute-provider.tf b/modules/runner-stack/variables.compute-provider.tf new file mode 100644 index 0000000000..afcec2dc3d --- /dev/null +++ b/modules/runner-stack/variables.compute-provider.tf @@ -0,0 +1,306 @@ +# Typed compute-provider input boundary between the common control plane and compute implementations. +variable "compute_provider" { + description = <<-EOT + Typed compute-provider configuration. Provider-owned settings remain inside the selected compute-provider block. + + Exactly one compute-provider block must be non-null. The populated block selects the provider, and its presence must be known during planning. Values inside the selected block may remain unknown until apply. + + - `ec2`: EC2 compute-provider configuration. + - `ec2.ami`: Optional AMI discovery or external AMI-parameter configuration. Null uses the operating-system and architecture defaults. + - `ec2.ami.filter`: EC2 AMI filters combined with the provider's default AMI-name filter. + - `ec2.ami.owners`: AWS account IDs or aliases allowed to own the selected AMI. + - `ec2.ami.id_ssm_parameter`: Optional externally managed SSM parameter containing the AMI ID. Null creates a provider-managed AMI-ID parameter. The wrapper's presence is the plan-time ownership discriminator, so keep the object literal even when its ARN comes from another resource. + - `ec2.ami.id_ssm_parameter.arn`: ARN of the externally managed SSM parameter. The ARN may be unknown until apply. + - `ec2.ami.kms_key`: Optional KMS key required to launch encrypted AMIs or snapshots. The wrapper's presence is the plan-time policy discriminator. + - `ec2.ami.kms_key.arn`: ARN of the KMS key. The ARN may be unknown until apply. + - `ec2.vpc_id`: VPC in which runner networking resources are created. + - `ec2.subnet_ids`: Subnets from which scale-up may launch runner instances. + - `ec2.overrides`: Optional resource-name overrides. + - `ec2.overrides.name_runner`: Name tag used for runner compute resources. An empty value uses the generated provider name. + - `ec2.overrides.name_sg`: Name tag used for the managed runner security group. An empty value uses the generated provider name. + - `ec2.instance_profile`: Optional externally managed instance profile used by the launch template. + - `ec2.instance_profile.name`: Name of the externally managed instance profile. + - `ec2.instance_profile_path`: IAM path for the provider-managed instance profile. Null uses a path derived from the stack prefix. + - `ec2.binaries_syncer`: Runner-distribution synchronization configuration. + - `ec2.binaries_syncer.enabled`: Enables use of a synchronized runner distribution from S3. + - `ec2.binaries_syncer.s3`: S3 object containing the synchronized runner distribution. Required when synchronization is enabled. + - `ec2.binaries_syncer.s3.arn`: ARN of the runner-distribution bucket, used by IAM policies. + - `ec2.binaries_syncer.s3.id`: Bucket name used to construct the runner-distribution S3 URI. + - `ec2.binaries_syncer.s3.key`: Object key of the runner distribution. + - `ec2.block_device_mappings`: EBS mappings added to the runner launch template. + - `ec2.block_device_mappings[].delete_on_termination`: Deletes the volume when its runner instance terminates. + - `ec2.block_device_mappings[].device_name`: Device name exposed to the runner instance. + - `ec2.block_device_mappings[].encrypted`: Enables EBS encryption. + - `ec2.block_device_mappings[].iops`: Provisioned IOPS for volume types that support it. + - `ec2.block_device_mappings[].kms_key_id`: KMS key ID or ARN used to encrypt the volume. + - `ec2.block_device_mappings[].snapshot_id`: Snapshot used to initialize the volume. + - `ec2.block_device_mappings[].throughput`: Provisioned throughput for volume types that support it. + - `ec2.block_device_mappings[].volume_initialization_rate`: Fixed initialization rate in MiB/s for supported snapshot-backed volumes. + - `ec2.block_device_mappings[].volume_size`: Volume size in GiB. + - `ec2.block_device_mappings[].volume_type`: EBS volume type. + - `ec2.ebs_optimized`: Requests EBS-optimized runner instances. + - `ec2.instance_target_capacity_type`: Primary capacity type, either `spot` or `on-demand`. + - `ec2.instance_allocation_strategy`: EC2 Fleet allocation strategy used to select instance capacity. + - `ec2.instance_type_priorities`: Optional numeric priorities keyed by instance type. + - `ec2.instance_max_spot_price`: Optional maximum hourly Spot price. + - `ec2.instance_types`: EC2 instance types available to the scale-up and pool functions. + - `ec2.user_data`: Runner bootstrap user-data configuration. + - `ec2.user_data.enabled`: Enables launch-template user data. + - `ec2.user_data.template`: Optional path to a custom user-data template. + - `ec2.user_data.content`: Optional complete user-data content. When set, it is used instead of rendering a template. + - `ec2.user_data.pre_install`: Script content inserted before runner installation in the default template. + - `ec2.user_data.post_install`: Script content inserted after runner installation in the default template. + - `ec2.user_data.debug_logging_enabled`: Enables verbose user-data tracing, which can expose secrets in logs. + - `ec2.ssm_enabled`: Attaches runner permissions and policies required for AWS Systems Manager access. + - `ec2.create_service_linked_role_spot`: Allows scale-up to create the EC2 Spot service-linked role. + - `ec2.cloudwatch_agent`: CloudWatch agent configuration for runner instances. + - `ec2.cloudwatch_agent.enabled`: Installs and configures the CloudWatch agent through the default bootstrap flow. + - `ec2.cloudwatch_agent.config`: Optional complete CloudWatch agent configuration. Null renders the provider default from `log_files`. + - `ec2.managed_security_group_enabled`: Creates and attaches the provider-managed runner security group. + - `ec2.log_files`: Optional log files collected by the CloudWatch agent. Null uses the provider defaults. + - `ec2.log_files[].log_group_name`: CloudWatch log-group name, before optional prefixing. + - `ec2.log_files[].prefix_log_group`: Prefixes the log-group name with the runner stack path when true. + - `ec2.log_files[].file_path`: File or glob read by the CloudWatch agent. + - `ec2.log_files[].log_stream_name`: CloudWatch log-stream name template. + - `ec2.log_files[].log_class`: CloudWatch log-group class for the collected file. + - `ec2.key_name`: Optional EC2 key-pair name added to the launch template. + - `ec2.additional_security_group_ids`: Existing security groups attached in addition to the managed security group. + - `ec2.detailed_monitoring_enabled`: Enables detailed EC2 monitoring for runner instances. + - `ec2.egress_rules`: Egress rules created on the managed runner security group. + - `ec2.egress_rules[].cidr_blocks`: IPv4 CIDR destinations. + - `ec2.egress_rules[].ipv6_cidr_blocks`: IPv6 CIDR destinations. + - `ec2.egress_rules[].prefix_list_ids`: AWS prefix-list destinations. + - `ec2.egress_rules[].from_port`: First destination port in the permitted range. + - `ec2.egress_rules[].protocol`: IP protocol name or number. Use `-1` for all protocols. + - `ec2.egress_rules[].security_groups`: Destination security-group IDs. + - `ec2.egress_rules[].self`: Allows traffic to the managed security group itself when true. + - `ec2.egress_rules[].to_port`: Last destination port in the permitted range. + - `ec2.egress_rules[].description`: Optional rule description. + - `ec2.tags`: Additional tags for runner instances, EBS volumes, network interfaces, and eligible Spot instance requests created from the launch template. They override module-level tags and the generated runner `Name`; the provider-managed `ghr:environment`, `ghr:ssm_config_path`, and `ghr:runner_name_prefix` bootstrap tags take final precedence. These tags do not apply to static provider resources such as the launch template, security group, IAM resources, SSM parameters, or log groups. + - `ec2.metadata_options`: Instance Metadata Service configuration in the launch template. + - `ec2.metadata_options.instance_metadata_tags`: Exposes instance tags through Instance Metadata Service when `enabled`. + - `ec2.metadata_options.http_endpoint`: Enables or disables the Instance Metadata Service endpoint. + - `ec2.metadata_options.http_tokens`: Controls whether IMDSv2 session tokens are optional or required. + - `ec2.metadata_options.http_put_response_hop_limit`: Network hop limit for Instance Metadata Service token responses. + - `ec2.credit_specification`: CPU credit mode for burstable instance types, either `standard` or `unlimited`. + - `ec2.cpu_options`: CPU topology and processor-feature configuration. + - `ec2.cpu_options.core_count`: Number of CPU cores exposed to the runner instance. + - `ec2.cpu_options.threads_per_core`: Number of hardware threads exposed per CPU core. + - `ec2.cpu_options.amd_sev_snp`: Enables or disables AMD SEV-SNP on supported instance types. + - `ec2.cpu_options.nested_virtualization`: Enables or disables nested virtualization on supported instance types. + - `ec2.placement`: EC2 placement configuration for runner instances. + - `ec2.placement.affinity`: Host affinity setting. + - `ec2.placement.availability_zone`: Availability Zone in which the instance is placed. + - `ec2.placement.group_id`: Placement-group ID. + - `ec2.placement.group_name`: Placement-group name. + - `ec2.placement.host_id`: Dedicated Host ID. + - `ec2.placement.host_resource_group_arn`: ARN of the host resource group used for placement. + - `ec2.placement.spread_domain`: Spread-domain placement value. + - `ec2.placement.tenancy`: Instance tenancy, such as `default`, `dedicated`, or `host`. + - `ec2.placement.partition_number`: Placement-group partition number. + - `ec2.license_specifications`: License Manager configurations added to the launch template. + - `ec2.license_specifications[].license_configuration_arn`: ARN of a License Manager license configuration. + - `ec2.associate_public_ipv4_address`: Associates a public IPv4 address with runner network interfaces. + - `ec2.enable_on_demand_failover_for_errors`: EC2 error codes that trigger an on-demand fallback after a Spot launch failure. + - `ec2.scale_errors`: EC2 error codes treated as retryable scale-up failures. + - `ec2.use_dedicated_host`: Enables the dedicated-host launch path, required for macOS runners. + - `microvm`: Lambda MicroVM compute-provider configuration. + - `microvm.image_identifier`: ARN or ID of the MicroVM image used to run GitHub runners. + - `microvm.image_version`: Optional MicroVM image version. + - `microvm.execution_role.arn`: Optional externally managed execution role assumed by MicroVMs. Null uses the common runner role. + - `microvm.egress_network_connectors`: Egress network connectors passed to RunMicrovm. + - `microvm.idle_policy`: Optional auto-suspend and auto-resume configuration passed to RunMicrovm. + - `microvm.logging`: Optional RunMicrovm logging union. Exactly one of `cloud_watch` or `disabled` must be selected when set. + - `microvm.run_hook_payload`: Optional payload delivered to the MicroVM `/run` hook. Maximum 16,384 characters. + - `microvm.maximum_duration_in_seconds`: Optional maximum MicroVM lifetime. Valid range is 1 through 28,800 seconds. + - `microvm.environment_variables`: Additional provider-specific Lambda environment variables merged into scale-up, scale-down, and pool. + - `microvm.tags`: Tags encoded into the MicroVM runner configuration. + - `microvm.iam`: Optional MicroVM control-plane IAM overrides and managed policy attachments. + EOT + + type = object({ + ec2 = optional(object({ + ami = optional(object({ + filter = optional(map(list(string)), { state = ["available"] }) + owners = optional(list(string), ["amazon"]) + id_ssm_parameter = optional(object({ + arn = string + }), null) + kms_key = optional(object({ + arn = string + }), null) + }), null) + vpc_id = string + subnet_ids = list(string) + overrides = optional(object({ + name_runner = optional(string, "") + name_sg = optional(string, "") + }), {}) + instance_profile = optional(object({ + name = string + }), null) + instance_profile_path = optional(string, null) + binaries_syncer = optional(object({ + enabled = optional(bool, true) + s3 = optional(object({ + arn = string + id = string + key = string + }), null) + }), {}) + block_device_mappings = optional(list(object({ + delete_on_termination = optional(bool, true) + device_name = optional(string, "/dev/xvda") + encrypted = optional(bool, true) + iops = optional(number) + kms_key_id = optional(string) + snapshot_id = optional(string) + throughput = optional(number) + volume_initialization_rate = optional(number) + volume_size = number + volume_type = optional(string, "gp3") + })), [{ volume_size = 30 }]) + ebs_optimized = optional(bool, false) + instance_target_capacity_type = optional(string, "spot") + instance_allocation_strategy = optional(string, "lowest-price") + instance_type_priorities = optional(map(number), null) + instance_max_spot_price = optional(string, null) + instance_types = list(string) + user_data = optional(object({ + enabled = optional(bool, true) + template = optional(string, null) + content = optional(string, null) + pre_install = optional(string, "") + post_install = optional(string, "") + debug_logging_enabled = optional(bool, false) + }), {}) + ssm_enabled = optional(bool, false) + create_service_linked_role_spot = optional(bool, false) + cloudwatch_agent = optional(object({ + enabled = optional(bool, true) + config = optional(string, null) + }), {}) + managed_security_group_enabled = optional(bool, true) + log_files = optional(list(object({ + log_group_name = string + prefix_log_group = bool + file_path = string + log_stream_name = string + log_class = optional(string, "STANDARD") + })), null) + key_name = optional(string, null) + additional_security_group_ids = optional(list(string), []) + detailed_monitoring_enabled = optional(bool, false) + egress_rules = optional(list(object({ + cidr_blocks = list(string) + ipv6_cidr_blocks = list(string) + prefix_list_ids = list(string) + from_port = number + protocol = string + security_groups = list(string) + self = bool + to_port = number + description = string + })), [{ + cidr_blocks = ["0.0.0.0/0"] + ipv6_cidr_blocks = ["::/0"] + prefix_list_ids = null + from_port = 0 + protocol = "-1" + security_groups = null + self = null + to_port = 0 + description = null + }]) + tags = optional(map(string), {}) + metadata_options = optional(object({ + instance_metadata_tags = optional(string, "enabled") + http_endpoint = optional(string, "enabled") + http_tokens = optional(string, "required") + http_put_response_hop_limit = optional(number, 1) + }), {}) + credit_specification = optional(string, null) + cpu_options = optional(object({ + core_count = optional(number) + threads_per_core = optional(number) + amd_sev_snp = optional(string) + nested_virtualization = optional(string) + }), null) + placement = optional(object({ + affinity = optional(string) + availability_zone = optional(string) + group_id = optional(string) + group_name = optional(string) + host_id = optional(string) + host_resource_group_arn = optional(string) + spread_domain = optional(string) + tenancy = optional(string) + partition_number = optional(number) + }), null) + license_specifications = optional(list(object({ + license_configuration_arn = string + })), []) + associate_public_ipv4_address = optional(bool, false) + enable_on_demand_failover_for_errors = optional(list(string), []) + scale_errors = optional(list(string), [ + "UnfulfillableCapacity", + "MaxSpotInstanceCountExceeded", + "TargetCapacityLimitExceededException", + "RequestLimitExceeded", + "ResourceLimitExceeded", + "MaxSpotInstanceCountExceeded", + "MaxSpotFleetRequestCountExceeded", + "InsufficientInstanceCapacity", + "InsufficientCapacityOnHost", + ]) + use_dedicated_host = optional(bool, false) + }), null) + + microvm = optional(object({ + image_identifier = string + image_version = optional(string, null) + execution_role = optional(object({ + arn = string + }), null) + egress_network_connectors = optional(list(string), []) + idle_policy = optional(object({ + max_idle_duration_seconds = number + suspended_duration_seconds = number + auto_resume_enabled = bool + }), null) + logging = optional(object({ + cloud_watch = optional(object({ + log_group = optional(string, null) + log_stream = optional(string, null) + }), null) + disabled = optional(bool, false) + }), null) + run_hook_payload = optional(string, null) + maximum_duration_in_seconds = optional(number, null) + environment_variables = optional(map(string), {}) + tags = optional(map(string), {}) + iam = optional(object({ + resource_arns = optional(list(string), ["*"]) + actions = optional(object({ + scale_up = optional(list(string), null) + scale_down = optional(list(string), null) + }), {}) + additional_policy_json = optional(object({ + scale_up = optional(string, null) + }), {}) + managed_policy_arns = optional(object({ + scale_up = optional(string, null) + pool = optional(string, null) + }), {}) + }), {}) + }), null) + }) + + validation { + condition = length([ + for provider_type, provider_config in var.compute_provider : provider_type + if provider_config != null + ]) == 1 + error_message = "Exactly one compute-provider block must be set. Supported compute-provider blocks: ec2, microvm." + } +} diff --git a/modules/runner-stack/variables.tf b/modules/runner-stack/variables.tf new file mode 100644 index 0000000000..6adb2e6fcd --- /dev/null +++ b/modules/runner-stack/variables.tf @@ -0,0 +1,441 @@ +variable "aws_region" { + description = "AWS region." + type = string +} + +variable "aws_partition" { + description = "AWS partition used to construct ARNs." + type = string + default = "aws" +} + +variable "prefix" { + description = "The prefix used for naming resources." + type = string + default = "github-actions" +} + +variable "tags" { + description = "Base tags added to taggable resources created by this stack. Shared, component, and compute-provider tag maps override matching keys within their documented resource scopes." + type = map(string) + default = {} +} + +variable "runner" { + description = <<-EOT + Provider-neutral GitHub runner configuration. + + - `os`: Runner operating system. Supported values are `linux`, `osx`, and `windows`. + - `architecture`: Runner distribution architecture, such as `x64` or `arm64`. + - `boot_time_in_minutes`: Expected instance boot duration used before a runner is considered stale. + - `disable_default_labels`: Prevents GitHub's default self-hosted, operating-system, and architecture labels from being registered. + - `labels`: Complete set of labels supplied to the control-plane functions. + - `group_name`: GitHub runner group used during registration. + - `name_prefix`: Prefix added to registered runner names. + - `run_as_root`: Runs the runner service as root when supported by the compute provider. + - `run_as`: Operating-system user used when `run_as_root` is false. + - `maximum_count`: Maximum number of runners that may exist for this stack. + - `ephemeral`: Registers runners in ephemeral mode. + - `jit_config_enabled`: Explicitly enables or disables just-in-time configuration. When null, runtime behavior follows `ephemeral`. + - `auto_update_disabled`: Disables the GitHub runner application's built-in updater. + - `tags`: Additional tags for common runner resources, currently the managed runner IAM role. These override module-level `tags` with the same key. + - `hooks.job_started`: Script content installed as the runner job-started hook. + - `hooks.job_completed`: Script content installed as the runner job-completed hook. + - `iam.role.arn`: ARN of an externally managed runner role. When set, this module does not create or modify that role. + - `iam.managed_policy_arns`: Named managed-policy ARNs attached to the module-managed runner role. + - `iam.additional_trust_policy_json`: Optional IAM policy document merged with the selected compute provider's default runner-role trust policy. + - `iam.path`: IAM path for the module-managed runner role. Defaults to a path derived from `prefix`. + - `iam.permissions_boundary`: Permissions-boundary ARN for the module-managed runner role. + EOT + type = object({ + os = optional(string, "linux") + architecture = optional(string, "x64") + boot_time_in_minutes = optional(number, 5) + disable_default_labels = optional(bool, false) + labels = list(string) + group_name = optional(string, "Default") + name_prefix = optional(string, "") + run_as_root = optional(bool, false) + run_as = optional(string, "ec2-user") + maximum_count = optional(number, 3) + ephemeral = optional(bool, false) + jit_config_enabled = optional(bool, null) + auto_update_disabled = optional(bool, false) + tags = optional(map(string), {}) + hooks = optional(object({ + job_started = optional(string, "") + job_completed = optional(string, "") + }), {}) + iam = optional(object({ + role = optional(object({ + arn = string + }), null) + managed_policy_arns = optional(map(string), {}) + additional_trust_policy_json = optional(string, null) + path = optional(string, null) + permissions_boundary = optional(string, null) + }), {}) + }) + + validation { + condition = contains(["linux", "osx", "windows"], var.runner.os) + error_message = "Valid values for runner.os are linux, osx, and windows." + } + + validation { + condition = length(var.runner.name_prefix) <= 45 + error_message = "runner.name_prefix must be at most 45 characters." + } + + validation { + condition = var.runner.iam.role == null ? true : trimspace(var.runner.iam.role.arn) != "" + error_message = "runner.iam.role.arn must be a non-empty ARN when set." + } + + validation { + condition = var.runner.iam.role == null || length(var.runner.iam.managed_policy_arns) == 0 + error_message = "runner.iam.managed_policy_arns cannot be set with an external runner.iam.role because external roles are not managed by this module." + } + + validation { + condition = var.runner.iam.additional_trust_policy_json == null ? true : can(jsondecode(var.runner.iam.additional_trust_policy_json)) + error_message = "runner.iam.additional_trust_policy_json must be valid JSON when set." + } + + validation { + condition = var.runner.iam.role == null || var.runner.iam.additional_trust_policy_json == null + error_message = "runner.iam.additional_trust_policy_json cannot be set with an external runner.iam.role because external role trust is not managed by this module." + } +} + +variable "github" { + description = <<-EOT + GitHub API and runner-registration configuration. + + - `app_parameters.key_base64`: Parameter Store reference for the GitHub App private key. + - `app_parameters.key_base64.name`: Name of the private-key parameter supplied to the control-plane functions. + - `app_parameters.key_base64.arn`: ARN of the private-key parameter used by IAM policies. + - `app_parameters.id`: Parameter Store reference for the GitHub App ID. + - `app_parameters.id.name`: Name of the App-ID parameter supplied to the control-plane functions. + - `app_parameters.id.arn`: ARN of the App-ID parameter used by IAM policies. + - `organization_runners`: Registers runners at organization scope when true; otherwise repository-scoped registration is used. + - `enterprise_server.url`: Optional GitHub Enterprise Server base URL. Null selects GitHub.com. + - `enterprise_server.ssl_verify`: Enables TLS certificate verification for GitHub Enterprise Server requests. + - `user_agent`: Optional User-Agent value added to GitHub API requests. + EOT + type = object({ + app_parameters = object({ + key_base64 = map(string) + id = map(string) + }) + organization_runners = bool + enterprise_server = optional(object({ + url = optional(string, null) + ssl_verify = optional(bool, true) + }), {}) + user_agent = optional(string, null) + }) +} + +variable "queue" { + description = <<-EOT + Build queue reference and queue-integrated Lambda configuration. + + - `build.arn`: ARN of the externally managed build queue consumed by scale-up. + - `build.url`: URL of the externally managed build queue used when messages are published. + - `event_source_mapping.batch_size`: Maximum records delivered to a Lambda invocation. + - `event_source_mapping.maximum_batching_window_in_seconds`: Maximum time Lambda may buffer records before invocation. + - `tags`: Shared tags for queue-related resources created by this stack, including event-source mappings and the optional job-retry queue. These override module-level `tags`; component `tags` override this map when keys conflict. The referenced build queue is not managed or tagged by this module. + EOT + type = object({ + build = object({ + arn = string + url = string + }) + event_source_mapping = optional(object({ + batch_size = optional(number, 10) + maximum_batching_window_in_seconds = optional(number, 0) + }), {}) + tags = optional(map(string), {}) + }) + + validation { + condition = var.queue.event_source_mapping.batch_size >= 1 && var.queue.event_source_mapping.batch_size <= 1000 + error_message = "queue.event_source_mapping.batch_size must be between 1 and 1000." + } + + validation { + condition = var.queue.event_source_mapping.maximum_batching_window_in_seconds >= 0 && var.queue.event_source_mapping.maximum_batching_window_in_seconds <= 300 + error_message = "queue.event_source_mapping.maximum_batching_window_in_seconds must be between 0 and 300." + } +} + +variable "lambda" { + description = <<-EOT + Configuration shared by the control-plane Lambda functions. + + - `zip`: Local control-plane archive. When null, the module's packaged runner archive is used. + - `s3.bucket`: Optional S3 bucket containing the Lambda archive. Setting this selects S3 instead of a local archive. + - `s3.key`: Object key of the Lambda archive in `s3.bucket`. + - `s3.object_version`: Optional version of the Lambda archive object. + - `runtime`: Runtime used by all control-plane Lambda functions. + - `architecture`: Instruction-set architecture used by all control-plane Lambda functions. Supported values are `arm64` and `x86_64`. + - `subnet_ids`: Subnets used for Lambda VPC configuration. + - `security_group_ids`: Security groups used for Lambda VPC configuration. + - `tags`: Shared tags applied to Lambda function resources only. These override module-level `tags`; component `tags` override this map when keys conflict. + - `role.path`: IAM path for module-managed Lambda execution roles. Defaults to a path derived from `prefix`. + - `role.permissions_boundary`: Permissions-boundary ARN applied to module-managed Lambda execution roles. + EOT + type = object({ + zip = optional(string, null) + s3 = optional(object({ + bucket = optional(string, null) + key = optional(string, null) + object_version = optional(string, null) + }), {}) + runtime = optional(string, "nodejs24.x") + architecture = optional(string, "arm64") + subnet_ids = optional(list(string), []) + security_group_ids = optional(list(string), []) + tags = optional(map(string), {}) + role = optional(object({ + path = optional(string, null) + permissions_boundary = optional(string, null) + }), {}) + }) + default = {} + + validation { + condition = contains(["arm64", "x86_64"], var.lambda.architecture) + error_message = "lambda.architecture must be arm64 or x86_64." + } +} + +variable "scale_up" { + description = <<-EOT + Scale-up component configuration. + + - `memory_size`: Memory allocated to the scale-up Lambda in MB. + - `timeout`: Scale-up Lambda timeout in seconds. + - `reserved_concurrent_executions`: Reserved concurrency for the scale-up Lambda. Use `-1` for unreserved concurrency. + - `job_queued_check_enabled`: Enables the queued-job verification before scaling. When null, the default is enabled for persistent runners and disabled for ephemeral runners. + - `tags`: Tags for scale-up resources, including the Lambda function, log group, event-source mapping, and IAM role. These override module-level tags and the shared `lambda.tags`, `queue.tags`, and `observability.logs.tags` maps when keys conflict. + EOT + type = object({ + memory_size = optional(number, 512) + timeout = optional(number, 60) + reserved_concurrent_executions = optional(number, 1) + job_queued_check_enabled = optional(bool, null) + tags = optional(map(string), {}) + }) + default = {} +} + +variable "scale_down" { + description = <<-EOT + Scale-down Lambda, schedule, and idle-runner configuration. + + - `memory_size`: Memory allocated to the scale-down Lambda in MB. + - `timeout`: Scale-down Lambda timeout in seconds. + - `schedule_expression`: EventBridge schedule expression that invokes scale-down. + - `minimum_running_time_in_minutes`: Minimum runner age before scale-down may terminate it. Null selects the operating-system default. + - `tags`: Tags for scale-down resources, including the Lambda function, log group, EventBridge rule, and IAM role. These override module-level tags and the shared `lambda.tags` and `observability.logs.tags` maps when keys conflict. + - `idle_config`: Time-based desired idle-runner configurations. + - `idle_config[].cron`: Cron expression identifying when the configuration applies. + - `idle_config[].timeZone`: IANA time zone used to evaluate `cron`. + - `idle_config[].idleCount`: Number of idle runners to retain during the matching period. + - `idle_config[].evictionStrategy`: Selection strategy used when excess idle runners are removed. + EOT + type = object({ + memory_size = optional(number, 512) + timeout = optional(number, 60) + schedule_expression = optional(string, "cron(*/5 * * * ? *)") + minimum_running_time_in_minutes = optional(number, null) + tags = optional(map(string), {}) + idle_config = optional(list(object({ + cron = string + timeZone = string + idleCount = number + evictionStrategy = optional(string, "oldest_first") + })), []) + }) + default = {} +} + +variable "pool" { + description = <<-EOT + Scheduled runner-pool configuration. The pool component is created only when `config` is non-empty. + + - `config`: Scheduled target pool sizes. + - `config[].schedule_expression`: Scheduler expression that activates the target size. + - `config[].schedule_expression_timezone`: Optional IANA time zone used to evaluate the schedule. + - `config[].size`: Desired number of runners for the schedule. + - `include_busy_runners`: Includes busy runners when calculating the current pool size. + - `runner_owner`: Optional GitHub organization or repository owner used when creating pooled runners. + - `tags`: Tags for pool resources, including the Lambda function, log group, IAM roles, and scheduler group. These override module-level tags and the shared `lambda.tags` and `observability.logs.tags` maps when keys conflict. + - `lambda.memory_size`: Memory allocated to the pool Lambda in MB. + - `lambda.timeout`: Pool Lambda timeout in seconds. + - `lambda.reserved_concurrent_executions`: Reserved concurrency for the pool Lambda. Use `-1` for unreserved concurrency. + EOT + type = object({ + config = optional(list(object({ + schedule_expression = string + schedule_expression_timezone = optional(string) + size = number + })), []) + include_busy_runners = optional(bool, false) + runner_owner = optional(string, null) + tags = optional(map(string), {}) + lambda = optional(object({ + memory_size = optional(number, 512) + timeout = optional(number, 60) + reserved_concurrent_executions = optional(number, 1) + }), {}) + }) + default = {} +} + +variable "job_retry" { + description = <<-EOT + Job-retry queue and Lambda configuration. + + - `enabled`: Creates the retry queue, Lambda function, event-source mapping, and related IAM resources. + - `delay_in_seconds`: Initial delay before a queued-job retry check. AWS SQS limits this value to 900 seconds. + - `delay_backoff`: Multiplier applied to the delay after each unsuccessful check. + - `max_attempts`: Maximum retry-check attempts before the message is no longer republished. + - `tags`: Tags for job-retry resources, including the Lambda function, log group, IAM role, retry queue, and event-source mapping. These override module-level tags and the shared `lambda.tags`, `queue.tags`, and `observability.logs.tags` maps when keys conflict. + - `lambda.memory_size`: Memory allocated to the job-retry Lambda in MB. + - `lambda.reserved_concurrent_executions`: Reserved concurrency for the job-retry Lambda. Use `-1` for unreserved concurrency. + - `lambda.timeout`: Job-retry Lambda timeout in seconds and visibility timeout for its retry queue. + EOT + type = object({ + enabled = optional(bool, false) + delay_in_seconds = optional(number, 300) + delay_backoff = optional(number, 2) + max_attempts = optional(number, 1) + tags = optional(map(string), {}) + lambda = optional(object({ + memory_size = optional(number, 256) + reserved_concurrent_executions = optional(number, 1) + timeout = optional(number, 30) + }), {}) + }) + default = {} + + validation { + condition = !var.job_retry.enabled || var.job_retry.delay_in_seconds <= 900 + error_message = "job_retry.delay_in_seconds cannot exceed the SQS maximum of 900 seconds." + } +} + +variable "ssm" { + description = <<-EOT + Parameter Store paths, encryption, tag scopes, and housekeeper configuration. + + - `paths.root`: Root Parameter Store path for this runner stack. + - `paths.tokens`: Path segment under `paths.root` used for registration tokens and just-in-time configuration. + - `paths.config`: Path segment under `paths.root` used for persistent runner configuration. + - `kms_key`: Optional customer-managed KMS key used to encrypt temporary registration parameters. The wrapper's presence is the plan-time policy discriminator. + - `kms_key.arn`: ARN of the customer-managed KMS key. The ARN may be unknown until apply. + - `tags`: Shared tags for SSM-related resources. These override module-level `tags` and are inherited by parameter and housekeeper resources. + - `parameters.tags`: Tags for Terraform-managed runner configuration parameters and temporary parameters created by the scale-up and pool Lambdas. These override module-level and `ssm.tags` values with the same key. + - `housekeeper.schedule_expression`: EventBridge schedule expression that invokes the SSM housekeeper. + - `housekeeper.state`: EventBridge rule state, such as `ENABLED` or `DISABLED`. + - `housekeeper.tags`: Tags for housekeeper resources, including the Lambda function, log group, EventBridge rule, and IAM role. These override module-level, `ssm.tags`, shared Lambda, and shared log tags when keys conflict. + - `housekeeper.lambda.memory_size`: Memory allocated to the SSM housekeeper Lambda in MB. + - `housekeeper.lambda.timeout`: SSM housekeeper Lambda timeout in seconds. + - `housekeeper.config.tokenPath`: Parameter Store token path cleaned by the housekeeper. When omitted, the configured runner token path is used. + - `housekeeper.config.minimumDaysOld`: Minimum parameter age in days before deletion is allowed. + - `housekeeper.config.dryRun`: Reports eligible parameters without deleting them when true. + EOT + type = object({ + paths = object({ + root = string + tokens = string + config = string + }) + kms_key = optional(object({ + arn = string + }), null) + tags = optional(map(string), {}) + parameters = optional(object({ + tags = optional(map(string), {}) + }), {}) + housekeeper = optional(object({ + schedule_expression = optional(string, "rate(1 day)") + state = optional(string, "ENABLED") + tags = optional(map(string), {}) + lambda = optional(object({ + memory_size = optional(number, 512) + timeout = optional(number, 60) + }), {}) + config = optional(object({ + tokenPath = optional(string) + minimumDaysOld = optional(number, 1) + dryRun = optional(bool, false) + }), {}) + }), {}) + }) +} + +variable "observability" { + description = <<-EOT + Logging, tracing, and metrics configuration for control-plane and provider resources. + + - `logs.level`: Application log level supplied to the control-plane functions. + - `logs.retention_in_days`: CloudWatch Logs retention period. + - `logs.kms_key_id`: Optional KMS key ID or ARN used to encrypt CloudWatch log groups. + - `logs.class`: CloudWatch log-group class. Supported values are `STANDARD` and `INFREQUENT_ACCESS`. + - `logs.tags`: Shared tags for CloudWatch log groups. These override module-level `tags`; component `tags` override this map when keys conflict. + - `tracing.mode`: Optional Lambda active-tracing mode. Null disables X-Ray tracing configuration. + - `tracing.capture_http_requests`: Enables HTTP request capture in the tracing helper. + - `tracing.capture_error`: Enables error capture in the tracing helper. + - `metrics.enable`: Enables module-emitted metrics. + - `metrics.namespace`: CloudWatch namespace used for emitted metrics. + - `metrics.metric.enable_github_app_rate_limit`: Emits GitHub App rate-limit metrics. + - `metrics.metric.enable_job_retry`: Emits job-retry metrics. + - `metrics.metric.enable_spot_termination_warning`: Emits spot-termination warning metrics where supported. + EOT + type = object({ + logs = optional(object({ + level = optional(string, "info") + retention_in_days = optional(number, 180) + kms_key_id = optional(string, null) + class = optional(string, "STANDARD") + tags = optional(map(string), {}) + }), {}) + tracing = optional(object({ + mode = optional(string, null) + capture_http_requests = optional(bool, false) + capture_error = optional(bool, false) + }), {}) + metrics = optional(object({ + enable = optional(bool, false) + namespace = optional(string, "GitHub Runners") + metric = optional(object({ + enable_github_app_rate_limit = optional(bool, true) + enable_job_retry = optional(bool, true) + enable_spot_termination_warning = optional(bool, true) + }), {}) + }), {}) + }) + default = {} + + validation { + condition = contains(["STANDARD", "INFREQUENT_ACCESS"], var.observability.logs.class) + error_message = "observability.logs.class must be STANDARD or INFREQUENT_ACCESS." + } + + validation { + condition = contains([ + "silly", + "trace", + "debug", + "info", + "warn", + "error", + "fatal", + ], var.observability.logs.level) + error_message = "observability.logs.level must be one of silly, trace, debug, info, warn, error, or fatal." + } +} diff --git a/modules/runner-stack/versions.tf b/modules/runner-stack/versions.tf new file mode 100644 index 0000000000..da9769f550 --- /dev/null +++ b/modules/runner-stack/versions.tf @@ -0,0 +1,10 @@ +terraform { + required_version = ">= 1.3.0" + + required_providers { + aws = { + source = "hashicorp/aws" + version = ">= 6.33" + } + } +} diff --git a/modules/runners/scale-down.tf b/modules/runners/scale-down.tf index 0775893d71..d99867243a 100644 --- a/modules/runners/scale-down.tf +++ b/modules/runners/scale-down.tf @@ -42,7 +42,7 @@ resource "aws_lambda_function" "scale_down" { POWERTOOLS_TRACE_ENABLED = var.tracing_config.mode != null ? true : false POWERTOOLS_TRACER_CAPTURE_HTTPS_REQUESTS = var.tracing_config.capture_http_requests POWERTOOLS_TRACER_CAPTURE_ERROR = var.tracing_config.capture_error - RUNNER_PROVIDER_TYPE = "ec2" + COMPUTE_PROVIDER_TYPE = "ec2" } } diff --git a/modules/runners/scale-up.tf b/modules/runners/scale-up.tf index bdda3c070f..16316b5a35 100644 --- a/modules/runners/scale-up.tf +++ b/modules/runners/scale-up.tf @@ -54,7 +54,7 @@ resource "aws_lambda_function" "scale_up" { RUNNER_LABELS = lower(join(",", var.runner_labels)) RUNNER_GROUP_NAME = var.runner_group_name RUNNER_NAME_PREFIX = var.runner_name_prefix - RUNNER_PROVIDER_TYPE = "ec2" + COMPUTE_PROVIDER_TYPE = "ec2" RUNNERS_MAXIMUM_COUNT = var.runners_maximum_count POWERTOOLS_SERVICE_NAME = "${var.prefix}-scale-up" SSM_TOKEN_PATH = local.token_path diff --git a/modules/webhook/README.md b/modules/webhook/README.md index e63a8c01b7..f104d69d15 100644 --- a/modules/webhook/README.md +++ b/modules/webhook/README.md @@ -89,7 +89,7 @@ yarn run dist | [repository\_white\_list](#input\_repository\_white\_list) | List of github repository full names (owner/repo\_name) that will be allowed to use the github app. Leave empty for no filtering. | `list(string)` | `[]` | no | | [role\_path](#input\_role\_path) | The path that will be added to the role; if not set, the environment name will be used. | `string` | `null` | no | | [role\_permissions\_boundary](#input\_role\_permissions\_boundary) | Permissions boundary that will be added to the created role for the lambda. | `string` | `null` | no | -| [runner\_matcher\_config](#input\_runner\_matcher\_config) | SQS queue to publish accepted build events based on the runner type. `runnerProvider` defaults to `ec2`; EC2 is the only provider currently implemented. When exact match is disabled the webhook accepts the event if one of the workflow job labels is part of the matcher. The priority defines the order the matchers are applied. Optional `matcherConfig.enableDynamicLabels` and `matcherConfig.awsDynamicLabelsPolicy` are evaluated by the dispatcher to gate provider dynamic labels per runner. The policy supports `blocked_keys = []` and `restricted_keys = { = { allowed = [globs], denied = [globs], max = number|string } }`; keys use the provider dynamic label suffix form, for example `instance-type` for `ghr-ec2-instance-type`. | 
map(object({
    arn            = string
    id             = string
    runnerProvider = optional(string, "ec2")
    matcherConfig = object({
      labelMatchers           = list(list(string))
      exactMatch              = bool
      bidirectionalLabelMatch = optional(bool, false)
      priority                = optional(number, 999)
      enableDynamicLabels     = optional(bool, false)
      awsDynamicLabelsPolicy  = optional(any, null)
    })
  }))
| n/a | yes | +| [runner\_matcher\_config](#input\_runner\_matcher\_config) | SQS queue to publish accepted build events based on the runner type. `computeProvider` defaults to `ec2`; supported values are `ec2` and `microvm`. When exact match is disabled the webhook accepts the event if one of the workflow job labels is part of the matcher. The priority defines the order the matchers are applied. Optional `matcherConfig.enableDynamicLabels` and `matcherConfig.awsDynamicLabelsPolicy` are evaluated by the dispatcher to gate provider dynamic labels per runner. The policy supports `blocked_keys = []` and `restricted_keys = { = { allowed = [globs], denied = [globs], max = number|string } }`; keys use the provider dynamic label suffix form, for example `instance-type` for `ghr-ec2-instance-type`. | 
map(object({
    arn             = string
    id              = string
    computeProvider = optional(string, "ec2")
    matcherConfig = object({
      labelMatchers           = list(list(string))
      exactMatch              = bool
      bidirectionalLabelMatch = optional(bool, false)
      priority                = optional(number, 999)
      enableDynamicLabels     = optional(bool, false)
      awsDynamicLabelsPolicy = optional(object({
        blocked_keys = optional(list(string), [])
        restricted_keys = optional(map(object({
          allowed = optional(list(string), [])
          denied  = optional(list(string), [])
          max     = optional(string, null)
        })), {})
      }), null)
    })
  }))
| n/a | yes | | [ssm\_paths](#input\_ssm\_paths) | The root path used in SSM to store configuration and secrets. | 
object({
    root    = string
    webhook = string
  })
| n/a | yes | | [tags](#input\_tags) | Map of tags that will be added to created resources. By default resources will be tagged with name and environment. | `map(string)` | `{}` | no | | [tracing\_config](#input\_tracing\_config) | Configuration for lambda tracing. | 
object({
    mode                  = optional(string, null)
    capture_http_requests = optional(bool, false)
    capture_error         = optional(bool, false)
  })
| `{}` | no | diff --git a/modules/webhook/variables.tf b/modules/webhook/variables.tf index fcdb7a7dcf..2983af59a2 100644 --- a/modules/webhook/variables.tf +++ b/modules/webhook/variables.tf @@ -23,31 +23,31 @@ variable "tags" { } variable "runner_matcher_config" { - description = "SQS queue to publish accepted build events based on the runner type. `runnerProvider` defaults to `ec2`; EC2 is the only provider currently implemented. When exact match is disabled the webhook accepts the event if one of the workflow job labels is part of the matcher. The priority defines the order the matchers are applied. Optional `matcherConfig.enableDynamicLabels` and `matcherConfig.awsDynamicLabelsPolicy` are evaluated by the dispatcher to gate provider dynamic labels per runner. The policy supports `blocked_keys = []` and `restricted_keys = { = { allowed = [globs], denied = [globs], max = number|string } }`; keys use the provider dynamic label suffix form, for example `instance-type` for `ghr-ec2-instance-type`." + description = "SQS queue to publish accepted build events based on the runner type. `computeProvider` defaults to `ec2`; supported values are `ec2` and `microvm`. When exact match is disabled the webhook accepts the event if one of the workflow job labels is part of the matcher. The priority defines the order the matchers are applied. Optional `matcherConfig.enableDynamicLabels` and `matcherConfig.awsDynamicLabelsPolicy` are evaluated by the dispatcher to gate provider dynamic labels per runner. The policy supports `blocked_keys = []` and `restricted_keys = { = { allowed = [globs], denied = [globs], max = number|string } }`; keys use the provider dynamic label suffix form, for example `instance-type` for `ghr-ec2-instance-type`." type = map(object({ - arn = string - id = string - runnerProvider = optional(string, "ec2") + arn = string + id = string + computeProvider = optional(string, "ec2") matcherConfig = object({ labelMatchers = list(list(string)) exactMatch = bool bidirectionalLabelMatch = optional(bool, false) priority = optional(number, 999) enableDynamicLabels = optional(bool, false) - awsDynamicLabelsPolicy = optional(any, null) + awsDynamicLabelsPolicy = optional(object({ + blocked_keys = optional(list(string), []) + restricted_keys = optional(map(object({ + allowed = optional(list(string), []) + denied = optional(list(string), []) + max = optional(string, null) + })), {}) + }), null) }) })) validation { condition = try(var.runner_matcher_config.matcherConfig.priority, 999) >= 0 && try(var.runner_matcher_config.matcherConfig.priority, 999) < 1000 error_message = "The priority of the matcher must be between 0 and 999." } - validation { - condition = alltrue([ - for config in values(var.runner_matcher_config) : - lower(trimspace(config.runnerProvider)) == "ec2" - ]) - error_message = "runnerProvider must be ec2." - } } variable "lambda_zip" { diff --git a/modules/webhook/webhook.tf b/modules/webhook/webhook.tf index 84a89bbc93..1c377bbcc0 100644 --- a/modules/webhook/webhook.tf +++ b/modules/webhook/webhook.tf @@ -2,8 +2,8 @@ locals { # config with combined key and order runner_matcher_config = { for k, v in var.runner_matcher_config : format("%03d-%s", v.matcherConfig.priority, k) => merge(v, { - key = k - runnerProvider = lower(trimspace(v.runnerProvider)) + key = k + computeProvider = lower(trimspace(v.computeProvider)) }) }