From ef14232c88f00251c2430622c786249fe5470ff8 Mon Sep 17 00:00:00 2001 From: Justin Gasper Date: Thu, 5 Nov 2026 07:32:12 +1100 Subject: [PATCH] Support reviewed first-release health checks and public token metadata --- README.md | 8 ++- docs/MIGRATION.md | 4 +- tcdeploy.py | 65 +++++++++++++++++++++--- tests/test_security.py | 111 ++++++++++++++++++++++++++++++++++++++++- 4 files changed, 174 insertions(+), 14 deletions(-) diff --git a/README.md b/README.md index 99480cf..1764841 100644 --- a/README.md +++ b/README.md @@ -11,7 +11,7 @@ Read [the migration guide](docs/MIGRATION.md), [the Auth0/AWS explanation](docs/ - A reviewed, digest-pinned Linux executor containing Python >=3.10 with the `tarfile` extraction-filter security backport, AWS CLI v2, Docker/Buildx, Git and system CA certificates. No package installation in the credential-bearing job. This deployer has no external Python dependencies. - CircleCI Cloud with `CIRCLE_OIDC_TOKEN_V3`, protected exact development (`develop` or `dev`) and production (`master` or legacy `prod`) refs, and restricted deployment contexts containing no broker/static AWS credentials. - Separate per-service/per-environment deploy, task and execution roles, and a reviewed non-secret manifest. -- Existing immutable ECR repositories with scan-on-push, ECS circuit-breaker rollback, positive desired capacity, explicit task health checks, and pre-provisioned runtime secret references. +- Existing immutable ECR repositories with scan-on-push, ECS circuit-breaker rollback, positive desired capacity, explicit task health checks (or a reviewed first-deployment health-check file), and pre-provisioned runtime secret references. ## Usage @@ -42,7 +42,11 @@ In a separately authorized deploy job, load the workflow-specific image archive, `awsconfiguration.sh` is an optional identity check, not a credential exporter. `master_deploy.sh` authenticates internally. Neither writes AWS credentials to the checkout/default profile, logs, or shell exports. Use the entire SHA-pinned suite, not the old four copied shell files. See `examples/circleci-v2.yml`; replace its explicit executor/commit placeholders before use. -The deployer verifies account/role, live service configuration, source label, ECR immutability, local/remote configuration digest, a scan completed within 24 hours with no MEDIUM/HIGH/CRITICAL/UNDEFINED findings, and the exact healthy running revision/digests. Only selected images change; roles, secret references, sidecars, resources, volumes and health checks remain intact. Non-root runtime images are required and `RESET_DB` cannot be enabled. The source label is a consistency check, not cryptographic build provenance; protect the pipeline and artifact writers. +The deployer verifies account/role, live service configuration, source label, ECR immutability, local/remote configuration digest, a scan completed within 24 hours with no MEDIUM/HIGH/CRITICAL/UNDEFINED findings, and the exact healthy running revision/digests. By default only selected images change; roles, secret references, sidecars, resources, volumes and existing health checks remain intact. Non-root runtime images are required and `RESET_DB` cannot be enabled. The source label is a consistency check, not cryptographic build provenance; protect the pipeline and artifact writers. + +For an existing service whose task definition has no health check, add a reviewed container-to-check JSON file in the application repository and pass `--bootstrap-health-checks deploy/dev-healthcheck.json`. The deployer adds the missing check in the same task revision as the new, scanned image. It never starts the old image with the new check, and never replaces a different existing check. Matching existing checks are accepted on later releases. Checks must use an explicit `CMD` argument list; all normal identity, configuration, scan, concurrency, readback, and healthy-running-digest gates still apply. This adds no IAM permissions. ECS circuit-breaker rollback retains the previous task revision; a rollback still fails the release. + +Plaintext runtime credentials remain forbidden. The specific public settings `M2M_TOKEN_ISSUER`, `M2M_TOKEN_AUDIENCE`, and `HUMAN_TOKEN_ISSUER` may contain HTTPS URLs without user information, query strings, or fragments; `TOKEN_CACHE_TIME` may contain an unsigned integer up to 2147483647. Other secret-like names still fail preflight. These exceptions preserve public application metadata and do not fetch tokens or grant access to secret stores. ECR may omit its optional severity-count map for a clean image. That response is accepted only after the same completion/freshness checks, with an exact registry/repository/digest match, explicitly empty findings lists, and no remaining page. An absent summary without that evidence, nonempty findings without a summary, or malformed counts fails closed. This requires no additional IAM permissions or changes to registry scanning. See the [ECR response schema](https://docs.aws.amazon.com/AmazonECR/latest/APIReference/API_ImageScanFindings.html). diff --git a/docs/MIGRATION.md b/docs/MIGRATION.md index b6409ff..cd9cb82 100644 --- a/docs/MIGRATION.md +++ b/docs/MIGRATION.md @@ -7,7 +7,7 @@ Merging this PR does not change pinned v1 consumers or installed IAM policies. * 1. Start with a straightforward one-container service, for example lookups. Have an administrator inspect its existing ECS service/task, ECR settings and IAM roles without exporting decrypted SSM values. Record the previous revision/digest in a restricted release record. 2. Create a manifest with exact identifiers from that inspection. Generate IAM documents using `tcdeploy.py policies`; install them through infrastructure review. Keep the explicit secret-access/role-chaining deny in the role policy and preferably use the generated permissions as its permissions boundary too. Remove unrelated policies and audit resource-based grants. The deployer needs no infrastructure-creation permissions. 3. Restrict PassRole to the exact task/execution roles (use JSON `null` for `taskRoleArn` when the existing task has no application role; do not add one just to deploy). Give the execution role only its runtime parameter/secret ARNs and necessary KMS keys. Pre-provision runtime bindings and defaults through administrator-owned infrastructure. v2 preserves references without fetching values/types: missing bindings fail at startup and trigger rollback. -4. Enable unconditional ECR tag immutability and supported basic/enhanced scan-on-push. Enable ECS circuit-breaker rollback, positive desired capacity and explicit task health checks. Remove plaintext secret environment values and `RESET_DB` secret bindings. Use non-root runtime images. Bootstrap/infrastructure updates remain separate from code deployment. +4. Enable unconditional ECR tag immutability and supported basic/enhanced scan-on-push. Enable ECS circuit-breaker rollback, positive desired capacity and explicit task health checks. When the first new image introduces the health endpoint, pass a reviewed `--bootstrap-health-checks` file to add a missing check atomically with that image; do not restart the old image with a check it cannot serve. Existing differing checks cannot be overwritten. Remove plaintext secret environment values and `RESET_DB` secret bindings. Use non-root runtime images. Bootstrap/infrastructure updates remain separate from code deployment. 5. Commit non-secret `deploy/dev.json`. Use an exact `develop` or `dev` branch for dev and the exact reviewed `master` or legacy `prod` branch for production source/config refs. Examples intentionally do not guess real targets. Do not put keys, client secrets, passwords or parameter values in manifests. 6. Prepare a reviewed executor with Python >=3.10 with the `tarfile` extraction-filter security backport, AWS CLI v2, Docker/Buildx, Git and trusted CA certificates. Pin its digest and the suite's full reviewed commit in service configuration. No runtime tool/package installation on the credential host. Replace template placeholders with literal reviewed values, not caller-overridable pipeline parameters. 7. Split build/test and deploy using `examples/circleci-v2.yml`. Remove `org-global`, old helper copies, all `source awsenvconf` / `source *_env`, SSM build/deploy/QA retrieval, and unpinned host installs. Use restricted dev/prod contexts with no broker/static AWS secrets, production approval and serialization. Verify OIDC v3 availability; restrict fork secret passing and SSH debugging. Pin/download the entire suite. @@ -20,7 +20,7 @@ For private packages, use a separate package-read-only identity and BuildKit sec - Run the offline tests and shell syntax checks. They do not prove installed IAM state. - With an administrator identity, validate/simulate the generated policies. SSM listing/reads, Secrets Manager reads, direct KMS decryption and role chaining must be explicitly denied. Verify only the intended ECS service, ECR repositories, task family and two PassRole ARNs are allowed. Check OIDC audience/project/checkout/config origin/ref; a wrong project/ref must not assume the role. Do not weaken project restrictions to make a test pass. - Run `awsconfiguration.sh --manifest deploy/dev.json` in the intended job. It must verify identity without files/credential output. Missing v3 token, wrong account/role and legacy `DEV` arguments must fail. -- Run a real dev release. Verify roles, runtime secret references, sidecars, resources, volumes and health checks remain unchanged. Compare reported image digests with healthy running tasks. CloudTrail should show no deployment-principal SSM/Secrets Manager calls; runtime retrieval belongs to the execution role. +- Run a real dev release. Verify roles, runtime secret references, sidecars, resources, volumes and existing health checks remain unchanged. If using health-check bootstrap, verify only the reviewed missing checks were added with the new image. Compare reported image digests with healthy running tasks. CloudTrail should show no deployment-principal SSM/Secrets Manager calls; runtime retrieval belongs to the execution role. - Test an intentionally unhealthy dev image and verify failed release plus ECS rollback to the previous revision. Wrong digest, failed scan, denied AWS call, concurrency conflict or wrong source label must stop promotion. Old steady-state events/zero capacity must not pass. - Inspect logs, workspace artifacts and image history/layers privately for accidental secrets. Remove stale broker/static credentials from project settings as well as contexts. Credential rotation/session revocation are separate incident actions. diff --git a/tcdeploy.py b/tcdeploy.py index 0cc6f23..99a0dcc 100644 --- a/tcdeploy.py +++ b/tcdeploy.py @@ -17,6 +17,7 @@ import sys import tempfile import time +from urllib.parse import urlsplit class DeployError(Exception): @@ -240,6 +241,7 @@ def authenticate(m, token, additional_actions=()): SECRET_ENV = re.compile(r'(PASSWORD|SECRET|TOKEN|ACCESS_KEY|PRIVATE_KEY|DATABASE_URL|POSTGRES_URL|IDENTITY_DB_URL)', re.I) +PUBLIC_TOKEN_URLS = {'M2M_TOKEN_ISSUER', 'M2M_TOKEN_AUDIENCE', 'HUMAN_TOKEN_ISSUER'} DIGEST = re.compile(r'sha256:[0-9a-f]{64}') TASK_FIELDS = {'family', 'taskRoleArn', 'executionRoleArn', 'networkMode', 'containerDefinitions', 'volumes', 'placementConstraints', 'requiresCompatibilities', 'cpu', 'memory', 'pidMode', 'ipcMode', @@ -248,10 +250,28 @@ def authenticate(m, token, additional_actions=()): 'registeredAt', 'registeredBy', 'deregisteredAt'} +def public_token_setting(name, value): + """Recognize specific public auth settings, never arbitrary TOKEN names.""" + if name == 'TOKEN_CACHE_TIME': + return bool(re.fullmatch(r'[0-9]{1,10}', value)) and int(value) <= 2147483647 + if name not in PUBLIC_TOKEN_URLS or len(value) > 2048 or any(c.isspace() for c in value): + return False + try: + url = urlsplit(value) + port = url.port + return (url.scheme == 'https' and bool(url.hostname) and url.username is None + and url.password is None and not url.query and not url.fragment + and (port is None or 1 <= port <= 65535)) + except ValueError: + return False + + def validate_environment(entries): for entry in entries: name, value = entry['name'], entry['value'] - require(not (SECRET_ENV.search(name) and value), 'Plaintext secret-like environment values are forbidden') + require(isinstance(name, str) and isinstance(value, str), 'Environment entries must contain strings') + require(not (SECRET_ENV.search(name) and value) or public_token_setting(name, value), + 'Plaintext secret-like environment values are forbidden') require(name != 'RESET_DB' or value == 'false', 'Database reset must be explicitly false') @@ -262,7 +282,24 @@ def non_root(user): return bool(name) and name != 'root' and not (name.isdecimal() and int(name) == 0) -def task_request(m, task, tags, images): +def validate_health_checks(m, checks): + require(isinstance(checks, dict) and checks and set(checks) <= set(m['images']), + 'Bootstrap health checks must name configured image containers only') + bounds = {'interval': (5, 300), 'timeout': (2, 60), 'retries': (1, 10), 'startPeriod': (0, 300)} + for check in checks.values(): + require(isinstance(check, dict) and 'command' in check and set(check) <= {'command', *bounds}, + 'Unexpected bootstrap health-check fields') + command = check['command'] + require(isinstance(command, list) and 2 <= len(command) <= 50 and command[0] == 'CMD' + and all(isinstance(arg, str) and arg and '\x00' not in arg for arg in command), + 'Bootstrap health checks require an explicit CMD argument list') + for key, (low, high) in bounds.items(): + require(key not in check or type(check[key]) is int and low <= check[key] <= high, + 'Bootstrap health-check timing is outside ECS bounds') + return checks + + +def task_request(m, task, tags, images, health_checks=None): require(set(task) <= TASK_FIELDS | TASK_METADATA, 'Unrecognized task fields; update the deployer before changing this definition') require(task.get('family') == m['taskFamily'] and task.get('status') == 'ACTIVE', 'Unexpected/inactive task family') require(task.get('taskRoleArn') == m['taskRoleArn'] and task.get('executionRoleArn') == m['executionRoleArn'], 'Task/execution role differs from reviewed manifest') @@ -270,6 +307,8 @@ def task_request(m, task, tags, images): platform = task.get('runtimePlatform', {}) require(platform.get('cpuArchitecture', 'X86_64') == 'X86_64' and platform.get('operatingSystemFamily', 'LINUX') == 'LINUX', 'This release supports Linux/amd64 tasks only') require(set(images) == set(m['images']), 'Supply exactly the manifest image set') + if health_checks is not None: + validate_health_checks(m, health_checks) result = {k: deepcopy(v) for k, v in task.items() if k in TASK_FIELDS} require(isinstance(tags, list), 'Task tags must be a list') # ECS rejects an explicitly empty tag list on registration. Omission keeps @@ -284,11 +323,16 @@ def task_request(m, task, tags, images): require(not container.get('privileged') and ('user' not in container or non_root(container['user'])), 'Application task must not override the image to root/privileged') counts[container['name']] += 1 require(DIGEST.fullmatch(images[container['name']].rsplit('@', 1)[-1]), 'Deploy only image digests') + if health_checks is not None and container['name'] in health_checks: + proposed = health_checks[container['name']] + require(not container.get('healthCheck') or container['healthCheck'] == proposed, + 'Bootstrap cannot replace an existing task health check') + container['healthCheck'] = deepcopy(proposed) require(container.get('healthCheck'), 'Updated containers require a task health check') container['image'] = images[container['name']] require(all(n == 1 for n in counts.values()), 'Each configured container must exist exactly once') - # All roles, secret references, sidecars, health checks, resources and volumes - # are preserved. Parameter values never enter this process. + # Existing configuration is preserved. Explicitly reviewed checks may be + # added where absent, with the new image, never by restarting the old image. return result @@ -447,7 +491,7 @@ def publish(self, name, local, tag, deadline): raise DeployError('Image scan timed out; service was not updated') -def deploy(aws, m, locals_by_name, release, publisher): +def deploy(aws, m, locals_by_name, release, publisher, health_checks=None): require(set(locals_by_name) == set(m['images']), 'Supply exactly one prebuilt image per configured container') require(re.fullmatch(r'[a-f0-9]{40}-[0-9]{1,12}', release), 'Release must be CIRCLE_SHA1-CIRCLE_BUILD_NUM') previous = read_service(aws, m) @@ -457,14 +501,14 @@ def deploy(aws, m, locals_by_name, release, publisher): task, tags = response['taskDefinition'], response.get('tags', []) # Validate configuration before publishing anything. Placeholder digests are # only used for validation and never sent to AWS. - task_request(m, task, tags, {name: 'preflight@sha256:' + '0' * 64 for name in m['images']}) + task_request(m, task, tags, {name: 'preflight@sha256:' + '0' * 64 for name in m['images']}, health_checks) images = {} scan_deadline = time.monotonic() + 900 for name, local in locals_by_name.items(): images[name] = publisher.publish(name, local, release, scan_deadline) current = read_service(aws, m) require(stable(current, definition), 'Service changed during image publication; retry after inspection') - request = task_request(m, task, tags, images) + request = task_request(m, task, tags, images, health_checks) registered = aws.call('ecs', 'register-task-definition', **request)['taskDefinition'] target = registered['taskDefinitionArn'] require(re.fullmatch(re.escape(arn(m, 'ecs', 'task-definition/' + m['taskFamily'])) + r':[0-9]+', target), 'Unexpected registered task ARN') @@ -499,8 +543,13 @@ def main(argv=None): p.add_argument('--manifest', required=True) if name == 'deploy': p.add_argument('--image', action='append', required=True, metavar='CONTAINER=LOCAL_IMAGE') + p.add_argument('--bootstrap-health-checks', metavar='REVIEWED_JSON') args = parser.parse_args(argv) m = read_manifest(args.manifest) + health_checks = None + if args.command == 'deploy' and args.bootstrap_health_checks: + health_checks = validate_health_checks(m, json.loads( + Path(args.bootstrap_health_checks).read_text(), object_pairs_hook=unique_object)) if args.command == 'policies': print(json.dumps({'trust': trust_policy(m), 'permissions': role_policy(m)}, indent=2)) return @@ -518,7 +567,7 @@ def main(argv=None): images[name] = image release = os.environ.get('CIRCLE_SHA1', '') + '-' + os.environ.get('CIRCLE_BUILD_NUM', '') with tempfile.TemporaryDirectory(prefix='tc-deploy-docker-') as directory: - result = deploy(aws, m, images, release, Publisher(aws, m, directory)) + result = deploy(aws, m, images, release, Publisher(aws, m, directory), health_checks) print(json.dumps(result, indent=2)) # Only non-secret rollout evidence. diff --git a/tests/test_security.py b/tests/test_security.py index 753c408..43ba67e 100644 --- a/tests/test_security.py +++ b/tests/test_security.py @@ -372,8 +372,8 @@ def setUp(self): self.aws = FakeAWS(self.m) self.publisher = FakePublisher(self.m) - def deploy(self): - return d.deploy(self.aws, self.m, {'example-v6': 'example:build'}, RELEASE, self.publisher) + def deploy(self, checks=None): + return d.deploy(self.aws, self.m, {'example-v6': 'example:build'}, RELEASE, self.publisher, checks) def test_full_release_never_calls_ssm_and_preserves_roles_secrets(self): original = deepcopy(self.aws.task) @@ -452,6 +452,113 @@ def test_plaintext_runtime_secret_rejected(self): with self.assertRaises(d.DeployError): self.deploy() + def test_public_token_metadata_is_preserved(self): + entries = [{'name': name, 'value': 'https://issuer.example.test/audience/'} + for name in d.PUBLIC_TOKEN_URLS] + entries.append({'name': 'TOKEN_CACHE_TIME', 'value': '86400'}) + self.aws.task['containerDefinitions'][0]['environment'] += entries + self.deploy() + self.assertEqual(self.aws.registered['containerDefinitions'][0]['environment'][-4:], entries) + + def test_public_metadata_does_not_allow_credentials(self): + values = ['opaque-token', 'http://issuer.test', 'https://user:pass@issuer.test', + 'https://@issuer.test', 'https://issuer.test/?token=secret', + 'https://issuer.test/#secret', 'https://issuer.test:bad', + 'https://issuer.test:0', 'https://issuer.test/ white', 'https://'] + entries = [(name, value) for name in d.PUBLIC_TOKEN_URLS for value in values] + entries += [('TOKEN_CACHE_TIME', value) for value in ['secret', '-1', '1.2', '2147483648']] + entries += [(name, 'https://issuer.test') for name in + ['AUTH0_CLIENT_SECRET', 'DB_PASSWORD', 'ACCESS_TOKEN', 'OTHER_TOKEN_ISSUER']] + for name, value in entries: + with self.subTest(name=name, value=value): + self.aws.task = task(self.m) + self.aws.task['containerDefinitions'][0]['environment'].append({'name': name, 'value': value}) + with self.assertRaises(d.DeployError): self.deploy() + self.assertEqual(self.publisher.calls, []) + + def test_missing_health_requires_explicit_bootstrap(self): + del self.aws.task['containerDefinitions'][0]['healthCheck'] + with self.assertRaises(d.DeployError): self.deploy() + self.assertEqual(self.publisher.calls, []) + self.assertIsNone(self.aws.registered) + + def test_bootstrap_adds_health_atomically_and_preserves_configuration(self): + del self.aws.task['containerDefinitions'][0]['healthCheck'] + self.aws.task['containerDefinitions'].append({'name': 'metrics', 'image': 'trusted@' + DIGEST, 'essential': False}) + original = deepcopy(self.aws.task) + checks = {'example-v6': {'command': ['CMD', 'node', 'health.js'], 'interval': 30, 'timeout': 5, 'retries': 3, 'startPeriod': 120}} + self.deploy(checks) + self.assertEqual(self.aws.task, original) + request = next(p for _, op, p in self.aws.calls if op == 'register-task-definition') + expected = {k: deepcopy(v) for k, v in original.items() if k in d.TASK_FIELDS} + expected['tags'] = self.aws.tags + expected['containerDefinitions'][0]['image'] = self.publisher.publish('example-v6', '', '', 0) + expected['containerDefinitions'][0]['healthCheck'] = checks['example-v6'] + self.assertEqual(request, expected) + operations = [op for _, op, _ in self.aws.calls] + self.assertEqual(operations.count('update-service'), 1) + self.assertLess(operations.index('register-task-definition'), operations.index('update-service')) + + def test_bootstrap_never_replaces_existing_health(self): + checks = {'example-v6': {'command': ['CMD', 'false']}} + with self.assertRaises(d.DeployError): self.deploy(checks) + self.assertEqual(self.publisher.calls, []) + checks['example-v6'] = deepcopy(self.aws.task['containerDefinitions'][0]['healthCheck']) + self.deploy(checks) + + def test_invalid_bootstrap_stops_before_publication(self): + invalid = [{}, [], {'sidecar': {'command': ['CMD', 'true']}}, + {'example-v6': {'command': ['CMD-SHELL', 'true']}}, + {'example-v6': {'command': ['CMD']}}, + {'example-v6': {'command': ['CMD', 'true'], 'unknown': 1}}] + for key, values in {'interval': [4, 301, True], 'timeout': [1, 61], + 'retries': [0, 11], 'startPeriod': [-1, 301]}.items(): + invalid += [{'example-v6': {'command': ['CMD', 'true'], key: value}} for value in values] + for checks in invalid: + with self.subTest(checks=checks): + with self.assertRaises(d.DeployError): self.deploy(checks) + self.assertEqual(self.publisher.calls, []) + self.assertIsNone(self.aws.registered) + + def test_bootstrap_scan_failure_leaves_old_revision_untouched(self): + del self.aws.task['containerDefinitions'][0]['healthCheck'] + original = deepcopy(self.aws.task) + with patch.object(self.publisher, 'publish', side_effect=d.DeployError('Scan failed')): + with self.assertRaises(d.DeployError): + self.deploy({'example-v6': {'command': ['CMD', 'true']}}) + self.assertEqual(self.aws.task, original) + self.assertIsNone(self.aws.registered) + self.assertNotIn('update-service', [op for _, op, _ in self.aws.calls]) + + def test_bootstrap_readback_tampering_stops_update(self): + del self.aws.task['containerDefinitions'][0]['healthCheck'] + def tamper(api, operation, params): + if operation == 'describe-task-definition' and self.aws.registered: + self.aws.registered['containerDefinitions'][0]['healthCheck'] = {'command': ['CMD', 'false']} + self.aws.before = tamper + with self.assertRaises(d.DeployError): + self.deploy({'example-v6': {'command': ['CMD', 'true']}}) + self.assertNotIn('update-service', [op for _, op, _ in self.aws.calls]) + + def test_bootstrap_native_rollback_remains_failed(self): + del self.aws.task['containerDefinitions'][0]['healthCheck'] + def rollback(api, operation, params): + if operation == 'describe-services' and any(op == 'update-service' for _, op, _ in self.aws.calls): + self.aws.service = service(self.m) + self.aws.before = rollback + with self.assertRaises(d.DeployError): + self.deploy({'example-v6': {'command': ['CMD', 'true']}}) + + def test_duplicate_bootstrap_fields_fail_before_authentication(self): + with tempfile.TemporaryDirectory() as temp: + p = Path(temp) / 'health.json' + p.write_text('{"example-v6":{"command":["CMD","true"],"command":["CMD","false"]}}') + with patch.object(d, 'authenticate') as authenticate: + with self.assertRaises(d.DeployError): + d.main(['deploy', '--manifest', str(ROOT / 'examples/deploy.dev.json'), + '--image', 'example-v6=example:build', '--bootstrap-health-checks', str(p)]) + authenticate.assert_not_called() + def test_task_unknown_fields_fail_closed(self): self.aws.task['newUnsupportedField'] = True with self.assertRaises(d.DeployError):