diff --git a/.circleci/config.yml b/.circleci/config.yml index 9f4097f..e7ef20d 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -1,190 +1,181 @@ version: 2.1 - -parameters: - run_automatedtesting: - default: false - type: boolean - run_basedeployment: - default: true - type: boolean - -defaults: &defaults - docker: - - image: cimg/python:3.12.2-browsers - -install_dependency: &install_dependency - name: Installation of build and deployment dependencies. - command: | - # The image already contains Chrome and the Docker client. Their optional - # repositories can retain obsolete signing-key configuration and block or - # warn during otherwise unrelated package installation. - sudo rm -f /etc/apt/sources.list.d/google-chrome.list - sudo rm -f /etc/apt/sources.list.d/archive_uri-https_download_docker_com_linux_ubuntu-jammy.list - sudo apt-get update - sudo apt-get install --yes jq python3-pip - sudo pip3 install awscli --upgrade - -install_deploysuite: &install_deploysuite - name: Installation of install_deploysuite. - command: | - git clone --branch v1.4.17 https://github.com/topcoder-platform/tc-deploy-scripts ../buildscript - cp ./../buildscript/master_deploy.sh . - cp ./../buildscript/buildenv.sh . - cp ./../buildscript/awsconfiguration.sh . - cp ./../buildscript/psvar-processor.sh . - -finger_print_add: &finger_print_add - fingerprints: - - SHA256:KJYSvQh9CxkY9yRXxp+Geo6cllnfSiiKAH3LFQ/X9f4 - -builddeploy_steps: &builddeploy_steps - - add_ssh_keys: *finger_print_add - - checkout - - setup_remote_docker - - run: *install_dependency - - run: *install_deploysuite - - run: - name: Build docker image - command: | - ./awsconfiguration.sh "${DEPLOY_ENV}" - source awsenvconf - ./psvar-processor.sh -t appenv -p "/config/${APPNAME}/buildvar" - source buildvar_env - rm -f awsenvconf - ./build.sh "${APPNAME}" - - deploy: - name: Running MasterScript. - command: | - ./awsconfiguration.sh "$DEPLOY_ENV" - source awsenvconf - ./psvar-processor.sh -t appenv -p "/config/${APPNAME}/deployvar" - source deployvar_env - ./master_deploy.sh \ - -d ECS \ - -e "$DEPLOY_ENV" \ - -t latest \ - -j "/config/common/global-appvar,/config/${APPNAME}/appvar" \ - -i "$APPNAME" \ - -p FARGATE - - ./psvar-processor.sh -t appenv -p /config/common/qa-appvar - source qa-appvar_env - curl --request POST \ - --url "https://circleci.com/api/v2/project/github/$CIRCLE_PROJECT_USERNAME/$CIRCLE_PROJECT_REPONAME/pipeline" \ - --header "Circle-Token: ${QA_USER_TOKEN}" \ - --header 'content-type: application/json' \ - --data '{"branch":"'"$CIRCLE_BRANCH"'","parameters":{"run_automatedtesting":true,"run_basedeployment":false}}' - +executors: + reviewed: + docker: + - image: cimg/aws@sha256:4cf3768db00ddf9bc439b8c84e8a60c11a14edf81c8baf76f74132144f8340d5 +commands: + checkout-source: + steps: + - run: + name: Check out the exact public source commit over HTTPS + command: | + set -euo pipefail + [[ "$CIRCLE_SHA1" =~ ^[0-9a-f]{40}$ ]] + export GIT_CONFIG_NOSYSTEM=1 GIT_CONFIG_GLOBAL=/dev/null GIT_TERMINAL_PROMPT=0 + git init --quiet . + git remote add origin https://github.com/topcoder-platform/resource-api-v6.git + git -c credential.helper= fetch --quiet --depth 1 origin "$CIRCLE_SHA1" + git checkout --quiet --detach FETCH_HEAD + test "$(git rev-parse HEAD)" = "$CIRCLE_SHA1" + checkout-suite: + steps: + - run: + name: Verify deployment suite v2.0.6 + command: | + set -euo pipefail + git init --quiet /tmp/tc-deploy-scripts + git -C /tmp/tc-deploy-scripts remote add origin https://github.com/topcoder-platform/tc-deploy-scripts.git + git -C /tmp/tc-deploy-scripts fetch --quiet --depth 1 origin refs/tags/v2.0.6 + git -C /tmp/tc-deploy-scripts checkout --quiet --detach FETCH_HEAD + test "$(git -C /tmp/tc-deploy-scripts rev-parse HEAD)" = 26aaf607463d4a271c0099854c1731214215a9c5 jobs: - build-dev: - <<: *defaults - environment: - DEPLOY_ENV: DEV - LOGICAL_ENV: dev - APPNAME: resources-api-v6 - steps: *builddeploy_steps - - build-prod: - <<: *defaults + build: + executor: reviewed + steps: + - checkout-source + - setup_remote_docker + - checkout-suite + - run: + name: Build committed source without AWS or broker credentials + command: | + set -euo pipefail + python3 -I /tmp/tc-deploy-scripts/build_image.py --dockerfile docker/Dockerfile --image resource-api-v6:candidate + mkdir -p /tmp/release + docker save --output /tmp/release/image.tar resource-api-v6:candidate + no_output_timeout: 30m + - persist_to_workspace: + root: /tmp/release + paths: + - image.tar + deploy: + executor: reviewed + parameters: + environment: + type: enum + enum: + - dev + - prod environment: - DEPLOY_ENV: PROD - LOGICAL_ENV: prod - APPNAME: resources-api-v6 - steps: *builddeploy_steps - - Run-Newman-Test: + DEPLOY_ENV: << parameters.environment >> + steps: + - checkout-source + - setup_remote_docker + - checkout-suite + - attach_workspace: + at: /tmp/release + - run: + name: Promote a scanned digest with repository-bound OIDC + command: | + set +x + set -euo pipefail + test -f "deploy/${DEPLOY_ENV}.json" + docker load --input /tmp/release/image.tar + python3 -I deploy/release.py "deploy/${DEPLOY_ENV}.json" + no_output_timeout: 45m + qa: docker: - - image: cimg/node:26.5.0 - environment: - DEPLOY_ENV: DEV - LOGICAL_ENV: dev - APPNAME: resources-api-v6 + - image: cimg/node@sha256:2c1320ee6a256628f698938038700911276310d40cc9887cfc8f66ec0daa4343 + environment: + DEPLOY_ENV: DEV + LOGICAL_ENV: dev + APPNAME: resources-api-v6 steps: - - checkout - - run: *install_dependency - - run: *install_deploysuite - - restore_cache: - keys: - - pnpm-store-v1-{{ checksum "pnpm-lock.yaml" }} - - pnpm-store-v1- - - run: - name: Install pnpm dependencies - command: | - sudo npm install --global pnpm@11.15.1 - pnpm install --frozen-lockfile - pnpm build - - save_cache: - key: pnpm-store-v1-{{ checksum "pnpm-lock.yaml" }} - paths: - - ~/.local/share/pnpm/store - - run: - name: Newman test - command: | - ./awsconfiguration.sh "$DEPLOY_ENV" - source awsenvconf - ./psvar-processor.sh -t appenv -p /config/resources-api-v6/qaappvar - source qaappvar_env - ./psvar-processor.sh -t appenv -p /config/common/global-appvar - source global-appvar_env + - checkout-source + - restore_cache: + keys: + - pnpm-store-v1-{{ checksum "pnpm-lock.yaml" }} + - pnpm-store-v1- + - run: + name: Install pnpm dependencies + command: | + sudo npm install --global pnpm@11.15.1 + pnpm install --frozen-lockfile + pnpm build + - save_cache: + key: pnpm-store-v1-{{ checksum "pnpm-lock.yaml" }} + paths: + - ~/.local/share/pnpm/store + - run: + name: Newman test + command: | + set +x + set -euo pipefail + : "${DATABASE_URL:?Set isolated QA database settings in the restricted QA context}" + : "${AUTH0_CLIENT_SECRET:?Set test-only Auth0 settings in the restricted QA context}" + pnpm run mock-challenge-api > /tmp/mock-challenge-api.log 2>&1 & + mock_pid=$! + NODE_ENV=test pnpm start > /tmp/resource-api.log 2>&1 & + app_pid=$! + trap 'kill "$mock_pid" "$app_pid" 2>/dev/null || true' EXIT - pnpm run mock-challenge-api > /tmp/mock-challenge-api.log 2>&1 & - mock_pid=$! - NODE_ENV=test pnpm start > /tmp/resource-api.log 2>&1 & - app_pid=$! - trap 'kill "$mock_pid" "$app_pid" 2>/dev/null || true' EXIT - - health_url="http://localhost:${PORT:-3000}/${API_VERSION:-v6}/resources/health" - for attempt in $(seq 1 30); do - if curl --fail --silent "$health_url" > /dev/null; then - break - fi - if [[ "$attempt" == 30 ]]; then - cat /tmp/mock-challenge-api.log - cat /tmp/resource-api.log - exit 1 - fi - sleep 1 - done - - pnpm run test:newman:clear - pnpm run test:newman - pnpm run test:newman:clear - - store_artifacts: - path: ./newman + health_url="http://localhost:${PORT:-3000}/${API_VERSION:-v6}/resources/health" + for attempt in $(seq 1 30); do + if curl --fail --silent "$health_url" > /dev/null; then + break + fi + if [[ "$attempt" == 30 ]]; then + exit 1 + fi + sleep 1 + done + pnpm run test:newman:clear + pnpm run test:newman + pnpm run test:newman:clear + - store_artifacts: + path: ./newman workflows: - version: 2 - build: - when: << pipeline.parameters.run_basedeployment >> - jobs: - - build-dev: - context: org-global - filters: - branches: - only: - - develop - - security_july_2026 - - hotfix/deploy - - PM-3458 - - PM-5919 - - build-prod: - context: org-global - filters: - branches: - only: - - master - - testingflow: - when: << pipeline.parameters.run_automatedtesting >> + release: jobs: - - Hold [Performance-Testing]: - type: approval - - Run-Newman-Test: - context: org-global - requires: - - Hold [Performance-Testing] - filters: - branches: - only: - - develop - - dev-circleci + - build + - deploy: + name: deploy-dev + environment: dev + requires: + - build + context: resource-api-v6-dev-deploy + serial-group: << pipeline.project.slug >>/deploy-dev + filters: + branches: + only: develop + tags: + ignore: /.*/ + - approve-prod: + type: approval + requires: + - build + filters: + branches: + only: master + tags: + ignore: /.*/ + - deploy: + name: deploy-prod + environment: prod + requires: + - approve-prod + context: resource-api-v6-prod-deploy + serial-group: << pipeline.project.slug >>/deploy-prod + filters: + branches: + only: master + tags: + ignore: /.*/ + - approve-qa: + type: approval + requires: + - deploy-dev + filters: + branches: + only: develop + tags: + ignore: /.*/ + - qa: + requires: + - approve-qa + context: resource-api-v6-qa + filters: + branches: + only: develop + tags: + ignore: /.*/ diff --git a/.gitignore b/.gitignore index 9748e37..9056621 100644 --- a/.gitignore +++ b/.gitignore @@ -72,3 +72,7 @@ src/generated/ # NestJS build output. dist/ + +# Infrastructure definitions and detailed AWS inventories are maintained privately. +/deploy/iam/ +/deploy/*-baseline.json diff --git a/deploy/MIGRATION.md b/deploy/MIGRATION.md new file mode 100644 index 0000000..d242d8b --- /dev/null +++ b/deploy/MIGRATION.md @@ -0,0 +1,43 @@ +# OIDC deployment migration + +This change replaces the Auth0 credential broker and SSM-to-shell deployment path with [tc-deploy-scripts v2.0.6](https://github.com/topcoder-platform/tc-deploy-scripts/releases/tag/v2.0.6), fetched by tag and verified against commit `26aaf607463d4a271c0099854c1731214215a9c5`. Build and deployment run in separate jobs. The deployer cannot read Parameter Store, Secrets Manager or decrypt KMS data, including through another assumed role. ECS continues resolving the existing runtime secret references using the execution role. + +## Production target + +The read-only inventory on 2026-11-03 verified account `409275337247`, cluster `tc-challenge-serverless`, service `resources-api-v6`, family `resources-api-v6`, and container `resources-api-v6`. `prod.json` preserves the observed task/execution roles and names the proposed service-specific deployment role. [prod-baseline.json](https://github.com/topcoder-platform/v6-services-iac/blob/main/services/resource-api-v6/deploy/prod-baseline.json) records references only; it contains no parameter values. Do not recreate the service or replace its complete task definition from this baseline. + +## Installed infrastructure and production cutover + +The dev deployment role/boundary, restricted empty CircleCI context, and immutable ECR tags are installed. Existing enhanced scanning is preserved. The production OIDC provider, deployment role/boundary and restricted context are also installed; production runtime and ECR changes remain gated on dev validation. Installation evidence and remaining service-specific steps are private in [PROVISIONING.txt](https://github.com/topcoder-platform/v6-services-iac/blob/main/services/resource-api-v6/deploy/PROVISIONING.txt). Deploy roles have explicit secret-store and KMS-decryption denies. Runtime values are retrieved only by ECS using the execution role. + +At production cutover, stop legacy publishers, enable unconditional tag immutability, adopt missing task health checks with the reviewed image, and use the approval-gated `master` workflow. Preserve capacity, roles, secret references and rollback. Do not run production before dev validation. + +## Development acceptance + +This PR targets `develop`. Administrator-installed dev prerequisites are recorded in the private IaC repository. The reviewed task health check was submitted only after confirming the existing image serves the identical healthy load-balancer endpoint. + +- Dev infrastructure prerequisites are installed; merge to run the application deployment, then validate functionality. + +Run the shared offline suite, validate the CircleCI config and IAM policies, and test a dev release plus an unhealthy-image rollback. Verify CloudTrail contains no SSM/Secrets Manager calls from the deployment principal. Remove its old broker grants only after migration. Check the runtime secret references and roles stayed unchanged. Production requires the manual approval job and a passing digest/scan/health verification. + +The pinned executor is `cimg/aws@sha256:4cf3768db00ddf9bc439b8c84e8a60c11a14edf81c8baf76f74132144f8340d5`. Python/AWS CLI/Buildx/Git compatibility was tested with the shared offline suite. No package installation or service build runs in the deployment job. The Dockerfile retains the existing in-image quality gates. This PR does not deploy, change IAM, rotate credentials, or claim a successful live rollout. + +## Newman integration tests + +The Newman workflow remains available after the dev release and a QA approval. It uses a separate restricted `resource-api-v6-qa` context containing only isolated test database and non-production test-user/Auth0 settings. An administrator must migrate those test settings before enabling the QA job. It does not receive the deployment role, `org-global`, CI broker credentials, or any Parameter Store permissions. The old recursive CircleCI API trigger and its QA_USER_TOKEN are removed; workflow dependencies now order the jobs. + +## Infrastructure source + +IAM templates and detailed AWS baselines are maintained in the private [v6-services-iac service directory](https://github.com/topcoder-platform/v6-services-iac/tree/main/services/resource-api-v6). Provision infrastructure separately with an administrator identity; application CI does not fetch that repository or install IAM. Required application manifests and health-check proposals remain here. + +## Dev branch refresh + +This PR targets `develop` and preserves its current application/dependency fixes. The shared suite is v2.0.6 (92 offline tests); service-specific adapters and their tests remain part of review. Use the current readiness record above for the remaining merge/deployment gates. Updating these files does not provision AWS or CircleCI resources. + +Source checkout uses the public HTTPS repository and verifies the exact CircleCI commit, avoiding dependency on failing SSH checkout keys. No GitHub token is added to build or deployment contexts. + +The credential-free builder rejects committed `.npmrc` files. Preserve the existing local compatibility dependencies with explicit `--install-links=true` npm options instead; do not reintroduce credential-capable npm configuration files. + +## Reviewed health-check bootstrap + +The release wrapper passes the environment-specific health-check JSON to v2.0.6. A missing check is added with the new scanned image; the old image is not restarted with a new probe. An identical existing check is preserved, and a different existing check stops the release. Role, runtime-secret reference, scan, concurrency and healthy-digest checks remain enforced. diff --git a/deploy/dev-healthcheck.json b/deploy/dev-healthcheck.json new file mode 100644 index 0000000..7e0face --- /dev/null +++ b/deploy/dev-healthcheck.json @@ -0,0 +1,14 @@ +{ + "resources-api-v6": { + "command": [ + "CMD", + "node", + "-e", + "const r=require('node:http').get(\"http://127.0.0.1:3000/v6/resources/health\",r=>{r.resume();process.exit(r.statusCode>=200&&r.statusCode<300?0:1)});r.setTimeout(4000,()=>{r.destroy();process.exit(1)});r.on('error',()=>process.exit(1))" + ], + "interval": 30, + "timeout": 5, + "retries": 3, + "startPeriod": 120 + } +} diff --git a/deploy/dev.json b/deploy/dev.json new file mode 100644 index 0000000..7eeadcd --- /dev/null +++ b/deploy/dev.json @@ -0,0 +1,19 @@ +{ + "schemaVersion": 2, + "environment": "dev", + "accountId": "811668436784", + "region": "us-east-1", + "deployRoleArn": "arn:aws:iam::811668436784:role/resource-api-v6-dev-deploy", + "taskRoleArn": "arn:aws:iam::811668436784:role/challenge-v5-container-role", + "executionRoleArn": "arn:aws:iam::811668436784:role/ecsTaskExecutionRole", + "cluster": "tc-challenge-serverless", + "service": "resources-api-v6", + "taskFamily": "resources-api-v6", + "circleciOrgId": "947bef5e-1669-415d-b83c-4fc85155b7df", + "circleciProjectId": "bd210f8f-832b-4cca-b7b5-2f55569f5292", + "repository": "github.com/topcoder-platform/resource-api-v6", + "branch": "develop", + "images": { + "resources-api-v6": "resources-api-v6" + } +} diff --git a/deploy/prod-healthcheck.json b/deploy/prod-healthcheck.json new file mode 100644 index 0000000..acca5d2 --- /dev/null +++ b/deploy/prod-healthcheck.json @@ -0,0 +1,14 @@ +{ + "resources-api-v6": { + "command": [ + "CMD", + "node", + "-e", + "const r=require('node:http').get('http://127.0.0.1:3000/v6/resources/health',r=>{r.resume();process.exit(r.statusCode>=200&&r.statusCode<300?0:1)});r.setTimeout(4000,()=>{r.destroy();process.exit(1)});r.on('error',()=>process.exit(1))" + ], + "interval": 30, + "timeout": 5, + "retries": 3, + "startPeriod": 120 + } +} diff --git a/deploy/prod.json b/deploy/prod.json new file mode 100644 index 0000000..5d6e323 --- /dev/null +++ b/deploy/prod.json @@ -0,0 +1,19 @@ +{ + "schemaVersion": 2, + "environment": "prod", + "accountId": "409275337247", + "region": "us-east-1", + "deployRoleArn": "arn:aws:iam::409275337247:role/resource-api-v6-prod-deploy", + "taskRoleArn": "arn:aws:iam::409275337247:role/challenge-v5-container-role", + "executionRoleArn": "arn:aws:iam::409275337247:role/ecsTaskExecutionRole", + "cluster": "tc-challenge-serverless", + "service": "resources-api-v6", + "taskFamily": "resources-api-v6", + "circleciOrgId": "947bef5e-1669-415d-b83c-4fc85155b7df", + "circleciProjectId": "bd210f8f-832b-4cca-b7b5-2f55569f5292", + "repository": "github.com/topcoder-platform/resource-api-v6", + "branch": "master", + "images": { + "resources-api-v6": "resources-api-v6" + } +} diff --git a/deploy/release.py b/deploy/release.py new file mode 100644 index 0000000..4aefefa --- /dev/null +++ b/deploy/release.py @@ -0,0 +1,14 @@ +#!/usr/bin/env python3 +"""Pass the reviewed manifest and this job's prebuilt images to the pinned suite.""" +import json +from pathlib import Path +import subprocess +import sys + +manifest = Path(sys.argv[1]) +data = json.loads(manifest.read_text()) +command = [sys.executable, '-I', '/tmp/tc-deploy-scripts/tcdeploy.py', 'deploy', '--manifest', str(manifest), + '--bootstrap-health-checks', str(manifest.with_name(data['environment'] + '-healthcheck.json'))] +for container in data['images']: + command.extend(['--image', container + '=resource-api-v6:candidate']) +subprocess.run(command, check=True) diff --git a/migrator/.npmrc b/migrator/.npmrc deleted file mode 100644 index c7d3517..0000000 --- a/migrator/.npmrc +++ /dev/null @@ -1 +0,0 @@ -install-links=true diff --git a/migrator/README.md b/migrator/README.md index 0aa401f..1d2e527 100644 --- a/migrator/README.md +++ b/migrator/README.md @@ -34,7 +34,7 @@ 1. Clone the repo and install dependencies: ```bash - npm install + npm install --install-links=true ``` 2. Start PostgreSQL with Docker Compose: