From 69ebd55c210672188c04adcd116f94d7af02dba6 Mon Sep 17 00:00:00 2001
From: Andrea Amorosi <dreamorosi@gmail.com>
Date: Tue, 1 Sep 2026 16:30:54 +0200
Subject: [PATCH] chore(ci): add partition layer balancer

Add a resumable workflow for copying contiguous commercial layer histories into China and GovCloud with separate partition credentials. Validate version position, content hashes, public permissions, and target identity before publishing.

Serialize existing partition writers with the balancer to protect layer version ordering.

Fixes #8418
---
 .github/workflows/layer_govcloud.yml          |   4 +
 .../workflows/layer_govcloud_python313.yml    |   4 +
 .../workflows/layers_partition_balance.yml    | 394 ++++++++++++++++++
 .github/workflows/layers_partitions.yml       |   4 +
 4 files changed, 406 insertions(+)
 create mode 100644 .github/workflows/layers_partition_balance.yml

diff --git a/.github/workflows/layer_govcloud.yml b/.github/workflows/layer_govcloud.yml
index bcb0005308d..e33204996ea 100644
--- a/.github/workflows/layer_govcloud.yml
+++ b/.github/workflows/layer_govcloud.yml
@@ -39,6 +39,10 @@ run-name: Layer Deployment (GovCloud) - ${{ inputs.environment }}
 permissions:
   contents: read
 
+concurrency:
+  group: layer-balancer-GovCloud-${{ inputs.environment }}
+  cancel-in-progress: false
+
 jobs:
   download:
     runs-on: ubuntu-latest
diff --git a/.github/workflows/layer_govcloud_python313.yml b/.github/workflows/layer_govcloud_python313.yml
index 285ed0081ea..b046203942b 100644
--- a/.github/workflows/layer_govcloud_python313.yml
+++ b/.github/workflows/layer_govcloud_python313.yml
@@ -39,6 +39,10 @@ run-name: Layer Deployment (GovCloud) - ${{ inputs.environment }}
 permissions:
   contents: read
 
+concurrency:
+  group: layer-balancer-GovCloud-${{ inputs.environment }}
+  cancel-in-progress: false
+
 jobs:
   download:
     runs-on: ubuntu-latest
diff --git a/.github/workflows/layers_partition_balance.yml b/.github/workflows/layers_partition_balance.yml
new file mode 100644
index 00000000000..abd23c42f08
--- /dev/null
+++ b/.github/workflows/layers_partition_balance.yml
@@ -0,0 +1,394 @@
+# Partition Layer Balancer
+# ---
+# This workflow copies a contiguous range of one Lambda layer from the commercial partition
+# into China or GovCloud while preserving layer version numbers.
+
+name: Layer Balancer (Partitions)
+run-name: Layer Balancer (${{ inputs.partition }}) - ${{ inputs.environment }} / ${{ inputs.python_version }} / ${{ inputs.start_version }}-${{ inputs.end_version }}
+
+on:
+  workflow_dispatch:
+    inputs:
+      environment:
+        description: Deployment environment
+        type: choice
+        options:
+          - Gamma
+          - Prod
+        required: true
+      partition:
+        description: Partition to balance
+        type: choice
+        options:
+          - China
+          - GovCloud
+        required: true
+      python_version:
+        description: Python layer suffix without a period, for example python314
+        type: string
+        required: true
+      start_version:
+        description: First commercial layer version to copy
+        type: string
+        required: true
+      end_version:
+        description: Last commercial layer version to copy
+        type: string
+        required: true
+      architecture:
+        description: Layer architecture to balance
+        type: choice
+        options:
+          - both
+          - arm64
+          - x86_64
+        default: both
+        required: true
+      dry_run:
+        description: Validate source artifacts and target position without publishing
+        type: boolean
+        default: true
+        required: true
+
+permissions: {}
+
+concurrency:
+  group: layer-balancer-${{ inputs.partition }}-${{ inputs.environment }}
+  cancel-in-progress: false
+
+jobs:
+  setup:
+    runs-on: ubuntu-latest
+    outputs:
+      regions: ${{ format('{0}{1}', steps.regions_china.outputs.regions, steps.regions_govcloud.outputs.regions) }}
+      partition: ${{ format('{0}{1}', steps.regions_china.outputs.partition, steps.regions_govcloud.outputs.partition) }}
+      audience: ${{ format('{0}{1}', steps.regions_china.outputs.audience, steps.regions_govcloud.outputs.audience) }}
+      layer: ${{ steps.inputs.outputs.layer }}
+      start_version: ${{ steps.inputs.outputs.start_version }}
+      end_version: ${{ steps.inputs.outputs.end_version }}
+      architectures: ${{ steps.inputs.outputs.architectures }}
+    steps:
+      - id: inputs
+        name: Validate inputs
+        env:
+          ARCHITECTURE: ${{ inputs.architecture }}
+          END_VERSION: ${{ inputs.end_version }}
+          PYTHON_VERSION: ${{ inputs.python_version }}
+          START_VERSION: ${{ inputs.start_version }}
+        run: |
+          if [[ ! "$PYTHON_VERSION" =~ ^python[0-9]+$ ]]; then
+            echo "python_version must match python followed by digits, for example python314"
+            exit 1
+          fi
+
+          if [[ ! "$START_VERSION" =~ ^[1-9][0-9]*$ ]] || [[ ! "$END_VERSION" =~ ^[1-9][0-9]*$ ]]; then
+            echo "start_version and end_version must be positive integers"
+            exit 1
+          fi
+
+          if (( START_VERSION > END_VERSION )); then
+            echo "start_version must not be greater than end_version"
+            exit 1
+          fi
+
+          if (( END_VERSION - START_VERSION >= 50 )); then
+            echo "A single balance run cannot contain more than 50 versions"
+            exit 1
+          fi
+
+          echo "layer=AWSLambdaPowertoolsPythonV3-${PYTHON_VERSION}" >> "$GITHUB_OUTPUT"
+          echo "start_version=$START_VERSION" >> "$GITHUB_OUTPUT"
+          echo "end_version=$END_VERSION" >> "$GITHUB_OUTPUT"
+
+          case "$ARCHITECTURE" in
+            both) echo 'architectures=["arm64","x86_64"]' >> "$GITHUB_OUTPUT" ;;
+            arm64) echo 'architectures=["arm64"]' >> "$GITHUB_OUTPUT" ;;
+            x86_64) echo 'architectures=["x86_64"]' >> "$GITHUB_OUTPUT" ;;
+            *) echo "Unsupported architecture: $ARCHITECTURE"; exit 1 ;;
+          esac
+      - id: regions_china
+        name: Partition (China)
+        if: ${{ inputs.partition == 'China' }}
+        run: |
+          echo 'regions=["cn-north-1"]' >> "$GITHUB_OUTPUT"
+          echo 'partition=aws-cn' >> "$GITHUB_OUTPUT"
+          echo 'audience=sts.amazonaws.com.cn' >> "$GITHUB_OUTPUT"
+      - id: regions_govcloud
+        name: Partition (GovCloud)
+        if: ${{ inputs.partition == 'GovCloud' }}
+        run: |
+          echo 'regions=["us-gov-east-1","us-gov-west-1"]' >> "$GITHUB_OUTPUT"
+          echo 'partition=aws-us-gov' >> "$GITHUB_OUTPUT"
+          echo 'audience=sts.amazonaws.com' >> "$GITHUB_OUTPUT"
+
+  download:
+    needs: setup
+    runs-on: ubuntu-latest
+    permissions:
+      id-token: write
+      contents: read
+    environment: Prod (Readonly)
+    env:
+      AWS_MAX_ATTEMPTS: "10"
+      AWS_RETRY_MODE: standard
+    strategy:
+      fail-fast: false
+      matrix:
+        architecture: ${{ fromJson(needs.setup.outputs.architectures) }}
+    steps:
+      - name: Configure AWS Credentials
+        uses: aws-actions/configure-aws-credentials@e6de054238d6b7531b4efff3b6587d9aade6a06c # v6.2.3
+        with:
+          role-to-assume: ${{ secrets.AWS_IAM_ROLE }}
+          aws-region: us-east-1
+          mask-aws-account-id: true
+      - name: Download commercial layer versions
+        env:
+          ARCHITECTURE: ${{ matrix.architecture }}
+          END_VERSION: ${{ needs.setup.outputs.end_version }}
+          LAYER: ${{ needs.setup.outputs.layer }}
+          START_VERSION: ${{ needs.setup.outputs.start_version }}
+        run: |
+          mkdir -p source
+
+          for (( VERSION=START_VERSION; VERSION<=END_VERSION; VERSION++ )); do
+            NAME="${LAYER}-${ARCHITECTURE}"
+            METADATA="source/${VERSION}.json"
+            ZIP="source/${VERSION}.zip"
+
+            aws --region us-east-1 lambda get-layer-version-by-arn \
+              --arn "arn:aws:lambda:us-east-1:017000801446:layer:${NAME}:${VERSION}" > "$METADATA"
+
+            LOCATION=$(jq -r '.Content.Location' "$METADATA")
+            curl --fail --location --retry 3 --retry-delay 2 --output "$ZIP" "$LOCATION"
+
+            EXPECTED_SHA=$(jq -r '.Content.CodeSha256' "$METADATA")
+            ACTUAL_SHA=$(openssl dgst -sha256 -binary "$ZIP" | openssl enc -base64)
+            if [[ "$ACTUAL_SHA" != "$EXPECTED_SHA" ]]; then
+              echo "SHA mismatch for ${NAME}:${VERSION}: expected ${EXPECTED_SHA}, received ${ACTUAL_SHA}"
+              exit 1
+            fi
+          done
+      - name: Store commercial layer versions
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+        with:
+          name: ${{ needs.setup.outputs.layer }}-${{ matrix.architecture }}-${{ needs.setup.outputs.start_version }}-${{ needs.setup.outputs.end_version }}
+          path: source
+          compression-level: 0
+          retention-days: 1
+          if-no-files-found: error
+          overwrite: true
+
+  balance:
+    needs:
+      - setup
+      - download
+    runs-on: ubuntu-latest
+    permissions:
+      id-token: write
+      contents: read
+    environment: ${{ inputs.partition }} ${{ inputs.environment }}
+    env:
+      AWS_MAX_ATTEMPTS: "10"
+      AWS_RETRY_MODE: standard
+    strategy:
+      fail-fast: false
+      matrix:
+        region: ${{ fromJson(needs.setup.outputs.regions) }}
+        architecture: ${{ fromJson(needs.setup.outputs.architectures) }}
+    steps:
+      - name: Download commercial layer versions
+        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
+        with:
+          name: ${{ needs.setup.outputs.layer }}-${{ matrix.architecture }}-${{ needs.setup.outputs.start_version }}-${{ needs.setup.outputs.end_version }}
+          path: source
+      - id: region
+        name: Normalize region
+        env:
+          REGION: ${{ matrix.region }}
+        run: |
+          NORMALIZED_REGION=${REGION^^}
+          echo "value=${NORMALIZED_REGION//-/_}" >> "$GITHUB_OUTPUT"
+      - name: Configure AWS Credentials
+        uses: aws-actions/configure-aws-credentials@e6de054238d6b7531b4efff3b6587d9aade6a06c # v6.2.3
+        with:
+          role-to-assume: ${{ secrets[format('IAM_ROLE_{0}', steps.region.outputs.value)] }}
+          aws-region: ${{ matrix.region }}
+          mask-aws-account-id: true
+          audience: ${{ needs.setup.outputs.audience }}
+      - name: Validate target account
+        env:
+          AWS_ACCOUNT: ${{ secrets[format('AWS_ACCOUNT_{0}', steps.region.outputs.value)] }}
+          PARTITION: ${{ needs.setup.outputs.partition }}
+          REGION: ${{ matrix.region }}
+        run: |
+          if [[ ! "$AWS_ACCOUNT" =~ ^[0-9]{12}$ ]]; then
+            echo "AWS account secret for ${REGION} is missing or invalid"
+            exit 1
+          fi
+
+          CALLER_ACCOUNT=$(aws --region "$REGION" sts get-caller-identity --query Account --output text)
+          CALLER_ARN=$(aws --region "$REGION" sts get-caller-identity --query Arn --output text)
+
+          if [[ "$CALLER_ACCOUNT" != "$AWS_ACCOUNT" ]] || [[ "$CALLER_ARN" != "arn:${PARTITION}:"* ]]; then
+            echo "Assumed role does not match the expected account and partition for ${REGION}"
+            exit 1
+          fi
+      - name: Balance layer versions
+        env:
+          ARCHITECTURE: ${{ matrix.architecture }}
+          AWS_ACCOUNT: ${{ secrets[format('AWS_ACCOUNT_{0}', steps.region.outputs.value)] }}
+          DRY_RUN: ${{ inputs.dry_run }}
+          END_VERSION: ${{ needs.setup.outputs.end_version }}
+          LAYER: ${{ needs.setup.outputs.layer }}
+          PARTITION: ${{ needs.setup.outputs.partition }}
+          REGION: ${{ matrix.region }}
+          START_VERSION: ${{ needs.setup.outputs.start_version }}
+        run: |
+          NAME="${LAYER}-${ARCHITECTURE}"
+          mkdir -p scratch
+          mkdir -p target
+
+          has_public_permission() {
+            jq -e '
+              (.Policy | if type == "string" then fromjson else . end)
+              | any(.Statement[]?;
+                  .Sid == "PublicLayer"
+                  and .Effect == "Allow"
+                  and ((.Action | if type == "array" then . else [.] end) | index("lambda:GetLayerVersion") != null)
+                  and (((.Principal | if type == "object" then .AWS // "" else . end) | if type == "array" then . else [.] end) | index("*") != null)
+                )
+            ' "$1" > /dev/null
+          }
+
+          if ! aws --region "$REGION" lambda list-layer-versions \
+            --layer-name "$NAME" \
+            --output json > scratch/versions.json 2> scratch/list-error.txt; then
+            if grep -q ResourceNotFoundException scratch/list-error.txt; then
+              echo '{"LayerVersions":[]}' > scratch/versions.json
+            else
+              cat scratch/list-error.txt
+              exit 1
+            fi
+          fi
+
+          CURRENT_POSITION=$(jq -r '[.LayerVersions[]?.Version] | max // 0' scratch/versions.json)
+
+          for (( VERSION=START_VERSION; VERSION<=END_VERSION; VERSION++ )); do
+            METADATA="source/${VERSION}.json"
+            ZIP="source/${VERSION}.zip"
+            TARGET_METADATA="target/${VERSION}.json"
+            TARGET_ARN="arn:${PARTITION}:lambda:${REGION}:${AWS_ACCOUNT}:layer:${NAME}:${VERSION}"
+
+            EXPECTED_SHA=$(jq -r '.Content.CodeSha256' "$METADATA")
+            ACTUAL_SHA=$(openssl dgst -sha256 -binary "$ZIP" | openssl enc -base64)
+            if [[ "$ACTUAL_SHA" != "$EXPECTED_SHA" ]]; then
+              echo "SHA mismatch for ${NAME}:${VERSION}: expected ${EXPECTED_SHA}, received ${ACTUAL_SHA}"
+              exit 1
+            fi
+
+            VERSION_EXISTS=false
+            if aws --region "$REGION" lambda get-layer-version-by-arn \
+              --arn "$TARGET_ARN" > "$TARGET_METADATA" 2> scratch/get-error.txt; then
+              VERSION_EXISTS=true
+              TARGET_SHA=$(jq -r '.Content.CodeSha256' "$TARGET_METADATA")
+              if [[ "$TARGET_SHA" != "$EXPECTED_SHA" ]]; then
+                echo "Existing ${NAME}:${VERSION} in ${REGION} has SHA ${TARGET_SHA}, expected ${EXPECTED_SHA}"
+                exit 1
+              fi
+            elif ! grep -q ResourceNotFoundException scratch/get-error.txt; then
+              cat scratch/get-error.txt
+              exit 1
+            fi
+
+            HAS_PUBLIC_PERMISSION=false
+            if aws --region "$REGION" lambda get-layer-version-policy \
+              --layer-name "$NAME" \
+              --version-number "$VERSION" > scratch/policy.json 2> scratch/policy-error.txt; then
+              if has_public_permission scratch/policy.json; then
+                HAS_PUBLIC_PERMISSION=true
+              fi
+            elif ! grep -q ResourceNotFoundException scratch/policy-error.txt; then
+              cat scratch/policy-error.txt
+              exit 1
+            fi
+
+            if [[ "$VERSION_EXISTS" == "true" ]]; then
+              echo "${NAME}:${VERSION} already exists in ${REGION} with the expected SHA"
+            else
+              EXPECTED_POSITION=$((VERSION - 1))
+              if (( CURRENT_POSITION != EXPECTED_POSITION )); then
+                echo "Cannot publish ${NAME}:${VERSION} in ${REGION}: latest version is ${CURRENT_POSITION}, expected ${EXPECTED_POSITION}"
+                exit 1
+              fi
+            fi
+
+            if [[ "$DRY_RUN" != "false" ]]; then
+              if [[ "$VERSION_EXISTS" == "false" ]]; then
+                echo "Would publish ${NAME}:${VERSION} to ${REGION}"
+                CURRENT_POSITION=$VERSION
+              elif [[ "$HAS_PUBLIC_PERMISSION" == "false" ]]; then
+                echo "Would add public permission to ${NAME}:${VERSION} in ${REGION}"
+              fi
+              continue
+            fi
+
+            if [[ "$VERSION_EXISTS" == "false" ]]; then
+              jq --arg layer_name "$NAME" \
+                '{LayerName: $layer_name, Description: .Description, CompatibleRuntimes: .CompatibleRuntimes, CompatibleArchitectures: .CompatibleArchitectures, LicenseInfo: .LicenseInfo} | with_entries(select(.value != null))' \
+                "$METADATA" > input.json
+
+              PUBLISHED_VERSION=$(aws --region "$REGION" lambda publish-layer-version \
+                --zip-file "fileb://${ZIP}" \
+                --cli-input-json file://input.json \
+                --query 'Version' \
+                --output text)
+
+              if (( PUBLISHED_VERSION != VERSION )); then
+                echo "Expected ${NAME} to publish as version ${VERSION}, received ${PUBLISHED_VERSION}"
+                exit 1
+              fi
+
+              CURRENT_POSITION=$PUBLISHED_VERSION
+            fi
+
+            if [[ "$HAS_PUBLIC_PERMISSION" == "false" ]]; then
+              if ! aws --region "$REGION" lambda add-layer-version-permission \
+                --layer-name "$NAME" \
+                --statement-id PublicLayer \
+                --action lambda:GetLayerVersion \
+                --principal '*' \
+                --version-number "$VERSION" 2> scratch/permission-error.txt; then
+                if ! grep -q ResourceConflictException scratch/permission-error.txt; then
+                  cat scratch/permission-error.txt
+                  exit 1
+                fi
+              fi
+            fi
+
+            aws --region "$REGION" lambda get-layer-version-by-arn \
+              --arn "$TARGET_ARN" > "$TARGET_METADATA"
+
+            TARGET_SHA=$(jq -r '.Content.CodeSha256' "$TARGET_METADATA")
+            if [[ "$TARGET_SHA" != "$EXPECTED_SHA" ]]; then
+              echo "Published ${NAME}:${VERSION} in ${REGION} has SHA ${TARGET_SHA}, expected ${EXPECTED_SHA}"
+              exit 1
+            fi
+
+            aws --region "$REGION" lambda get-layer-version-policy \
+              --layer-name "$NAME" \
+              --version-number "$VERSION" > scratch/policy.json
+            if ! has_public_permission scratch/policy.json; then
+              echo "${NAME}:${VERSION} in ${REGION} is missing the expected public permission"
+              exit 1
+            fi
+          done
+      - name: Store partition layer metadata
+        if: ${{ !inputs.dry_run && !cancelled() }}
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+        with:
+          name: ${{ needs.setup.outputs.layer }}-${{ matrix.architecture }}-${{ matrix.region }}-${{ needs.setup.outputs.start_version }}-${{ needs.setup.outputs.end_version }}
+          path: target
+          retention-days: 1
+          if-no-files-found: warn
+          overwrite: true
diff --git a/.github/workflows/layers_partitions.yml b/.github/workflows/layers_partitions.yml
index f6284aeb461..65b80f30748 100644
--- a/.github/workflows/layers_partitions.yml
+++ b/.github/workflows/layers_partitions.yml
@@ -43,6 +43,10 @@ run-name: Layer Deployment (${{ inputs.partition }}) - ${{ inputs.environment }}
 permissions:
   contents: read
 
+concurrency:
+  group: layer-balancer-${{ inputs.partition }}-${{ inputs.environment }}
+  cancel-in-progress: false
+
 jobs:
   setup:
     runs-on: ubuntu-latest