diff --git a/.gitlab/datasources/flavors.yaml b/.gitlab/datasources/flavors.yaml index e1e2d2097..992a10396 100644 --- a/.gitlab/datasources/flavors.yaml +++ b/.gitlab/datasources/flavors.yaml @@ -8,6 +8,9 @@ flavors: layer_name_base_suffix: "" max_layer_compressed_size_mb: 27 max_layer_uncompressed_size_mb: 54 + # Also build an unstripped copy on main (job "bottlecap symbols"), for size, + # ELF and benchmark analysis. The copy is a CI artifact only, never shipped. + keep_symbols: true - name: arm64 arch: arm64 @@ -18,6 +21,7 @@ flavors: layer_name_base_suffix: "-ARM" max_layer_compressed_size_mb: 24 max_layer_uncompressed_size_mb: 50 + keep_symbols: true - name: amd64, alpine arch: amd64 diff --git a/.gitlab/scripts/compile_bottlecap.sh b/.gitlab/scripts/compile_bottlecap.sh index 59d9779ff..e11643bda 100755 --- a/.gitlab/scripts/compile_bottlecap.sh +++ b/.gitlab/scripts/compile_bottlecap.sh @@ -38,6 +38,19 @@ else printf "Debug mode requested: ${DEBUG}\n" fi +if [ "$KEEP_SYMBOLS" != "1" ]; then + OUTPUT_NAME=bottlecap-$FILE_SUFFIX + KEEP_SYMBOLS_ARG=() +else + if [ "$ALPINE" != "0" ]; then + printf "[ERROR]: KEEP_SYMBOLS=1 is only supported with Dockerfile.bottlecap.compile (ALPINE=0)\n" + exit 1 + fi + OUTPUT_NAME=bottlecap-$FILE_SUFFIX-unstripped + KEEP_SYMBOLS_ARG=(--build-arg KEEP_SYMBOLS=1) + printf "Keeping symbols: the output is %s and is not shipped\n" "$OUTPUT_NAME" +fi + if [ "$ALPINE" = "0" ]; then COMPILE_IMAGE=Dockerfile.bottlecap.compile else @@ -53,7 +66,7 @@ cd $ROOT_DIR BINARY_DIR=".binaries" TARGET_DIR=$(pwd)/$BINARY_DIR -BINARY_PATH=$TARGET_DIR/compiled-bottlecap-$FILE_SUFFIX +BINARY_PATH=$TARGET_DIR/compiled-$OUTPUT_NAME mkdir -p $BINARY_DIR @@ -75,11 +88,12 @@ docker_build() { --build-arg PLATFORM=$PLATFORM \ --build-arg FIPS="${FIPS}" \ --build-arg DEBUG="${DEBUG}" \ + "${KEEP_SYMBOLS_ARG[@]}" \ . -o $BINARY_PATH # Copy the compiled binary to the target directory with the expected name # If it already exist, it will be replaced - cp $BINARY_PATH/bottlecap $TARGET_DIR/bottlecap-$FILE_SUFFIX + cp "$BINARY_PATH/bottlecap" "$TARGET_DIR/$OUTPUT_NAME" } docker_build $ARCHITECTURE $COMPILE_IMAGE diff --git a/.gitlab/templates/pipeline.yaml.tpl b/.gitlab/templates/pipeline.yaml.tpl index db89d1069..4fcd41768 100644 --- a/.gitlab/templates/pipeline.yaml.tpl +++ b/.gitlab/templates/pipeline.yaml.tpl @@ -95,6 +95,51 @@ bottlecap ({{ $flavor.name }}): script: - .gitlab/scripts/compile_bottlecap.sh +{{ if index $flavor "keep_symbols" }} +# Per-commit release binaries for analysis. Runs on main and tags (manual elsewhere), +# never blocks the pipeline. Bundles the shipped stripped binary from +# `bottlecap ({{ $flavor.name }})` with a second compile that keeps the symbol table, +# under one artifact named after the commit, kept for 90 days. The shipped layer does not change. +# latest main: $CI_API_V4_URL/projects/$CI_PROJECT_ID/jobs/artifacts/main/raw/.binaries/bottlecap-{{ $flavor.suffix }}-unstripped?job=bottlecap+symbols+({{ $flavor.name }}) +bottlecap symbols ({{ $flavor.name }}): + stage: compile + image: registry.ddbuild.io/images/docker:20.10 + tags: ["arch:{{ $flavor.arch }}"] + rules: + - if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH' + allow_failure: true + - if: '$CI_COMMIT_TAG' + allow_failure: true + - when: manual + allow_failure: true + needs: + - bottlecap ({{ $flavor.name }}) + timeout: 20m + retry: + max: 2 + when: + - stuck_or_timeout_failure + - runner_system_failure + artifacts: + name: bottlecap-{{ $flavor.suffix }}-${CI_COMMIT_SHORT_SHA} + expire_in: 90 days + paths: + - .binaries/bottlecap-{{ $flavor.suffix }} + - .binaries/bottlecap-{{ $flavor.suffix }}-unstripped + - .binaries/bottlecap-{{ $flavor.suffix }}.build-info + variables: + ARCHITECTURE: {{ $flavor.arch }} + ALPINE: {{ $flavor.alpine }} + FIPS: {{ $flavor.fips }} + FILE_SUFFIX: {{ $flavor.suffix }} + KEEP_SYMBOLS: "1" + script: + - .gitlab/scripts/compile_bottlecap.sh + - printf 'commit=%s\nref=%s\npipeline=%s\njob=%s\n' "$CI_COMMIT_SHA" "$CI_COMMIT_REF_NAME" "$CI_PIPELINE_URL" "$CI_JOB_URL" > .binaries/bottlecap-{{ $flavor.suffix }}.build-info + - ls -la .binaries/ + - echo "unstripped binary for ${CI_COMMIT_SHA}:" "${CI_JOB_URL}/artifacts/raw/.binaries/bottlecap-{{ $flavor.suffix }}-unstripped" +{{ end }}{{/* end keep_symbols */}} + layer ({{ $flavor.name }}): stage: build image: registry.ddbuild.io/images/docker:27.3.1@sha256:966b6cd18c53e3730e7aaea2c0ca6b6b9877dcff11d9446f25ee5bc2293585d0 diff --git a/images/Dockerfile.bottlecap.compile b/images/Dockerfile.bottlecap.compile index 877e6e21e..403d986c2 100644 --- a/images/Dockerfile.bottlecap.compile +++ b/images/Dockerfile.bottlecap.compile @@ -31,6 +31,10 @@ ENV AWS_LC_FIPS_SYS_CC=clang ENV AWS_LC_FIPS_SYS_CXX=clang++ WORKDIR /tmp/dd/bottlecap +# KEEP_SYMBOLS=1 builds the same source and profile with strip disabled, so the +# linker keeps the symbol table. Only the `bottlecap symbols` CI job sets it; +# that output is never shipped. Declared here so both variants share the layers above. +ARG KEEP_SYMBOLS=0 RUN --mount=type=cache,target=/usr/local/cargo/git \ --mount=type=cache,target=/usr/local/cargo/registry \ --mount=type=bind,source=.,target=/tmp/dd,rw \ @@ -49,6 +53,9 @@ RUN --mount=type=cache,target=/usr/local/cargo/git \ export BUILD_MODE=debug; \ export BUILD_FLAG=""; \ fi; \ + if [ "${KEEP_SYMBOLS}" = "1" ]; then \ + export CARGO_PROFILE_RELEASE_STRIP=false; \ + fi; \ # The `libddwaf` crate links against static objects that require `libclang_rt.builtins`, but # this is not presented to the linker by default on this platform, so we force it in. export RUSTFLAGS="${RUSTFLAGS:-} -Clinker=clang -L$(dirname $(clang --print-file-name="libclang_rt.builtins-$(uname -m).a")) -lclang_rt.builtins-$(uname -m)"; \