diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 42041e0..8817c44 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -72,31 +72,42 @@ jobs: sudo mv /tmp/cyclonedx /usr/local/bin/cyclonedx sudo chmod +x /usr/local/bin/cyclonedx - # Mock S3 using the official MinIO image (S3-compatible). Started as a - # plain `docker run` step rather than a service container because the - # image needs `server /data` passed as command, and GitHub Actions - # service containers can't override a container's command/entrypoint. - # LocalStack would have been the prior choice but its :latest now - # requires a paid auth token. - # Pulled from quay.io: docker.io/minio/minio is no longer publicly - # available ("pull access denied"). - - name: ๐Ÿชฃ Start MinIO + # Mock S3 with RustFS (Apache-2.0, MinIO-compatible API): same port 9000, + # same /minio/health/live probe, static root credentials via env and no + # command argument, so the test env below is unchanged. Started as a + # plain `docker run` step rather than a service container so the explicit + # readiness loop can gate the bucket step and dump the log on failure. + # Why not MinIO: github.com/minio/minio is archived and source-only since + # RELEASE.2025-10-15; docker.io/minio/minio was deleted in September 2026 + # and quay.io/minio/minio stopped serving anonymous pulls between + # 2026-09-21 and 2026-09-24 ("unauthorized"), which broke this job on + # 2026-09-28. Why not LocalStack: its community edition is archived and + # the remaining image needs an account plus auth token. + # The tag is pinned on purpose: Dependabot does not track images in + # `run:` steps and a CI fixture should not move underneath us. Bump it + # deliberately. + - name: ๐Ÿชฃ Start RustFS (S3-compatible) run: | - docker run -d --name minio \ + docker run -d --name rustfs \ -p 9000:9000 \ - -e MINIO_ROOT_USER=minioadmin \ - -e MINIO_ROOT_PASSWORD=minioadmin \ - quay.io/minio/minio:latest server /data - # Wait up to 60s for MinIO to be live before continuing. + -e RUSTFS_ACCESS_KEY=minioadmin \ + -e RUSTFS_SECRET_KEY=minioadmin \ + rustfs/rustfs:1.0.0 + # Wait up to 60s for RustFS to be live before continuing. for i in $(seq 1 60); do - if curl --silent --fail http://localhost:9000/minio/health/live; then - echo "MinIO ready after ${i}s" + if curl --silent --fail --output /dev/null http://localhost:9000/minio/health/live; then + echo "RustFS ready after ${i}s" break fi sleep 1 done + # Fail here, with the server log, instead of in the bucket step. + # RustFS logs to /logs inside the container, not to stdout; `docker cp` + # (unlike `docker exec`) still works when the container has exited. + curl --silent --fail --output /dev/null http://localhost:9000/minio/health/live \ + || { docker logs rustfs; docker cp rustfs:/logs/rustfs.log /tmp/rustfs.log && cat /tmp/rustfs.log; exit 1; } - - name: โš™๏ธ Create MinIO bucket + - name: โš™๏ธ Create S3 bucket run: | aws --endpoint-url=http://localhost:9000 s3 mb s3://test-bucket env: @@ -105,7 +116,7 @@ jobs: AWS_DEFAULT_REGION: us-east-1 # The S3 client switches to path-style addressing whenever - # AWS_ENDPOINT_URL is set, which is what MinIO requires. + # AWS_ENDPOINT_URL is set, which is what RustFS (like MinIO) requires. - name: ๐Ÿงช Run integration tests run: go test -v -tags=integration ./... env: diff --git a/CLAUDE.md b/CLAUDE.md index 777a7c2..5e67e50 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -25,9 +25,9 @@ go test -v -race -coverprofile=coverage.out -covermode=atomic ./... # Single test go test -v -run TestName ./internal/sbom -# Integration tests (build-tagged; require MinIO (or any S3 API) + ClickHouse + the cyclonedx CLI on PATH) +# Integration tests (build-tagged; require an S3-compatible store (CI uses RustFS), ClickHouse, and the cyclonedx CLI on PATH) go test -v -tags=integration ./... -# docker run -d -p 9000:9000 -e MINIO_ROOT_USER=minioadmin -e MINIO_ROOT_PASSWORD=minioadmin quay.io/minio/minio server /data +# docker run -d -p 9000:9000 -e RUSTFS_ACCESS_KEY=minioadmin -e RUSTFS_SECRET_KEY=minioadmin rustfs/rustfs:1.0.0 # docker run -d -p 8123:8123 -e CLICKHOUSE_USER=clickbom -e CLICKHOUSE_PASSWORD=clickbom clickhouse/clickhouse-server # AWS_ENDPOINT_URL=http://localhost:9000 CLICKHOUSE_URL=http://localhost:8123 \ # CLICKHOUSE_USERNAME=clickbom CLICKHOUSE_PASSWORD=clickbom \ @@ -68,7 +68,7 @@ Format conversion shells out to the `cyclonedx` CLI (installed in the Dockerfile - `merge.go` โ€” `MergeSBOMs`, `ExtractSourceReference` (multi-strategy: SPDX doc name โ†’ component name โ†’ bom-ref โ†’ filename). - `filter.go` โ€” `filepath.Match` glob filtering for merge mode. - `license_mapper.go` + [license-mappings.json](license-mappings.json) โ€” overrides "unknown"/missing licenses by component name. The mapping file is baked into the Docker image at `/app/license-mappings.json`; override with `LICENSE_MAPPING_FILE`. -- [internal/storage](internal/storage) โ€” `S3Client` (AWS SDK v2) and `ClickHouseClient`. `S3Client` resolves each bucket's home region once (HeadBucket โ†’ `x-amz-bucket-region`, which S3 returns even on 301/403) and caches a per-region client, so the job's `AWS_REGION` need not match the bucket; when `AWS_ENDPOINT_URL` is set (MinIO/LocalStack) it uses path-style addressing and skips region discovery. ClickHouse uses raw HTTP POST queries (only HTTP is supported โ€” no native protocol); the URL is stored without a trailing slash. `SetupTable` auto-migrates older tables by adding a `source LowCardinality(String)` column when missing. +- [internal/storage](internal/storage) โ€” `S3Client` (AWS SDK v2) and `ClickHouseClient`. `S3Client` resolves each bucket's home region once (HeadBucket โ†’ `x-amz-bucket-region`, which S3 returns even on 301/403) and caches a per-region client, so the job's `AWS_REGION` need not match the bucket; when `AWS_ENDPOINT_URL` is set (RustFS/MinIO/LocalStack) it uses path-style addressing and skips region discovery. ClickHouse uses raw HTTP POST queries (only HTTP is supported โ€” no native protocol); the URL is stored without a trailing slash. `SetupTable` auto-migrates older tables by adding a `source LowCardinality(String)` column when missing. - [pkg/logger](pkg/logger) โ€” colorized leveled logger gated by the `DEBUG` env var or `SetDebug(true)`. ### Table-name generation @@ -77,12 +77,12 @@ Format conversion shells out to the `cyclonedx` CLI (installed in the Dockerfile ### Runtime image -The Dockerfile is multi-stage and ends on `gcr.io/distroless/cc-debian12:nonroot`. The runtime image contains the static `clickbom` binary, two external tools copied from the `tools` stage (`cyclonedx`, `trivy`; both version-pinned via `ARG` and SHA-256 verified at build time), `libz.so.1` copied from a `debian:12-slim` `libs` stage (the .NET host inside cyclonedx-cli fails at startup without it โ€” keep that stage on the same Debian release as the distroless base), and `license-mappings.json`. Anything that needs to shell out must be one of those two tools (or added to the tools stage). The base image must stay a glibc variant (`cc`, or `base` + libstdc++): `cyclonedx-cli` is a dynamically linked .NET single-file app, and on `distroless/static` it fails with `fork/exec /usr/local/bin/cyclonedx: no such file or directory` (missing ELF interpreter) โ€” that regression shipped once and broke every GitHub-sourced run. `DOTNET_SYSTEM_GLOBALIZATION_INVARIANT=1` is set so .NET starts without ICU. Tool versions and checksums are pinned via `ARG` (`CYCLONEDX_CLI_VERSION` + `CYCLONEDX_CLI_SHA256`, `TRIVY_VERSION`; Trivy is checked against its published `_checksums.txt`); Dependabot does not bump them, so update version and hash together. The `test_docker` CI job loads the built image and actually runs `cyclonedx convert` inside it โ€” keep that smoke test if you touch the runtime stage. +The Dockerfile is multi-stage and ends on `gcr.io/distroless/cc-debian13:nonroot`. The runtime image contains the static `clickbom` binary, two external tools copied from the `tools` stage (`cyclonedx`, `trivy`; both version-pinned via `ARG` and SHA-256 verified at build time), `libz.so.1` copied from a `debian:13-slim` `libs` stage (the .NET host inside cyclonedx-cli fails at startup without it; `cc-debian12` did not ship zlib1g, `cc-debian13` does, so on Debian 13 the copy is redundant but harmless โ€” keep that stage on the same Debian release as the distroless base; Dependabot bumps the `debian:N-slim` tag but cannot bump the distroless image, whose Debian release is part of the image name rather than the tag, so when a `debian` bump lands, move `cc-debianN` by hand in the same PR โ€” #126 moved `libs` to 13 and left the runtime on 12), and `license-mappings.json`. Anything that needs to shell out must be one of those two tools (or added to the tools stage). The base image must stay a glibc variant (`cc`, or `base` + libstdc++): `cyclonedx-cli` is a dynamically linked .NET single-file app, and on `distroless/static` it fails with `fork/exec /usr/local/bin/cyclonedx: no such file or directory` (missing ELF interpreter) โ€” that regression shipped once and broke every GitHub-sourced run. `DOTNET_SYSTEM_GLOBALIZATION_INVARIANT=1` is set so .NET starts without ICU. Tool versions and checksums are pinned via `ARG` (`CYCLONEDX_CLI_VERSION` + `CYCLONEDX_CLI_SHA256`, `TRIVY_VERSION`; Trivy is checked against its published `_checksums.txt`); Dependabot does not bump them, so update version and hash together. The `test_docker` CI job loads the built image and actually runs `cyclonedx convert` inside it โ€” keep that smoke test if you touch the runtime stage. ## Conventions worth knowing - `goimports` uses `-local github.com/ClickHouse/ClickBOM`, so internal imports go in their own block. -- Integration tests use `//go:build integration` and are excluded from the default `go test ./...` run. CI runs them in the `test_integration` job against MinIO (`quay.io/minio/minio` โ€” the Docker Hub image is no longer pullable) and a `clickhouse/clickhouse-server` service container, with the `cyclonedx` CLI installed on the runner. Locally: start the same two containers (ClickHouse with `CLICKHOUSE_USER=clickbom CLICKHOUSE_PASSWORD=clickbom`; current images refuse the passwordless `default` user from outside the container), create `test-bucket`, put a `cyclonedx` binary on `PATH`, and export `AWS_ENDPOINT_URL`, `CLICKHOUSE_URL`, `CLICKHOUSE_USERNAME`/`CLICKHOUSE_PASSWORD`, `AWS_ACCESS_KEY_ID`/`AWS_SECRET_ACCESS_KEY`, `AWS_REGION` (see the command block above). +- Integration tests use `//go:build integration` and are excluded from the default `go test ./...` run. CI runs them in the `test_integration` job against RustFS (`rustfs/rustfs:1.0.0`, an Apache-2.0 store with a MinIO-compatible API: port 9000, `/minio/health/live`, root credentials via `RUSTFS_ACCESS_KEY`/`RUSTFS_SECRET_KEY`) and a `clickhouse/clickhouse-server` service container, with the `cyclonedx` CLI installed on the runner. Do not go back to MinIO: the project went source-only in October 2025, was archived in April 2026, and its Docker Hub and quay.io images were withdrawn in September 2026 (quay.io started answering `unauthorized` to anonymous pulls between 2026-09-21 and 2026-09-24, which is what broke the job). LocalStack's community image was discontinued and the remaining image needs an account and auth token. The RustFS tag is pinned by hand because Dependabot does not track images referenced in `run:` steps. Locally: start the same two containers (ClickHouse with `CLICKHOUSE_USER=clickbom CLICKHOUSE_PASSWORD=clickbom`; current images refuse the passwordless `default` user from outside the container), create `test-bucket`, put a `cyclonedx` binary on `PATH`, and export `AWS_ENDPOINT_URL`, `CLICKHOUSE_URL`, `CLICKHOUSE_USERNAME`/`CLICKHOUSE_PASSWORD`, `AWS_ACCESS_KEY_ID`/`AWS_SECRET_ACCESS_KEY`, `AWS_REGION` (see the command block above). - Consumers: [ClickHouse/sbom](https://github.com/ClickHouse/sbom) (`private-government.yml`, `clickhouse-cloud.yml`) and, until fully migrated, [ClickHouse/security-integrations](https://github.com/ClickHouse/security-integrations) `clickbom.yml`. The sbom workflows pin the `v2.0.0` tag, so a fix reaches them only after a new tag and a ref bump; security-integrations pins `@main`. Never point a consumer at a feature branch: the ref breaks the moment the branch is deleted. - Pre-commit blocks direct commits to `main`/`master` and enforces conventional commit messages. - `gocyclo -over 26` is the hard cyclomatic-complexity ceiling; `handleNormalMode` and `handleMergeMode` are close to it โ€” prefer extracting helpers when adding branches. diff --git a/Dockerfile b/Dockerfile index 5bfce1b..a84ad79 100644 --- a/Dockerfile +++ b/Dockerfile @@ -72,25 +72,32 @@ RUN wget -qO trivy_checksums.txt "https://github.com/aquasecurity/trivy/releases chmod +x /usr/local/bin/trivy && \ rm -f trivy_checksums.txt "trivy_${TRIVY_VERSION}_Linux-64bit.tar.gz" -# Shared libraries the .NET runtime embedded in cyclonedx-cli needs beyond -# what distroless `cc` ships. Sourced from the same Debian release as the -# runtime image so the glibc ABI matches exactly. -# - libz.so.1 (zlib): loaded at startup by the .NET host; without it -# cyclonedx fails with "error while loading shared libraries: libz.so.1". +# Shared libraries the .NET runtime embedded in cyclonedx-cli needs at startup. +# Sourced from the same Debian release as the runtime image so the glibc ABI +# matches exactly (see the lockstep note on the runtime stage below). +# - libz.so.1 (zlib): loaded at startup by the .NET host. `cc-debian12` did +# not ship it and cyclonedx failed with "error while loading shared +# libraries: libz.so.1". `cc-debian13` ships zlib1g itself, so on Debian 13 +# this copy is redundant: it only overwrites the base image's libz.so.1 +# with the `debian:13-slim` build of the same package. FROM debian:13-slim AS libs # Runtime stage - Distroless. # -# `cc-debian12`, NOT `static-debian12`: cyclonedx-cli needs glibc + libgcc + +# `cc-debian13`, NOT `static-debian13`: cyclonedx-cli needs glibc + libgcc + # libstdc++ at runtime. On `static` (which ships none of them) every # `cyclonedx convert` failed with # fork/exec /usr/local/bin/cyclonedx: no such file or directory # because the kernel could not find the ELF interpreter /lib64/ld-linux-x86-64.so.2. -FROM gcr.io/distroless/cc-debian12:nonroot +# +# Keep the Debian release in lockstep with the `libs` stage above. Dependabot +# bumps the `debian:N-slim` tag there but cannot bump this image, whose release +# is part of the image name rather than the tag, so move both by hand. +FROM gcr.io/distroless/cc-debian13:nonroot LABEL maintainer="ClickHouse Security Team" \ description="ClickBOM - SBOM Management Tool" \ - version="2.0.0" \ + version="2.0.2" \ security.scan="enabled" # Copy from tools stage diff --git a/README.md b/README.md index 130d5e0..1d1acb8 100644 --- a/README.md +++ b/README.md @@ -95,7 +95,7 @@ Downloads SBOMs from GitHub, Mend, and Wiz, or generates them from container ima - It is recommended that an S3 bucket be created for the purposes of ClickBOM. - The `aws-*` inputs are kept for backward compatibility with the bash version of this action. The recommended path is to use [`aws-actions/configure-aws-credentials`](https://github.com/aws-actions/configure-aws-credentials) with GitHub OIDC; that action exports `AWS_ACCESS_KEY_ID` / `AWS_SECRET_ACCESS_KEY` / `AWS_SESSION_TOKEN` / `AWS_REGION` as job-level env vars, and the Actions runner passes job env through to the ClickBOM container unchanged, so nothing needs to be passed as an input. (Do not pass `steps..outputs.aws-access-key-id`: those outputs are empty unless `output-credentials: true` is set.) - The bucket does not have to be in the job's `aws-region`. ClickBOM resolves each bucket's home region up front (via `HeadBucket`'s `x-amz-bucket-region` header) and talks to the right regional endpoint, so a mismatch no longer fails with `301 PermanentRedirect`. -- Setting `AWS_ENDPOINT_URL` (e.g. to MinIO or LocalStack) switches the client to path-style addressing and disables region discovery. +- Setting `AWS_ENDPOINT_URL` (e.g. to RustFS, MinIO or another S3-compatible store) switches the client to path-style addressing and disables region discovery. ### ClickHouse @@ -637,7 +637,7 @@ jobs: ## Runtime Image -The action runs as a Docker container built from this repository's `Dockerfile`: a static Go binary plus two external tools, `cyclonedx` (format conversion) and `trivy` (image scanning), on `gcr.io/distroless/cc-debian12:nonroot`. The `cc` variant is required because `cyclonedx-cli` is a dynamically linked .NET application; on `distroless/static` it cannot execute at all. CI builds the image and runs a conversion inside it on every push. +The action runs as a Docker container built from this repository's `Dockerfile`: a static Go binary plus two external tools, `cyclonedx` (format conversion) and `trivy` (image scanning), on `gcr.io/distroless/cc-debian13:nonroot`. The `cc` variant is required because `cyclonedx-cli` is a dynamically linked .NET application; on `distroless/static` it cannot execute at all. CI builds the image and runs a conversion inside it on every push. ## Creating a GitHub App diff --git a/internal/storage/s3.go b/internal/storage/s3.go index a2b640b..c21776f 100644 --- a/internal/storage/s3.go +++ b/internal/storage/s3.go @@ -57,7 +57,7 @@ type S3Client struct { // (AWS_REGION / AWS_DEFAULT_REGION / shared config). defaultClient *s3.Client // customEndpoint is true when AWS_ENDPOINT_URL (or AWS_ENDPOINT_URL_S3) - // points at a non-AWS S3-compatible endpoint such as MinIO or LocalStack. + // points at a non-AWS S3-compatible endpoint such as RustFS or MinIO. // Region discovery is skipped in that case: there is only one endpoint. customEndpoint bool