From 1de65224ec465925147d7c88eb92053aff4ff37e Mon Sep 17 00:00:00 2001 From: Stefan Wiedemann Date: Tue, 3 Nov 2026 13:48:16 +0100 Subject: [PATCH 1/2] Validate custom resources against the CRDs a chart ships kubeconform validated every custom resource against the datreeio CRDs catalog, which is generated from whichever CRD version it was last refreshed with. A chart that deploys a newer operator then fails on a field its own CRD allows: postgres-operator 1.15.1 sets aws_or_gcp.enable_ebs_gp3_migration on its OperatorConfiguration, and the catalog schema for acid.zalan.do/v1 declares additionalProperties: false over a property set that predates it. eval-charts.sh now extracts the openAPIV3Schema of every CRD version a chart ships - its own crds/ and those of its dependencies, unpacked from the archives helm dependency build leaves behind - and gives kubeconform that schema location ahead of the catalog, which still covers the kinds a chart defines no CRD for. Only a crds/ directory that sits directly in a chart is read, the way Helm reads them: opentelemetry-operator keeps Helm templates in conf/crds, which are not parseable YAML. --- README.md | 4 +- scripts/eval-charts.sh | 91 ++++++++++++++++++++++++++++++++---- tests/lint-eval-charts.bats | 92 +++++++++++++++++++++++++++++++++++++ 3 files changed, 178 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index 4f4223d..a28306f 100644 --- a/README.md +++ b/README.md @@ -15,7 +15,9 @@ Together, the chart workflows cover the life of a chart: a pull request is linte ## Chart lint (`chart-lint.yml`) -Lints every chart under `charts/` (the directories with a `Chart.yaml`) with `helm lint`, then builds its dependencies, renders it and validates the result with `kubeconform -strict`, against the Kubernetes schemas and the [CRDs catalog](https://github.com/datreeio/CRDs-catalog). Each `/ci/*-values.yaml` is rendered and validated as well, to cover what the default values leave off; with those, custom resources that are not in the catalog are skipped. Every chart is checked before the result is reported. +Lints every chart under `charts/` (the directories with a `Chart.yaml`) with `helm lint`, then builds its dependencies, renders it and validates the result with `kubeconform -strict` against the Kubernetes schemas. Each `/ci/*-values.yaml` is rendered and validated as well, to cover what the default values leave off. Every chart is checked before the result is reported. + +A custom resource is validated against the CRD the chart itself ships — its own `crds/` and those of its dependencies — and, for a kind that no CRD in the chart defines, against the [CRDs catalog](https://github.com/datreeio/CRDs-catalog); with the ci values, a kind in neither is skipped. The chart's own CRD comes first because it is the authority for its custom resources: the catalog is generated from whichever CRD version it was last refreshed with, so a field that the operator a chart deploys accepts today can be rejected by a schema built from an older one. ```yaml jobs: diff --git a/scripts/eval-charts.sh b/scripts/eval-charts.sh index 9b205e0..4de1895 100755 --- a/scripts/eval-charts.sh +++ b/scripts/eval-charts.sh @@ -1,15 +1,21 @@ #!/usr/bin/env bash # Renders every Helm chart under CHARTS_DIR and validates the result with kubeconform: with the # default values, and with each /ci/*-values.yaml, which can turn on what the defaults -# leave off. Custom resources are validated against the CRDs catalog -# (https://github.com/datreeio/CRDs-catalog); with the ci values, those that are not in it are -# skipped, every other resource is still validated strictly. Every chart is evaluated before the -# result is reported. +# leave off. Custom resources are validated against the CRDs the chart itself ships - its own +# crds/ and those of its dependencies - and, for the kinds it defines none, against the CRDs +# catalog (https://github.com/datreeio/CRDs-catalog); with the ci values, those that are in +# neither are skipped, every other resource is still validated strictly. Every chart is evaluated +# before the result is reported. +# +# A chart's own CRD comes first because it is the authority for its custom resources: the catalog +# is generated from whichever CRD version it was last refreshed with, so a field that the operator +# the chart deploys accepts today can be rejected by a schema built from an older one. # # Environment: # CHARTS_DIR directory that holds one directory per chart (default: charts) # HELM helm binary (default: helm) # KUBECONFORM kubeconform binary (default: kubeconform) +# YQ yq binary (mikefarah, v4; default: yq) # # Run add-helm-repos.sh first for charts with dependencies. @@ -18,27 +24,96 @@ set -uo pipefail CHARTS_DIR="${CHARTS_DIR:-charts}" HELM="${HELM:-helm}" KUBECONFORM="${KUBECONFORM:-kubeconform}" -SCHEMAS=(-schema-location default - -schema-location 'https://raw.githubusercontent.com/datreeio/CRDs-catalog/main/{{.Group}}/{{.ResourceKind}}_{{.ResourceAPIVersion}}.json') +YQ="${YQ:-yq}" + +# How kubeconform addresses a schema. It renders the kind in lower case, which is the name the +# catalog uses and the name write_crd_schemas writes. +SCHEMA_PATH='{{.Group}}/{{.ResourceKind}}_{{.ResourceAPIVersion}}.json' +CATALOG="https://raw.githubusercontent.com/datreeio/CRDs-catalog/main/$SCHEMA_PATH" + +workdir=$(mktemp -d) +trap 'rm -rf "$workdir"' EXIT + +# write_crd_schemas +# Writes the openAPIV3Schema of every CustomResourceDefinition version the chart ships to +# //_.json. A dependency's CRDs are inside the archive that +# helm dependency build leaves in /charts, so those are unpacked first; the chart's own +# crds/, and a dependency that is already unpacked, are read where they are. +write_crd_schemas() { + local chart=$1 schema_dir=$2 unpack_dir=$3 + local archive crd doc group kind index version + + # one line " " per schema to write. + # shellcheck disable=SC2016 # $crd is a yq variable, not a shell one + local versions_query=' + select(.kind == "CustomResourceDefinition") | + [documentIndex, .spec.group, .spec.names.kind] as $crd | + .spec.versions | to_entries[] | + select(.value.schema.openAPIV3Schema != null) | + ($crd + [.key, .value.name]) | join(" ")' + + mkdir -p "$unpack_dir" + for archive in "$chart"/charts/*.tgz; do + [ -f "$archive" ] || continue + tar -xzf "$archive" -C "$unpack_dir" || return 1 + done + + while IFS= read -r crd; do + while read -r doc group kind index version; do + [ -n "${version:-}" ] || continue + mkdir -p "$schema_dir/$group" + "$YQ" e -N -o=json \ + "select(documentIndex == $doc) | .spec.versions[$index].schema.openAPIV3Schema" \ + "$crd" > "$schema_dir/$group/${kind,,}_$version.json" || return 1 + done < <("$YQ" e -N "$versions_query" "$crd") + done < <(crd_files "$chart" "$unpack_dir") +} + +# crd_files ... +# Prints every CRD file under the given directories: the *.yaml in a crds/ directory that sits +# directly in a chart, which is where Helm reads them from. A crds/ elsewhere in a chart is not +# one - opentelemetry-operator, for instance, keeps Helm templates in conf/crds - and templates +# are not parseable YAML. +crd_files() { + local crd_dir + while IFS= read -r crd_dir; do + [ -f "$(dirname "$crd_dir")/Chart.yaml" ] || continue + find "$crd_dir" -type f \( -name '*.yaml' -o -name '*.yml' \) + done < <(find "$@" -type d -name crds 2>/dev/null) +} status=0 +chart_index=0 for chart_file in "$CHARTS_DIR"/*/Chart.yaml; do [ -f "$chart_file" ] || continue chart=$(dirname "$chart_file") + chart_index=$((chart_index + 1)) echo "Evaluating $chart" if ! "$HELM" dependency build "$chart"; then status=1 continue fi - if ! "$HELM" template "$chart" | "$KUBECONFORM" -strict -summary "${SCHEMAS[@]}"; then + + # the dependencies are only on disk once they are built, so their CRDs are collected after + schema_dir="$workdir/schemas/$chart_index" + if ! write_crd_schemas "$chart" "$schema_dir" "$workdir/unpacked/$chart_index"; then + echo "Could not read the CRDs of $chart" >&2 + status=1 + continue + fi + schemas=(-schema-location default + -schema-location "$schema_dir/$SCHEMA_PATH" + -schema-location "$CATALOG") + + if ! "$HELM" template "$chart" | "$KUBECONFORM" -strict -summary "${schemas[@]}"; then status=1 fi for values in "$chart"/ci/*-values.yaml; do [ -f "$values" ] || continue echo "Evaluating $chart with $(basename "$values")" - if ! "$HELM" template "$chart" -f "$values" | "$KUBECONFORM" -strict -summary -ignore-missing-schemas "${SCHEMAS[@]}"; then + if ! "$HELM" template "$chart" -f "$values" | "$KUBECONFORM" -strict -summary -ignore-missing-schemas "${schemas[@]}"; then status=1 fi done diff --git a/tests/lint-eval-charts.bats b/tests/lint-eval-charts.bats index 72ff40b..2a479c3 100644 --- a/tests/lint-eval-charts.bats +++ b/tests/lint-eval-charts.bats @@ -76,3 +76,95 @@ EOF [ "$status" -eq 1 ] [[ "$output" == *"Evaluating charts/good with extra-values.yaml"* ]] } + +# crd [] +# gives the chart a CRD for widgets.example.test, whose spec takes a "size" integer and, when +# asked for, one more string property. The group is not in the CRDs catalog, so a widget can only +# validate against the CRD the chart itself ships. +crd() { + local extra="" + [ -z "${2:-}" ] || extra=$(printf '\n %s:\n type: string' "$2") + mkdir -p "charts/$1/crds" + cat > "charts/$1/crds/widget.yaml" < +widget() { + cat > "charts/$1/templates/widget.yaml" < charts/good/conf/crds/templated.yaml + run "$EVAL" + [ "$status" -eq 0 ] + [[ "$output" != *"bad file"* ]] +} + +@test "the CRDs a dependency ships are read from the archive helm builds" { + chart dep 1.0.0 + crd dep + # the widget is in the chart that depends on dep, so its schema can only come from the + # dep-1.0.0.tgz that helm dependency build puts in charts/user/charts + chart user 1.0.0 dep 1.0.0 "file://../dep" + widget user " size: 3" + run "$EVAL" + [ "$status" -eq 0 ] + [[ "$output" == *"Chart evaluation successful"* ]] +} From 32a7fbf368ce1aac3957284e812c2a5fb9c05ece Mon Sep 17 00:00:00 2001 From: Stefan Wiedemann Date: Wed, 4 Nov 2026 07:02:56 +0100 Subject: [PATCH 2/2] pr fixes --- .github/workflows/chart-lint.yml | 15 +++++++++++++++ README.md | 1 + scripts/eval-charts.sh | 11 ++++++++--- tests/lint-eval-charts.bats | 10 ++++++++++ 4 files changed, 34 insertions(+), 3 deletions(-) diff --git a/.github/workflows/chart-lint.yml b/.github/workflows/chart-lint.yml index 041427d..27ba784 100644 --- a/.github/workflows/chart-lint.yml +++ b/.github/workflows/chart-lint.yml @@ -18,6 +18,10 @@ on: description: kubeconform version to validate the rendered charts with type: string default: v0.7.0 + yq-version: + description: yq version to read the CRDs a chart ships with (mikefarah v4.45.1 or newer) + type: string + default: v4.54.1 permissions: contents: read @@ -82,6 +86,17 @@ jobs: | sudo tar -xz -C /usr/local/bin kubeconform kubeconform -v + # the runner image ships a yq of its own; the one the script reads the CRDs with is pinned + # here so that the validation does not depend on which version that happens to be + - name: Install yq + env: + YQ_VERSION: ${{ inputs.yq-version }} + run: | + sudo curl -sSfL "https://github.com/mikefarah/yq/releases/download/$YQ_VERSION/yq_linux_amd64" \ + -o /usr/local/bin/yq + sudo chmod +x /usr/local/bin/yq + yq --version + - name: Add the dependency repositories working-directory: repo env: diff --git a/README.md b/README.md index a28306f..19ea5b5 100644 --- a/README.md +++ b/README.md @@ -30,6 +30,7 @@ jobs: | `charts-dir` | `charts` | Directory that holds one directory per chart | | `helm-version` | `v3.17.3` | Helm version | | `kubeconform-version` | `v0.7.0` | kubeconform version | +| `yq-version` | `v4.54.1` | yq version the CRDs of a chart are read with (mikefarah v4.45.1 or newer) | ## Chart version (`chart-pr.yml`) diff --git a/scripts/eval-charts.sh b/scripts/eval-charts.sh index 4de1895..6143001 100755 --- a/scripts/eval-charts.sh +++ b/scripts/eval-charts.sh @@ -15,7 +15,8 @@ # CHARTS_DIR directory that holds one directory per chart (default: charts) # HELM helm binary (default: helm) # KUBECONFORM kubeconform binary (default: kubeconform) -# YQ yq binary (mikefarah, v4; default: yq) +# YQ yq binary (mikefarah v4.45.1 or newer - an older one cannot parse the query that +# reads the CRD versions, and v4.35.2 rejects it outright; default: yq) # # Run add-helm-repos.sh first for charts with dependencies. @@ -41,7 +42,7 @@ trap 'rm -rf "$workdir"' EXIT # crds/, and a dependency that is already unpacked, are read where they are. write_crd_schemas() { local chart=$1 schema_dir=$2 unpack_dir=$3 - local archive crd doc group kind index version + local archive crd entries doc group kind index version # one line " " per schema to write. # shellcheck disable=SC2016 # $crd is a yq variable, not a shell one @@ -58,14 +59,18 @@ write_crd_schemas() { tar -xzf "$archive" -C "$unpack_dir" || return 1 done + # the query is read into a variable rather than piped in: the exit status of a process + # substitution is lost, and an unreadable CRD would then leave no schema and silently send + # kubeconform back to the catalog - the very failure this works around while IFS= read -r crd; do + entries=$("$YQ" e -N "$versions_query" "$crd") || return 1 while read -r doc group kind index version; do [ -n "${version:-}" ] || continue mkdir -p "$schema_dir/$group" "$YQ" e -N -o=json \ "select(documentIndex == $doc) | .spec.versions[$index].schema.openAPIV3Schema" \ "$crd" > "$schema_dir/$group/${kind,,}_$version.json" || return 1 - done < <("$YQ" e -N "$versions_query" "$crd") + done <<< "$entries" done < <(crd_files "$chart" "$unpack_dir") } diff --git a/tests/lint-eval-charts.bats b/tests/lint-eval-charts.bats index 2a479c3..941ede9 100644 --- a/tests/lint-eval-charts.bats +++ b/tests/lint-eval-charts.bats @@ -157,6 +157,16 @@ EOF [[ "$output" != *"bad file"* ]] } +@test "a CRD that cannot be read fails, instead of falling back to the catalog" { + crd good + widget good " size: 3" + # a yq that fails stands for an unparseable CRD, or one a yq too old for the query chokes on: + # the schema is then missing, and the resource must not simply be validated without it + YQ=false run "$EVAL" + [ "$status" -eq 1 ] + [[ "$output" == *"Could not read the CRDs of charts/good"* ]] +} + @test "the CRDs a dependency ships are read from the archive helm builds" { chart dep 1.0.0 crd dep