Validate custom resources against the CRDs a chart ships

.github/workflows/chart-lint.yml

@@ -18,6 +18,10 @@ on:
         description: kubeconform version to validate the rendered charts with
         type: string
         default: v0.7.0
+      yq-version:
+        description: yq version to read the CRDs a chart ships with (mikefarah v4.45.1 or newer)
+        type: string
+        default: v4.54.1
 
 permissions:
   contents: read
@@ -82,6 +86,17 @@ jobs:
             | sudo tar -xz -C /usr/local/bin kubeconform
           kubeconform -v
 
+      # the runner image ships a yq of its own; the one the script reads the CRDs with is pinned
+      # here so that the validation does not depend on which version that happens to be
+      - name: Install yq
+        env:
+          YQ_VERSION: ${{ inputs.yq-version }}
+        run: |
+          sudo curl -sSfL "https://github.com/mikefarah/yq/releases/download/$YQ_VERSION/yq_linux_amd64" \
+            -o /usr/local/bin/yq
+          sudo chmod +x /usr/local/bin/yq
+          yq --version
+
       - name: Add the dependency repositories
         working-directory: repo
         env:

README.md

@@ -15,7 +15,9 @@ Together, the chart workflows cover the life of a chart: a pull request is linte
 
 ## Chart lint (`chart-lint.yml`)
 
-Lints every chart under `charts/` (the directories with a `Chart.yaml`) with `helm lint`, then builds its dependencies, renders it and validates the result with `kubeconform -strict`, against the Kubernetes schemas and the [CRDs catalog](https://github.com/datreeio/CRDs-catalog). Each `<chart>/ci/*-values.yaml` is rendered and validated as well, to cover what the default values leave off; with those, custom resources that are not in the catalog are skipped. Every chart is checked before the result is reported.
+Lints every chart under `charts/` (the directories with a `Chart.yaml`) with `helm lint`, then builds its dependencies, renders it and validates the result with `kubeconform -strict` against the Kubernetes schemas. Each `<chart>/ci/*-values.yaml` is rendered and validated as well, to cover what the default values leave off. Every chart is checked before the result is reported.
+
+A custom resource is validated against the CRD the chart itself ships — its own `crds/` and those of its dependencies — and, for a kind that no CRD in the chart defines, against the [CRDs catalog](https://github.com/datreeio/CRDs-catalog); with the ci values, a kind in neither is skipped. The chart's own CRD comes first because it is the authority for its custom resources: the catalog is generated from whichever CRD version it was last refreshed with, so a field that the operator a chart deploys accepts today can be rejected by a schema built from an older one.
 
 ```yaml
 jobs:
@@ -28,6 +30,7 @@ jobs:
 | `charts-dir` | `charts` | Directory that holds one directory per chart |
 | `helm-version` | `v3.17.3` | Helm version |
 | `kubeconform-version` | `v0.7.0` | kubeconform version |
+| `yq-version` | `v4.54.1` | yq version the CRDs of a chart are read with (mikefarah v4.45.1 or newer) |
 
 ## Chart version (`chart-pr.yml`)
 

scripts/eval-charts.sh

@@ -1,15 +1,22 @@
 #!/usr/bin/env bash
 # Renders every Helm chart under CHARTS_DIR and validates the result with kubeconform: with the
 # default values, and with each <chart>/ci/*-values.yaml, which can turn on what the defaults
-# leave off. Custom resources are validated against the CRDs catalog
-# (https://github.com/datreeio/CRDs-catalog); with the ci values, those that are not in it are
-# skipped, every other resource is still validated strictly. Every chart is evaluated before the
-# result is reported.
+# leave off. Custom resources are validated against the CRDs the chart itself ships - its own
+# crds/ and those of its dependencies - and, for the kinds it defines none, against the CRDs
+# catalog (https://github.com/datreeio/CRDs-catalog); with the ci values, those that are in
+# neither are skipped, every other resource is still validated strictly. Every chart is evaluated
+# before the result is reported.
+#
+# A chart's own CRD comes first because it is the authority for its custom resources: the catalog
+# is generated from whichever CRD version it was last refreshed with, so a field that the operator
+# the chart deploys accepts today can be rejected by a schema built from an older one.
 #
 # Environment:
 #   CHARTS_DIR   directory that holds one directory per chart (default: charts)
 #   HELM         helm binary (default: helm)
 #   KUBECONFORM  kubeconform binary (default: kubeconform)
+#   YQ           yq binary (mikefarah v4.45.1 or newer - an older one cannot parse the query that
+#                reads the CRD versions, and v4.35.2 rejects it outright; default: yq)
 #
 # Run add-helm-repos.sh first for charts with dependencies.
 
@@ -18,27 +25,100 @@ set -uo pipefail
 CHARTS_DIR="${CHARTS_DIR:-charts}"
 HELM="${HELM:-helm}"
 KUBECONFORM="${KUBECONFORM:-kubeconform}"
-SCHEMAS=(-schema-location default
-         -schema-location 'https://raw.githubusercontent.com/datreeio/CRDs-catalog/main/{{.Group}}/{{.ResourceKind}}_{{.ResourceAPIVersion}}.json')
+YQ="${YQ:-yq}"
+
+# How kubeconform addresses a schema. It renders the kind in lower case, which is the name the
+# catalog uses and the name write_crd_schemas writes.
+SCHEMA_PATH='{{.Group}}/{{.ResourceKind}}_{{.ResourceAPIVersion}}.json'
+CATALOG="https://raw.githubusercontent.com/datreeio/CRDs-catalog/main/$SCHEMA_PATH"
+
+workdir=$(mktemp -d)
+trap 'rm -rf "$workdir"' EXIT
+
+# write_crd_schemas <chart> <schema dir> <unpack dir>
+# Writes the openAPIV3Schema of every CustomResourceDefinition version the chart ships to
+# <schema dir>/<group>/<kind>_<version>.json. A dependency's CRDs are inside the archive that
+# helm dependency build leaves in <chart>/charts, so those are unpacked first; the chart's own
+# crds/, and a dependency that is already unpacked, are read where they are.
+write_crd_schemas() {
+    local chart=$1 schema_dir=$2 unpack_dir=$3
+    local archive crd entries doc group kind index version
+
+    # one line "<document> <group> <kind> <version index> <version>" per schema to write.
+    # shellcheck disable=SC2016 # $crd is a yq variable, not a shell one
+    local versions_query='
+        select(.kind == "CustomResourceDefinition") |
+        [documentIndex, .spec.group, .spec.names.kind] as $crd |
+        .spec.versions | to_entries[] |
+        select(.value.schema.openAPIV3Schema != null) |
+        ($crd + [.key, .value.name]) | join(" ")'
+
+    mkdir -p "$unpack_dir"
+    for archive in "$chart"/charts/*.tgz; do
+        [ -f "$archive" ] || continue
+        tar -xzf "$archive" -C "$unpack_dir" || return 1
+    done
+
+    # the query is read into a variable rather than piped in: the exit status of a process
+    # substitution is lost, and an unreadable CRD would then leave no schema and silently send
+    # kubeconform back to the catalog - the very failure this works around
+    while IFS= read -r crd; do
+        entries=$("$YQ" e -N "$versions_query" "$crd") || return 1
+        while read -r doc group kind index version; do
+            [ -n "${version:-}" ] || continue
+            mkdir -p "$schema_dir/$group"
+            "$YQ" e -N -o=json \
+                "select(documentIndex == $doc) | .spec.versions[$index].schema.openAPIV3Schema" \
+                "$crd" > "$schema_dir/$group/${kind,,}_$version.json" || return 1
+        done <<< "$entries"
+    done < <(crd_files "$chart" "$unpack_dir")
+}
+
+# crd_files <dir>...
+# Prints every CRD file under the given directories: the *.yaml in a crds/ directory that sits
+# directly in a chart, which is where Helm reads them from. A crds/ elsewhere in a chart is not
+# one - opentelemetry-operator, for instance, keeps Helm templates in conf/crds - and templates
+# are not parseable YAML.
+crd_files() {
+    local crd_dir
+    while IFS= read -r crd_dir; do
+        [ -f "$(dirname "$crd_dir")/Chart.yaml" ] || continue
+        find "$crd_dir" -type f \( -name '*.yaml' -o -name '*.yml' \)
+    done < <(find "$@" -type d -name crds 2>/dev/null)
+}
 
 status=0
+chart_index=0
 for chart_file in "$CHARTS_DIR"/*/Chart.yaml; do
     [ -f "$chart_file" ] || continue
     chart=$(dirname "$chart_file")
+    chart_index=$((chart_index + 1))
 
     echo "Evaluating $chart"
     if ! "$HELM" dependency build "$chart"; then
         status=1
         continue
     fi
-    if ! "$HELM" template "$chart" | "$KUBECONFORM" -strict -summary "${SCHEMAS[@]}"; then
+
+    # the dependencies are only on disk once they are built, so their CRDs are collected after
+    schema_dir="$workdir/schemas/$chart_index"
+    if ! write_crd_schemas "$chart" "$schema_dir" "$workdir/unpacked/$chart_index"; then
+        echo "Could not read the CRDs of $chart" >&2
+        status=1
+        continue
+    fi
+    schemas=(-schema-location default
+             -schema-location "$schema_dir/$SCHEMA_PATH"
+             -schema-location "$CATALOG")
+
+    if ! "$HELM" template "$chart" | "$KUBECONFORM" -strict -summary "${schemas[@]}"; then
         status=1
     fi
 
     for values in "$chart"/ci/*-values.yaml; do
         [ -f "$values" ] || continue
         echo "Evaluating $chart with $(basename "$values")"
-        if ! "$HELM" template "$chart" -f "$values" | "$KUBECONFORM" -strict -summary -ignore-missing-schemas "${SCHEMAS[@]}"; then
+        if ! "$HELM" template "$chart" -f "$values" | "$KUBECONFORM" -strict -summary -ignore-missing-schemas "${schemas[@]}"; then
             status=1
         fi
     done

tests/lint-eval-charts.bats

@@ -76,3 +76,105 @@ EOF
     [ "$status" -eq 1 ]
     [[ "$output" == *"Evaluating charts/good with extra-values.yaml"* ]]
 }
+
+# crd <chart> [<extra string property>]
+# gives the chart a CRD for widgets.example.test, whose spec takes a "size" integer and, when
+# asked for, one more string property. The group is not in the CRDs catalog, so a widget can only
+# validate against the CRD the chart itself ships.
+crd() {
+    local extra=""
+    [ -z "${2:-}" ] || extra=$(printf '\n                %s:\n                  type: string' "$2")
+    mkdir -p "charts/$1/crds"
+    cat > "charts/$1/crds/widget.yaml" <<EOF
+apiVersion: apiextensions.k8s.io/v1
+kind: CustomResourceDefinition
+metadata:
+  name: widgets.example.test
+spec:
+  group: example.test
+  names:
+    kind: Widget
+    plural: widgets
+  scope: Namespaced
+  versions:
+    - name: v1
+      served: true
+      storage: true
+      schema:
+        openAPIV3Schema:
+          type: object
+          properties:
+            spec:
+              type: object
+              properties:
+                size:
+                  type: integer$extra
+EOF
+}
+
+# widget <chart> <spec body>
+widget() {
+    cat > "charts/$1/templates/widget.yaml" <<EOF
+apiVersion: example.test/v1
+kind: Widget
+metadata:
+  name: widget
+spec:
+$2
+EOF
+}
+
+@test "a custom resource is validated against the CRD its own chart ships" {
+    crd good
+    widget good "  size: 3"
+    run "$EVAL"
+    [ "$status" -eq 0 ]
+    [[ "$output" == *"Chart evaluation successful"* ]]
+}
+
+@test "a custom resource that the chart's own CRD rejects fails" {
+    crd good
+    widget good "  size: big"
+    run "$EVAL"
+    [ "$status" -eq 1 ]
+}
+
+@test "a property the chart's CRD adds is accepted, where the catalog alone would reject it" {
+    crd good colour
+    widget good "  size: 3
+  colour: red"
+    run "$EVAL"
+    [ "$status" -eq 0 ]
+}
+
+@test "a crds directory that is not a chart's own is ignored" {
+    # opentelemetry-operator keeps Helm templates in conf/crds; they are not parseable YAML and
+    # must not be read as CRDs
+    mkdir -p charts/good/conf/crds
+    echo '{{- if .Values.x }}' > charts/good/conf/crds/templated.yaml
+    run "$EVAL"
+    [ "$status" -eq 0 ]
+    [[ "$output" != *"bad file"* ]]
+}
+
+@test "a CRD that cannot be read fails, instead of falling back to the catalog" {
+    crd good
+    widget good "  size: 3"
+    # a yq that fails stands for an unparseable CRD, or one a yq too old for the query chokes on:
+    # the schema is then missing, and the resource must not simply be validated without it
+    YQ=false run "$EVAL"
+    [ "$status" -eq 1 ]
+    [[ "$output" == *"Could not read the CRDs of charts/good"* ]]
+}
+
+@test "the CRDs a dependency ships are read from the archive helm builds" {
+    chart dep 1.0.0
+    crd dep
+    # the widget is in the chart that depends on dep, so its schema can only come from the
+    # dep-1.0.0.tgz that helm dependency build puts in charts/user/charts
+    chart user 1.0.0 dep 1.0.0 "file://../dep"
+    widget user "  size: 3"
+    run "$EVAL"
+    [ "$status" -eq 0 ]
+    [[ "$output" == *"Chart evaluation successful"* ]]
+}