From ba420f514e66ed0b300e6f9c0b4997b551ebd7ab Mon Sep 17 00:00:00 2001 From: Alice Frosi Date: Fri, 30 Oct 2026 08:41:45 +0000 Subject: [PATCH] e2e: Install the released chart in the upgrade test Since the move to Helm, TestOperatorUpgrade installed the released operator image with the chart from the current checkout. That only works while the current chart stays compatible with the released binary: a chart that passes a new controller flag or narrows the RBAC makes the released controller crash or fail to sync, and the test never gets to the upgrade. Install the released version from its published chart instead, the way users upgrading from it have it deployed, then upgrade to the local chart as before. The chart comes from RELEASED_OPERATOR_CHART, so downstream builds can point elsewhere, and its version is RELEASED_OPERATOR_TAG without the leading "v", matching how the release workflow publishes it. The released image is now the one the chart deploys, so RELEASED_OPERATOR_IMG is gone. On bink, push-released-operator-image reads that image from the chart and still copies it to the local registry, which keeps the test fast and avoids pulls from a remote registry timing out; the test overrides the chart image with that copy. The restore cleanup is now registered right after the uninstall: when the released operator failed to come up, the cleanup was never registered and every later test ran without an operator. Assisted-by: AI Signed-off-by: Alice Frosi (cherry picked from commit 490d90304dd3082ba39afac29787cbb2168add25) # Conflicts: # Makefile --- Makefile | 34 +++++++++++++++------ test/e2e/upgrade_test.go | 65 ++++++++++++++++++++++++++-------------- 2 files changed, 68 insertions(+), 31 deletions(-) diff --git a/Makefile b/Makefile index f9c7ec06..0cba1490 100644 --- a/Makefile +++ b/Makefile @@ -17,10 +17,19 @@ BINK_NODE_DISK_IMAGE ?= ghcr.io/bootc-dev/bink/node:v$(DEFAULT_KUBE_MINOR)-fedor BINK_LOCAL_REGISTRY_NODE_IMAGE ?= registry.cluster.local:5000/node E2E_REGISTRY_USER ?= e2e-user E2E_REGISTRY_PASSWORD ?= e2e-password +<<<<<<< HEAD # Released operator for upgrade testing (TestOperatorUpgrade). # Override RELEASED_OPERATOR_IMG for downstream images. RELEASED_OPERATOR_TAG ?= v0.2.1 RELEASED_OPERATOR_IMG ?= ghcr.io/bootc-dev/bootc-operator:$(RELEASED_OPERATOR_TAG) +======= +# Released operator for upgrade testing (TestOperatorUpgrade): the published +# chart and the release tag, whose version without the leading "v" is the +# chart version. The released image is the one the chart deploys. Override +# RELEASED_OPERATOR_CHART for downstream charts. +RELEASED_OPERATOR_TAG ?= v0.2.2 +RELEASED_OPERATOR_CHART ?= oci://ghcr.io/bootc-dev/bootc-operator/charts/bootc-operator +>>>>>>> 490d903 (e2e: Install the released chart in the upgrade test) IMG_BINK_RELEASED ?= registry.cluster.local:5000/bootc-operator-released:latest # YEAR defines the year value used for substituting the YEAR placeholder in the boilerplate header. YEAR ?= $(shell date +%Y) @@ -120,8 +129,14 @@ e2e: ## Run e2e tests (requires: make deploy-bink). V=1 for verbose. RUN= E2E_NODE_IMAGE_UPDATE2_DIGEST=$$(skopeo inspect --tls-verify=false docker://localhost:5000/node:update2 | jq -r '.Digest') \ E2E_REGISTRY_USER=$(E2E_REGISTRY_USER) E2E_REGISTRY_PASSWORD=$(E2E_REGISTRY_PASSWORD) \ $(if $(RELEASED_OPERATOR_TAG),E2E_OPERATOR_RELEASE_TAG=$(RELEASED_OPERATOR_TAG)) \ +<<<<<<< HEAD $(if $(RELEASED_OPERATOR_IMG),E2E_OPERATOR_RELEASED_IMG=$(IMG_BINK_RELEASED)) \ go test -timeout 60m -count=1 -parallel 2 $(if $(V),-v) $(if $(RUN),-run '$(RUN)') . +======= + $(if $(RELEASED_OPERATOR_TAG),E2E_OPERATOR_RELEASED_IMG=$(IMG_BINK_RELEASED)) \ + $(if $(RELEASED_OPERATOR_CHART),E2E_OPERATOR_RELEASED_CHART=$(RELEASED_OPERATOR_CHART)) \ + go test -timeout 90m -count=1 -parallel 2 $(if $(V),-v) $(if $(RUN),-run '$(RUN)') . +>>>>>>> 490d903 (e2e: Install the released chart in the upgrade test) # EKS e2e settings EKS_CLUSTER_NAME ?= @@ -152,7 +167,7 @@ e2e-eks: helm ## Run e2e tests against EKS. V=1 for verbose. RUN= to filt E2E_REGISTRY_USER="$(EKS_REGISTRY_USER)" \ E2E_REGISTRY_PASSWORD="$(EKS_REGISTRY_PASSWORD)" \ $(if $(RELEASED_OPERATOR_TAG),E2E_OPERATOR_RELEASE_TAG=$(RELEASED_OPERATOR_TAG)) \ - $(if $(RELEASED_OPERATOR_IMG),E2E_OPERATOR_RELEASED_IMG=$(RELEASED_OPERATOR_IMG)) \ + $(if $(RELEASED_OPERATOR_CHART),E2E_OPERATOR_RELEASED_CHART=$(RELEASED_OPERATOR_CHART)) \ $(if $(EKS_IMAGE_PULL_SECRET),E2E_IMAGE_PULL_SECRET=$(EKS_IMAGE_PULL_SECRET)) \ E2E_NODE_PULL_SECRET="$(EKS_NODE_PULL_SECRET)" \ ARTIFACTS="$(ARTIFACTS)" \ @@ -192,13 +207,14 @@ build-update-image: ## Build derived node images for update testing and push to podman push --tls-verify=false localhost:5000/node:update2 .PHONY: push-released-operator-image -push-released-operator-image: ## Pull released operator image and push to bink registry for upgrade testing. - @if [ -z "$(RELEASED_OPERATOR_IMG)" ]; then \ - echo "Error: RELEASED_OPERATOR_IMG must be set (e.g. ghcr.io/bootc-dev/bootc-operator:v0.1.0)"; \ - exit 1; \ - fi - podman pull $(RELEASED_OPERATOR_IMG) - podman push --tls-verify=false $(RELEASED_OPERATOR_IMG) localhost:5000/bootc-operator-released:latest +push-released-operator-image: yq ## Copy the image of the released chart to the bink registry for upgrade testing. + img=$$(helm template bootc-operator $(RELEASED_OPERATOR_CHART) \ + --version $(RELEASED_OPERATOR_TAG:v%=%) \ + --kube-version $(DEFAULT_KUBE_MINOR).0 | \ + $(YQ) 'select(.kind == "Deployment") | .spec.template.spec.containers[] | select(.name == "manager") | .image') && \ + test -n "$$img" && \ + podman pull "$$img" && \ + podman push --tls-verify=false "$$img" localhost:5000/bootc-operator-released:latest ##@ Deployment @@ -256,7 +272,7 @@ start-bink: seed-node-image ## Start a bink cluster (idempotent). kubectl --kubeconfig $(KUBECONFIG_BINK) wait --for=condition=Ready node/controller --timeout=5m .PHONY: deploy-bink -deploy-bink: start-bink build-update-image $(if $(RELEASED_OPERATOR_IMG),push-released-operator-image) helm ## Deploy to a bink cluster (requires: buildimg). +deploy-bink: start-bink build-update-image $(if $(RELEASED_OPERATOR_TAG),push-released-operator-image) helm ## Deploy to a bink cluster (requires: buildimg). podman push --tls-verify=false $(IMG) localhost:5000/bootc-operator-e2e:latest helm dependency build ./chart/bootc-operator img="$(IMG_BINK)" && \ diff --git a/test/e2e/upgrade_test.go b/test/e2e/upgrade_test.go index da7ff675..957539b1 100644 --- a/test/e2e/upgrade_test.go +++ b/test/e2e/upgrade_test.go @@ -32,9 +32,9 @@ var operatorCRDNames = []string{ } // TestOperatorUpgrade installs the released version of the operator -// via Helm, verifies basic functionality, then upgrades by running -// helm upgrade with the current image and verifies the operator still -// works. +// from its published Helm chart, verifies basic functionality, then +// upgrades by running helm upgrade with the current chart and image and +// verifies the operator still works. func TestOperatorUpgrade(t *testing.T) { e2eutil.Providers(t, "bink", "eks") @@ -42,10 +42,14 @@ func TestOperatorUpgrade(t *testing.T) { if releaseTag == "" { t.Skip("E2E_OPERATOR_RELEASE_TAG not set") } + // Optional: overrides the image the released chart deploys, e.g. + // with a copy in a local registry. releasedImg := os.Getenv("E2E_OPERATOR_RELEASED_IMG") - if releasedImg == "" { - t.Skip("E2E_OPERATOR_RELEASED_IMG not set") + releasedChart := os.Getenv("E2E_OPERATOR_RELEASED_CHART") + if releasedChart == "" { + t.Skip("E2E_OPERATOR_RELEASED_CHART not set") } + releasedChartVersion := strings.TrimPrefix(releaseTag, "v") g := NewWithT(t) g.SetDefaultEventuallyTimeout(pollTimeout) @@ -64,16 +68,17 @@ func TestOperatorUpgrade(t *testing.T) { currentImg := containerImage(t, origDeploy.Spec.Template.Spec.Containers, "manager") extraArgs := controllerExtraArgs(t, origDeploy.Spec.Template.Spec.Containers) t.Logf("Current operator image: %s", currentImg) - t.Logf("Released operator tag: %s, image: %s", releaseTag, releasedImg) + t.Logf("Released operator tag: %s, image: %s, chart: %s", + releaseTag, releasedImg, releasedChart) - // Phase 1: Uninstall the current operator and re-install with - // the released image via Helm. + // Phase 1: Uninstall the current operator and re-install the + // released version from its published chart, the way users + // upgrading from it have it deployed. The current chart may not work + // with the released image. uninstallOperator(t, g, ctx, env) - helmInstallOperator(t, chartPath, releasedImg, extraArgs) - waitForOperatorReady(t, g, ctx, env.Client) - - t.Logf("Released operator %s installed via Helm", releaseTag) + // Restore the current operator even if the released one fails to + // install, so the tests that run after this one have an operator. t.Cleanup(func() { t.Logf("Restoring operator to current version...") helmUpgradeOperator(t, chartPath, currentImg, extraArgs) @@ -81,6 +86,12 @@ func TestOperatorUpgrade(t *testing.T) { t.Logf("Operator restored") }) + helmInstallOperator(t, releasedChart, releasedChartVersion, + releasedImg, extraArgs) + waitForOperatorReady(t, g, ctx, env.Client) + + t.Logf("Released operator %s installed via Helm", releaseTag) + // Phase 2: Verify the released version works. nodeName := env.AddNode(t) @@ -181,12 +192,17 @@ func uninstallOperator( t.Logf("Operator uninstalled") } -// helmInstallOperator installs the operator via helm install. -func helmInstallOperator(t *testing.T, chartPath, img string, extraArgs []string) { +// helmInstallOperator installs the given version of a published chart +// via helm install. +func helmInstallOperator( + t *testing.T, + chartRef, chartVersion, img string, + extraArgs []string, +) { t.Helper() - helmRun(t, "dependency", "build", chartPath) args := []string{ - "install", helmReleaseName, chartPath, + "install", helmReleaseName, chartRef, + "--version", chartVersion, "--namespace", testutil.OperatorNamespaceName, "--create-namespace", } @@ -194,9 +210,11 @@ func helmInstallOperator(t *testing.T, chartPath, img string, extraArgs []string helmRun(t, args...) } -// helmUpgradeOperator upgrades the operator via helm upgrade. +// helmUpgradeOperator upgrades the operator to the local chart via helm +// upgrade. func helmUpgradeOperator(t *testing.T, chartPath, img string, extraArgs []string) { t.Helper() + helmRun(t, "dependency", "build", chartPath) args := []string{ "upgrade", "--install", helmReleaseName, chartPath, "--namespace", testutil.OperatorNamespaceName, @@ -206,13 +224,16 @@ func helmUpgradeOperator(t *testing.T, chartPath, img string, extraArgs []string } // helmImageArgs returns the --set flags for image and controller args. +// An empty img keeps the image of the chart. func helmImageArgs(t *testing.T, img string, extraArgs []string) []string { t.Helper() - repo, tag := splitImage(img) - args := []string{ - "--set-string", "image.repository=" + repo, - "--set-string", "image.tag=" + tag, - "--set-string", "image.pullPolicy=Always", + args := []string{"--set-string", "image.pullPolicy=Always"} + if img != "" { + repo, tag := splitImage(img) + args = append(args, + "--set-string", "image.repository="+repo, + "--set-string", "image.tag="+tag, + ) } if secret := os.Getenv("E2E_IMAGE_PULL_SECRET"); secret != "" { args = append(args, "--set", "imagePullSecrets[0].name="+secret)