From 604f652de99d3319dba05188a5e341ed6708f072 Mon Sep 17 00:00:00 2001 From: Alice Frosi Date: Fri, 6 Nov 2026 13:23:41 +0000 Subject: [PATCH 1/4] test: add Quarantine helper for flaky tests Quarantine skips a test with a given reason unless the E2E_RUN_QUARANTINED env var is set to a truthy value, so known-flaky tests can stop blocking CI while their underlying issue is investigated and still run on demand. The reason must reference a tracking issue, since that is what tells us when the quarantine can be lifted; quarantine is temporary by design. Assisted-by: AI Signed-off-by: Alice Frosi (cherry picked from commit 0e707beaf9c253aed0edc53da969ad1239e116ce) --- test/util/quarantine.go | 58 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 58 insertions(+) create mode 100644 test/util/quarantine.go diff --git a/test/util/quarantine.go b/test/util/quarantine.go new file mode 100644 index 00000000..fb20f5c7 --- /dev/null +++ b/test/util/quarantine.go @@ -0,0 +1,58 @@ +// SPDX-License-Identifier: Apache-2.0 + +package testutil + +import ( + "fmt" + "os" + "strconv" + "testing" +) + +// runQuarantinedEnv is the environment variable that opts in to running +// quarantined tests. +const runQuarantinedEnv = "E2E_RUN_QUARANTINED" + +// Quarantine skips t unless the E2E_RUN_QUARANTINED environment variable +// is set to a truthy value. It marks a test as quarantined: known-flaky +// or failing for reasons outside the operator's control, so it does not +// block CI while the underlying issue is investigated and fixed. reason +// must describe the failure and reference the tracking issue, since that +// is what tells us when the quarantine can be lifted. +// +// Quarantine is temporary by design: remove the call once the test is +// reliably green again. To run quarantined tests on demand, e.g. +// +// E2E_RUN_QUARANTINED=1 make e2e RUN=TestMidRolloutImageChange +func Quarantine(t *testing.T, reason string) { + t.Helper() + + run, err := parseQuarantineOverride(os.Getenv(runQuarantinedEnv)) + if err != nil { + t.Fatal(err) + } + if !run { + t.Skipf( + "quarantined: %s (set %s=1 to run it anyway)", + reason, + runQuarantinedEnv, + ) + } +} + +// parseQuarantineOverride interprets the E2E_RUN_QUARANTINED value. An +// empty value means quarantined tests are skipped. +func parseQuarantineOverride(value string) (bool, error) { + if value == "" { + return false, nil + } + run, err := strconv.ParseBool(value) + if err != nil { + return false, fmt.Errorf( + "invalid %s value %q: must be a boolean (e.g. 1 or true)", + runQuarantinedEnv, + value, + ) + } + return run, nil +} From b78dd5a79cfb2b0b78f4298cbcfb03a1f0f83162 Mon Sep 17 00:00:00 2001 From: Alice Frosi Date: Fri, 6 Nov 2026 14:35:27 +0100 Subject: [PATCH 2/4] make: define E2E_RUN_QUARANTINED The E2E_RUN_QUARANTINED runs the quarantined tests. Signed-off-by: Alice Frosi (cherry picked from commit dbd60b852b87da57c32b0cc8421314d73f5eb4a5) --- Makefile | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 9e0ba18a..83ec5fbe 100644 --- a/Makefile +++ b/Makefile @@ -110,8 +110,17 @@ go-lines-fix: get-golines vulncheck: govulncheck ## Check Go code for known vulnerabilities (requires network access to the Go vulnerability database). "$(GOVULNCHECK)" ./... +# Give external-status observation room for provisioning without changing the daemon default. +E2E_BOOTC_POLL_INTERVAL ?= 10m + +# Tests quarantined via testutil.Quarantine are skipped unless this is +# set to a truthy value (e.g. 1). Works both as a make variable +# (`make e2e E2E_RUN_QUARANTINED=1`) and via the environment. See +# CONTRIBUTING.md, "Quarantining Flaky Tests". +E2E_RUN_QUARANTINED ?= + .PHONY: e2e -e2e: ## Run e2e tests (requires: make deploy-bink). V=1 for verbose. RUN= to filter. +e2e: ## Run e2e tests (requires: make deploy-bink). V=1 for verbose. RUN= to filter. Set E2E_RUN_QUARANTINED=1 to also run quarantined tests. # NB: we `cd` here instead of passing a package path to `go test` so that `-v` # actually gives us streaming output (otherwise, it spawns a subprocess for # each package, even though we just have one here--but I really like streaming @@ -119,6 +128,8 @@ e2e: ## Run e2e tests (requires: make deploy-bink). V=1 for verbose. RUN= rm -rf $(ARTIFACTS) cd test/e2e && KUBECONFIG=$(abspath $(KUBECONFIG_BINK)) \ E2E_PROVIDER=bink \ + E2E_BOOTC_POLL_INTERVAL=$(E2E_BOOTC_POLL_INTERVAL) \ + E2E_RUN_QUARANTINED=$(E2E_RUN_QUARANTINED) \ BINK_CLUSTER_NAME=$(BINK_CLUSTER_NAME) \ BINK_NODE_DISK_IMAGE=$(BINK_NODE_DISK_IMAGE) \ E2E_NODE_IMAGE_REGISTRY=$(BINK_LOCAL_REGISTRY_NODE_IMAGE) \ @@ -151,6 +162,7 @@ e2e-eks: helm ## Run e2e tests against EKS. V=1 for verbose. RUN= to filt rm -rf $(ARTIFACTS) cd test/e2e && KUBECONFIG="$(KUBECONFIG)" \ E2E_PROVIDER=eks \ + E2E_RUN_QUARANTINED=$(E2E_RUN_QUARANTINED) \ EKS_CLUSTER_NAME="$(EKS_CLUSTER_NAME)" \ EKS_NODE_GROUP="$(EKS_NODE_GROUP)" \ AWS_REGION="$(AWS_REGION)" \ From d6e507564d955366a756c1a4ba8b93f098a71018 Mon Sep 17 00:00:00 2001 From: Alice Frosi Date: Fri, 6 Nov 2026 14:36:39 +0100 Subject: [PATCH 3/4] Document how to quarantine e2e tests Signed-off-by: Alice Frosi (cherry picked from commit 33ebd1fac269d7cf6f300b8ff7be5157452537ee) --- CONTRIBUTING.md | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 779e1365..cd6f0f13 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -148,6 +148,35 @@ make e2e RUN=Foo # Run tests matching "Foo" > every code change. Run `make buildimg` followed by `make deploy-bink` to > pick up your latest changes in the cluster. +### Quarantining Flaky Tests + +A test that is intermittently failing for reasons outside the operator's +control can be quarantined so it stops blocking CI while the underlying +issue is investigated: + +```go +func TestFlaky(t *testing.T) { + testutil.Quarantine(t, "flaky: , ") + + // ... rest of the test, unchanged +} +``` + +Call `Quarantine` first thing in the test, before any provisioning, so a +skipped test costs nothing. The reason must reference an open tracking +issue: that is what tells us when the quarantine can be lifted. + +Quarantined tests are skipped via `t.Skip` unless `E2E_RUN_QUARANTINED` is +set to a truthy value, so they still run on demand, for example to check +whether the underlying issue has been fixed: + +```shell +E2E_RUN_QUARANTINED=1 make e2e RUN=TestMidRolloutImageChange +``` + +CI leaves the variable unset. Quarantine is temporary by design: remove +the call once the test is reliably green again. + ### Internal Registry The bink cluster includes an internal container registry. From the host, push From 668e181b83555c3f17eb236e5c819cc72a6c3e22 Mon Sep 17 00:00:00 2001 From: Alice Frosi Date: Fri, 6 Nov 2026 14:37:24 +0100 Subject: [PATCH 4/4] Quarantine the TestMidRolloutImageChange test( The TestMidRolloutImageChange causes spurious restages of the image. The investigation results are reported here: https://github.com/bootc-dev/bootc-operator/issues/256 Signed-off-by: Alice Frosi (cherry picked from commit 235a020324bfd857a3e05d5b1567a6ae2c400f86) --- test/e2e/bootcnode_test.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/test/e2e/bootcnode_test.go b/test/e2e/bootcnode_test.go index 3c56d491..c89ad2da 100644 --- a/test/e2e/bootcnode_test.go +++ b/test/e2e/bootcnode_test.go @@ -343,6 +343,10 @@ func TestTagResolution(t *testing.T) { // image and that the non-rebooting node does not wastefully reboot into the // first update image. func TestMidRolloutImageChange(t *testing.T) { + testutil.Quarantine(t, "flaky: intermittent spurious second reboot "+ + "after a mid-rollout image change, "+ + "https://github.com/bootc-dev/bootc-operator/issues/256") + e2eutil.Providers(t, "bink") t.Parallel() g := NewWithT(t)