diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 779e1365..cd6f0f13 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -148,6 +148,35 @@ make e2e RUN=Foo # Run tests matching "Foo" > every code change. Run `make buildimg` followed by `make deploy-bink` to > pick up your latest changes in the cluster. +### Quarantining Flaky Tests + +A test that is intermittently failing for reasons outside the operator's +control can be quarantined so it stops blocking CI while the underlying +issue is investigated: + +```go +func TestFlaky(t *testing.T) { + testutil.Quarantine(t, "flaky: , ") + + // ... rest of the test, unchanged +} +``` + +Call `Quarantine` first thing in the test, before any provisioning, so a +skipped test costs nothing. The reason must reference an open tracking +issue: that is what tells us when the quarantine can be lifted. + +Quarantined tests are skipped via `t.Skip` unless `E2E_RUN_QUARANTINED` is +set to a truthy value, so they still run on demand, for example to check +whether the underlying issue has been fixed: + +```shell +E2E_RUN_QUARANTINED=1 make e2e RUN=TestMidRolloutImageChange +``` + +CI leaves the variable unset. Quarantine is temporary by design: remove +the call once the test is reliably green again. + ### Internal Registry The bink cluster includes an internal container registry. From the host, push diff --git a/Makefile b/Makefile index 9e0ba18a..83ec5fbe 100644 --- a/Makefile +++ b/Makefile @@ -110,8 +110,17 @@ go-lines-fix: get-golines vulncheck: govulncheck ## Check Go code for known vulnerabilities (requires network access to the Go vulnerability database). "$(GOVULNCHECK)" ./... +# Give external-status observation room for provisioning without changing the daemon default. +E2E_BOOTC_POLL_INTERVAL ?= 10m + +# Tests quarantined via testutil.Quarantine are skipped unless this is +# set to a truthy value (e.g. 1). Works both as a make variable +# (`make e2e E2E_RUN_QUARANTINED=1`) and via the environment. See +# CONTRIBUTING.md, "Quarantining Flaky Tests". +E2E_RUN_QUARANTINED ?= + .PHONY: e2e -e2e: ## Run e2e tests (requires: make deploy-bink). V=1 for verbose. RUN= to filter. +e2e: ## Run e2e tests (requires: make deploy-bink). V=1 for verbose. RUN= to filter. Set E2E_RUN_QUARANTINED=1 to also run quarantined tests. # NB: we `cd` here instead of passing a package path to `go test` so that `-v` # actually gives us streaming output (otherwise, it spawns a subprocess for # each package, even though we just have one here--but I really like streaming @@ -119,6 +128,8 @@ e2e: ## Run e2e tests (requires: make deploy-bink). V=1 for verbose. RUN= rm -rf $(ARTIFACTS) cd test/e2e && KUBECONFIG=$(abspath $(KUBECONFIG_BINK)) \ E2E_PROVIDER=bink \ + E2E_BOOTC_POLL_INTERVAL=$(E2E_BOOTC_POLL_INTERVAL) \ + E2E_RUN_QUARANTINED=$(E2E_RUN_QUARANTINED) \ BINK_CLUSTER_NAME=$(BINK_CLUSTER_NAME) \ BINK_NODE_DISK_IMAGE=$(BINK_NODE_DISK_IMAGE) \ E2E_NODE_IMAGE_REGISTRY=$(BINK_LOCAL_REGISTRY_NODE_IMAGE) \ @@ -151,6 +162,7 @@ e2e-eks: helm ## Run e2e tests against EKS. V=1 for verbose. RUN= to filt rm -rf $(ARTIFACTS) cd test/e2e && KUBECONFIG="$(KUBECONFIG)" \ E2E_PROVIDER=eks \ + E2E_RUN_QUARANTINED=$(E2E_RUN_QUARANTINED) \ EKS_CLUSTER_NAME="$(EKS_CLUSTER_NAME)" \ EKS_NODE_GROUP="$(EKS_NODE_GROUP)" \ AWS_REGION="$(AWS_REGION)" \ diff --git a/test/e2e/bootcnode_test.go b/test/e2e/bootcnode_test.go index 3c56d491..c89ad2da 100644 --- a/test/e2e/bootcnode_test.go +++ b/test/e2e/bootcnode_test.go @@ -343,6 +343,10 @@ func TestTagResolution(t *testing.T) { // image and that the non-rebooting node does not wastefully reboot into the // first update image. func TestMidRolloutImageChange(t *testing.T) { + testutil.Quarantine(t, "flaky: intermittent spurious second reboot "+ + "after a mid-rollout image change, "+ + "https://github.com/bootc-dev/bootc-operator/issues/256") + e2eutil.Providers(t, "bink") t.Parallel() g := NewWithT(t) diff --git a/test/util/quarantine.go b/test/util/quarantine.go new file mode 100644 index 00000000..fb20f5c7 --- /dev/null +++ b/test/util/quarantine.go @@ -0,0 +1,58 @@ +// SPDX-License-Identifier: Apache-2.0 + +package testutil + +import ( + "fmt" + "os" + "strconv" + "testing" +) + +// runQuarantinedEnv is the environment variable that opts in to running +// quarantined tests. +const runQuarantinedEnv = "E2E_RUN_QUARANTINED" + +// Quarantine skips t unless the E2E_RUN_QUARANTINED environment variable +// is set to a truthy value. It marks a test as quarantined: known-flaky +// or failing for reasons outside the operator's control, so it does not +// block CI while the underlying issue is investigated and fixed. reason +// must describe the failure and reference the tracking issue, since that +// is what tells us when the quarantine can be lifted. +// +// Quarantine is temporary by design: remove the call once the test is +// reliably green again. To run quarantined tests on demand, e.g. +// +// E2E_RUN_QUARANTINED=1 make e2e RUN=TestMidRolloutImageChange +func Quarantine(t *testing.T, reason string) { + t.Helper() + + run, err := parseQuarantineOverride(os.Getenv(runQuarantinedEnv)) + if err != nil { + t.Fatal(err) + } + if !run { + t.Skipf( + "quarantined: %s (set %s=1 to run it anyway)", + reason, + runQuarantinedEnv, + ) + } +} + +// parseQuarantineOverride interprets the E2E_RUN_QUARANTINED value. An +// empty value means quarantined tests are skipped. +func parseQuarantineOverride(value string) (bool, error) { + if value == "" { + return false, nil + } + run, err := strconv.ParseBool(value) + if err != nil { + return false, fmt.Errorf( + "invalid %s value %q: must be a boolean (e.g. 1 or true)", + runQuarantinedEnv, + value, + ) + } + return run, nil +}