Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
29 changes: 29 additions & 0 deletions CONTRIBUTING.md
Original file line number Diff line number Diff line change
Expand Up @@ -148,6 +148,35 @@ make e2e RUN=Foo # Run tests matching "Foo"
> every code change. Run `make buildimg` followed by `make deploy-bink` to
> pick up your latest changes in the cluster.

### Quarantining Flaky Tests

A test that is intermittently failing for reasons outside the operator's
control can be quarantined so it stops blocking CI while the underlying
issue is investigated:

```go
func TestFlaky(t *testing.T) {
testutil.Quarantine(t, "flaky: <what fails>, <tracking issue URL>")

// ... rest of the test, unchanged
}
```

Call `Quarantine` first thing in the test, before any provisioning, so a
skipped test costs nothing. The reason must reference an open tracking
issue: that is what tells us when the quarantine can be lifted.

Quarantined tests are skipped via `t.Skip` unless `E2E_RUN_QUARANTINED` is
set to a truthy value, so they still run on demand, for example to check
whether the underlying issue has been fixed:

```shell
E2E_RUN_QUARANTINED=1 make e2e RUN=TestMidRolloutImageChange
```

CI leaves the variable unset. Quarantine is temporary by design: remove
the call once the test is reliably green again.

### Internal Registry

The bink cluster includes an internal container registry. From the host, push
Expand Down
10 changes: 9 additions & 1 deletion Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -113,8 +113,14 @@ vulncheck: govulncheck ## Check Go code for known vulnerabilities (requires netw
# Give external-status observation room for provisioning without changing the daemon default.
E2E_BOOTC_POLL_INTERVAL ?= 10m

# Tests quarantined via testutil.Quarantine are skipped unless this is
# set to a truthy value (e.g. 1). Works both as a make variable
# (`make e2e E2E_RUN_QUARANTINED=1`) and via the environment. See
# CONTRIBUTING.md, "Quarantining Flaky Tests".
E2E_RUN_QUARANTINED ?=

.PHONY: e2e
e2e: ## Run e2e tests (requires: make deploy-bink). V=1 for verbose. RUN=<regex> to filter.
e2e: ## Run e2e tests (requires: make deploy-bink). V=1 for verbose. RUN=<regex> to filter. Set E2E_RUN_QUARANTINED=1 to also run quarantined tests.
# NB: we `cd` here instead of passing a package path to `go test` so that `-v`
# actually gives us streaming output (otherwise, it spawns a subprocess for
# each package, even though we just have one here--but I really like streaming
Expand All @@ -123,6 +129,7 @@ e2e: ## Run e2e tests (requires: make deploy-bink). V=1 for verbose. RUN=<regex>
cd test/e2e && KUBECONFIG=$(abspath $(KUBECONFIG_BINK)) \
E2E_PROVIDER=bink \
E2E_BOOTC_POLL_INTERVAL=$(E2E_BOOTC_POLL_INTERVAL) \
E2E_RUN_QUARANTINED=$(E2E_RUN_QUARANTINED) \
BINK_CLUSTER_NAME=$(BINK_CLUSTER_NAME) \
BINK_NODE_DISK_IMAGE=$(BINK_NODE_DISK_IMAGE) \
E2E_NODE_IMAGE_REGISTRY=$(BINK_LOCAL_REGISTRY_NODE_IMAGE) \
Expand Down Expand Up @@ -155,6 +162,7 @@ e2e-eks: helm ## Run e2e tests against EKS. V=1 for verbose. RUN=<regex> to filt
rm -rf $(ARTIFACTS)
cd test/e2e && KUBECONFIG="$(KUBECONFIG)" \
E2E_PROVIDER=eks \
E2E_RUN_QUARANTINED=$(E2E_RUN_QUARANTINED) \
EKS_CLUSTER_NAME="$(EKS_CLUSTER_NAME)" \
EKS_NODE_GROUP="$(EKS_NODE_GROUP)" \
AWS_REGION="$(AWS_REGION)" \
Expand Down
4 changes: 4 additions & 0 deletions test/e2e/bootcnode_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -344,6 +344,10 @@ func TestTagResolution(t *testing.T) {
// image and that the non-rebooting node does not wastefully reboot into the
// first update image.
func TestMidRolloutImageChange(t *testing.T) {
testutil.Quarantine(t, "flaky: intermittent spurious second reboot "+
"after a mid-rollout image change, "+
"https://github.com/bootc-dev/bootc-operator/issues/256")

e2eutil.Providers(t, "bink")
t.Parallel()
g := NewWithT(t)
Expand Down
58 changes: 58 additions & 0 deletions test/util/quarantine.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,58 @@
// SPDX-License-Identifier: Apache-2.0

package testutil

import (
"fmt"
"os"
"strconv"
"testing"
)

// runQuarantinedEnv is the environment variable that opts in to running
// quarantined tests.
const runQuarantinedEnv = "E2E_RUN_QUARANTINED"

// Quarantine skips t unless the E2E_RUN_QUARANTINED environment variable
// is set to a truthy value. It marks a test as quarantined: known-flaky
// or failing for reasons outside the operator's control, so it does not
// block CI while the underlying issue is investigated and fixed. reason
// must describe the failure and reference the tracking issue, since that
// is what tells us when the quarantine can be lifted.
//
// Quarantine is temporary by design: remove the call once the test is
// reliably green again. To run quarantined tests on demand, e.g.
//
// E2E_RUN_QUARANTINED=1 make e2e RUN=TestMidRolloutImageChange
func Quarantine(t *testing.T, reason string) {
t.Helper()

run, err := parseQuarantineOverride(os.Getenv(runQuarantinedEnv))
if err != nil {
t.Fatal(err)
}
if !run {
t.Skipf(
"quarantined: %s (set %s=1 to run it anyway)",
reason,
runQuarantinedEnv,
)
}
}

// parseQuarantineOverride interprets the E2E_RUN_QUARANTINED value. An
// empty value means quarantined tests are skipped.
func parseQuarantineOverride(value string) (bool, error) {
if value == "" {
return false, nil
}
run, err := strconv.ParseBool(value)
if err != nil {
return false, fmt.Errorf(
"invalid %s value %q: must be a boolean (e.g. 1 or true)",
runQuarantinedEnv,
value,
)
}
return run, nil
}
Loading