HYPERFLEET-1493 - fix: eliminate CPU-contention flakes in perf latency specs

docs/development.md

@@ -328,10 +328,12 @@ Valid reasons to mark a spec `Serial`:
 
 - **Mutates shared infrastructure** (e.g., scales deployments, deletes shared resources)
 - **Deploys temporary adapters** that subscribe to all events, causing cross-talk with concurrent specs
+- **Asserts a millisecond-scale latency threshold**, where CPU contention from other parallel procs would inflate the measurement enough to flake the assertion
+- **Waits on a real timeout-bound condition** (e.g., reconciliation) whose duration is sensitive to concurrent create/reconcile load elsewhere in the suite - contention here risks an outright timeout failure, not just a skewed measurement
 
-Specs already marked `Serial`: sentinel scale-down, force-delete, stuck-deletion, crash-recovery, maestro-unavailability, adapter-failover, adapter-failure, maestro negative scenarios.
+Specs already marked `Serial`: sentinel scale-down, force-delete, stuck-deletion, crash-recovery, maestro-unavailability, adapter-failover, adapter-failure, maestro negative scenarios, and the 18 ms-scale channel/version/wifconfig/cluster perf spec files (list/create/update/delete/read latency; 22 individual specs, since the cluster list-with-filters file holds three and the cluster read-by-entity-size file holds three).
 
-Performance specs (`labels.Performance`) are a separate category - they carry no tier label and run in their own dedicated CI job (`--label-filter="perf"`) on a quiet system. They are not marked `Serial` because they never run alongside functional tests.
+Performance specs (`labels.Performance`) carry `labels.Tier1` like any other tier1 spec and run inside `tier1-nightly` alongside functional tests - there is no dedicated perf CI job. Ms-scale API perf specs (list/create/update/delete/read latency) are marked `Serial` for exactly this reason: without it, Ginkgo's parallel procs (`--procs=8`) contend for CPU and inflate latency measurements enough to flake the threshold assertions. The cluster read-by-entity-size specs carry a second reason: each waits on cluster reconciliation before reading, and reconciliation duration itself is sensitive to concurrent cluster create/reconcile load elsewhere in the suite. Second-scale reconciliation perf specs (which assert reconciliation time itself, not read latency) stay parallel since their larger thresholds already carry enough margin to absorb that contention.
 
 ## Adding New Tests
 

e2e/channel/perf_create_latency.go

@@ -2,7 +2,6 @@ package channel
 
 import (
 	"context"
-	"time"
 
 	"github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega" //nolint:staticcheck // dot import for test readability
@@ -14,6 +13,7 @@ import (
 
 var _ = ginkgo.Describe("[Suite: channel][perf] Create latency",
 	ginkgo.Label(labels.Tier1, labels.Performance),
+	ginkgo.Serial,
 	func() {
 		var h *helper.Helper
 
@@ -22,25 +22,16 @@ var _ = ginkgo.Describe("[Suite: channel][perf] Create latency",
 		})
 
 		ginkgo.It("should create a channel within acceptable latency", func(ctx context.Context) {
-			ginkgo.By("creating a channel and timing the response")
-			start := time.Now()
-
-			channel, err := h.Client.CreateChannelFromPayload(ctx, h.TestDataPath("payloads/channels/channel-request.json"))
-			if channel != nil && channel.Id != nil {
-				id := *channel.Id
-				ginkgo.DeferCleanup(func(ctx context.Context) {
-					if err := h.CleanupTestChannel(ctx, id); err != nil {
-						ginkgo.GinkgoWriter.Printf("Warning: failed to cleanup channel %s: %v\n", id, err)
+			helper.MeasureMedianLatency("POST /channels", config.ThresholdAPICreate, helper.DefaultSamples,
+				func(int) {
+					channel, err := h.Client.CreateChannelFromPayload(ctx, h.TestDataPath("payloads/channels/channel-request.json"))
+					if channel != nil && channel.Id != nil {
+						h.DeferChannelCleanup(*channel.Id)
 					}
-				})
-			}
-			Expect(err).NotTo(HaveOccurred())
-			Expect(channel.Id).NotTo(BeNil(), "channel ID should be set")
-			elapsed := time.Since(start)
-
-			ginkgo.GinkgoWriter.Printf("[PERF] POST /channels latency: %v\n", elapsed)
-			Expect(elapsed).To(BeNumerically("<", config.ThresholdAPICreate),
-				"channel create exceeded threshold")
+					Expect(err).NotTo(HaveOccurred())
+					Expect(channel.Id).NotTo(BeNil(), "channel ID should be set")
+				},
+			)
 		})
 	},
 )

e2e/channel/perf_delete_latency.go

@@ -2,7 +2,6 @@ package channel
 
 import (
 	"context"
-	"time"
 
 	"github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega" //nolint:staticcheck // dot import for test readability
@@ -14,37 +13,31 @@ import (
 
 var _ = ginkgo.Describe("[Suite: channel][perf] Delete latency",
 	ginkgo.Label(labels.Tier1, labels.Performance),
+	ginkgo.Serial,
 	func() {
 		var h *helper.Helper
-		var channelID string
 
 		ginkgo.BeforeEach(func(ctx context.Context) {
 			h = helper.New()
-
-			channel, err := h.Client.CreateChannelFromPayload(ctx, h.TestDataPath("payloads/channels/channel-request.json"))
-			Expect(err).NotTo(HaveOccurred())
-			Expect(channel.Id).NotTo(BeNil(), "channel ID should be set")
-			channelID = *channel.Id
-
-			ginkgo.DeferCleanup(func(ctx context.Context) {
-				if err := h.CleanupTestChannel(ctx, channelID); err != nil {
-					ginkgo.GinkgoWriter.Printf("Warning: failed to cleanup channel %s: %v\n", channelID, err)
-				}
-			})
 		})
 
 		ginkgo.It("should delete a channel within acceptable latency", func(ctx context.Context) {
-			ginkgo.By("deleting channel and timing the response")
-			start := time.Now()
-
-			deleted, err := h.Client.DeleteChannel(ctx, channelID)
-			Expect(err).NotTo(HaveOccurred())
-			Expect(deleted.DeletedTime).NotTo(BeNil(), "deleted channel should have deleted_time set")
-			elapsed := time.Since(start)
-
-			ginkgo.GinkgoWriter.Printf("[PERF] DELETE /channels/%s latency: %v\n", channelID, elapsed)
-			Expect(elapsed).To(BeNumerically("<", config.ThresholdAPIDelete),
-				"channel delete exceeded threshold")
+			channelIDs := make([]string, helper.DefaultSamples)
+			for i := range channelIDs {
+				channel, err := h.Client.CreateChannelFromPayload(ctx, h.TestDataPath("payloads/channels/channel-request.json"))
+				Expect(err).NotTo(HaveOccurred())
+				Expect(channel.Id).NotTo(BeNil(), "channel ID should be set")
+				channelIDs[i] = *channel.Id
+				h.DeferChannelCleanup(*channel.Id)
+			}
+
+			helper.MeasureMedianLatency("DELETE /channels/{id}", config.ThresholdAPIDelete, len(channelIDs),
+				func(i int) {
+					deleted, err := h.Client.DeleteChannel(ctx, channelIDs[i])
+					Expect(err).NotTo(HaveOccurred())
+					Expect(deleted.DeletedTime).NotTo(BeNil(), "deleted channel should have deleted_time set")
+				},
+			)
 		})
 	},
 )

e2e/channel/perf_get_latency.go

@@ -2,8 +2,6 @@ package channel
 
 import (
 	"context"
-	"slices"
-	"time"
 
 	"github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega" //nolint:staticcheck // dot import for test readability
@@ -15,6 +13,7 @@ import (
 
 var _ = ginkgo.Describe("[Suite: channel][perf] API read latency",
 	ginkgo.Label(labels.Tier1, labels.Performance),
+	ginkgo.Serial,
 	func() {
 		var h *helper.Helper
 		var channelID string
@@ -35,24 +34,12 @@ var _ = ginkgo.Describe("[Suite: channel][perf] API read latency",
 		})
 
 		ginkgo.It("should read a channel within acceptable latency", func(ctx context.Context) {
-			ginkgo.By("warming up with untimed read")
-			_, err := h.Client.GetChannel(ctx, channelID)
-			Expect(err).NotTo(HaveOccurred())
-
-			ginkgo.By("measuring GET /channels/{id} response time")
-			const samples = 5
-			durations := make([]time.Duration, samples)
-			for i := range samples {
-				start := time.Now()
-				_, err = h.Client.GetChannel(ctx, channelID)
-				Expect(err).NotTo(HaveOccurred())
-				durations[i] = time.Since(start)
-			}
-			slices.Sort(durations)
-			median := durations[samples/2]
-			ginkgo.GinkgoWriter.Printf("[PERF] GET /channels/%s latency: %v (median of %d samples)\n", channelID, median, samples)
-			Expect(median).To(BeNumerically("<", config.ThresholdAPIRead),
-				"GET /channels/{id} exceeded threshold")
+			helper.MeasureMedianLatency("GET /channels/{id}", config.ThresholdAPIRead, helper.DefaultSamples,
+				func(int) {
+					_, err := h.Client.GetChannel(ctx, channelID)
+					Expect(err).NotTo(HaveOccurred())
+				},
+			)
 		})
 	},
 )

e2e/channel/perf_list_latency.go

@@ -2,7 +2,6 @@ package channel
 
 import (
 	"context"
-	"time"
 
 	"github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega" //nolint:staticcheck // dot import for test readability
@@ -14,6 +13,7 @@ import (
 
 var _ = ginkgo.Describe("[Suite: channel][perf] API list latency",
 	ginkgo.Label(labels.Tier1, labels.Performance),
+	ginkgo.Serial,
 	func() {
 		var h *helper.Helper
 		var channelID string
@@ -34,14 +34,12 @@ var _ = ginkgo.Describe("[Suite: channel][perf] API list latency",
 		})
 
 		ginkgo.It("should list channels within acceptable latency", func(ctx context.Context) {
-			ginkgo.By("measuring GET /channels response time")
-			start := time.Now()
-			_, err := h.Client.ListChannels(ctx, "")
-			Expect(err).NotTo(HaveOccurred())
-			elapsed := time.Since(start)
-			ginkgo.GinkgoWriter.Printf("[PERF] GET /channels latency: %v\n", elapsed)
-			Expect(elapsed).To(BeNumerically("<", config.ThresholdAPIList),
-				"GET /channels exceeded threshold")
+			helper.MeasureMedianLatency("GET /channels", config.ThresholdAPIList, helper.DefaultSamples,
+				func(int) {
+					_, err := h.Client.ListChannels(ctx, "")
+					Expect(err).NotTo(HaveOccurred())
+				},
+			)
 		})
 	},
 )

e2e/channel/perf_update_latency.go

@@ -2,7 +2,7 @@ package channel
 
 import (
 	"context"
-	"time"
+	"fmt"
 
 	"github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega" //nolint:staticcheck // dot import for test readability
@@ -15,6 +15,7 @@ import (
 
 var _ = ginkgo.Describe("[Suite: channel][perf] Update latency",
 	ginkgo.Label(labels.Tier1, labels.Performance),
+	ginkgo.Serial,
 	func() {
 		var h *helper.Helper
 		var channelID string
@@ -35,22 +36,17 @@ var _ = ginkgo.Describe("[Suite: channel][perf] Update latency",
 		})
 
 		ginkgo.It("should update a channel within acceptable latency", func(ctx context.Context) {
-			ginkgo.By("patching channel and timing the response")
-			start := time.Now()
-
-			patched, err := h.Client.PatchChannel(ctx, channelID, client.ResourcePatchRequest{
-				Spec: map[string]any{
-					"is_default":    true,
-					"enabled_regex": ".*",
+			helper.MeasureMedianLatency("PATCH /channels/{id}", config.ThresholdAPIUpdate, helper.DefaultSamples,
+				func(i int) {
+					_, err := h.Client.PatchChannel(ctx, channelID, client.ResourcePatchRequest{
+						Spec: map[string]any{
+							"is_default":    true,
+							"enabled_regex": fmt.Sprintf("^v%d\\..*$", i),
+						},
+					})
+					Expect(err).NotTo(HaveOccurred())
 				},
-			})
-			Expect(err).NotTo(HaveOccurred())
-			Expect(patched.Generation).To(Equal(int32(2)), "generation should increment after PATCH")
-			elapsed := time.Since(start)
-
-			ginkgo.GinkgoWriter.Printf("[PERF] PATCH /channels/%s latency: %v\n", channelID, elapsed)
-			Expect(elapsed).To(BeNumerically("<", config.ThresholdAPIUpdate),
-				"channel update exceeded threshold")
+			)
 		})
 	},
 )

e2e/cluster/perf_list_filtered_latency.go

@@ -3,7 +3,6 @@ package cluster
 import (
 	"context"
 	"net/url"
-	"time"
 
 	"github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega" //nolint:staticcheck // dot import for test readability
@@ -15,6 +14,7 @@ import (
 
 var _ = ginkgo.Describe("[Suite: cluster][perf] API list latency with filters and pagination",
 	ginkgo.Label(labels.Tier1, labels.Performance),
+	ginkgo.Serial,
 	func() {
 		var h *helper.Helper
 		var clusterID string
@@ -35,37 +35,31 @@ var _ = ginkgo.Describe("[Suite: cluster][perf] API list latency with filters an
 		})
 
 		ginkgo.It("should list clusters with search filter within acceptable latency", func(ctx context.Context) {
-			ginkgo.By("measuring GET /clusters?search=... response time")
 			filter := "labels.environment='test'"
-			start := time.Now()
-			_, err := h.Client.ListClustersWithParams(ctx, url.Values{"search": {filter}})
-			Expect(err).NotTo(HaveOccurred())
-			elapsed := time.Since(start)
-			ginkgo.GinkgoWriter.Printf("[PERF] GET /clusters (search filter) latency: %v\n", elapsed)
-			Expect(elapsed).To(BeNumerically("<", config.ThresholdAPIList),
-				"GET /clusters with search filter exceeded threshold")
+			helper.MeasureMedianLatency("GET /clusters (search filter)", config.ThresholdAPIList, helper.DefaultSamples,
+				func(int) {
+					_, err := h.Client.ListClustersWithParams(ctx, url.Values{"search": {filter}})
+					Expect(err).NotTo(HaveOccurred())
+				},
+			)
 		})
 
 		ginkgo.It("should list clusters with page size limit within acceptable latency", func(ctx context.Context) {
-			ginkgo.By("measuring GET /clusters?size=10 response time")
-			start := time.Now()
-			_, err := h.Client.ListClustersWithParams(ctx, url.Values{"size": {"10"}})
-			Expect(err).NotTo(HaveOccurred())
-			elapsed := time.Since(start)
-			ginkgo.GinkgoWriter.Printf("[PERF] GET /clusters (size=10) latency: %v\n", elapsed)
-			Expect(elapsed).To(BeNumerically("<", config.ThresholdAPIList),
-				"GET /clusters with page size limit exceeded threshold")
+			helper.MeasureMedianLatency("GET /clusters (size=10)", config.ThresholdAPIList, helper.DefaultSamples,
+				func(int) {
+					_, err := h.Client.ListClustersWithParams(ctx, url.Values{"size": {"10"}})
+					Expect(err).NotTo(HaveOccurred())
+				},
+			)
 		})
 
 		ginkgo.It("should list clusters with pagination within acceptable latency", func(ctx context.Context) {
-			ginkgo.By("measuring GET /clusters?page=1&size=10 response time")
-			start := time.Now()
-			_, err := h.Client.ListClustersWithParams(ctx, url.Values{"page": {"1"}, "size": {"10"}})
-			Expect(err).NotTo(HaveOccurred())
-			elapsed := time.Since(start)
-			ginkgo.GinkgoWriter.Printf("[PERF] GET /clusters (page=1, size=10) latency: %v\n", elapsed)
-			Expect(elapsed).To(BeNumerically("<", config.ThresholdAPIList),
-				"GET /clusters with pagination exceeded threshold")
+			helper.MeasureMedianLatency("GET /clusters (page=1, size=10)", config.ThresholdAPIList, helper.DefaultSamples,
+				func(int) {
+					_, err := h.Client.ListClustersWithParams(ctx, url.Values{"page": {"1"}, "size": {"10"}})
+					Expect(err).NotTo(HaveOccurred())
+				},
+			)
 		})
 	},
 )

e2e/cluster/perf_list_latency.go

@@ -2,7 +2,6 @@ package cluster
 
 import (
 	"context"
-	"time"
 
 	"github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega" //nolint:staticcheck // dot import for test readability
@@ -14,6 +13,7 @@ import (
 
 var _ = ginkgo.Describe("[Suite: cluster][perf] API list latency",
 	ginkgo.Label(labels.Tier1, labels.Performance),
+	ginkgo.Serial,
 	func() {
 		var h *helper.Helper
 		var clusterID string
@@ -34,14 +34,12 @@ var _ = ginkgo.Describe("[Suite: cluster][perf] API list latency",
 		})
 
 		ginkgo.It("should list clusters within acceptable latency", func(ctx context.Context) {
-			ginkgo.By("measuring GET /clusters response time")
-			start := time.Now()
-			_, err := h.Client.ListClusters(ctx)
-			Expect(err).NotTo(HaveOccurred())
-			elapsed := time.Since(start)
-			ginkgo.GinkgoWriter.Printf("[PERF] GET /clusters latency: %v\n", elapsed)
-			Expect(elapsed).To(BeNumerically("<", config.ThresholdAPIList),
-				"GET /clusters exceeded threshold")
+			helper.MeasureMedianLatency("GET /clusters", config.ThresholdAPIList, helper.DefaultSamples,
+				func(int) {
+					_, err := h.Client.ListClusters(ctx)
+					Expect(err).NotTo(HaveOccurred())
+				},
+			)
 		})
 	},
 )