From 4d92809f6c1d5d01bbe0f0b6c1bca9f07cda70c0 Mon Sep 17 00:00:00 2001 From: Dmitrii Andreev Date: Mon, 10 Aug 2026 18:59:10 -0500 Subject: [PATCH] HYPERFLEET-1493 - fix: eliminate CPU-contention flakes in perf latency specs - Mark ms-scale perf latency specs (channel/version/wifconfig/cluster create/update/list/delete/read) ginkgo.Serial so Ginkgo's parallel procs don't contend for CPU and skew the threshold assertions - Extract shared median-of-N sampling into helper.MeasureMedianLatency, replacing hand-rolled timing code duplicated across 18 spec files - Split the median/threshold assertion out of MeasureMedianLatency into assertMedianBelowThreshold so its unit tests exercise the math with exact durations instead of time.Sleep, removing the last source of CPU-contention flakiness in this package's own test suite - Mint the suite JWT once via SynchronizedBeforeSuite instead of once per parallel process - Add DeferChannelCleanup/DeferWifConfigCleanup helpers alongside the existing DeferClusterCleanup --- docs/development.md | 6 +- e2e/channel/perf_create_latency.go | 29 ++--- e2e/channel/perf_delete_latency.go | 41 +++--- e2e/channel/perf_get_latency.go | 27 ++-- e2e/channel/perf_list_latency.go | 16 ++- e2e/channel/perf_update_latency.go | 28 ++--- e2e/cluster/perf_list_filtered_latency.go | 44 +++---- e2e/cluster/perf_list_latency.go | 16 ++- e2e/cluster/perf_read_entity_size_latency.go | 30 ++--- e2e/version/perf_create_latency.go | 21 ++-- e2e/version/perf_delete_latency.go | 36 +++--- e2e/version/perf_get_latency.go | 27 ++-- e2e/version/perf_list_latency.go | 16 ++- e2e/version/perf_update_latency.go | 32 +++-- e2e/wifconfig/perf_create_latency.go | 29 ++--- e2e/wifconfig/perf_delete_latency.go | 41 +++--- e2e/wifconfig/perf_get_latency.go | 27 ++-- e2e/wifconfig/perf_list_latency.go | 16 ++- e2e/wifconfig/perf_update_latency.go | 28 ++--- pkg/config/thresholds.go | 4 +- pkg/e2e/suite.go | 65 ++++++---- pkg/helper/helper.go | 20 +++ pkg/helper/perf.go | 56 +++++++++ pkg/helper/perf_test.go | 124 +++++++++++++++++++ 24 files changed, 444 insertions(+), 335 deletions(-) create mode 100644 pkg/helper/perf.go create mode 100644 pkg/helper/perf_test.go diff --git a/docs/development.md b/docs/development.md index cf7cdfc9..d9b643da 100644 --- a/docs/development.md +++ b/docs/development.md @@ -328,10 +328,12 @@ Valid reasons to mark a spec `Serial`: - **Mutates shared infrastructure** (e.g., scales deployments, deletes shared resources) - **Deploys temporary adapters** that subscribe to all events, causing cross-talk with concurrent specs +- **Asserts a millisecond-scale latency threshold**, where CPU contention from other parallel procs would inflate the measurement enough to flake the assertion +- **Waits on a real timeout-bound condition** (e.g., reconciliation) whose duration is sensitive to concurrent create/reconcile load elsewhere in the suite - contention here risks an outright timeout failure, not just a skewed measurement -Specs already marked `Serial`: sentinel scale-down, force-delete, stuck-deletion, crash-recovery, maestro-unavailability, adapter-failover, adapter-failure, maestro negative scenarios. +Specs already marked `Serial`: sentinel scale-down, force-delete, stuck-deletion, crash-recovery, maestro-unavailability, adapter-failover, adapter-failure, maestro negative scenarios, and the 18 ms-scale channel/version/wifconfig/cluster perf spec files (list/create/update/delete/read latency; 22 individual specs, since the cluster list-with-filters file holds three and the cluster read-by-entity-size file holds three). -Performance specs (`labels.Performance`) are a separate category - they carry no tier label and run in their own dedicated CI job (`--label-filter="perf"`) on a quiet system. They are not marked `Serial` because they never run alongside functional tests. +Performance specs (`labels.Performance`) carry `labels.Tier1` like any other tier1 spec and run inside `tier1-nightly` alongside functional tests - there is no dedicated perf CI job. Ms-scale API perf specs (list/create/update/delete/read latency) are marked `Serial` for exactly this reason: without it, Ginkgo's parallel procs (`--procs=8`) contend for CPU and inflate latency measurements enough to flake the threshold assertions. The cluster read-by-entity-size specs carry a second reason: each waits on cluster reconciliation before reading, and reconciliation duration itself is sensitive to concurrent cluster create/reconcile load elsewhere in the suite. Second-scale reconciliation perf specs (which assert reconciliation time itself, not read latency) stay parallel since their larger thresholds already carry enough margin to absorb that contention. ## Adding New Tests diff --git a/e2e/channel/perf_create_latency.go b/e2e/channel/perf_create_latency.go index 8f6ea05f..146aa982 100644 --- a/e2e/channel/perf_create_latency.go +++ b/e2e/channel/perf_create_latency.go @@ -2,7 +2,6 @@ package channel import ( "context" - "time" "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" //nolint:staticcheck // dot import for test readability @@ -14,6 +13,7 @@ import ( var _ = ginkgo.Describe("[Suite: channel][perf] Create latency", ginkgo.Label(labels.Tier1, labels.Performance), + ginkgo.Serial, func() { var h *helper.Helper @@ -22,25 +22,16 @@ var _ = ginkgo.Describe("[Suite: channel][perf] Create latency", }) ginkgo.It("should create a channel within acceptable latency", func(ctx context.Context) { - ginkgo.By("creating a channel and timing the response") - start := time.Now() - - channel, err := h.Client.CreateChannelFromPayload(ctx, h.TestDataPath("payloads/channels/channel-request.json")) - if channel != nil && channel.Id != nil { - id := *channel.Id - ginkgo.DeferCleanup(func(ctx context.Context) { - if err := h.CleanupTestChannel(ctx, id); err != nil { - ginkgo.GinkgoWriter.Printf("Warning: failed to cleanup channel %s: %v\n", id, err) + helper.MeasureMedianLatency("POST /channels", config.ThresholdAPICreate, helper.DefaultSamples, + func(int) { + channel, err := h.Client.CreateChannelFromPayload(ctx, h.TestDataPath("payloads/channels/channel-request.json")) + if channel != nil && channel.Id != nil { + h.DeferChannelCleanup(*channel.Id) } - }) - } - Expect(err).NotTo(HaveOccurred()) - Expect(channel.Id).NotTo(BeNil(), "channel ID should be set") - elapsed := time.Since(start) - - ginkgo.GinkgoWriter.Printf("[PERF] POST /channels latency: %v\n", elapsed) - Expect(elapsed).To(BeNumerically("<", config.ThresholdAPICreate), - "channel create exceeded threshold") + Expect(err).NotTo(HaveOccurred()) + Expect(channel.Id).NotTo(BeNil(), "channel ID should be set") + }, + ) }) }, ) diff --git a/e2e/channel/perf_delete_latency.go b/e2e/channel/perf_delete_latency.go index cf82e815..b7504140 100644 --- a/e2e/channel/perf_delete_latency.go +++ b/e2e/channel/perf_delete_latency.go @@ -2,7 +2,6 @@ package channel import ( "context" - "time" "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" //nolint:staticcheck // dot import for test readability @@ -14,37 +13,31 @@ import ( var _ = ginkgo.Describe("[Suite: channel][perf] Delete latency", ginkgo.Label(labels.Tier1, labels.Performance), + ginkgo.Serial, func() { var h *helper.Helper - var channelID string ginkgo.BeforeEach(func(ctx context.Context) { h = helper.New() - - channel, err := h.Client.CreateChannelFromPayload(ctx, h.TestDataPath("payloads/channels/channel-request.json")) - Expect(err).NotTo(HaveOccurred()) - Expect(channel.Id).NotTo(BeNil(), "channel ID should be set") - channelID = *channel.Id - - ginkgo.DeferCleanup(func(ctx context.Context) { - if err := h.CleanupTestChannel(ctx, channelID); err != nil { - ginkgo.GinkgoWriter.Printf("Warning: failed to cleanup channel %s: %v\n", channelID, err) - } - }) }) ginkgo.It("should delete a channel within acceptable latency", func(ctx context.Context) { - ginkgo.By("deleting channel and timing the response") - start := time.Now() - - deleted, err := h.Client.DeleteChannel(ctx, channelID) - Expect(err).NotTo(HaveOccurred()) - Expect(deleted.DeletedTime).NotTo(BeNil(), "deleted channel should have deleted_time set") - elapsed := time.Since(start) - - ginkgo.GinkgoWriter.Printf("[PERF] DELETE /channels/%s latency: %v\n", channelID, elapsed) - Expect(elapsed).To(BeNumerically("<", config.ThresholdAPIDelete), - "channel delete exceeded threshold") + channelIDs := make([]string, helper.DefaultSamples) + for i := range channelIDs { + channel, err := h.Client.CreateChannelFromPayload(ctx, h.TestDataPath("payloads/channels/channel-request.json")) + Expect(err).NotTo(HaveOccurred()) + Expect(channel.Id).NotTo(BeNil(), "channel ID should be set") + channelIDs[i] = *channel.Id + h.DeferChannelCleanup(*channel.Id) + } + + helper.MeasureMedianLatency("DELETE /channels/{id}", config.ThresholdAPIDelete, len(channelIDs), + func(i int) { + deleted, err := h.Client.DeleteChannel(ctx, channelIDs[i]) + Expect(err).NotTo(HaveOccurred()) + Expect(deleted.DeletedTime).NotTo(BeNil(), "deleted channel should have deleted_time set") + }, + ) }) }, ) diff --git a/e2e/channel/perf_get_latency.go b/e2e/channel/perf_get_latency.go index 76c9690b..52cabec7 100644 --- a/e2e/channel/perf_get_latency.go +++ b/e2e/channel/perf_get_latency.go @@ -2,8 +2,6 @@ package channel import ( "context" - "slices" - "time" "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" //nolint:staticcheck // dot import for test readability @@ -15,6 +13,7 @@ import ( var _ = ginkgo.Describe("[Suite: channel][perf] API read latency", ginkgo.Label(labels.Tier1, labels.Performance), + ginkgo.Serial, func() { var h *helper.Helper var channelID string @@ -35,24 +34,12 @@ var _ = ginkgo.Describe("[Suite: channel][perf] API read latency", }) ginkgo.It("should read a channel within acceptable latency", func(ctx context.Context) { - ginkgo.By("warming up with untimed read") - _, err := h.Client.GetChannel(ctx, channelID) - Expect(err).NotTo(HaveOccurred()) - - ginkgo.By("measuring GET /channels/{id} response time") - const samples = 5 - durations := make([]time.Duration, samples) - for i := range samples { - start := time.Now() - _, err = h.Client.GetChannel(ctx, channelID) - Expect(err).NotTo(HaveOccurred()) - durations[i] = time.Since(start) - } - slices.Sort(durations) - median := durations[samples/2] - ginkgo.GinkgoWriter.Printf("[PERF] GET /channels/%s latency: %v (median of %d samples)\n", channelID, median, samples) - Expect(median).To(BeNumerically("<", config.ThresholdAPIRead), - "GET /channels/{id} exceeded threshold") + helper.MeasureMedianLatency("GET /channels/{id}", config.ThresholdAPIRead, helper.DefaultSamples, + func(int) { + _, err := h.Client.GetChannel(ctx, channelID) + Expect(err).NotTo(HaveOccurred()) + }, + ) }) }, ) diff --git a/e2e/channel/perf_list_latency.go b/e2e/channel/perf_list_latency.go index dd3d62b2..277f0a96 100644 --- a/e2e/channel/perf_list_latency.go +++ b/e2e/channel/perf_list_latency.go @@ -2,7 +2,6 @@ package channel import ( "context" - "time" "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" //nolint:staticcheck // dot import for test readability @@ -14,6 +13,7 @@ import ( var _ = ginkgo.Describe("[Suite: channel][perf] API list latency", ginkgo.Label(labels.Tier1, labels.Performance), + ginkgo.Serial, func() { var h *helper.Helper var channelID string @@ -34,14 +34,12 @@ var _ = ginkgo.Describe("[Suite: channel][perf] API list latency", }) ginkgo.It("should list channels within acceptable latency", func(ctx context.Context) { - ginkgo.By("measuring GET /channels response time") - start := time.Now() - _, err := h.Client.ListChannels(ctx, "") - Expect(err).NotTo(HaveOccurred()) - elapsed := time.Since(start) - ginkgo.GinkgoWriter.Printf("[PERF] GET /channels latency: %v\n", elapsed) - Expect(elapsed).To(BeNumerically("<", config.ThresholdAPIList), - "GET /channels exceeded threshold") + helper.MeasureMedianLatency("GET /channels", config.ThresholdAPIList, helper.DefaultSamples, + func(int) { + _, err := h.Client.ListChannels(ctx, "") + Expect(err).NotTo(HaveOccurred()) + }, + ) }) }, ) diff --git a/e2e/channel/perf_update_latency.go b/e2e/channel/perf_update_latency.go index d7040098..52cbbaf5 100644 --- a/e2e/channel/perf_update_latency.go +++ b/e2e/channel/perf_update_latency.go @@ -2,7 +2,7 @@ package channel import ( "context" - "time" + "fmt" "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" //nolint:staticcheck // dot import for test readability @@ -15,6 +15,7 @@ import ( var _ = ginkgo.Describe("[Suite: channel][perf] Update latency", ginkgo.Label(labels.Tier1, labels.Performance), + ginkgo.Serial, func() { var h *helper.Helper var channelID string @@ -35,22 +36,17 @@ var _ = ginkgo.Describe("[Suite: channel][perf] Update latency", }) ginkgo.It("should update a channel within acceptable latency", func(ctx context.Context) { - ginkgo.By("patching channel and timing the response") - start := time.Now() - - patched, err := h.Client.PatchChannel(ctx, channelID, client.ResourcePatchRequest{ - Spec: map[string]any{ - "is_default": true, - "enabled_regex": ".*", + helper.MeasureMedianLatency("PATCH /channels/{id}", config.ThresholdAPIUpdate, helper.DefaultSamples, + func(i int) { + _, err := h.Client.PatchChannel(ctx, channelID, client.ResourcePatchRequest{ + Spec: map[string]any{ + "is_default": true, + "enabled_regex": fmt.Sprintf("^v%d\\..*$", i), + }, + }) + Expect(err).NotTo(HaveOccurred()) }, - }) - Expect(err).NotTo(HaveOccurred()) - Expect(patched.Generation).To(Equal(int32(2)), "generation should increment after PATCH") - elapsed := time.Since(start) - - ginkgo.GinkgoWriter.Printf("[PERF] PATCH /channels/%s latency: %v\n", channelID, elapsed) - Expect(elapsed).To(BeNumerically("<", config.ThresholdAPIUpdate), - "channel update exceeded threshold") + ) }) }, ) diff --git a/e2e/cluster/perf_list_filtered_latency.go b/e2e/cluster/perf_list_filtered_latency.go index 9645e8c3..f0c039ca 100644 --- a/e2e/cluster/perf_list_filtered_latency.go +++ b/e2e/cluster/perf_list_filtered_latency.go @@ -3,7 +3,6 @@ package cluster import ( "context" "net/url" - "time" "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" //nolint:staticcheck // dot import for test readability @@ -15,6 +14,7 @@ import ( var _ = ginkgo.Describe("[Suite: cluster][perf] API list latency with filters and pagination", ginkgo.Label(labels.Tier1, labels.Performance), + ginkgo.Serial, func() { var h *helper.Helper var clusterID string @@ -35,37 +35,31 @@ var _ = ginkgo.Describe("[Suite: cluster][perf] API list latency with filters an }) ginkgo.It("should list clusters with search filter within acceptable latency", func(ctx context.Context) { - ginkgo.By("measuring GET /clusters?search=... response time") filter := "labels.environment='test'" - start := time.Now() - _, err := h.Client.ListClustersWithParams(ctx, url.Values{"search": {filter}}) - Expect(err).NotTo(HaveOccurred()) - elapsed := time.Since(start) - ginkgo.GinkgoWriter.Printf("[PERF] GET /clusters (search filter) latency: %v\n", elapsed) - Expect(elapsed).To(BeNumerically("<", config.ThresholdAPIList), - "GET /clusters with search filter exceeded threshold") + helper.MeasureMedianLatency("GET /clusters (search filter)", config.ThresholdAPIList, helper.DefaultSamples, + func(int) { + _, err := h.Client.ListClustersWithParams(ctx, url.Values{"search": {filter}}) + Expect(err).NotTo(HaveOccurred()) + }, + ) }) ginkgo.It("should list clusters with page size limit within acceptable latency", func(ctx context.Context) { - ginkgo.By("measuring GET /clusters?size=10 response time") - start := time.Now() - _, err := h.Client.ListClustersWithParams(ctx, url.Values{"size": {"10"}}) - Expect(err).NotTo(HaveOccurred()) - elapsed := time.Since(start) - ginkgo.GinkgoWriter.Printf("[PERF] GET /clusters (size=10) latency: %v\n", elapsed) - Expect(elapsed).To(BeNumerically("<", config.ThresholdAPIList), - "GET /clusters with page size limit exceeded threshold") + helper.MeasureMedianLatency("GET /clusters (size=10)", config.ThresholdAPIList, helper.DefaultSamples, + func(int) { + _, err := h.Client.ListClustersWithParams(ctx, url.Values{"size": {"10"}}) + Expect(err).NotTo(HaveOccurred()) + }, + ) }) ginkgo.It("should list clusters with pagination within acceptable latency", func(ctx context.Context) { - ginkgo.By("measuring GET /clusters?page=1&size=10 response time") - start := time.Now() - _, err := h.Client.ListClustersWithParams(ctx, url.Values{"page": {"1"}, "size": {"10"}}) - Expect(err).NotTo(HaveOccurred()) - elapsed := time.Since(start) - ginkgo.GinkgoWriter.Printf("[PERF] GET /clusters (page=1, size=10) latency: %v\n", elapsed) - Expect(elapsed).To(BeNumerically("<", config.ThresholdAPIList), - "GET /clusters with pagination exceeded threshold") + helper.MeasureMedianLatency("GET /clusters (page=1, size=10)", config.ThresholdAPIList, helper.DefaultSamples, + func(int) { + _, err := h.Client.ListClustersWithParams(ctx, url.Values{"page": {"1"}, "size": {"10"}}) + Expect(err).NotTo(HaveOccurred()) + }, + ) }) }, ) diff --git a/e2e/cluster/perf_list_latency.go b/e2e/cluster/perf_list_latency.go index 2bae9439..9d9a4b5d 100644 --- a/e2e/cluster/perf_list_latency.go +++ b/e2e/cluster/perf_list_latency.go @@ -2,7 +2,6 @@ package cluster import ( "context" - "time" "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" //nolint:staticcheck // dot import for test readability @@ -14,6 +13,7 @@ import ( var _ = ginkgo.Describe("[Suite: cluster][perf] API list latency", ginkgo.Label(labels.Tier1, labels.Performance), + ginkgo.Serial, func() { var h *helper.Helper var clusterID string @@ -34,14 +34,12 @@ var _ = ginkgo.Describe("[Suite: cluster][perf] API list latency", }) ginkgo.It("should list clusters within acceptable latency", func(ctx context.Context) { - ginkgo.By("measuring GET /clusters response time") - start := time.Now() - _, err := h.Client.ListClusters(ctx) - Expect(err).NotTo(HaveOccurred()) - elapsed := time.Since(start) - ginkgo.GinkgoWriter.Printf("[PERF] GET /clusters latency: %v\n", elapsed) - Expect(elapsed).To(BeNumerically("<", config.ThresholdAPIList), - "GET /clusters exceeded threshold") + helper.MeasureMedianLatency("GET /clusters", config.ThresholdAPIList, helper.DefaultSamples, + func(int) { + _, err := h.Client.ListClusters(ctx) + Expect(err).NotTo(HaveOccurred()) + }, + ) }) }, ) diff --git a/e2e/cluster/perf_read_entity_size_latency.go b/e2e/cluster/perf_read_entity_size_latency.go index 84b143c3..993b4bd3 100644 --- a/e2e/cluster/perf_read_entity_size_latency.go +++ b/e2e/cluster/perf_read_entity_size_latency.go @@ -2,8 +2,6 @@ package cluster import ( "context" - "slices" - "time" "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" //nolint:staticcheck // dot import for test readability @@ -16,6 +14,10 @@ import ( var _ = ginkgo.Describe("[Suite: cluster][perf] API read latency by entity size", ginkgo.Label(labels.Tier1, labels.Performance), + // Serial: the ms-scale latency assertion needs CPU contention-free timing, + // and the Eventually(...Reconciled...) wait needs reconciliation itself + // free of concurrent create/reconcile load elsewhere in the suite. + ginkgo.Serial, func() { var h *helper.Helper @@ -50,24 +52,12 @@ var _ = ginkgo.Describe("[Suite: cluster][perf] API read latency by entity size" Eventually(h.PollCluster(ctx, clusterID), h.Cfg.Timeouts.Cluster.Reconciled, h.Cfg.Polling.Interval). Should(helper.HaveResourceCondition(client.ConditionTypeReconciled, client.ResourceConditionStatusTrue)) - ginkgo.By("warming up with untimed read") - _, err = h.Client.GetCluster(ctx, clusterID) - Expect(err).NotTo(HaveOccurred()) - - ginkgo.By("measuring GET /clusters/{id} response time for " + size.name + " entity") - const samples = 5 - durations := make([]time.Duration, samples) - for i := range samples { - start := time.Now() - _, err = h.Client.GetCluster(ctx, clusterID) - Expect(err).NotTo(HaveOccurred()) - durations[i] = time.Since(start) - } - slices.Sort(durations) - median := durations[samples/2] - ginkgo.GinkgoWriter.Printf("[PERF] GET /clusters/%s (%s entity) latency: %v (median of %d samples)\n", clusterID, size.name, median, samples) - Expect(median).To(BeNumerically("<", config.ThresholdAPIRead), - "GET /clusters/{id} (%s entity) exceeded threshold", size.name) + helper.MeasureMedianLatency("GET /clusters/{id} ("+size.name+" entity)", config.ThresholdAPIRead, helper.DefaultSamples, + func(int) { + _, err := h.Client.GetCluster(ctx, clusterID) + Expect(err).NotTo(HaveOccurred()) + }, + ) }) } }, diff --git a/e2e/version/perf_create_latency.go b/e2e/version/perf_create_latency.go index b4e49438..de1d0e71 100644 --- a/e2e/version/perf_create_latency.go +++ b/e2e/version/perf_create_latency.go @@ -2,7 +2,6 @@ package version import ( "context" - "time" "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" //nolint:staticcheck // dot import for test readability @@ -14,6 +13,7 @@ import ( var _ = ginkgo.Describe("[Suite: version][perf] Create latency", ginkgo.Label(labels.Tier1, labels.Performance), + ginkgo.Serial, func() { var h *helper.Helper var channelID string @@ -35,17 +35,14 @@ var _ = ginkgo.Describe("[Suite: version][perf] Create latency", }) ginkgo.It("should create a version within acceptable latency", func(ctx context.Context) { - ginkgo.By("creating a version and timing the response") - start := time.Now() - - version, err := h.Client.CreateVersionFromPayload(ctx, channelID, h.TestDataPath("payloads/versions/version-request.json")) - Expect(err).NotTo(HaveOccurred()) - Expect(version.Id).NotTo(BeNil(), "version ID should be set") - elapsed := time.Since(start) - - ginkgo.GinkgoWriter.Printf("[PERF] POST /channels/%s/versions latency: %v\n", channelID, elapsed) - Expect(elapsed).To(BeNumerically("<", config.ThresholdAPICreate), - "version create exceeded threshold") + helper.MeasureMedianLatency("POST /channels/{parent_id}/versions", config.ThresholdAPICreate, helper.DefaultSamples, + func(int) { + // No per-version cleanup: the channel's DeferCleanup above sweeps all its versions. + version, err := h.Client.CreateVersionFromPayload(ctx, channelID, h.TestDataPath("payloads/versions/version-request.json")) + Expect(err).NotTo(HaveOccurred()) + Expect(version.Id).NotTo(BeNil(), "version ID should be set") + }, + ) }) }, ) diff --git a/e2e/version/perf_delete_latency.go b/e2e/version/perf_delete_latency.go index eb47f5d8..eee6b113 100644 --- a/e2e/version/perf_delete_latency.go +++ b/e2e/version/perf_delete_latency.go @@ -2,7 +2,6 @@ package version import ( "context" - "time" "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" //nolint:staticcheck // dot import for test readability @@ -14,10 +13,10 @@ import ( var _ = ginkgo.Describe("[Suite: version][perf] Delete latency", ginkgo.Label(labels.Tier1, labels.Performance), + ginkgo.Serial, func() { var h *helper.Helper var channelID string - var versionID string ginkgo.BeforeEach(func(ctx context.Context) { h = helper.New() @@ -33,26 +32,25 @@ var _ = ginkgo.Describe("[Suite: version][perf] Delete latency", ginkgo.GinkgoWriter.Printf("Warning: failed to cleanup channel %s: %v\n", channelID, err) } }) - - ginkgo.By("creating version under channel") - version, err := h.Client.CreateVersionFromPayload(ctx, channelID, h.TestDataPath("payloads/versions/version-request.json")) - Expect(err).NotTo(HaveOccurred()) - Expect(version.Id).NotTo(BeNil(), "version ID should be set") - versionID = *version.Id }) ginkgo.It("should delete a version within acceptable latency", func(ctx context.Context) { - ginkgo.By("deleting version and timing the response") - start := time.Now() - - deleted, err := h.Client.DeleteVersion(ctx, channelID, versionID) - Expect(err).NotTo(HaveOccurred()) - Expect(deleted.DeletedTime).NotTo(BeNil(), "deleted version should have deleted_time set") - elapsed := time.Since(start) - - ginkgo.GinkgoWriter.Printf("[PERF] DELETE /channels/%s/versions/%s latency: %v\n", channelID, versionID, elapsed) - Expect(elapsed).To(BeNumerically("<", config.ThresholdAPIDelete), - "version delete exceeded threshold") + // No per-version cleanup: the channel's DeferCleanup above sweeps all its versions. + versionIDs := make([]string, helper.DefaultSamples) + for i := range versionIDs { + version, err := h.Client.CreateVersionFromPayload(ctx, channelID, h.TestDataPath("payloads/versions/version-request.json")) + Expect(err).NotTo(HaveOccurred()) + Expect(version.Id).NotTo(BeNil(), "version ID should be set") + versionIDs[i] = *version.Id + } + + helper.MeasureMedianLatency("DELETE /channels/{parent_id}/versions/{id}", config.ThresholdAPIDelete, len(versionIDs), + func(i int) { + deleted, err := h.Client.DeleteVersion(ctx, channelID, versionIDs[i]) + Expect(err).NotTo(HaveOccurred()) + Expect(deleted.DeletedTime).NotTo(BeNil(), "deleted version should have deleted_time set") + }, + ) }) }, ) diff --git a/e2e/version/perf_get_latency.go b/e2e/version/perf_get_latency.go index 1ec27ade..4d99a7f5 100644 --- a/e2e/version/perf_get_latency.go +++ b/e2e/version/perf_get_latency.go @@ -2,8 +2,6 @@ package version import ( "context" - "slices" - "time" "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" //nolint:staticcheck // dot import for test readability @@ -15,6 +13,7 @@ import ( var _ = ginkgo.Describe("[Suite: version][perf] API read latency", ginkgo.Label(labels.Tier1, labels.Performance), + ginkgo.Serial, func() { var h *helper.Helper var channelID string @@ -43,24 +42,12 @@ var _ = ginkgo.Describe("[Suite: version][perf] API read latency", }) ginkgo.It("should read a version within acceptable latency", func(ctx context.Context) { - ginkgo.By("warming up with untimed read") - _, err := h.Client.GetVersion(ctx, channelID, versionID) - Expect(err).NotTo(HaveOccurred()) - - ginkgo.By("measuring GET /channels/{parent_id}/versions/{id} response time") - const samples = 5 - durations := make([]time.Duration, samples) - for i := range samples { - start := time.Now() - _, err = h.Client.GetVersion(ctx, channelID, versionID) - Expect(err).NotTo(HaveOccurred()) - durations[i] = time.Since(start) - } - slices.Sort(durations) - median := durations[samples/2] - ginkgo.GinkgoWriter.Printf("[PERF] GET /channels/%s/versions/%s latency: %v (median of %d samples)\n", channelID, versionID, median, samples) - Expect(median).To(BeNumerically("<", config.ThresholdAPIRead), - "GET /channels/{parent_id}/versions/{id} exceeded threshold") + helper.MeasureMedianLatency("GET /channels/{parent_id}/versions/{id}", config.ThresholdAPIRead, helper.DefaultSamples, + func(int) { + _, err := h.Client.GetVersion(ctx, channelID, versionID) + Expect(err).NotTo(HaveOccurred()) + }, + ) }) }, ) diff --git a/e2e/version/perf_list_latency.go b/e2e/version/perf_list_latency.go index d298309f..20fcb5a6 100644 --- a/e2e/version/perf_list_latency.go +++ b/e2e/version/perf_list_latency.go @@ -2,7 +2,6 @@ package version import ( "context" - "time" "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" //nolint:staticcheck // dot import for test readability @@ -14,6 +13,7 @@ import ( var _ = ginkgo.Describe("[Suite: version][perf] API list latency", ginkgo.Label(labels.Tier1, labels.Performance), + ginkgo.Serial, func() { var h *helper.Helper var channelID string @@ -40,14 +40,12 @@ var _ = ginkgo.Describe("[Suite: version][perf] API list latency", }) ginkgo.It("should list versions within acceptable latency", func(ctx context.Context) { - ginkgo.By("measuring GET /channels/{parent_id}/versions response time") - start := time.Now() - _, err := h.Client.ListVersions(ctx, channelID, "") - Expect(err).NotTo(HaveOccurred()) - elapsed := time.Since(start) - ginkgo.GinkgoWriter.Printf("[PERF] GET /channels/%s/versions latency: %v\n", channelID, elapsed) - Expect(elapsed).To(BeNumerically("<", config.ThresholdAPIList), - "GET /channels/{parent_id}/versions exceeded threshold") + helper.MeasureMedianLatency("GET /channels/{parent_id}/versions", config.ThresholdAPIList, helper.DefaultSamples, + func(int) { + _, err := h.Client.ListVersions(ctx, channelID, "") + Expect(err).NotTo(HaveOccurred()) + }, + ) }) }, ) diff --git a/e2e/version/perf_update_latency.go b/e2e/version/perf_update_latency.go index 64fc6b07..3e70bf3a 100644 --- a/e2e/version/perf_update_latency.go +++ b/e2e/version/perf_update_latency.go @@ -2,7 +2,7 @@ package version import ( "context" - "time" + "fmt" "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" //nolint:staticcheck // dot import for test readability @@ -15,6 +15,7 @@ import ( var _ = ginkgo.Describe("[Suite: version][perf] Update latency", ginkgo.Label(labels.Tier1, labels.Performance), + ginkgo.Serial, func() { var h *helper.Helper var channelID string @@ -43,24 +44,19 @@ var _ = ginkgo.Describe("[Suite: version][perf] Update latency", }) ginkgo.It("should update a version within acceptable latency", func(ctx context.Context) { - ginkgo.By("patching version and timing the response") - start := time.Now() - - patched, err := h.Client.PatchVersion(ctx, channelID, versionID, client.ResourcePatchRequest{ - Spec: map[string]any{ - "raw_version": "4.18.0", - "enabled": true, - "is_default": true, - "release_image": "quay.io/openshift-release-dev/ocp-release:4.18.0", + helper.MeasureMedianLatency("PATCH /channels/{parent_id}/versions/{id}", config.ThresholdAPIUpdate, helper.DefaultSamples, + func(i int) { + _, err := h.Client.PatchVersion(ctx, channelID, versionID, client.ResourcePatchRequest{ + Spec: map[string]any{ + "raw_version": fmt.Sprintf("4.18.%d", i), + "enabled": true, + "is_default": true, + "release_image": fmt.Sprintf("quay.io/openshift-release-dev/ocp-release:4.18.%d", i), + }, + }) + Expect(err).NotTo(HaveOccurred()) }, - }) - Expect(err).NotTo(HaveOccurred()) - Expect(patched.Generation).To(Equal(int32(2)), "generation should increment after PATCH") - elapsed := time.Since(start) - - ginkgo.GinkgoWriter.Printf("[PERF] PATCH /channels/%s/versions/%s latency: %v\n", channelID, versionID, elapsed) - Expect(elapsed).To(BeNumerically("<", config.ThresholdAPIUpdate), - "version update exceeded threshold") + ) }) }, ) diff --git a/e2e/wifconfig/perf_create_latency.go b/e2e/wifconfig/perf_create_latency.go index 246c1fda..c6681fad 100644 --- a/e2e/wifconfig/perf_create_latency.go +++ b/e2e/wifconfig/perf_create_latency.go @@ -2,7 +2,6 @@ package wifconfig import ( "context" - "time" "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" //nolint:staticcheck // dot import for test readability @@ -14,6 +13,7 @@ import ( var _ = ginkgo.Describe("[Suite: wifconfig][perf] Create latency", ginkgo.Label(labels.Tier1, labels.Performance), + ginkgo.Serial, func() { var h *helper.Helper @@ -22,25 +22,16 @@ var _ = ginkgo.Describe("[Suite: wifconfig][perf] Create latency", }) ginkgo.It("should create a wifconfig within acceptable latency", func(ctx context.Context) { - ginkgo.By("creating a wifconfig and timing the response") - start := time.Now() - - wifConfig, err := h.Client.CreateWifConfigFromPayload(ctx, h.TestDataPath("payloads/wifconfigs/wifconfig-request.json")) - if wifConfig != nil && wifConfig.Id != nil { - id := *wifConfig.Id - ginkgo.DeferCleanup(func(ctx context.Context) { - if err := h.CleanupTestWifConfig(ctx, id); err != nil { - ginkgo.GinkgoWriter.Printf("Warning: failed to cleanup wifconfig %s: %v\n", id, err) + helper.MeasureMedianLatency("POST /wifconfigs", config.ThresholdAPICreate, helper.DefaultSamples, + func(int) { + wifConfig, err := h.Client.CreateWifConfigFromPayload(ctx, h.TestDataPath("payloads/wifconfigs/wifconfig-request.json")) + if wifConfig != nil && wifConfig.Id != nil { + h.DeferWifConfigCleanup(*wifConfig.Id) } - }) - } - Expect(err).NotTo(HaveOccurred()) - Expect(wifConfig.Id).NotTo(BeNil(), "wifconfig ID should be set") - elapsed := time.Since(start) - - ginkgo.GinkgoWriter.Printf("[PERF] POST /wifconfigs latency: %v\n", elapsed) - Expect(elapsed).To(BeNumerically("<", config.ThresholdAPICreate), - "wifconfig create exceeded threshold") + Expect(err).NotTo(HaveOccurred()) + Expect(wifConfig.Id).NotTo(BeNil(), "wifconfig ID should be set") + }, + ) }) }, ) diff --git a/e2e/wifconfig/perf_delete_latency.go b/e2e/wifconfig/perf_delete_latency.go index 9ee35700..4e4e0128 100644 --- a/e2e/wifconfig/perf_delete_latency.go +++ b/e2e/wifconfig/perf_delete_latency.go @@ -2,7 +2,6 @@ package wifconfig import ( "context" - "time" "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" //nolint:staticcheck // dot import for test readability @@ -14,37 +13,31 @@ import ( var _ = ginkgo.Describe("[Suite: wifconfig][perf] Delete latency", ginkgo.Label(labels.Tier1, labels.Performance), + ginkgo.Serial, func() { var h *helper.Helper - var wifConfigID string ginkgo.BeforeEach(func(ctx context.Context) { h = helper.New() - - wifConfig, err := h.Client.CreateWifConfigFromPayload(ctx, h.TestDataPath("payloads/wifconfigs/wifconfig-request.json")) - Expect(err).NotTo(HaveOccurred()) - Expect(wifConfig.Id).NotTo(BeNil(), "wifconfig ID should be set") - wifConfigID = *wifConfig.Id - - ginkgo.DeferCleanup(func(ctx context.Context) { - if err := h.CleanupTestWifConfig(ctx, wifConfigID); err != nil { - ginkgo.GinkgoWriter.Printf("Warning: failed to cleanup wifconfig %s: %v\n", wifConfigID, err) - } - }) }) ginkgo.It("should delete a wifconfig within acceptable latency", func(ctx context.Context) { - ginkgo.By("deleting wifconfig and timing the response") - start := time.Now() - - deleted, err := h.Client.DeleteWifConfig(ctx, wifConfigID) - Expect(err).NotTo(HaveOccurred()) - Expect(deleted.DeletedTime).NotTo(BeNil(), "deleted wifconfig should have deleted_time set") - elapsed := time.Since(start) - - ginkgo.GinkgoWriter.Printf("[PERF] DELETE /wifconfigs/%s latency: %v\n", wifConfigID, elapsed) - Expect(elapsed).To(BeNumerically("<", config.ThresholdAPIDelete), - "wifconfig delete exceeded threshold") + wifConfigIDs := make([]string, helper.DefaultSamples) + for i := range wifConfigIDs { + wifConfig, err := h.Client.CreateWifConfigFromPayload(ctx, h.TestDataPath("payloads/wifconfigs/wifconfig-request.json")) + Expect(err).NotTo(HaveOccurred()) + Expect(wifConfig.Id).NotTo(BeNil(), "wifconfig ID should be set") + wifConfigIDs[i] = *wifConfig.Id + h.DeferWifConfigCleanup(*wifConfig.Id) + } + + helper.MeasureMedianLatency("DELETE /wifconfigs/{id}", config.ThresholdAPIDelete, len(wifConfigIDs), + func(i int) { + deleted, err := h.Client.DeleteWifConfig(ctx, wifConfigIDs[i]) + Expect(err).NotTo(HaveOccurred()) + Expect(deleted.DeletedTime).NotTo(BeNil(), "deleted wifconfig should have deleted_time set") + }, + ) }) }, ) diff --git a/e2e/wifconfig/perf_get_latency.go b/e2e/wifconfig/perf_get_latency.go index d80c136d..befebf22 100644 --- a/e2e/wifconfig/perf_get_latency.go +++ b/e2e/wifconfig/perf_get_latency.go @@ -2,8 +2,6 @@ package wifconfig import ( "context" - "slices" - "time" "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" //nolint:staticcheck // dot import for test readability @@ -15,6 +13,7 @@ import ( var _ = ginkgo.Describe("[Suite: wifconfig][perf] API read latency", ginkgo.Label(labels.Tier1, labels.Performance), + ginkgo.Serial, func() { var h *helper.Helper var wifConfigID string @@ -35,24 +34,12 @@ var _ = ginkgo.Describe("[Suite: wifconfig][perf] API read latency", }) ginkgo.It("should read a wifconfig within acceptable latency", func(ctx context.Context) { - ginkgo.By("warming up with untimed read") - _, err := h.Client.GetWifConfig(ctx, wifConfigID) - Expect(err).NotTo(HaveOccurred()) - - ginkgo.By("measuring GET /wifconfigs/{id} response time") - const samples = 5 - durations := make([]time.Duration, samples) - for i := range samples { - start := time.Now() - _, err = h.Client.GetWifConfig(ctx, wifConfigID) - Expect(err).NotTo(HaveOccurred()) - durations[i] = time.Since(start) - } - slices.Sort(durations) - median := durations[samples/2] - ginkgo.GinkgoWriter.Printf("[PERF] GET /wifconfigs/%s latency: %v (median of %d samples)\n", wifConfigID, median, samples) - Expect(median).To(BeNumerically("<", config.ThresholdAPIRead), - "GET /wifconfigs/{id} exceeded threshold") + helper.MeasureMedianLatency("GET /wifconfigs/{id}", config.ThresholdAPIRead, helper.DefaultSamples, + func(int) { + _, err := h.Client.GetWifConfig(ctx, wifConfigID) + Expect(err).NotTo(HaveOccurred()) + }, + ) }) }, ) diff --git a/e2e/wifconfig/perf_list_latency.go b/e2e/wifconfig/perf_list_latency.go index 1fafdcf0..38299399 100644 --- a/e2e/wifconfig/perf_list_latency.go +++ b/e2e/wifconfig/perf_list_latency.go @@ -2,7 +2,6 @@ package wifconfig import ( "context" - "time" "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" //nolint:staticcheck // dot import for test readability @@ -14,6 +13,7 @@ import ( var _ = ginkgo.Describe("[Suite: wifconfig][perf] API list latency", ginkgo.Label(labels.Tier1, labels.Performance), + ginkgo.Serial, func() { var h *helper.Helper var wifConfigID string @@ -34,14 +34,12 @@ var _ = ginkgo.Describe("[Suite: wifconfig][perf] API list latency", }) ginkgo.It("should list wifconfigs within acceptable latency", func(ctx context.Context) { - ginkgo.By("measuring GET /wifconfigs response time") - start := time.Now() - _, err := h.Client.ListWifConfigs(ctx, "") - Expect(err).NotTo(HaveOccurred()) - elapsed := time.Since(start) - ginkgo.GinkgoWriter.Printf("[PERF] GET /wifconfigs latency: %v\n", elapsed) - Expect(elapsed).To(BeNumerically("<", config.ThresholdAPIList), - "GET /wifconfigs exceeded threshold") + helper.MeasureMedianLatency("GET /wifconfigs", config.ThresholdAPIList, helper.DefaultSamples, + func(int) { + _, err := h.Client.ListWifConfigs(ctx, "") + Expect(err).NotTo(HaveOccurred()) + }, + ) }) }, ) diff --git a/e2e/wifconfig/perf_update_latency.go b/e2e/wifconfig/perf_update_latency.go index 8925a60b..80d60f85 100644 --- a/e2e/wifconfig/perf_update_latency.go +++ b/e2e/wifconfig/perf_update_latency.go @@ -2,7 +2,7 @@ package wifconfig import ( "context" - "time" + "fmt" "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" //nolint:staticcheck // dot import for test readability @@ -15,6 +15,7 @@ import ( var _ = ginkgo.Describe("[Suite: wifconfig][perf] Update latency", ginkgo.Label(labels.Tier1, labels.Performance), + ginkgo.Serial, func() { var h *helper.Helper var wifConfigID string @@ -35,22 +36,17 @@ var _ = ginkgo.Describe("[Suite: wifconfig][perf] Update latency", }) ginkgo.It("should update a wifconfig within acceptable latency", func(ctx context.Context) { - ginkgo.By("patching wifconfig and timing the response") - start := time.Now() - - patched, err := h.Client.PatchWifConfig(ctx, wifConfigID, client.ResourcePatchRequest{ - Spec: map[string]any{ - "projectId": "updated-project", - "version": "4.18", + helper.MeasureMedianLatency("PATCH /wifconfigs/{id}", config.ThresholdAPIUpdate, helper.DefaultSamples, + func(i int) { + _, err := h.Client.PatchWifConfig(ctx, wifConfigID, client.ResourcePatchRequest{ + Spec: map[string]any{ + "projectId": fmt.Sprintf("updated-project-%d", i), + "version": "4.18", + }, + }) + Expect(err).NotTo(HaveOccurred()) }, - }) - Expect(err).NotTo(HaveOccurred()) - Expect(patched.Generation).To(Equal(int32(2)), "generation should increment after PATCH") - elapsed := time.Since(start) - - ginkgo.GinkgoWriter.Printf("[PERF] PATCH /wifconfigs/%s latency: %v\n", wifConfigID, elapsed) - Expect(elapsed).To(BeNumerically("<", config.ThresholdAPIUpdate), - "wifconfig update exceeded threshold") + ) }) }, ) diff --git a/pkg/config/thresholds.go b/pkg/config/thresholds.go index cf914a50..1a66c4fc 100644 --- a/pkg/config/thresholds.go +++ b/pkg/config/thresholds.go @@ -31,8 +31,8 @@ const ( // Calibrated from a GKE-dev baseline run (~1k seeded rows per kind, 2026-07-27); // observed latencies were 3-10ms across all three kinds, well under this shared // threshold — see hyperfleet/docs/performance-baselines.md in the architecture repo. -// No Prow tier1-nightly baseline exists yet for these specific operations (the specs -// are new); revisit this value once the first post-merge nightly run lands. +// No Prow tier1-nightly baseline yet; recalibrate once 5+ clean nightly runs +// are available now that these specs run Serial. const ( ThresholdAPICreate = 50 * time.Millisecond ThresholdAPIUpdate = 50 * time.Millisecond diff --git a/pkg/e2e/suite.go b/pkg/e2e/suite.go index bf853826..b5b9599c 100644 --- a/pkg/e2e/suite.go +++ b/pkg/e2e/suite.go @@ -27,20 +27,19 @@ func GetSuiteConfig() *config.Config { return suiteConfig } -var _ = ginkgo.BeforeSuite(func(ctx ginkgo.SpecContext) { - cfg := GetSuiteConfig() - if cfg == nil { - log.Fatalf("Suite config not initialized") - } - - if err := logger.Init(&cfg.Log, "dev"); err != nil { - log.Fatalf("Failed to initialize logger: %v", err) - } +var _ = ginkgo.SynchronizedBeforeSuite( + // Process 1 only: mint the JWT token once and share it, so parallel + // processes don't each burn a redundant TokenRequest call at startup. + func(ctx ginkgo.SpecContext) []byte { + cfg := GetSuiteConfig() + if cfg == nil { + log.Fatalf("Suite config not initialized") + } - cfg.Display() - logger.Info("starting hyperfleet-e2e test suite - creating resources with", "run-id", cfg.RunID) + if !cfg.Identity.TokenRequest.IsEnabled() { + return nil + } - if cfg.Identity.TokenRequest.IsEnabled() { k8s, err := k8sclient.NewClient() if err != nil { log.Fatalf("Failed to create K8s client for token acquisition: %v", err) @@ -55,19 +54,39 @@ var _ = ginkgo.BeforeSuite(func(ctx ginkgo.SpecContext) { if err != nil { log.Fatalf("Failed to acquire JWT via TokenRequest: %v", err) } - cfg.Identity.SetToken(token) - logger.Info("acquired JWT for suite", - "service-account", cfg.Identity.TokenRequest.Namespace+"/"+cfg.Identity.TokenRequest.ServiceAccountName, - "audience", cfg.Identity.TokenRequest.Audience, - "expires-seconds", cfg.Identity.TokenRequest.ExpirationSeconds) - } + return []byte(token) + }, + // Runs on every process: apply the token minted above (if any) and finish suite setup. + func(ctx ginkgo.SpecContext, tokenBytes []byte) { + cfg := GetSuiteConfig() + if cfg == nil { + log.Fatalf("Suite config not initialized") + } - // Initialize adapter deployment list - for test tiers that deploy temporary adapters - adapterDeploymentList := helper.InitAdapterDeploymentList() - helper.SetAdapterDeploymentList(adapterDeploymentList) + if err := logger.Init(&cfg.Log, "dev"); err != nil { + log.Fatalf("Failed to initialize logger: %v", err) + } - logger.Info("starting hyperfleet-e2e test suite - each test creates temporary resources") -}) + if ginkgo.GinkgoParallelProcess() == 1 { + cfg.Display() + } + logger.Info("starting hyperfleet-e2e test suite - creating resources with", "run-id", cfg.RunID) + + if len(tokenBytes) > 0 { + cfg.Identity.SetToken(string(tokenBytes)) + logger.Info("acquired JWT for suite", + "service-account", cfg.Identity.TokenRequest.Namespace+"/"+cfg.Identity.TokenRequest.ServiceAccountName, + "audience", cfg.Identity.TokenRequest.Audience, + "expires-seconds", cfg.Identity.TokenRequest.ExpirationSeconds) + } + + // Initialize adapter deployment list - for test tiers that deploy temporary adapters. + adapterDeploymentList := helper.InitAdapterDeploymentList() + helper.SetAdapterDeploymentList(adapterDeploymentList) + + logger.Info("starting hyperfleet-e2e test suite - each test creates temporary resources") + }, +) var _ = ginkgo.SynchronizedAfterSuite( // Per-process: sweep Pub/Sub resources. Safe to call from every process diff --git a/pkg/helper/helper.go b/pkg/helper/helper.go index b9340b61..b437608d 100644 --- a/pkg/helper/helper.go +++ b/pkg/helper/helper.go @@ -183,6 +183,26 @@ func (h *Helper) CleanupTestWifConfig(ctx context.Context, wifConfigID string) e return nil } +// DeferChannelCleanup registers a DeferCleanup that will delete the channel after the test, +// regardless of pass/fail. If cleanup fails, it logs a warning. +func (h *Helper) DeferChannelCleanup(channelID string) { + ginkgo.DeferCleanup(func(ctx context.Context) { + if err := h.CleanupTestChannel(ctx, channelID); err != nil { + ginkgo.GinkgoWriter.Printf("Warning: failed to cleanup channel %s: %v\n", channelID, err) + } + }) +} + +// DeferWifConfigCleanup registers a DeferCleanup that will delete the wifconfig after the test, +// regardless of pass/fail. If cleanup fails, it logs a warning. +func (h *Helper) DeferWifConfigCleanup(wifConfigID string) { + ginkgo.DeferCleanup(func(ctx context.Context) { + if err := h.CleanupTestWifConfig(ctx, wifConfigID); err != nil { + ginkgo.GinkgoWriter.Printf("Warning: failed to cleanup wifconfig %s: %v\n", wifConfigID, err) + } + }) +} + // ExpectedIdentity returns the configured expected audit identity value. // Returns empty string if not configured, signalling callers to skip audit assertions. func (h *Helper) ExpectedIdentity() string { diff --git a/pkg/helper/perf.go b/pkg/helper/perf.go new file mode 100644 index 00000000..52c707be --- /dev/null +++ b/pkg/helper/perf.go @@ -0,0 +1,56 @@ +package helper + +import ( + "slices" + "time" + + "github.com/onsi/ginkgo/v2" + "github.com/onsi/gomega" +) + +// init registers a fail handler so gomega.Expect works under +// `go test ./pkg/helper/...`, outside the e2e binary's own registration in +// pkg/e2e/e2e.go. Double registration is harmless. +func init() { + gomega.RegisterFailHandler(ginkgo.Fail) +} + +// DefaultSamples is the sample count for perf specs measuring a single, +// self-contained operation. +const DefaultSamples = 5 + +// MeasureMedianLatency calls sampleFn n times with its index, records each +// call's duration, and asserts the median is below threshold. +// +// ASSUMPTION: the HTTP connection pool is already warm, since callers run +// ginkgo.Serial after other specs have used it. +func MeasureMedianLatency(name string, threshold time.Duration, n int, sampleFn func(i int)) { + gomega.Expect(n).To(gomega.BeNumerically(">", 0), "MeasureMedianLatency: sample count must be positive, got %d", n) + + durations := make([]time.Duration, n) + for i := range n { + start := time.Now() + sampleFn(i) + durations[i] = time.Since(start) + } + + assertMedianBelowThreshold(name, threshold, durations) +} + +// assertMedianBelowThreshold computes the median of durations (averaging the +// two middle values for an even count) and asserts it's below threshold. +// durations must be non-empty; callers are responsible for that (see the +// n > 0 check in MeasureMedianLatency). +func assertMedianBelowThreshold(name string, threshold time.Duration, durations []time.Duration) { + slices.Sort(durations) + + n := len(durations) + median := durations[n/2] + if n%2 == 0 { + median = (durations[n/2-1] + durations[n/2]) / 2 + } + ginkgo.GinkgoWriter.Printf("[PERF] [proc %d] %s: median=%v (n=%d)\n", ginkgo.GinkgoParallelProcess(), name, median, n) + + gomega.Expect(median).To(gomega.BeNumerically("<", threshold), + "%s exceeded threshold: median=%v threshold=%v", name, median, threshold) +} diff --git a/pkg/helper/perf_test.go b/pkg/helper/perf_test.go new file mode 100644 index 00000000..d2d77b54 --- /dev/null +++ b/pkg/helper/perf_test.go @@ -0,0 +1,124 @@ +package helper + +import ( + "fmt" + "testing" + "time" + + "github.com/onsi/gomega" +) + +// TestAssertMedianBelowThreshold exercises the median/threshold math with +// exact durations, independent of real elapsed time. +func TestAssertMedianBelowThreshold(t *testing.T) { + tests := []struct { + name string + threshold time.Duration + durations []time.Duration + wantFailure bool + }{ + { + name: "below-threshold", + threshold: 50 * time.Millisecond, + durations: []time.Duration{1 * time.Millisecond, 2 * time.Millisecond, 3 * time.Millisecond}, + }, + { + name: "above-threshold", + threshold: 50 * time.Millisecond, + durations: []time.Duration{60 * time.Millisecond, 70 * time.Millisecond, 80 * time.Millisecond}, + wantFailure: true, + }, + { + // Strict '<': an exact match must still fail. + name: "equals-threshold", + threshold: 50 * time.Millisecond, + durations: []time.Duration{50 * time.Millisecond}, + wantFailure: true, + }, + { + // 100ms outlier among four 1ms samples: mean (~20.8ms) exceeds + // the threshold, median (1ms) doesn't. + name: "median-not-mean", + threshold: 15 * time.Millisecond, + durations: []time.Duration{ + 1 * time.Millisecond, 1 * time.Millisecond, 100 * time.Millisecond, 1 * time.Millisecond, 1 * time.Millisecond, + }, + }, + { + // 10/20/40/50ms averages to 30ms; lower-middle (20ms) would + // wrongly pass this threshold. + name: "even-sample-count-lower-middle", + threshold: 25 * time.Millisecond, + durations: []time.Duration{10 * time.Millisecond, 20 * time.Millisecond, 40 * time.Millisecond, 50 * time.Millisecond}, + wantFailure: true, + }, + { + // 10/20/40/50ms averages to 30ms; upper-middle (40ms) would + // wrongly fail this threshold. + name: "even-sample-count-upper-middle", + threshold: 35 * time.Millisecond, + durations: []time.Duration{10 * time.Millisecond, 20 * time.Millisecond, 40 * time.Millisecond, 50 * time.Millisecond}, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + failure := gomega.InterceptGomegaFailure(func() { + assertMedianBelowThreshold(tt.name, tt.threshold, tt.durations) + }) + + if tt.wantFailure && failure == nil { + t.Fatal("expected a failure, got none") + } + if !tt.wantFailure && failure != nil { + t.Fatalf("expected no failure, got: %v", failure) + } + }) + } +} + +// TestMeasureMedianLatency exercises the real-timing wrapper: call count, +// index passing, and non-positive n rejection. The threshold is generous +// since no case here depends on the median/threshold comparison itself. +func TestMeasureMedianLatency(t *testing.T) { + t.Run("calls sampleFn exactly n times with its index", func(t *testing.T) { + const n = 7 + var gotIndexes []int + + failure := gomega.InterceptGomegaFailure(func() { + MeasureMedianLatency("sample-n-times", time.Second, n, func(i int) { + gotIndexes = append(gotIndexes, i) + }) + }) + + if failure != nil { + t.Fatalf("unexpected failure: %v", failure) + } + if len(gotIndexes) != n { + t.Fatalf("expected sampleFn to be called %d times, got %d", n, len(gotIndexes)) + } + for i, got := range gotIndexes { + if got != i { + t.Fatalf("expected call %d to receive index %d, got %d", i, i, got) + } + } + }) + + t.Run("rejects non-positive sample counts", func(t *testing.T) { + for _, n := range []int{0, -1} { + t.Run(fmt.Sprintf("n=%d", n), func(t *testing.T) { + called := false + failure := gomega.InterceptGomegaFailure(func() { + MeasureMedianLatency("invalid-n", 50*time.Millisecond, n, func(int) { called = true }) + }) + + if failure == nil { + t.Fatalf("expected a validation failure for n=%d, got none", n) + } + if called { + t.Fatalf("expected sampleFn not to be called for n=%d", n) + } + }) + } + }) +}