From d01196cc9a0979130197a596b76581cfab4e9875 Mon Sep 17 00:00:00 2001 From: Malte <140147670+umswmayj@users.noreply.github.com> Date: Wed, 19 Aug 2026 10:37:50 +0200 Subject: [PATCH 1/2] feat: apply jitter to failover revalidation requeue interval Add a RandFloat64 field to FailoverReservationController and a revalidationIntervalWithJitter() method that returns the base interval jittered uniformly within [base/2, 3*base/2]. All four call sites that previously used the raw RevalidationInterval duration now use this method. Tests cover the lower bound, upper bound, midpoint, and the nil (no-op) random source fallback. Signed-off-by: Malte <140147670+umswmayj@users.noreply.github.com> --- .../reservations/failover/controller.go | 23 ++++++-- .../reservations/failover/controller_test.go | 54 +++++++++++++++++++ 2 files changed, 73 insertions(+), 4 deletions(-) diff --git a/internal/scheduling/reservations/failover/controller.go b/internal/scheduling/reservations/failover/controller.go index 73e471f1d..d12eba4cc 100644 --- a/internal/scheduling/reservations/failover/controller.go +++ b/internal/scheduling/reservations/failover/controller.go @@ -45,6 +45,10 @@ type FailoverReservationController struct { Recorder events.EventRecorder // Event recorder for emitting Kubernetes events Monitor *FailoverMonitor reconcileCount int64 // Track reconciliation count for rotating VM selection + + // RandFloat64 returns a value in [0.0, 1.0). Used for revalidation interval jitter. + // If nil, math/rand/v2 rand.Float64 is used. Overridable in tests. + RandFloat64 func() float64 } func NewFailoverReservationController(c client.Client, vmSource reservations.VMSource, config FailoverConfig, schedulerClient *reservations.SchedulerClient, monitor *FailoverMonitor) *FailoverReservationController { @@ -94,7 +98,7 @@ func (c *FailoverReservationController) Reconcile(ctx context.Context, req ctrl. // Skip if no failover status (reservation not yet initialized by periodic controller) if res.Status.FailoverReservation == nil { logger.V(1).Info("skipping reservation without failover status") - return ctrl.Result{RequeueAfter: c.Config.RevalidationInterval.Duration}, nil + return ctrl.Result{RequeueAfter: c.revalidationIntervalWithJitter()}, nil } // Validate and acknowledge the reservation @@ -126,7 +130,7 @@ func (c *FailoverReservationController) reconcileValidateAndAcknowledge(ctx cont return ctrl.Result{}, patchErr } - return ctrl.Result{RequeueAfter: c.Config.RevalidationInterval.Duration}, nil + return ctrl.Result{RequeueAfter: c.revalidationIntervalWithJitter()}, nil } // Validate the reservation @@ -134,7 +138,7 @@ func (c *FailoverReservationController) reconcileValidateAndAcknowledge(ctx cont if validationErr != nil { logger.Error(validationErr, "transient error during reservation validation, will retry", "host", res.Status.Host) - return ctrl.Result{RequeueAfter: c.Config.RevalidationInterval.Duration}, nil + return ctrl.Result{RequeueAfter: c.revalidationIntervalWithJitter()}, nil } if !valid { @@ -172,7 +176,18 @@ func (c *FailoverReservationController) reconcileValidateAndAcknowledge(ctx cont logger.V(1).Info("reservation validation passed (no new changes to acknowledge)", "host", res.Status.Host) } - return ctrl.Result{RequeueAfter: c.Config.RevalidationInterval.Duration}, nil + return ctrl.Result{RequeueAfter: c.revalidationIntervalWithJitter()}, nil +} + +// revalidationIntervalWithJitter returns RevalidationInterval jittered uniformly +// within [Duration/2, 3*Duration/2] to spread requeues over time. +func (c *FailoverReservationController) revalidationIntervalWithJitter() time.Duration { + rnd := c.RandFloat64 + if rnd == nil { + rnd = rand.Float64 + } + base := c.Config.RevalidationInterval.Duration + return base/2 + time.Duration(rnd()*float64(base)) } // validateReservation validates that a reservation is still valid for all its allocated VMs. diff --git a/internal/scheduling/reservations/failover/controller_test.go b/internal/scheduling/reservations/failover/controller_test.go index 55a09cc03..1da614006 100644 --- a/internal/scheduling/reservations/failover/controller_test.go +++ b/internal/scheduling/reservations/failover/controller_test.go @@ -5,7 +5,9 @@ package failover import ( "context" + "math" "testing" + "time" "github.com/cobaltcore-dev/cortex/api/v1alpha1" "github.com/cobaltcore-dev/cortex/internal/scheduling/reservations" @@ -981,3 +983,55 @@ func TestSelectVMsToProcess(t *testing.T) { } }) } + +// ============================================================================ +// Test: revalidationIntervalWithJitter +// ============================================================================ + +func TestRevalidationIntervalWithJitter(t *testing.T) { + const base = 30 * time.Minute + c := &FailoverReservationController{ + Config: FailoverConfig{ + RevalidationInterval: metav1.Duration{Duration: base}, + }, + } + + t.Run("bounds at rnd=0 return base/2", func(t *testing.T) { + c.RandFloat64 = func() float64 { return 0 } + got := c.revalidationIntervalWithJitter() + if got != base/2 { + t.Errorf("rnd=0: got %v, want %v", got, base/2) + } + }) + + t.Run("bounds near rnd=1 return ~3*base/2", func(t *testing.T) { + // rand.Float64 returns values in [0.0, 1.0), so the true max is exclusive. + // Use a value close to 1 and assert the result is within a small delta of 3*base/2. + c.RandFloat64 = func() float64 { return 0.9999999 } + got := c.revalidationIntervalWithJitter() + want := 3 * base / 2 + epsilon := time.Millisecond + if math.Abs(float64(got-want)) > float64(epsilon) { + t.Errorf("rnd~1: got %v, want %v (±%v)", got, want, epsilon) + } + }) + + t.Run("midpoint rnd=0.5 returns base", func(t *testing.T) { + c.RandFloat64 = func() float64 { return 0.5 } + got := c.revalidationIntervalWithJitter() + if got != base { + t.Errorf("rnd=0.5: got %v, want %v", got, base) + } + }) + + t.Run("default (nil) source stays within [base/2, 3*base/2]", func(t *testing.T) { + c.RandFloat64 = nil + lo, hi := base/2, 3*base/2 + for i := range 1000 { + got := c.revalidationIntervalWithJitter() + if got < lo || got > hi { + t.Fatalf("iteration %d: got %v, want in [%v, %v]", i, got, lo, hi) + } + } + }) +} From 77015359225f70cb87bf5fcca0a8fcf96428d4d0 Mon Sep 17 00:00:00 2001 From: Malte <140147670+umswmayj@users.noreply.github.com> Date: Wed, 19 Aug 2026 10:56:09 +0200 Subject: [PATCH 2/2] simplify base on comment Signed-off-by: Malte <140147670+umswmayj@users.noreply.github.com> --- .../reservations/failover/controller.go | 24 ++------- .../reservations/failover/controller_test.go | 54 ------------------- 2 files changed, 5 insertions(+), 73 deletions(-) diff --git a/internal/scheduling/reservations/failover/controller.go b/internal/scheduling/reservations/failover/controller.go index d12eba4cc..c9bd443ba 100644 --- a/internal/scheduling/reservations/failover/controller.go +++ b/internal/scheduling/reservations/failover/controller.go @@ -18,6 +18,7 @@ import ( "github.com/cobaltcore-dev/cortex/pkg/multicluster" hv1 "github.com/cobaltcore-dev/openstack-hypervisor-operator/api/v1" "github.com/google/uuid" + "github.com/sapcc/go-bits/jobloop" corev1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/api/meta" @@ -45,10 +46,6 @@ type FailoverReservationController struct { Recorder events.EventRecorder // Event recorder for emitting Kubernetes events Monitor *FailoverMonitor reconcileCount int64 // Track reconciliation count for rotating VM selection - - // RandFloat64 returns a value in [0.0, 1.0). Used for revalidation interval jitter. - // If nil, math/rand/v2 rand.Float64 is used. Overridable in tests. - RandFloat64 func() float64 } func NewFailoverReservationController(c client.Client, vmSource reservations.VMSource, config FailoverConfig, schedulerClient *reservations.SchedulerClient, monitor *FailoverMonitor) *FailoverReservationController { @@ -98,7 +95,7 @@ func (c *FailoverReservationController) Reconcile(ctx context.Context, req ctrl. // Skip if no failover status (reservation not yet initialized by periodic controller) if res.Status.FailoverReservation == nil { logger.V(1).Info("skipping reservation without failover status") - return ctrl.Result{RequeueAfter: c.revalidationIntervalWithJitter()}, nil + return ctrl.Result{RequeueAfter: jobloop.DefaultJitter(c.Config.RevalidationInterval.Duration)}, nil } // Validate and acknowledge the reservation @@ -130,7 +127,7 @@ func (c *FailoverReservationController) reconcileValidateAndAcknowledge(ctx cont return ctrl.Result{}, patchErr } - return ctrl.Result{RequeueAfter: c.revalidationIntervalWithJitter()}, nil + return ctrl.Result{RequeueAfter: jobloop.DefaultJitter(c.Config.RevalidationInterval.Duration)}, nil } // Validate the reservation @@ -138,7 +135,7 @@ func (c *FailoverReservationController) reconcileValidateAndAcknowledge(ctx cont if validationErr != nil { logger.Error(validationErr, "transient error during reservation validation, will retry", "host", res.Status.Host) - return ctrl.Result{RequeueAfter: c.revalidationIntervalWithJitter()}, nil + return ctrl.Result{RequeueAfter: jobloop.DefaultJitter(c.Config.RevalidationInterval.Duration)}, nil } if !valid { @@ -176,18 +173,7 @@ func (c *FailoverReservationController) reconcileValidateAndAcknowledge(ctx cont logger.V(1).Info("reservation validation passed (no new changes to acknowledge)", "host", res.Status.Host) } - return ctrl.Result{RequeueAfter: c.revalidationIntervalWithJitter()}, nil -} - -// revalidationIntervalWithJitter returns RevalidationInterval jittered uniformly -// within [Duration/2, 3*Duration/2] to spread requeues over time. -func (c *FailoverReservationController) revalidationIntervalWithJitter() time.Duration { - rnd := c.RandFloat64 - if rnd == nil { - rnd = rand.Float64 - } - base := c.Config.RevalidationInterval.Duration - return base/2 + time.Duration(rnd()*float64(base)) + return ctrl.Result{RequeueAfter: jobloop.DefaultJitter(c.Config.RevalidationInterval.Duration)}, nil } // validateReservation validates that a reservation is still valid for all its allocated VMs. diff --git a/internal/scheduling/reservations/failover/controller_test.go b/internal/scheduling/reservations/failover/controller_test.go index 1da614006..55a09cc03 100644 --- a/internal/scheduling/reservations/failover/controller_test.go +++ b/internal/scheduling/reservations/failover/controller_test.go @@ -5,9 +5,7 @@ package failover import ( "context" - "math" "testing" - "time" "github.com/cobaltcore-dev/cortex/api/v1alpha1" "github.com/cobaltcore-dev/cortex/internal/scheduling/reservations" @@ -983,55 +981,3 @@ func TestSelectVMsToProcess(t *testing.T) { } }) } - -// ============================================================================ -// Test: revalidationIntervalWithJitter -// ============================================================================ - -func TestRevalidationIntervalWithJitter(t *testing.T) { - const base = 30 * time.Minute - c := &FailoverReservationController{ - Config: FailoverConfig{ - RevalidationInterval: metav1.Duration{Duration: base}, - }, - } - - t.Run("bounds at rnd=0 return base/2", func(t *testing.T) { - c.RandFloat64 = func() float64 { return 0 } - got := c.revalidationIntervalWithJitter() - if got != base/2 { - t.Errorf("rnd=0: got %v, want %v", got, base/2) - } - }) - - t.Run("bounds near rnd=1 return ~3*base/2", func(t *testing.T) { - // rand.Float64 returns values in [0.0, 1.0), so the true max is exclusive. - // Use a value close to 1 and assert the result is within a small delta of 3*base/2. - c.RandFloat64 = func() float64 { return 0.9999999 } - got := c.revalidationIntervalWithJitter() - want := 3 * base / 2 - epsilon := time.Millisecond - if math.Abs(float64(got-want)) > float64(epsilon) { - t.Errorf("rnd~1: got %v, want %v (±%v)", got, want, epsilon) - } - }) - - t.Run("midpoint rnd=0.5 returns base", func(t *testing.T) { - c.RandFloat64 = func() float64 { return 0.5 } - got := c.revalidationIntervalWithJitter() - if got != base { - t.Errorf("rnd=0.5: got %v, want %v", got, base) - } - }) - - t.Run("default (nil) source stays within [base/2, 3*base/2]", func(t *testing.T) { - c.RandFloat64 = nil - lo, hi := base/2, 3*base/2 - for i := range 1000 { - got := c.revalidationIntervalWithJitter() - if got < lo || got > hi { - t.Fatalf("iteration %d: got %v, want in [%v, %v]", i, got, lo, hi) - } - } - }) -}