diff --git a/stackit/internal/services/objectstorage/bucket/datasource.go b/stackit/internal/services/objectstorage/bucket/datasource.go index 9130ef6c9..33786d171 100644 --- a/stackit/internal/services/objectstorage/bucket/datasource.go +++ b/stackit/internal/services/objectstorage/bucket/datasource.go @@ -130,7 +130,11 @@ func (r *bucketDataSource) Read(ctx context.Context, req datasource.ReadRequest, ctx = tflog.SetField(ctx, "name", bucketName) ctx = tflog.SetField(ctx, "region", region) - bucketResp, err := r.client.DefaultAPI.GetBucket(ctx, projectId, region, bucketName).Execute() + bucketResp, err := utils.RetryRequest( + ctx, + r.client.DefaultAPI.GetBucket(ctx, projectId, region, bucketName).Execute, + objectstorageUtils.RateLimitRetryConfig, + ) if err != nil { utils.LogError( ctx, diff --git a/stackit/internal/services/objectstorage/bucket/resource.go b/stackit/internal/services/objectstorage/bucket/resource.go index 7bcb1cab5..44cd178e7 100644 --- a/stackit/internal/services/objectstorage/bucket/resource.go +++ b/stackit/internal/services/objectstorage/bucket/resource.go @@ -218,7 +218,11 @@ func (r *bucketResource) Create(ctx context.Context, req resource.CreateRequest, } // Create new bucket - _, err = r.client.DefaultAPI.CreateBucket(ctx, projectId, region, bucketName).ObjectLockEnabled(model.ObjectLock.ValueBool()).Execute() + _, err = utils.RetryRequest( + ctx, + r.client.DefaultAPI.CreateBucket(ctx, projectId, region, bucketName).ObjectLockEnabled(model.ObjectLock.ValueBool()).Execute, + objectstorageUtils.RateLimitRetryConfig, + ) if err != nil { core.LogAndAddError(ctx, &resp.Diagnostics, "Error creating bucket", fmt.Sprintf("Calling API: %v", err)) return @@ -275,7 +279,11 @@ func (r *bucketResource) Read(ctx context.Context, req resource.ReadRequest, res ctx = tflog.SetField(ctx, "name", bucketName) ctx = tflog.SetField(ctx, "region", region) - bucketResp, err := r.client.DefaultAPI.GetBucket(ctx, projectId, region, bucketName).Execute() + bucketResp, err := utils.RetryRequest( + ctx, + r.client.DefaultAPI.GetBucket(ctx, projectId, region, bucketName).Execute, + objectstorageUtils.RateLimitRetryConfig, + ) if err != nil { var oapiErr *oapierror.GenericOpenAPIError if errors.As(err, &oapiErr) && oapiErr.StatusCode == http.StatusNotFound { @@ -330,7 +338,11 @@ func (r *bucketResource) Delete(ctx context.Context, req resource.DeleteRequest, ctx = tflog.SetField(ctx, "region", region) // Delete existing bucket - _, err := r.client.DefaultAPI.DeleteBucket(ctx, projectId, region, bucketName).Execute() + _, err := utils.RetryRequest( + ctx, + r.client.DefaultAPI.DeleteBucket(ctx, projectId, region, bucketName).Execute, + objectstorageUtils.RateLimitRetryConfig, + ) if err != nil { var oapiErr *oapierror.GenericOpenAPIError if errors.As(err, &oapiErr) { diff --git a/stackit/internal/services/objectstorage/credential/datasource.go b/stackit/internal/services/objectstorage/credential/datasource.go index 9d8b96a3b..d9efdd8c5 100644 --- a/stackit/internal/services/objectstorage/credential/datasource.go +++ b/stackit/internal/services/objectstorage/credential/datasource.go @@ -133,7 +133,11 @@ func (r *credentialDataSource) Read(ctx context.Context, req datasource.ReadRequ ctx = tflog.SetField(ctx, "credential_id", credentialId) ctx = tflog.SetField(ctx, "region", region) - credentialsGroupResp, err := r.client.DefaultAPI.ListAccessKeys(ctx, projectId, region).CredentialsGroup(credentialsGroupId).Execute() + credentialsGroupResp, err := utils.RetryRequest( + ctx, + r.client.DefaultAPI.ListAccessKeys(ctx, projectId, region).CredentialsGroup(credentialsGroupId).Execute, + objectstorageUtils.RateLimitRetryConfig, + ) if err != nil { utils.LogError( ctx, diff --git a/stackit/internal/services/objectstorage/credential/resource.go b/stackit/internal/services/objectstorage/credential/resource.go index 6321d5e15..79037ad14 100644 --- a/stackit/internal/services/objectstorage/credential/resource.go +++ b/stackit/internal/services/objectstorage/credential/resource.go @@ -294,7 +294,11 @@ func (r *credentialResource) Create(ctx context.Context, req resource.CreateRequ return } // Create new credential - credentialResp, err := r.client.DefaultAPI.CreateAccessKey(ctx, projectId, region).CredentialsGroup(credentialsGroupId).CreateAccessKeyPayload(*payload).Execute() + credentialResp, err := utils.RetryRequest( + ctx, + r.client.DefaultAPI.CreateAccessKey(ctx, projectId, region).CredentialsGroup(credentialsGroupId).CreateAccessKeyPayload(*payload).Execute, + objectstorageUtils.RateLimitRetryConfig, + ) if err != nil { core.LogAndAddError(ctx, &resp.Diagnostics, "Error creating credential", fmt.Sprintf("Calling API: %v", err)) return @@ -454,7 +458,11 @@ func (r *credentialResource) Delete(ctx context.Context, req resource.DeleteRequ ctx = tflog.SetField(ctx, "region", region) // Delete existing credential - _, err := r.client.DefaultAPI.DeleteAccessKey(ctx, projectId, region, credentialId).CredentialsGroup(credentialsGroupId).Execute() + _, err := utils.RetryRequest( + ctx, + r.client.DefaultAPI.DeleteAccessKey(ctx, projectId, region, credentialId).CredentialsGroup(credentialsGroupId).Execute, + objectstorageUtils.RateLimitRetryConfig, + ) if err != nil { var oapiErr *oapierror.GenericOpenAPIError if errors.As(err, &oapiErr) && oapiErr.StatusCode == http.StatusNotFound { @@ -560,7 +568,11 @@ func readCredentials(ctx context.Context, model *Model, region string, client *o credentialsGroupId := model.CredentialsGroupId.ValueString() credentialId := model.CredentialId.ValueString() - credentialsGroupResp, err := client.DefaultAPI.ListAccessKeys(ctx, projectId, region).CredentialsGroup(credentialsGroupId).Execute() + credentialsGroupResp, err := utils.RetryRequest( + ctx, + client.DefaultAPI.ListAccessKeys(ctx, projectId, region).CredentialsGroup(credentialsGroupId).Execute, + objectstorageUtils.RateLimitRetryConfig, + ) if err != nil { var oapiErr *oapierror.GenericOpenAPIError if errors.As(err, &oapiErr) && oapiErr.StatusCode == http.StatusNotFound { diff --git a/stackit/internal/services/objectstorage/credentialsgroup/resource.go b/stackit/internal/services/objectstorage/credentialsgroup/resource.go index 6d69d0413..e9a967856 100644 --- a/stackit/internal/services/objectstorage/credentialsgroup/resource.go +++ b/stackit/internal/services/objectstorage/credentialsgroup/resource.go @@ -199,7 +199,11 @@ func (r *credentialsGroupResource) Create(ctx context.Context, req resource.Crea } // Create new credentials group - got, err := r.client.DefaultAPI.CreateCredentialsGroup(ctx, projectId, region).CreateCredentialsGroupPayload(createCredentialsGroupPayload).Execute() + got, err := utils.RetryRequest( + ctx, + r.client.DefaultAPI.CreateCredentialsGroup(ctx, projectId, region).CreateCredentialsGroupPayload(createCredentialsGroupPayload).Execute, + objectstorageUtils.RateLimitRetryConfig, + ) if err != nil { core.LogAndAddError(ctx, &resp.Diagnostics, "Error creating credentials group", fmt.Sprintf("Calling API: %v", err)) return @@ -310,7 +314,11 @@ func (r *credentialsGroupResource) Delete(ctx context.Context, req resource.Dele ctx = tflog.SetField(ctx, "region", region) // Delete existing credentials group - _, err := r.client.DefaultAPI.DeleteCredentialsGroup(ctx, projectId, region, credentialsGroupId).Execute() + _, err := utils.RetryRequest( + ctx, + r.client.DefaultAPI.DeleteCredentialsGroup(ctx, projectId, region, credentialsGroupId).Execute, + objectstorageUtils.RateLimitRetryConfig, + ) if err != nil { var oapiErr *oapierror.GenericOpenAPIError if errors.As(err, &oapiErr) && oapiErr.StatusCode == http.StatusNotFound { @@ -390,7 +398,11 @@ func readCredentialsGroups(ctx context.Context, model *Model, region string, cli return found, fmt.Errorf("missing configuration: either name or credentials group id must be provided") } - credentialsGroupsResp, err := client.ListCredentialsGroups(ctx, model.ProjectId.ValueString(), region).Execute() + credentialsGroupsResp, err := utils.RetryRequest( + ctx, + client.ListCredentialsGroups(ctx, model.ProjectId.ValueString(), region).Execute, + objectstorageUtils.RateLimitRetryConfig, + ) if err != nil { var oapiErr *oapierror.GenericOpenAPIError if errors.As(err, &oapiErr) && oapiErr.StatusCode == http.StatusNotFound { diff --git a/stackit/internal/services/objectstorage/utils/util.go b/stackit/internal/services/objectstorage/utils/util.go index 8e3ab1a8a..f27d0fe3a 100644 --- a/stackit/internal/services/objectstorage/utils/util.go +++ b/stackit/internal/services/objectstorage/utils/util.go @@ -20,6 +20,35 @@ const ( enableProjectRetryDelay = 2 * time.Second ) +// RateLimitRetryConfig retries on HTTP 429 with exponential backoff. +// +// The Object Storage Control Plane is rate-limited to 80 req/min (~1.33 req/s). +// Large states trigger this during the parallel refresh phase (terraform plan/apply) +// and during bulk creates/deletes in a single apply. +// +// Practical example — 300 buckets in a single state: +// - Minimum time to process all requests at the rate limit: 300/80*60 = 225s (~3.75 min). +// - The first ~80 requests succeed immediately; the remaining ~220 receive 429 and retry. +// - With Terraform's default parallelism of 10, the retry waves clear roughly every 7.5s +// (10 goroutines / 1.33 req/s), so most goroutines need only 2–3 attempts. +// - Total retry budget of ~435s comfortably exceeds the 225s floor. +// +// Design rationale: +// - Starting delay of 5s: at 1.33 req/s refill, 500ms returns less than 1 new token — +// all goroutines would immediately fail again, burning attempts without progress. +// 5s refills ~6.7 tokens, enough for the majority of competing goroutines to succeed. +// - Cap of 60s: covers a full fixed-window rate-limit reset so goroutines do not exhaust +// their budget before the 1-minute window clears. +// - 10 attempts: backoff schedule 5s+10s+20s+40s+(5×60s) = 435s total budget. +var RateLimitRetryConfig = utils.RetryConfig{ + Attempts: 10, + Backoff: func(attempt int) time.Duration { + // Exponential backoff: 5s, 10s, 20s, 40s, 60s (capped) + return min(5*time.Second*(1<