Skip to content

chore(objectstorage): add retry on 429 for objectstorage control plane #1764

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
h3adex wants to merge 1 commit into
base: main
Choose a base branch
from
Draft

chore(objectstorage): add retry on 429 for objectstorage control plane #1764

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 5 additions & 1 deletion stackit/internal/services/objectstorage/bucket/datasource.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -130,7 +130,11 @@ func (r *bucketDataSource) Read(ctx context.Context, req datasource.ReadRequest,
ctx = tflog.SetField(ctx, "name", bucketName)
ctx = tflog.SetField(ctx, "region", region)

bucketResp, err := r.client.DefaultAPI.GetBucket(ctx, projectId, region, bucketName).Execute()
bucketResp, err := utils.RetryRequest(
ctx,
r.client.DefaultAPI.GetBucket(ctx, projectId, region, bucketName).Execute,
objectstorageUtils.RateLimitRetryConfig,
)
if err != nil {
utils.LogError(
ctx,
Expand Down
18 changes: 15 additions & 3 deletions stackit/internal/services/objectstorage/bucket/resource.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -218,7 +218,11 @@ func (r *bucketResource) Create(ctx context.Context, req resource.CreateRequest,
}

// Create new bucket
_, err = r.client.DefaultAPI.CreateBucket(ctx, projectId, region, bucketName).ObjectLockEnabled(model.ObjectLock.ValueBool()).Execute()
_, err = utils.RetryRequest(
ctx,
r.client.DefaultAPI.CreateBucket(ctx, projectId, region, bucketName).ObjectLockEnabled(model.ObjectLock.ValueBool()).Execute,
objectstorageUtils.RateLimitRetryConfig,
)
if err != nil {
core.LogAndAddError(ctx, &resp.Diagnostics, "Error creating bucket", fmt.Sprintf("Calling API: %v", err))
return
Expand Down Expand Up @@ -275,7 +279,11 @@ func (r *bucketResource) Read(ctx context.Context, req resource.ReadRequest, res
ctx = tflog.SetField(ctx, "name", bucketName)
ctx = tflog.SetField(ctx, "region", region)

bucketResp, err := r.client.DefaultAPI.GetBucket(ctx, projectId, region, bucketName).Execute()
bucketResp, err := utils.RetryRequest(
ctx,
r.client.DefaultAPI.GetBucket(ctx, projectId, region, bucketName).Execute,
objectstorageUtils.RateLimitRetryConfig,
)
if err != nil {
var oapiErr *oapierror.GenericOpenAPIError
if errors.As(err, &oapiErr) && oapiErr.StatusCode == http.StatusNotFound {
Expand Down Expand Up @@ -330,7 +338,11 @@ func (r *bucketResource) Delete(ctx context.Context, req resource.DeleteRequest,
ctx = tflog.SetField(ctx, "region", region)

// Delete existing bucket
_, err := r.client.DefaultAPI.DeleteBucket(ctx, projectId, region, bucketName).Execute()
_, err := utils.RetryRequest(
ctx,
r.client.DefaultAPI.DeleteBucket(ctx, projectId, region, bucketName).Execute,
objectstorageUtils.RateLimitRetryConfig,
)
if err != nil {
var oapiErr *oapierror.GenericOpenAPIError
if errors.As(err, &oapiErr) {
Expand Down
6 changes: 5 additions & 1 deletion stackit/internal/services/objectstorage/credential/datasource.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -133,7 +133,11 @@ func (r *credentialDataSource) Read(ctx context.Context, req datasource.ReadRequ
ctx = tflog.SetField(ctx, "credential_id", credentialId)
ctx = tflog.SetField(ctx, "region", region)

credentialsGroupResp, err := r.client.DefaultAPI.ListAccessKeys(ctx, projectId, region).CredentialsGroup(credentialsGroupId).Execute()
credentialsGroupResp, err := utils.RetryRequest(
ctx,
r.client.DefaultAPI.ListAccessKeys(ctx, projectId, region).CredentialsGroup(credentialsGroupId).Execute,
objectstorageUtils.RateLimitRetryConfig,
)
if err != nil {
utils.LogError(
ctx,
Expand Down
18 changes: 15 additions & 3 deletions stackit/internal/services/objectstorage/credential/resource.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -294,7 +294,11 @@ func (r *credentialResource) Create(ctx context.Context, req resource.CreateRequ
return
}
// Create new credential
credentialResp, err := r.client.DefaultAPI.CreateAccessKey(ctx, projectId, region).CredentialsGroup(credentialsGroupId).CreateAccessKeyPayload(*payload).Execute()
credentialResp, err := utils.RetryRequest(
ctx,
r.client.DefaultAPI.CreateAccessKey(ctx, projectId, region).CredentialsGroup(credentialsGroupId).CreateAccessKeyPayload(*payload).Execute,
objectstorageUtils.RateLimitRetryConfig,
)
if err != nil {
core.LogAndAddError(ctx, &resp.Diagnostics, "Error creating credential", fmt.Sprintf("Calling API: %v", err))
return
Expand Down Expand Up @@ -454,7 +458,11 @@ func (r *credentialResource) Delete(ctx context.Context, req resource.DeleteRequ
ctx = tflog.SetField(ctx, "region", region)

// Delete existing credential
_, err := r.client.DefaultAPI.DeleteAccessKey(ctx, projectId, region, credentialId).CredentialsGroup(credentialsGroupId).Execute()
_, err := utils.RetryRequest(
ctx,
r.client.DefaultAPI.DeleteAccessKey(ctx, projectId, region, credentialId).CredentialsGroup(credentialsGroupId).Execute,
objectstorageUtils.RateLimitRetryConfig,
)
if err != nil {
var oapiErr *oapierror.GenericOpenAPIError
if errors.As(err, &oapiErr) && oapiErr.StatusCode == http.StatusNotFound {
Expand Down Expand Up @@ -560,7 +568,11 @@ func readCredentials(ctx context.Context, model *Model, region string, client *o
credentialsGroupId := model.CredentialsGroupId.ValueString()
credentialId := model.CredentialId.ValueString()

credentialsGroupResp, err := client.DefaultAPI.ListAccessKeys(ctx, projectId, region).CredentialsGroup(credentialsGroupId).Execute()
credentialsGroupResp, err := utils.RetryRequest(
ctx,
client.DefaultAPI.ListAccessKeys(ctx, projectId, region).CredentialsGroup(credentialsGroupId).Execute,
objectstorageUtils.RateLimitRetryConfig,
)
if err != nil {
var oapiErr *oapierror.GenericOpenAPIError
if errors.As(err, &oapiErr) && oapiErr.StatusCode == http.StatusNotFound {
Expand Down
18 changes: 15 additions & 3 deletions stackit/internal/services/objectstorage/credentialsgroup/resource.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -199,7 +199,11 @@ func (r *credentialsGroupResource) Create(ctx context.Context, req resource.Crea
}

// Create new credentials group
got, err := r.client.DefaultAPI.CreateCredentialsGroup(ctx, projectId, region).CreateCredentialsGroupPayload(createCredentialsGroupPayload).Execute()
got, err := utils.RetryRequest(
ctx,
r.client.DefaultAPI.CreateCredentialsGroup(ctx, projectId, region).CreateCredentialsGroupPayload(createCredentialsGroupPayload).Execute,
objectstorageUtils.RateLimitRetryConfig,
)
if err != nil {
core.LogAndAddError(ctx, &resp.Diagnostics, "Error creating credentials group", fmt.Sprintf("Calling API: %v", err))
return
Expand Down Expand Up @@ -310,7 +314,11 @@ func (r *credentialsGroupResource) Delete(ctx context.Context, req resource.Dele
ctx = tflog.SetField(ctx, "region", region)

// Delete existing credentials group
_, err := r.client.DefaultAPI.DeleteCredentialsGroup(ctx, projectId, region, credentialsGroupId).Execute()
_, err := utils.RetryRequest(
ctx,
r.client.DefaultAPI.DeleteCredentialsGroup(ctx, projectId, region, credentialsGroupId).Execute,
objectstorageUtils.RateLimitRetryConfig,
)
if err != nil {
var oapiErr *oapierror.GenericOpenAPIError
if errors.As(err, &oapiErr) && oapiErr.StatusCode == http.StatusNotFound {
Expand Down Expand Up @@ -390,7 +398,11 @@ func readCredentialsGroups(ctx context.Context, model *Model, region string, cli
return found, fmt.Errorf("missing configuration: either name or credentials group id must be provided")
}

credentialsGroupsResp, err := client.ListCredentialsGroups(ctx, model.ProjectId.ValueString(), region).Execute()
credentialsGroupsResp, err := utils.RetryRequest(
ctx,
client.ListCredentialsGroups(ctx, model.ProjectId.ValueString(), region).Execute,
objectstorageUtils.RateLimitRetryConfig,
)
if err != nil {
var oapiErr *oapierror.GenericOpenAPIError
if errors.As(err, &oapiErr) && oapiErr.StatusCode == http.StatusNotFound {
Expand Down
29 changes: 29 additions & 0 deletions stackit/internal/services/objectstorage/utils/util.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,35 @@ const (
enableProjectRetryDelay = 2 * time.Second
)

// RateLimitRetryConfig retries on HTTP 429 with exponential backoff.
//
// The Object Storage Control Plane is rate-limited to 80 req/min (~1.33 req/s).
// Large states trigger this during the parallel refresh phase (terraform plan/apply)
// and during bulk creates/deletes in a single apply.
//
// Practical example — 300 buckets in a single state:
// - Minimum time to process all requests at the rate limit: 300/80*60 = 225s (~3.75 min).
// - The first ~80 requests succeed immediately; the remaining ~220 receive 429 and retry.
// - With Terraform's default parallelism of 10, the retry waves clear roughly every 7.5s
// (10 goroutines / 1.33 req/s), so most goroutines need only 2–3 attempts.
// - Total retry budget of ~435s comfortably exceeds the 225s floor.
//
// Design rationale:
// - Starting delay of 5s: at 1.33 req/s refill, 500ms returns less than 1 new token —
// all goroutines would immediately fail again, burning attempts without progress.
// 5s refills ~6.7 tokens, enough for the majority of competing goroutines to succeed.
// - Cap of 60s: covers a full fixed-window rate-limit reset so goroutines do not exhaust
// their budget before the 1-minute window clears.
// - 10 attempts: backoff schedule 5s+10s+20s+40s+(5×60s) = 435s total budget.
var RateLimitRetryConfig = utils.RetryConfig{
Attempts: 10,
Backoff: func(attempt int) time.Duration {
// Exponential backoff: 5s, 10s, 20s, 40s, 60s (capped)
return min(5*time.Second*(1<<uint(attempt-1)), 60*time.Second)
},
RetryStatusCodes: []int{http.StatusTooManyRequests},
}

// EnableProject enables object storage for the specified project. If the project is already enabled, nothing happens.
// Two resources created in the same apply call this concurrently and the API rejects the losing call with
// 409 project.create_conflict; retrying is safe, since enabling an already enabled project succeeds.
Expand Down
Loading