From 4d5b4ce5ecc4b5442e6e713848ee9fdcaf1dfdc6 Mon Sep 17 00:00:00 2001 From: Mariem Baccari Date: Wed, 7 Oct 2026 14:40:36 +0200 Subject: [PATCH 1/3] [deploy] Make CockroachDB cache and max-sql-memory configurable --- NEXT_RELEASE_NOTES.md | 1 + .../terraform-commons-dss/helm.tf | 8 ++-- .../terraform-commons-dss/tanka.tf | 2 + .../templates/main.jsonnet.tmp | 2 + .../terraform-commons-dss/variables.gen.tf | 26 +++++++++++++ .../modules/terraform-aws-dss/TFVARS.gen.md | 12 ++++++ .../modules/terraform-aws-dss/main.tf | 2 + .../terraform-aws-dss/variables.gen.tf | 26 +++++++++++++ .../terraform-google-dss/TFVARS.gen.md | 12 ++++++ .../modules/terraform-google-dss/main.tf | 2 + .../terraform-google-dss/variables.gen.tf | 26 +++++++++++++ .../utils/definitions/crdb_cache.tf | 11 ++++++ .../utils/definitions/crdb_max_sql_memory.tf | 11 ++++++ deploy/infrastructure/utils/variables.py | 2 + deploy/operations/ci/aws-1/variables.gen.tf | 26 +++++++++++++ deploy/services/tanka/cockroachdb.libsonnet | 4 +- deploy/services/tanka/metadata_base.libsonnet | 2 + docs/operations/performance.md | 37 +++++++++++++++++++ 18 files changed, 207 insertions(+), 5 deletions(-) create mode 100644 deploy/infrastructure/utils/definitions/crdb_cache.tf create mode 100644 deploy/infrastructure/utils/definitions/crdb_max_sql_memory.tf diff --git a/NEXT_RELEASE_NOTES.md b/NEXT_RELEASE_NOTES.md index 9df11430a..c68b7cf16 100644 --- a/NEXT_RELEASE_NOTES.md +++ b/NEXT_RELEASE_NOTES.md @@ -51,6 +51,7 @@ The release notes should contain at least the following sections: * [deploy] AWS NLB TLS listeners (DSS gateway and Prometheus, for both Helm and Tanka) now use the `ELBSecurityPolicy-TLS13-1-2-Res-2021-06` SSL negotiation policy, which enforces a minimum of TLS 1.2 (aligning with the Google deployment) and restricts TLS 1.2 to AEAD (GCM/ChaCha20) cipher suites. Previously the AWS Load Balancer Controller default (`ELBSecurityPolicy-2016-08`) applied, which allowed TLS 1.0 and 1.1. Clients limited to TLS 1.0 or 1.1, or to CBC cipher suites, will no longer be able to connect. Unlike the Google deployment, no resources need to be provisioned by Terraform or manually to set up the policy. * [evict] RID cleanup (`db-manager evict`, including the RID cleanup default `CronJob`) now evicts the expired ISAs and subscriptions of all the DSS instances of the pool, instead of only the ones written by the DSS instance identified by `--locality`. The `--locality` flag of `db-manager evict` is deprecated and has no effect. It will be removed in a future release. +* [deploy] The CockroachDB `--cache` and `--max-sql-memory` flags can now be configured with Terraform, Tanka and Helm. See [CPU and memory allocation](docs/operations/performance.md#cpu-and-memory-allocation). ## Minimal database schema version diff --git a/deploy/infrastructure/dependencies/terraform-commons-dss/helm.tf b/deploy/infrastructure/dependencies/terraform-commons-dss/helm.tf index 50b801b87..8674c9b70 100644 --- a/deploy/infrastructure/dependencies/terraform-commons-dss/helm.tf +++ b/deploy/infrastructure/dependencies/terraform-commons-dss/helm.tf @@ -18,9 +18,11 @@ resource "local_file" "helm_chart_values" { fullnameOverride = local.helm_crdb_statefulset_name conf = { - join = var.crdb_external_nodes - cluster-name = var.crdb_cluster_name - single-node = false # Always false. Even with 1 replica, we would expect to keep the ability to pool it with another cluster. + join = var.crdb_external_nodes + cluster-name = var.crdb_cluster_name + cache = var.crdb_cache + max-sql-memory = var.crdb_max_sql_memory + single-node = false # Always false. Even with 1 replica, we would expect to keep the ability to pool it with another cluster. } statefulset = { diff --git a/deploy/infrastructure/dependencies/terraform-commons-dss/tanka.tf b/deploy/infrastructure/dependencies/terraform-commons-dss/tanka.tf index 788123feb..9c6d67a69 100644 --- a/deploy/infrastructure/dependencies/terraform-commons-dss/tanka.tf +++ b/deploy/infrastructure/dependencies/terraform-commons-dss/tanka.tf @@ -22,6 +22,8 @@ resource "local_file" "tanka_config_main" { VAR_DOCKER_IMAGE_NAME = var.image VAR_CRDB_DOCKER_IMAGE_NAME = "cockroachdb/cockroach:${var.crdb_image_tag}" VAR_CRDB_CLUSTER_NAME = var.crdb_cluster_name + VAR_CRDB_CACHE = var.crdb_cache + VAR_CRDB_MAX_SQL_MEMORY = var.crdb_max_sql_memory VAR_YUGABYTE_CLOUD = var.yugabyte_cloud VAR_YUGABYTE_REGION = var.yugabyte_region VAR_YUGABYTE_ZONE = var.yugabyte_zone diff --git a/deploy/infrastructure/dependencies/terraform-commons-dss/templates/main.jsonnet.tmp b/deploy/infrastructure/dependencies/terraform-commons-dss/templates/main.jsonnet.tmp index eeca5f9f1..03bc78b47 100644 --- a/deploy/infrastructure/dependencies/terraform-commons-dss/templates/main.jsonnet.tmp +++ b/deploy/infrastructure/dependencies/terraform-commons-dss/templates/main.jsonnet.tmp @@ -25,6 +25,8 @@ local metadata = metadataBase { JoinExisting: [${VAR_CRDB_EXTERNAL_NODES}], storageClass: '${VAR_STORAGE_CLASS}', clusterName: '${VAR_CRDB_CLUSTER_NAME}', + cache: '${VAR_CRDB_CACHE}', + maxSqlMemory: '${VAR_CRDB_MAX_SQL_MEMORY}', }, yugabyte+: { image: '${VAR_YUGABYTE_DOCKER_IMAGE_NAME}', diff --git a/deploy/infrastructure/dependencies/terraform-commons-dss/variables.gen.tf b/deploy/infrastructure/dependencies/terraform-commons-dss/variables.gen.tf index d4012be3a..d7d0af311 100644 --- a/deploy/infrastructure/dependencies/terraform-commons-dss/variables.gen.tf +++ b/deploy/infrastructure/dependencies/terraform-commons-dss/variables.gen.tf @@ -284,6 +284,32 @@ variable "crdb_cluster_name" { } +variable "crdb_cache" { + type = string + description = <<-EOT + Size of the CockroachDB storage engine cache, passed to the `--cache` flag of `cockroach start`. + Either a percentage of the memory available to the container or an absolute size. + + Example: `25%` or `2GiB` + EOT + + default = "25%" +} + + +variable "crdb_max_sql_memory" { + type = string + description = <<-EOT + Maximum memory CockroachDB can use for SQL queries, passed to the `--max-sql-memory` flag of `cockroach start`. + Either a percentage of the memory available to the container or an absolute size. + + Example: `25%` or `2GiB` + EOT + + default = "25%" +} + + variable "locality" { type = string description = <<-EOT diff --git a/deploy/infrastructure/modules/terraform-aws-dss/TFVARS.gen.md b/deploy/infrastructure/modules/terraform-aws-dss/TFVARS.gen.md index 15a42ee72..a71e6d4bb 100644 --- a/deploy/infrastructure/modules/terraform-aws-dss/TFVARS.gen.md +++ b/deploy/infrastructure/modules/terraform-aws-dss/TFVARS.gen.md @@ -119,6 +119,12 @@ Leave empty to disable record creation.

Name of the kubernetes cluster that will host this DSS instance (should generally describe the DSS instance being hosted)

Example: dss-che-1

+ + crdb_cache (string) +

Size of the CockroachDB storage engine cache, passed to the --cache flag of cockroach start. +Either a percentage of the memory available to the container or an absolute size.

+

Example: 25% or 2GiB

+
Default value: "25%" crdb_cluster_name (string)

A string that specifies a CRDB cluster name. This is used together to ensure that all newly created @@ -148,6 +154,12 @@ From v.17, the recommended CockroachDB version is v24.1.3.

crdb_locality (string)

This variable has been renamed to locality and is left to warn users about migration.


Default value: "" + + crdb_max_sql_memory (string) +

Maximum memory CockroachDB can use for SQL queries, passed to the --max-sql-memory flag of cockroach start. +Either a percentage of the memory available to the container or an absolute size.

+

Example: 25% or 2GiB

+
Default value: "25%" datastore_max_open_conns (number)

Maximum number of open connections to the datastore.

diff --git a/deploy/infrastructure/modules/terraform-aws-dss/main.tf b/deploy/infrastructure/modules/terraform-aws-dss/main.tf index 23d577b1b..09fa58d7e 100644 --- a/deploy/infrastructure/modules/terraform-aws-dss/main.tf +++ b/deploy/infrastructure/modules/terraform-aws-dss/main.tf @@ -25,6 +25,8 @@ module "terraform-commons-dss" { app_hostname = var.app_hostname crdb_image_tag = var.crdb_image_tag crdb_cluster_name = var.crdb_cluster_name + crdb_cache = var.crdb_cache + crdb_max_sql_memory = var.crdb_max_sql_memory db_hostname_suffix = var.db_hostname_suffix datastore_type = var.datastore_type datastore_max_open_conns = var.datastore_max_open_conns diff --git a/deploy/infrastructure/modules/terraform-aws-dss/variables.gen.tf b/deploy/infrastructure/modules/terraform-aws-dss/variables.gen.tf index c20e043de..1fbbb3d07 100644 --- a/deploy/infrastructure/modules/terraform-aws-dss/variables.gen.tf +++ b/deploy/infrastructure/modules/terraform-aws-dss/variables.gen.tf @@ -411,6 +411,32 @@ variable "crdb_cluster_name" { } +variable "crdb_cache" { + type = string + description = <<-EOT + Size of the CockroachDB storage engine cache, passed to the `--cache` flag of `cockroach start`. + Either a percentage of the memory available to the container or an absolute size. + + Example: `25%` or `2GiB` + EOT + + default = "25%" +} + + +variable "crdb_max_sql_memory" { + type = string + description = <<-EOT + Maximum memory CockroachDB can use for SQL queries, passed to the `--max-sql-memory` flag of `cockroach start`. + Either a percentage of the memory available to the container or an absolute size. + + Example: `25%` or `2GiB` + EOT + + default = "25%" +} + + variable "locality" { type = string description = <<-EOT diff --git a/deploy/infrastructure/modules/terraform-google-dss/TFVARS.gen.md b/deploy/infrastructure/modules/terraform-google-dss/TFVARS.gen.md index e757c4e67..0958eef17 100644 --- a/deploy/infrastructure/modules/terraform-google-dss/TFVARS.gen.md +++ b/deploy/infrastructure/modules/terraform-google-dss/TFVARS.gen.md @@ -76,6 +76,12 @@ Example:

Name of the kubernetes cluster that will host this DSS instance (should generally describe the DSS instance being hosted)

Example: dss-che-1

+ + crdb_cache (string) +

Size of the CockroachDB storage engine cache, passed to the --cache flag of cockroach start. +Either a percentage of the memory available to the container or an absolute size.

+

Example: 25% or 2GiB

+
Default value: "25%" crdb_cluster_name (string)

A string that specifies a CRDB cluster name. This is used together to ensure that all newly created @@ -105,6 +111,12 @@ From v.17, the recommended CockroachDB version is v24.1.3.

crdb_locality (string)

This variable has been renamed to locality and is left to warn users about migration.


Default value: "" + + crdb_max_sql_memory (string) +

Maximum memory CockroachDB can use for SQL queries, passed to the --max-sql-memory flag of cockroach start. +Either a percentage of the memory available to the container or an absolute size.

+

Example: 25% or 2GiB

+
Default value: "25%" datastore_max_open_conns (number)

Maximum number of open connections to the datastore.

diff --git a/deploy/infrastructure/modules/terraform-google-dss/main.tf b/deploy/infrastructure/modules/terraform-google-dss/main.tf index 8cf9088db..8ffd534f4 100644 --- a/deploy/infrastructure/modules/terraform-google-dss/main.tf +++ b/deploy/infrastructure/modules/terraform-google-dss/main.tf @@ -25,6 +25,8 @@ module "terraform-commons-dss" { app_hostname = var.app_hostname crdb_image_tag = var.crdb_image_tag crdb_cluster_name = var.crdb_cluster_name + crdb_cache = var.crdb_cache + crdb_max_sql_memory = var.crdb_max_sql_memory db_hostname_suffix = var.db_hostname_suffix datastore_type = var.datastore_type datastore_max_open_conns = var.datastore_max_open_conns diff --git a/deploy/infrastructure/modules/terraform-google-dss/variables.gen.tf b/deploy/infrastructure/modules/terraform-google-dss/variables.gen.tf index ad306b72d..543ec91c6 100644 --- a/deploy/infrastructure/modules/terraform-google-dss/variables.gen.tf +++ b/deploy/infrastructure/modules/terraform-google-dss/variables.gen.tf @@ -400,6 +400,32 @@ variable "crdb_cluster_name" { } +variable "crdb_cache" { + type = string + description = <<-EOT + Size of the CockroachDB storage engine cache, passed to the `--cache` flag of `cockroach start`. + Either a percentage of the memory available to the container or an absolute size. + + Example: `25%` or `2GiB` + EOT + + default = "25%" +} + + +variable "crdb_max_sql_memory" { + type = string + description = <<-EOT + Maximum memory CockroachDB can use for SQL queries, passed to the `--max-sql-memory` flag of `cockroach start`. + Either a percentage of the memory available to the container or an absolute size. + + Example: `25%` or `2GiB` + EOT + + default = "25%" +} + + variable "locality" { type = string description = <<-EOT diff --git a/deploy/infrastructure/utils/definitions/crdb_cache.tf b/deploy/infrastructure/utils/definitions/crdb_cache.tf new file mode 100644 index 000000000..6a733f8c4 --- /dev/null +++ b/deploy/infrastructure/utils/definitions/crdb_cache.tf @@ -0,0 +1,11 @@ +variable "crdb_cache" { + type = string + description = <<-EOT + Size of the CockroachDB storage engine cache, passed to the `--cache` flag of `cockroach start`. + Either a percentage of the memory available to the container or an absolute size. + + Example: `25%` or `2GiB` + EOT + + default = "25%" +} diff --git a/deploy/infrastructure/utils/definitions/crdb_max_sql_memory.tf b/deploy/infrastructure/utils/definitions/crdb_max_sql_memory.tf new file mode 100644 index 000000000..64f04bc5c --- /dev/null +++ b/deploy/infrastructure/utils/definitions/crdb_max_sql_memory.tf @@ -0,0 +1,11 @@ +variable "crdb_max_sql_memory" { + type = string + description = <<-EOT + Maximum memory CockroachDB can use for SQL queries, passed to the `--max-sql-memory` flag of `cockroach start`. + Either a percentage of the memory available to the container or an absolute size. + + Example: `25%` or `2GiB` + EOT + + default = "25%" +} diff --git a/deploy/infrastructure/utils/variables.py b/deploy/infrastructure/utils/variables.py index be8457e76..905aa5d47 100755 --- a/deploy/infrastructure/utils/variables.py +++ b/deploy/infrastructure/utils/variables.py @@ -48,6 +48,8 @@ "desired_aux_db_version", "crdb_image_tag", "crdb_cluster_name", + "crdb_cache", + "crdb_max_sql_memory", "locality", "datastore_max_open_conns", "crdb_external_nodes", diff --git a/deploy/operations/ci/aws-1/variables.gen.tf b/deploy/operations/ci/aws-1/variables.gen.tf index c20e043de..1fbbb3d07 100644 --- a/deploy/operations/ci/aws-1/variables.gen.tf +++ b/deploy/operations/ci/aws-1/variables.gen.tf @@ -411,6 +411,32 @@ variable "crdb_cluster_name" { } +variable "crdb_cache" { + type = string + description = <<-EOT + Size of the CockroachDB storage engine cache, passed to the `--cache` flag of `cockroach start`. + Either a percentage of the memory available to the container or an absolute size. + + Example: `25%` or `2GiB` + EOT + + default = "25%" +} + + +variable "crdb_max_sql_memory" { + type = string + description = <<-EOT + Maximum memory CockroachDB can use for SQL queries, passed to the `--max-sql-memory` flag of `cockroach start`. + Either a percentage of the memory available to the container or an absolute size. + + Example: `25%` or `2GiB` + EOT + + default = "25%" +} + + variable "locality" { type = string description = <<-EOT diff --git a/deploy/services/tanka/cockroachdb.libsonnet b/deploy/services/tanka/cockroachdb.libsonnet index c43f257cb..6788e3d47 100644 --- a/deploy/services/tanka/cockroachdb.libsonnet +++ b/deploy/services/tanka/cockroachdb.libsonnet @@ -90,8 +90,8 @@ local volumes = import 'volumes.libsonnet'; locality: 'zone=' + metadata.locality, 'locality-advertise-addr': 'zone=' + metadata.locality + '@$(hostname -f)', 'http-addr': '0.0.0.0', - cache: '25%', - 'max-sql-memory': '25%', + cache: metadata.cockroach.cache, + 'max-sql-memory': metadata.cockroach.maxSqlMemory, } + if metadata.cockroach.clusterName != "" then { 'cluster-name': metadata.cockroach.clusterName, } else {}, diff --git a/deploy/services/tanka/metadata_base.libsonnet b/deploy/services/tanka/metadata_base.libsonnet index 4122b2423..1f9d1ca76 100644 --- a/deploy/services/tanka/metadata_base.libsonnet +++ b/deploy/services/tanka/metadata_base.libsonnet @@ -25,6 +25,8 @@ JoinExisting: [], storageClass: 'standard', clusterName: '', + cache: '25%', // Value of the --cache flag of `cockroach start` + maxSqlMemory: '25%', // Value of the --max-sql-memory flag of `cockroach start` }, yugabyte: { image: error 'must specify yugabyte db image', diff --git a/docs/operations/performance.md b/docs/operations/performance.md index 78a58b73c..bb06ccda4 100644 --- a/docs/operations/performance.md +++ b/docs/operations/performance.md @@ -6,6 +6,43 @@ Overtime, old entries should be removed to ensure the system is not overloaded p See the detailed [section about cleanup](cleanup.md). +## CPU and memory allocation + +The following settings are available: + +* CockroachDB `--cache` and `--max-sql-memory` flags (both `25%` by default). See [CockroachDB recommendations](https://www.cockroachlabs.com/docs/stable/recommended-production-settings#cache-and-sql-memory-size) before changing them. + +To set them: + +1. Set the values according to the deployment tool: + + === "Terraform" + Set the following variables (see `TFVARS.gen.md` for details): + + * `crdb_cache` and `crdb_max_sql_memory`, e.g. `"25%"` + + Then run `terraform apply` to regenerate the Tanka and Helm configuration. + + === "Tanka" + Set the following fields of the metadata in your `main.jsonnet`: + + ```jsonnet + cockroach+: { + cache: '25%', + maxSqlMemory: '25%', + }, + ``` + + === "Helm" + Set the following values: + + ```yaml + cockroachdb: + conf: + cache: 25% + max-sql-memory: 25% + ``` + ## The SCD global lock option !!! danger From 0e43468b82cef18c37c19e69dea8d96d660f3e4c Mon Sep 17 00:00:00 2001 From: Mariem Baccari Date: Wed, 7 Oct 2026 14:41:05 +0200 Subject: [PATCH 2/3] [deploy] Make CockroachDB and core-service resources configurable --- NEXT_RELEASE_NOTES.md | 2 +- .../terraform-commons-dss/helm.tf | 7 ++-- .../terraform-commons-dss/tanka.tf | 2 ++ .../templates/main.jsonnet.tmp | 4 ++- .../terraform-commons-dss/variables.gen.tf | 36 +++++++++++++++++++ .../modules/terraform-aws-dss/TFVARS.gen.md | 12 +++++++ .../modules/terraform-aws-dss/main.tf | 2 ++ .../terraform-aws-dss/variables.gen.tf | 36 +++++++++++++++++++ .../terraform-google-dss/TFVARS.gen.md | 12 +++++++ .../modules/terraform-google-dss/main.tf | 2 ++ .../terraform-google-dss/variables.gen.tf | 36 +++++++++++++++++++ .../definitions/core_service_resources.tf | 16 +++++++++ .../utils/definitions/crdb_resources.tf | 16 +++++++++ deploy/infrastructure/utils/variables.py | 2 ++ deploy/operations/ci/aws-1/variables.gen.tf | 36 +++++++++++++++++++ .../dss/templates/dss-core-service.yaml | 4 +++ .../helm-charts/dss/values.schema.json | 4 +++ deploy/services/helm-charts/dss/values.yaml | 1 + deploy/services/tanka/cockroachdb.libsonnet | 1 + deploy/services/tanka/core-service.libsonnet | 1 + deploy/services/tanka/metadata_base.libsonnet | 2 ++ docs/operations/performance.md | 14 ++++++++ 22 files changed, 244 insertions(+), 4 deletions(-) create mode 100644 deploy/infrastructure/utils/definitions/core_service_resources.tf create mode 100644 deploy/infrastructure/utils/definitions/crdb_resources.tf diff --git a/NEXT_RELEASE_NOTES.md b/NEXT_RELEASE_NOTES.md index c68b7cf16..44ba60342 100644 --- a/NEXT_RELEASE_NOTES.md +++ b/NEXT_RELEASE_NOTES.md @@ -51,7 +51,7 @@ The release notes should contain at least the following sections: * [deploy] AWS NLB TLS listeners (DSS gateway and Prometheus, for both Helm and Tanka) now use the `ELBSecurityPolicy-TLS13-1-2-Res-2021-06` SSL negotiation policy, which enforces a minimum of TLS 1.2 (aligning with the Google deployment) and restricts TLS 1.2 to AEAD (GCM/ChaCha20) cipher suites. Previously the AWS Load Balancer Controller default (`ELBSecurityPolicy-2016-08`) applied, which allowed TLS 1.0 and 1.1. Clients limited to TLS 1.0 or 1.1, or to CBC cipher suites, will no longer be able to connect. Unlike the Google deployment, no resources need to be provisioned by Terraform or manually to set up the policy. * [evict] RID cleanup (`db-manager evict`, including the RID cleanup default `CronJob`) now evicts the expired ISAs and subscriptions of all the DSS instances of the pool, instead of only the ones written by the DSS instance identified by `--locality`. The `--locality` flag of `db-manager evict` is deprecated and has no effect. It will be removed in a future release. -* [deploy] The CockroachDB `--cache` and `--max-sql-memory` flags can now be configured with Terraform, Tanka and Helm. See [CPU and memory allocation](docs/operations/performance.md#cpu-and-memory-allocation). +* [deploy] CPU and memory requests and limits of the CockroachDB and core-service containers and the CockroachDB `--cache` and `--max-sql-memory` flags can now be configured with Terraform, Tanka and Helm. See [CPU and memory allocation](docs/operations/performance.md#cpu-and-memory-allocation). ## Minimal database schema version diff --git a/deploy/infrastructure/dependencies/terraform-commons-dss/helm.tf b/deploy/infrastructure/dependencies/terraform-commons-dss/helm.tf index 8674c9b70..37c8d8bf4 100644 --- a/deploy/infrastructure/dependencies/terraform-commons-dss/helm.tf +++ b/deploy/infrastructure/dependencies/terraform-commons-dss/helm.tf @@ -31,6 +31,7 @@ resource "local_file" "helm_chart_values" { "--locality-advertise-addr=zone=${var.locality}@$(hostname -f)", "--advertise-addr=$${HOSTNAME##*-}.${var.db_hostname_suffix}" ] + resources = var.crdb_resources } storage = { @@ -58,7 +59,8 @@ resource "local_file" "helm_chart_values" { } dss = { - image = var.image + image = var.image + resources = var.core_service_resources conf = { pubKeys = [ @@ -273,7 +275,8 @@ resource "local_file" "helm_chart_values" { } dss = { - image = var.image + image = var.image + resources = var.core_service_resources conf = { pubKeys = [ diff --git a/deploy/infrastructure/dependencies/terraform-commons-dss/tanka.tf b/deploy/infrastructure/dependencies/terraform-commons-dss/tanka.tf index 9c6d67a69..f463ab594 100644 --- a/deploy/infrastructure/dependencies/terraform-commons-dss/tanka.tf +++ b/deploy/infrastructure/dependencies/terraform-commons-dss/tanka.tf @@ -20,10 +20,12 @@ resource "local_file" "tanka_config_main" { VAR_CRDB_EXTERNAL_NODES = join(",", [for a in var.crdb_external_nodes : "'${a}'"]) VAR_STORAGE_CLASS = var.kubernetes_storage_class VAR_DOCKER_IMAGE_NAME = var.image + VAR_CORE_SERVICE_RESOURCES = jsonencode(var.core_service_resources) VAR_CRDB_DOCKER_IMAGE_NAME = "cockroachdb/cockroach:${var.crdb_image_tag}" VAR_CRDB_CLUSTER_NAME = var.crdb_cluster_name VAR_CRDB_CACHE = var.crdb_cache VAR_CRDB_MAX_SQL_MEMORY = var.crdb_max_sql_memory + VAR_CRDB_RESOURCES = jsonencode(var.crdb_resources) VAR_YUGABYTE_CLOUD = var.yugabyte_cloud VAR_YUGABYTE_REGION = var.yugabyte_region VAR_YUGABYTE_ZONE = var.yugabyte_zone diff --git a/deploy/infrastructure/dependencies/terraform-commons-dss/templates/main.jsonnet.tmp b/deploy/infrastructure/dependencies/terraform-commons-dss/templates/main.jsonnet.tmp index 03bc78b47..d9b18df7d 100644 --- a/deploy/infrastructure/dependencies/terraform-commons-dss/templates/main.jsonnet.tmp +++ b/deploy/infrastructure/dependencies/terraform-commons-dss/templates/main.jsonnet.tmp @@ -27,6 +27,7 @@ local metadata = metadataBase { clusterName: '${VAR_CRDB_CLUSTER_NAME}', cache: '${VAR_CRDB_CACHE}', maxSqlMemory: '${VAR_CRDB_MAX_SQL_MEMORY}', + resources: ${VAR_CRDB_RESOURCES}, }, yugabyte+: { image: '${VAR_YUGABYTE_DOCKER_IMAGE_NAME}', @@ -64,7 +65,8 @@ local metadata = metadataBase { hostname: '${VAR_APP_HOSTNAME}', publicEndpoint: 'https://${VAR_APP_HOSTNAME}', dumpRequests: false, - sslPolicy: '${VAR_SSL_POLICY}' + sslPolicy: '${VAR_SSL_POLICY}', + resources: ${VAR_CORE_SERVICE_RESOURCES}, }, schema_manager+: { enable: true, diff --git a/deploy/infrastructure/dependencies/terraform-commons-dss/variables.gen.tf b/deploy/infrastructure/dependencies/terraform-commons-dss/variables.gen.tf index d7d0af311..5fda715ce 100644 --- a/deploy/infrastructure/dependencies/terraform-commons-dss/variables.gen.tf +++ b/deploy/infrastructure/dependencies/terraform-commons-dss/variables.gen.tf @@ -310,6 +310,42 @@ variable "crdb_max_sql_memory" { } +variable "crdb_resources" { + type = map(map(string)) + description = <<-EOT + Kubernetes CPU and memory requests and limits of the CockroachDB containers. + Leave empty to not set any. + + Example: `{ requests = { cpu = "2", memory = "10Gi" }, limits = { cpu = "2", memory = "10Gi" } }` + EOT + + default = {} + + validation { + condition = alltrue([for k in keys(var.crdb_resources) : contains(["requests", "limits"], k)]) + error_message = "crdb_resources keys must be `requests` or `limits`." + } +} + + +variable "core_service_resources" { + type = map(map(string)) + description = <<-EOT + Kubernetes CPU and memory requests and limits of the core-service containers. + Leave empty to not set any. + + Example: `{ requests = { cpu = "1", memory = "2Gi" }, limits = { memory = "2Gi" } }` + EOT + + default = {} + + validation { + condition = alltrue([for k in keys(var.core_service_resources) : contains(["requests", "limits"], k)]) + error_message = "core_service_resources keys must be `requests` or `limits`." + } +} + + variable "locality" { type = string description = <<-EOT diff --git a/deploy/infrastructure/modules/terraform-aws-dss/TFVARS.gen.md b/deploy/infrastructure/modules/terraform-aws-dss/TFVARS.gen.md index a71e6d4bb..26b44cc47 100644 --- a/deploy/infrastructure/modules/terraform-aws-dss/TFVARS.gen.md +++ b/deploy/infrastructure/modules/terraform-aws-dss/TFVARS.gen.md @@ -119,6 +119,12 @@ Leave empty to disable record creation.

Name of the kubernetes cluster that will host this DSS instance (should generally describe the DSS instance being hosted)

Example: dss-che-1

+ + core_service_resources (map(map(string))) +

Kubernetes CPU and memory requests and limits of the core-service containers. +Leave empty to not set any.

+

Example: { requests = { cpu = "1", memory = "2Gi" }, limits = { memory = "2Gi" } }

+
Default value: {} crdb_cache (string)

Size of the CockroachDB storage engine cache, passed to the --cache flag of cockroach start. @@ -160,6 +166,12 @@ From v.17, the recommended CockroachDB version is v24.1.3.

Either a percentage of the memory available to the container or an absolute size.

Example: 25% or 2GiB


Default value: "25%" + + crdb_resources (map(map(string))) +

Kubernetes CPU and memory requests and limits of the CockroachDB containers. +Leave empty to not set any.

+

Example: { requests = { cpu = "2", memory = "10Gi" }, limits = { cpu = "2", memory = "10Gi" } }

+
Default value: {} datastore_max_open_conns (number)

Maximum number of open connections to the datastore.

diff --git a/deploy/infrastructure/modules/terraform-aws-dss/main.tf b/deploy/infrastructure/modules/terraform-aws-dss/main.tf index 09fa58d7e..7da8e0d98 100644 --- a/deploy/infrastructure/modules/terraform-aws-dss/main.tf +++ b/deploy/infrastructure/modules/terraform-aws-dss/main.tf @@ -27,6 +27,8 @@ module "terraform-commons-dss" { crdb_cluster_name = var.crdb_cluster_name crdb_cache = var.crdb_cache crdb_max_sql_memory = var.crdb_max_sql_memory + crdb_resources = var.crdb_resources + core_service_resources = var.core_service_resources db_hostname_suffix = var.db_hostname_suffix datastore_type = var.datastore_type datastore_max_open_conns = var.datastore_max_open_conns diff --git a/deploy/infrastructure/modules/terraform-aws-dss/variables.gen.tf b/deploy/infrastructure/modules/terraform-aws-dss/variables.gen.tf index 1fbbb3d07..296767e52 100644 --- a/deploy/infrastructure/modules/terraform-aws-dss/variables.gen.tf +++ b/deploy/infrastructure/modules/terraform-aws-dss/variables.gen.tf @@ -437,6 +437,42 @@ variable "crdb_max_sql_memory" { } +variable "crdb_resources" { + type = map(map(string)) + description = <<-EOT + Kubernetes CPU and memory requests and limits of the CockroachDB containers. + Leave empty to not set any. + + Example: `{ requests = { cpu = "2", memory = "10Gi" }, limits = { cpu = "2", memory = "10Gi" } }` + EOT + + default = {} + + validation { + condition = alltrue([for k in keys(var.crdb_resources) : contains(["requests", "limits"], k)]) + error_message = "crdb_resources keys must be `requests` or `limits`." + } +} + + +variable "core_service_resources" { + type = map(map(string)) + description = <<-EOT + Kubernetes CPU and memory requests and limits of the core-service containers. + Leave empty to not set any. + + Example: `{ requests = { cpu = "1", memory = "2Gi" }, limits = { memory = "2Gi" } }` + EOT + + default = {} + + validation { + condition = alltrue([for k in keys(var.core_service_resources) : contains(["requests", "limits"], k)]) + error_message = "core_service_resources keys must be `requests` or `limits`." + } +} + + variable "locality" { type = string description = <<-EOT diff --git a/deploy/infrastructure/modules/terraform-google-dss/TFVARS.gen.md b/deploy/infrastructure/modules/terraform-google-dss/TFVARS.gen.md index 0958eef17..bad1024a2 100644 --- a/deploy/infrastructure/modules/terraform-google-dss/TFVARS.gen.md +++ b/deploy/infrastructure/modules/terraform-google-dss/TFVARS.gen.md @@ -76,6 +76,12 @@ Example:

Name of the kubernetes cluster that will host this DSS instance (should generally describe the DSS instance being hosted)

Example: dss-che-1

+ + core_service_resources (map(map(string))) +

Kubernetes CPU and memory requests and limits of the core-service containers. +Leave empty to not set any.

+

Example: { requests = { cpu = "1", memory = "2Gi" }, limits = { memory = "2Gi" } }

+
Default value: {} crdb_cache (string)

Size of the CockroachDB storage engine cache, passed to the --cache flag of cockroach start. @@ -117,6 +123,12 @@ From v.17, the recommended CockroachDB version is v24.1.3.

Either a percentage of the memory available to the container or an absolute size.

Example: 25% or 2GiB


Default value: "25%" + + crdb_resources (map(map(string))) +

Kubernetes CPU and memory requests and limits of the CockroachDB containers. +Leave empty to not set any.

+

Example: { requests = { cpu = "2", memory = "10Gi" }, limits = { cpu = "2", memory = "10Gi" } }

+
Default value: {} datastore_max_open_conns (number)

Maximum number of open connections to the datastore.

diff --git a/deploy/infrastructure/modules/terraform-google-dss/main.tf b/deploy/infrastructure/modules/terraform-google-dss/main.tf index 8ffd534f4..937581e06 100644 --- a/deploy/infrastructure/modules/terraform-google-dss/main.tf +++ b/deploy/infrastructure/modules/terraform-google-dss/main.tf @@ -27,6 +27,8 @@ module "terraform-commons-dss" { crdb_cluster_name = var.crdb_cluster_name crdb_cache = var.crdb_cache crdb_max_sql_memory = var.crdb_max_sql_memory + crdb_resources = var.crdb_resources + core_service_resources = var.core_service_resources db_hostname_suffix = var.db_hostname_suffix datastore_type = var.datastore_type datastore_max_open_conns = var.datastore_max_open_conns diff --git a/deploy/infrastructure/modules/terraform-google-dss/variables.gen.tf b/deploy/infrastructure/modules/terraform-google-dss/variables.gen.tf index 543ec91c6..9410fd135 100644 --- a/deploy/infrastructure/modules/terraform-google-dss/variables.gen.tf +++ b/deploy/infrastructure/modules/terraform-google-dss/variables.gen.tf @@ -426,6 +426,42 @@ variable "crdb_max_sql_memory" { } +variable "crdb_resources" { + type = map(map(string)) + description = <<-EOT + Kubernetes CPU and memory requests and limits of the CockroachDB containers. + Leave empty to not set any. + + Example: `{ requests = { cpu = "2", memory = "10Gi" }, limits = { cpu = "2", memory = "10Gi" } }` + EOT + + default = {} + + validation { + condition = alltrue([for k in keys(var.crdb_resources) : contains(["requests", "limits"], k)]) + error_message = "crdb_resources keys must be `requests` or `limits`." + } +} + + +variable "core_service_resources" { + type = map(map(string)) + description = <<-EOT + Kubernetes CPU and memory requests and limits of the core-service containers. + Leave empty to not set any. + + Example: `{ requests = { cpu = "1", memory = "2Gi" }, limits = { memory = "2Gi" } }` + EOT + + default = {} + + validation { + condition = alltrue([for k in keys(var.core_service_resources) : contains(["requests", "limits"], k)]) + error_message = "core_service_resources keys must be `requests` or `limits`." + } +} + + variable "locality" { type = string description = <<-EOT diff --git a/deploy/infrastructure/utils/definitions/core_service_resources.tf b/deploy/infrastructure/utils/definitions/core_service_resources.tf new file mode 100644 index 000000000..aba580207 --- /dev/null +++ b/deploy/infrastructure/utils/definitions/core_service_resources.tf @@ -0,0 +1,16 @@ +variable "core_service_resources" { + type = map(map(string)) + description = <<-EOT + Kubernetes CPU and memory requests and limits of the core-service containers. + Leave empty to not set any. + + Example: `{ requests = { cpu = "1", memory = "2Gi" }, limits = { memory = "2Gi" } }` + EOT + + default = {} + + validation { + condition = alltrue([for k in keys(var.core_service_resources) : contains(["requests", "limits"], k)]) + error_message = "core_service_resources keys must be `requests` or `limits`." + } +} diff --git a/deploy/infrastructure/utils/definitions/crdb_resources.tf b/deploy/infrastructure/utils/definitions/crdb_resources.tf new file mode 100644 index 000000000..775ed90e7 --- /dev/null +++ b/deploy/infrastructure/utils/definitions/crdb_resources.tf @@ -0,0 +1,16 @@ +variable "crdb_resources" { + type = map(map(string)) + description = <<-EOT + Kubernetes CPU and memory requests and limits of the CockroachDB containers. + Leave empty to not set any. + + Example: `{ requests = { cpu = "2", memory = "10Gi" }, limits = { cpu = "2", memory = "10Gi" } }` + EOT + + default = {} + + validation { + condition = alltrue([for k in keys(var.crdb_resources) : contains(["requests", "limits"], k)]) + error_message = "crdb_resources keys must be `requests` or `limits`." + } +} diff --git a/deploy/infrastructure/utils/variables.py b/deploy/infrastructure/utils/variables.py index 905aa5d47..07357048e 100755 --- a/deploy/infrastructure/utils/variables.py +++ b/deploy/infrastructure/utils/variables.py @@ -50,6 +50,8 @@ "crdb_cluster_name", "crdb_cache", "crdb_max_sql_memory", + "crdb_resources", + "core_service_resources", "locality", "datastore_max_open_conns", "crdb_external_nodes", diff --git a/deploy/operations/ci/aws-1/variables.gen.tf b/deploy/operations/ci/aws-1/variables.gen.tf index 1fbbb3d07..296767e52 100644 --- a/deploy/operations/ci/aws-1/variables.gen.tf +++ b/deploy/operations/ci/aws-1/variables.gen.tf @@ -437,6 +437,42 @@ variable "crdb_max_sql_memory" { } +variable "crdb_resources" { + type = map(map(string)) + description = <<-EOT + Kubernetes CPU and memory requests and limits of the CockroachDB containers. + Leave empty to not set any. + + Example: `{ requests = { cpu = "2", memory = "10Gi" }, limits = { cpu = "2", memory = "10Gi" } }` + EOT + + default = {} + + validation { + condition = alltrue([for k in keys(var.crdb_resources) : contains(["requests", "limits"], k)]) + error_message = "crdb_resources keys must be `requests` or `limits`." + } +} + + +variable "core_service_resources" { + type = map(map(string)) + description = <<-EOT + Kubernetes CPU and memory requests and limits of the core-service containers. + Leave empty to not set any. + + Example: `{ requests = { cpu = "1", memory = "2Gi" }, limits = { memory = "2Gi" } }` + EOT + + default = {} + + validation { + condition = alltrue([for k in keys(var.core_service_resources) : contains(["requests", "limits"], k)]) + error_message = "core_service_resources keys must be `requests` or `limits`." + } +} + + variable "locality" { type = string description = <<-EOT diff --git a/deploy/services/helm-charts/dss/templates/dss-core-service.yaml b/deploy/services/helm-charts/dss/templates/dss-core-service.yaml index 9902a46ff..78a400908 100644 --- a/deploy/services/helm-charts/dss/templates/dss-core-service.yaml +++ b/deploy/services/helm-charts/dss/templates/dss-core-service.yaml @@ -92,6 +92,10 @@ spec: {{- end }} command: - core-service + {{- with $dss.resources }} + resources: + {{- toYaml . | nindent 12 }} + {{- end }} image: {{ $dss.image }} imagePullPolicy: {{ $dss.imagePullPolicy | default "Always" }} name: core-service diff --git a/deploy/services/helm-charts/dss/values.schema.json b/deploy/services/helm-charts/dss/values.schema.json index 5e9fa6401..a6e9d7650 100644 --- a/deploy/services/helm-charts/dss/values.schema.json +++ b/deploy/services/helm-charts/dss/values.schema.json @@ -242,6 +242,10 @@ "type": "string", "description": "Image of the DSS. Please note that the usage of the `latest` tag is discouraged to prevent accidental upgrades in case of restart. Example: `docker.io/interuss/dss:v0.15.0`. Official image releases: https://hub.docker.com/r/interuss/dss/tags" }, + "resources": { + "type": "object", + "description": "Kubernetes CPU and memory requests and limits of the core-service containers. Example: `{requests: {cpu: '1', memory: 2Gi}, limits: {memory: 2Gi}}`" + }, "conf": { "type": "object", "properties": { diff --git a/deploy/services/helm-charts/dss/values.yaml b/deploy/services/helm-charts/dss/values.yaml index e9f72b9ad..0b2c7f924 100644 --- a/deploy/services/helm-charts/dss/values.yaml +++ b/deploy/services/helm-charts/dss/values.yaml @@ -1,6 +1,7 @@ # Default configuration dss: + resources: {} conf: enableScd: true evict: diff --git a/deploy/services/tanka/cockroachdb.libsonnet b/deploy/services/tanka/cockroachdb.libsonnet index 6788e3d47..db87333ca 100644 --- a/deploy/services/tanka/cockroachdb.libsonnet +++ b/deploy/services/tanka/cockroachdb.libsonnet @@ -55,6 +55,7 @@ local volumes = import 'volumes.libsonnet'; value: 'kubernetes-multiregion', }, ], + resources: metadata.cockroach.resources, livenessProbe: { httpGet: { path: '/health', diff --git a/deploy/services/tanka/core-service.libsonnet b/deploy/services/tanka/core-service.libsonnet index fd3bf56a5..820e488ad 100644 --- a/deploy/services/tanka/core-service.libsonnet +++ b/deploy/services/tanka/core-service.libsonnet @@ -118,6 +118,7 @@ local awsLoadBalancer(metadata) = base.AWSLoadBalancerWithManagedCert(metadata, }, ] else [], volumeMounts: volumes.all(metadata).backendMounts, + resources: metadata.backend.resources, command: ['core-service'], args_:: { addr: ':' + metadata.backend.port, diff --git a/deploy/services/tanka/metadata_base.libsonnet b/deploy/services/tanka/metadata_base.libsonnet index 1f9d1ca76..bb7e30e03 100644 --- a/deploy/services/tanka/metadata_base.libsonnet +++ b/deploy/services/tanka/metadata_base.libsonnet @@ -27,6 +27,7 @@ clusterName: '', cache: '25%', // Value of the --cache flag of `cockroach start` maxSqlMemory: '25%', // Value of the --max-sql-memory flag of `cockroach start` + resources: {}, // Kubernetes container resources, e.g. { requests: { cpu: '2', memory: '10Gi' }, limits: { cpu: '2', memory: '10Gi' } } }, yugabyte: { image: error 'must specify yugabyte db image', @@ -69,6 +70,7 @@ dumpRequests: false, certName: if $.cloud_provider == "aws" then error 'must specify certName for AWS cloud provider', # Only used by AWS sslPolicy: '', # SSL Policy Name. Only used by Google Cloud. + resources: {}, // Kubernetes container resources, e.g. { requests: { cpu: '1', memory: '2Gi' }, limits: { memory: '2Gi' } } }, alert: { enable: false, diff --git a/docs/operations/performance.md b/docs/operations/performance.md index bb06ccda4..a6e547b82 100644 --- a/docs/operations/performance.md +++ b/docs/operations/performance.md @@ -8,8 +8,12 @@ See the detailed [section about cleanup](cleanup.md). ## CPU and memory allocation +By default, the deployment tooling does not set any Kubernetes CPU or memory requests or limits on the CockroachDB and core-service containers. +Kubernetes may then place them on nodes without enough free memory and kill them when the node runs out of memory (see issue [#1731](https://github.com/interuss/dss/issues/1731)). + The following settings are available: +* CPU and memory requests and limits of the CockroachDB and core-service containers. * CockroachDB `--cache` and `--max-sql-memory` flags (both `25%` by default). See [CockroachDB recommendations](https://www.cockroachlabs.com/docs/stable/recommended-production-settings#cache-and-sql-memory-size) before changing them. To set them: @@ -19,7 +23,9 @@ To set them: === "Terraform" Set the following variables (see `TFVARS.gen.md` for details): + * `crdb_resources`, e.g. `{ requests = { cpu = "2", memory = "10Gi" }, limits = { cpu = "2", memory = "10Gi" } }` * `crdb_cache` and `crdb_max_sql_memory`, e.g. `"25%"` + * `core_service_resources`, e.g. `{ requests = { cpu = "1", memory = "2Gi" }, limits = { memory = "2Gi" } }` Then run `terraform apply` to regenerate the Tanka and Helm configuration. @@ -28,9 +34,13 @@ To set them: ```jsonnet cockroach+: { + resources: { requests: { cpu: '2', memory: '10Gi' }, limits: { cpu: '2', memory: '10Gi' } }, cache: '25%', maxSqlMemory: '25%', }, + backend+: { + resources: { requests: { cpu: '1', memory: '2Gi' }, limits: { memory: '2Gi' } }, + }, ``` === "Helm" @@ -41,6 +51,10 @@ To set them: conf: cache: 25% max-sql-memory: 25% + statefulset: + resources: {requests: {cpu: "2", memory: 10Gi}, limits: {cpu: "2", memory: 10Gi}} + dss: + resources: {requests: {cpu: "1", memory: 2Gi}, limits: {memory: 2Gi}} ``` ## The SCD global lock option From d10ce8b38d7cb0726505295ac39b2d63a8f88015 Mon Sep 17 00:00:00 2001 From: Mariem Baccari Date: Wed, 7 Oct 2026 16:32:55 +0200 Subject: [PATCH 3/3] update resource examples --- .../terraform-commons-dss/variables.gen.tf | 4 ++-- .../modules/terraform-aws-dss/TFVARS.gen.md | 4 ++-- .../modules/terraform-aws-dss/variables.gen.tf | 4 ++-- .../modules/terraform-google-dss/TFVARS.gen.md | 4 ++-- .../modules/terraform-google-dss/variables.gen.tf | 4 ++-- .../utils/definitions/core_service_resources.tf | 2 +- .../utils/definitions/crdb_resources.tf | 2 +- deploy/operations/ci/aws-1/variables.gen.tf | 4 ++-- deploy/services/helm-charts/dss/values.schema.json | 2 +- deploy/services/tanka/metadata_base.libsonnet | 4 ++-- docs/operations/performance.md | 12 ++++++------ 11 files changed, 23 insertions(+), 23 deletions(-) diff --git a/deploy/infrastructure/dependencies/terraform-commons-dss/variables.gen.tf b/deploy/infrastructure/dependencies/terraform-commons-dss/variables.gen.tf index 5fda715ce..a1f000701 100644 --- a/deploy/infrastructure/dependencies/terraform-commons-dss/variables.gen.tf +++ b/deploy/infrastructure/dependencies/terraform-commons-dss/variables.gen.tf @@ -316,7 +316,7 @@ variable "crdb_resources" { Kubernetes CPU and memory requests and limits of the CockroachDB containers. Leave empty to not set any. - Example: `{ requests = { cpu = "2", memory = "10Gi" }, limits = { cpu = "2", memory = "10Gi" } }` + Example: `{ requests = { cpu = "3.5", memory = "14Gi" }, limits = { cpu = "3.5", memory = "14Gi" } }` EOT default = {} @@ -334,7 +334,7 @@ variable "core_service_resources" { Kubernetes CPU and memory requests and limits of the core-service containers. Leave empty to not set any. - Example: `{ requests = { cpu = "1", memory = "2Gi" }, limits = { memory = "2Gi" } }` + Example: `{ requests = { cpu = "0.5", memory = "2Gi" }, limits = { memory = "2Gi" } }` EOT default = {} diff --git a/deploy/infrastructure/modules/terraform-aws-dss/TFVARS.gen.md b/deploy/infrastructure/modules/terraform-aws-dss/TFVARS.gen.md index 26b44cc47..48c8bb7a0 100644 --- a/deploy/infrastructure/modules/terraform-aws-dss/TFVARS.gen.md +++ b/deploy/infrastructure/modules/terraform-aws-dss/TFVARS.gen.md @@ -123,7 +123,7 @@ Leave empty to disable record creation.

core_service_resources (map(map(string)))

Kubernetes CPU and memory requests and limits of the core-service containers. Leave empty to not set any.

-

Example: { requests = { cpu = "1", memory = "2Gi" }, limits = { memory = "2Gi" } }

+

Example: { requests = { cpu = "0.5", memory = "2Gi" }, limits = { memory = "2Gi" } }


Default value: {} crdb_cache (string) @@ -170,7 +170,7 @@ Either a percentage of the memory available to the container or an absolute size crdb_resources (map(map(string)))

Kubernetes CPU and memory requests and limits of the CockroachDB containers. Leave empty to not set any.

-

Example: { requests = { cpu = "2", memory = "10Gi" }, limits = { cpu = "2", memory = "10Gi" } }

+

Example: { requests = { cpu = "3.5", memory = "14Gi" }, limits = { cpu = "3.5", memory = "14Gi" } }


Default value: {} datastore_max_open_conns (number) diff --git a/deploy/infrastructure/modules/terraform-aws-dss/variables.gen.tf b/deploy/infrastructure/modules/terraform-aws-dss/variables.gen.tf index 296767e52..aab65c002 100644 --- a/deploy/infrastructure/modules/terraform-aws-dss/variables.gen.tf +++ b/deploy/infrastructure/modules/terraform-aws-dss/variables.gen.tf @@ -443,7 +443,7 @@ variable "crdb_resources" { Kubernetes CPU and memory requests and limits of the CockroachDB containers. Leave empty to not set any. - Example: `{ requests = { cpu = "2", memory = "10Gi" }, limits = { cpu = "2", memory = "10Gi" } }` + Example: `{ requests = { cpu = "3.5", memory = "14Gi" }, limits = { cpu = "3.5", memory = "14Gi" } }` EOT default = {} @@ -461,7 +461,7 @@ variable "core_service_resources" { Kubernetes CPU and memory requests and limits of the core-service containers. Leave empty to not set any. - Example: `{ requests = { cpu = "1", memory = "2Gi" }, limits = { memory = "2Gi" } }` + Example: `{ requests = { cpu = "0.5", memory = "2Gi" }, limits = { memory = "2Gi" } }` EOT default = {} diff --git a/deploy/infrastructure/modules/terraform-google-dss/TFVARS.gen.md b/deploy/infrastructure/modules/terraform-google-dss/TFVARS.gen.md index bad1024a2..de72ba59a 100644 --- a/deploy/infrastructure/modules/terraform-google-dss/TFVARS.gen.md +++ b/deploy/infrastructure/modules/terraform-google-dss/TFVARS.gen.md @@ -80,7 +80,7 @@ Example: core_service_resources (map(map(string)))

Kubernetes CPU and memory requests and limits of the core-service containers. Leave empty to not set any.

-

Example: { requests = { cpu = "1", memory = "2Gi" }, limits = { memory = "2Gi" } }

+

Example: { requests = { cpu = "0.5", memory = "2Gi" }, limits = { memory = "2Gi" } }


Default value: {} crdb_cache (string) @@ -127,7 +127,7 @@ Either a percentage of the memory available to the container or an absolute size crdb_resources (map(map(string)))

Kubernetes CPU and memory requests and limits of the CockroachDB containers. Leave empty to not set any.

-

Example: { requests = { cpu = "2", memory = "10Gi" }, limits = { cpu = "2", memory = "10Gi" } }

+

Example: { requests = { cpu = "3.5", memory = "14Gi" }, limits = { cpu = "3.5", memory = "14Gi" } }


Default value: {} datastore_max_open_conns (number) diff --git a/deploy/infrastructure/modules/terraform-google-dss/variables.gen.tf b/deploy/infrastructure/modules/terraform-google-dss/variables.gen.tf index 9410fd135..e3a46e319 100644 --- a/deploy/infrastructure/modules/terraform-google-dss/variables.gen.tf +++ b/deploy/infrastructure/modules/terraform-google-dss/variables.gen.tf @@ -432,7 +432,7 @@ variable "crdb_resources" { Kubernetes CPU and memory requests and limits of the CockroachDB containers. Leave empty to not set any. - Example: `{ requests = { cpu = "2", memory = "10Gi" }, limits = { cpu = "2", memory = "10Gi" } }` + Example: `{ requests = { cpu = "3.5", memory = "14Gi" }, limits = { cpu = "3.5", memory = "14Gi" } }` EOT default = {} @@ -450,7 +450,7 @@ variable "core_service_resources" { Kubernetes CPU and memory requests and limits of the core-service containers. Leave empty to not set any. - Example: `{ requests = { cpu = "1", memory = "2Gi" }, limits = { memory = "2Gi" } }` + Example: `{ requests = { cpu = "0.5", memory = "2Gi" }, limits = { memory = "2Gi" } }` EOT default = {} diff --git a/deploy/infrastructure/utils/definitions/core_service_resources.tf b/deploy/infrastructure/utils/definitions/core_service_resources.tf index aba580207..ea9679cd8 100644 --- a/deploy/infrastructure/utils/definitions/core_service_resources.tf +++ b/deploy/infrastructure/utils/definitions/core_service_resources.tf @@ -4,7 +4,7 @@ variable "core_service_resources" { Kubernetes CPU and memory requests and limits of the core-service containers. Leave empty to not set any. - Example: `{ requests = { cpu = "1", memory = "2Gi" }, limits = { memory = "2Gi" } }` + Example: `{ requests = { cpu = "0.5", memory = "2Gi" }, limits = { memory = "2Gi" } }` EOT default = {} diff --git a/deploy/infrastructure/utils/definitions/crdb_resources.tf b/deploy/infrastructure/utils/definitions/crdb_resources.tf index 775ed90e7..fd2360d02 100644 --- a/deploy/infrastructure/utils/definitions/crdb_resources.tf +++ b/deploy/infrastructure/utils/definitions/crdb_resources.tf @@ -4,7 +4,7 @@ variable "crdb_resources" { Kubernetes CPU and memory requests and limits of the CockroachDB containers. Leave empty to not set any. - Example: `{ requests = { cpu = "2", memory = "10Gi" }, limits = { cpu = "2", memory = "10Gi" } }` + Example: `{ requests = { cpu = "3.5", memory = "14Gi" }, limits = { cpu = "3.5", memory = "14Gi" } }` EOT default = {} diff --git a/deploy/operations/ci/aws-1/variables.gen.tf b/deploy/operations/ci/aws-1/variables.gen.tf index 296767e52..aab65c002 100644 --- a/deploy/operations/ci/aws-1/variables.gen.tf +++ b/deploy/operations/ci/aws-1/variables.gen.tf @@ -443,7 +443,7 @@ variable "crdb_resources" { Kubernetes CPU and memory requests and limits of the CockroachDB containers. Leave empty to not set any. - Example: `{ requests = { cpu = "2", memory = "10Gi" }, limits = { cpu = "2", memory = "10Gi" } }` + Example: `{ requests = { cpu = "3.5", memory = "14Gi" }, limits = { cpu = "3.5", memory = "14Gi" } }` EOT default = {} @@ -461,7 +461,7 @@ variable "core_service_resources" { Kubernetes CPU and memory requests and limits of the core-service containers. Leave empty to not set any. - Example: `{ requests = { cpu = "1", memory = "2Gi" }, limits = { memory = "2Gi" } }` + Example: `{ requests = { cpu = "0.5", memory = "2Gi" }, limits = { memory = "2Gi" } }` EOT default = {} diff --git a/deploy/services/helm-charts/dss/values.schema.json b/deploy/services/helm-charts/dss/values.schema.json index a6e9d7650..9841372d8 100644 --- a/deploy/services/helm-charts/dss/values.schema.json +++ b/deploy/services/helm-charts/dss/values.schema.json @@ -244,7 +244,7 @@ }, "resources": { "type": "object", - "description": "Kubernetes CPU and memory requests and limits of the core-service containers. Example: `{requests: {cpu: '1', memory: 2Gi}, limits: {memory: 2Gi}}`" + "description": "Kubernetes CPU and memory requests and limits of the core-service containers. Example: `{requests: {cpu: '0.5', memory: 2Gi}, limits: {memory: 2Gi}}`" }, "conf": { "type": "object", diff --git a/deploy/services/tanka/metadata_base.libsonnet b/deploy/services/tanka/metadata_base.libsonnet index bb7e30e03..775f05d60 100644 --- a/deploy/services/tanka/metadata_base.libsonnet +++ b/deploy/services/tanka/metadata_base.libsonnet @@ -27,7 +27,7 @@ clusterName: '', cache: '25%', // Value of the --cache flag of `cockroach start` maxSqlMemory: '25%', // Value of the --max-sql-memory flag of `cockroach start` - resources: {}, // Kubernetes container resources, e.g. { requests: { cpu: '2', memory: '10Gi' }, limits: { cpu: '2', memory: '10Gi' } } + resources: {}, // Kubernetes container resources, e.g. { requests: { cpu: '3.5', memory: '14Gi' }, limits: { cpu: '3.5', memory: '14Gi' } } }, yugabyte: { image: error 'must specify yugabyte db image', @@ -70,7 +70,7 @@ dumpRequests: false, certName: if $.cloud_provider == "aws" then error 'must specify certName for AWS cloud provider', # Only used by AWS sslPolicy: '', # SSL Policy Name. Only used by Google Cloud. - resources: {}, // Kubernetes container resources, e.g. { requests: { cpu: '1', memory: '2Gi' }, limits: { memory: '2Gi' } } + resources: {}, // Kubernetes container resources, e.g. { requests: { cpu: '0.5', memory: '2Gi' }, limits: { memory: '2Gi' } } }, alert: { enable: false, diff --git a/docs/operations/performance.md b/docs/operations/performance.md index a6e547b82..2c31882b4 100644 --- a/docs/operations/performance.md +++ b/docs/operations/performance.md @@ -23,9 +23,9 @@ To set them: === "Terraform" Set the following variables (see `TFVARS.gen.md` for details): - * `crdb_resources`, e.g. `{ requests = { cpu = "2", memory = "10Gi" }, limits = { cpu = "2", memory = "10Gi" } }` + * `crdb_resources`, e.g. `{ requests = { cpu = "3.5", memory = "14Gi" }, limits = { cpu = "3.5", memory = "14Gi" } }` * `crdb_cache` and `crdb_max_sql_memory`, e.g. `"25%"` - * `core_service_resources`, e.g. `{ requests = { cpu = "1", memory = "2Gi" }, limits = { memory = "2Gi" } }` + * `core_service_resources`, e.g. `{ requests = { cpu = "0.5", memory = "2Gi" }, limits = { memory = "2Gi" } }` Then run `terraform apply` to regenerate the Tanka and Helm configuration. @@ -34,12 +34,12 @@ To set them: ```jsonnet cockroach+: { - resources: { requests: { cpu: '2', memory: '10Gi' }, limits: { cpu: '2', memory: '10Gi' } }, + resources: { requests: { cpu: '3.5', memory: '14Gi' }, limits: { cpu: '3.5', memory: '14Gi' } }, cache: '25%', maxSqlMemory: '25%', }, backend+: { - resources: { requests: { cpu: '1', memory: '2Gi' }, limits: { memory: '2Gi' } }, + resources: { requests: { cpu: '0.5', memory: '2Gi' }, limits: { memory: '2Gi' } }, }, ``` @@ -52,9 +52,9 @@ To set them: cache: 25% max-sql-memory: 25% statefulset: - resources: {requests: {cpu: "2", memory: 10Gi}, limits: {cpu: "2", memory: 10Gi}} + resources: {requests: {cpu: "3.5", memory: 14Gi}, limits: {cpu: "3.5", memory: 14Gi}} dss: - resources: {requests: {cpu: "1", memory: 2Gi}, limits: {memory: 2Gi}} + resources: {requests: {cpu: "0.5", memory: 2Gi}, limits: {memory: 2Gi}} ``` ## The SCD global lock option