[deploy] Make CockroachDB and core-service kubernetes resources configurable

NEXT_RELEASE_NOTES.md

@@ -51,6 +51,7 @@ The release notes should contain at least the following sections:
 
 * [deploy] AWS NLB TLS listeners (DSS gateway and Prometheus, for both Helm and Tanka) now use the `ELBSecurityPolicy-TLS13-1-2-Res-2021-06` SSL negotiation policy, which enforces a minimum of TLS 1.2 (aligning with the Google deployment) and restricts TLS 1.2 to AEAD (GCM/ChaCha20) cipher suites. Previously the AWS Load Balancer Controller default (`ELBSecurityPolicy-2016-08`) applied, which allowed TLS 1.0 and 1.1. Clients limited to TLS 1.0 or 1.1, or to CBC cipher suites, will no longer be able to connect. Unlike the Google deployment, no resources need to be provisioned by Terraform or manually to set up the policy.
 * [evict] RID cleanup (`db-manager evict`, including the RID cleanup default `CronJob`) now evicts the expired ISAs and subscriptions of all the DSS instances of the pool, instead of only the ones written by the DSS instance identified by `--locality`. The `--locality` flag of `db-manager evict` is deprecated and has no effect. It will be removed in a future release.
+* [deploy] CPU and memory requests and limits of the CockroachDB and core-service containers and the CockroachDB `--cache` and `--max-sql-memory` flags can now be configured with Terraform, Tanka and Helm. See [CPU and memory allocation](docs/operations/performance.md#cpu-and-memory-allocation).
 
 ## Minimal database schema version
 

deploy/infrastructure/dependencies/terraform-commons-dss/helm.tf

@@ -18,9 +18,11 @@ resource "local_file" "helm_chart_values" {
       fullnameOverride = local.helm_crdb_statefulset_name
 
       conf = {
-        join         = var.crdb_external_nodes
-        cluster-name = var.crdb_cluster_name
-        single-node  = false # Always false. Even with 1 replica, we would expect to keep the ability to pool it with another cluster.
+        join           = var.crdb_external_nodes
+        cluster-name   = var.crdb_cluster_name
+        cache          = var.crdb_cache
+        max-sql-memory = var.crdb_max_sql_memory
+        single-node    = false # Always false. Even with 1 replica, we would expect to keep the ability to pool it with another cluster.
       }
 
       statefulset = {
@@ -29,6 +31,7 @@ resource "local_file" "helm_chart_values" {
           "--locality-advertise-addr=zone=${var.locality}@$(hostname -f)",
           "--advertise-addr=$${HOSTNAME##*-}.${var.db_hostname_suffix}"
         ]
+        resources = var.crdb_resources
       }
 
       storage = {
@@ -56,7 +59,8 @@ resource "local_file" "helm_chart_values" {
     }
 
     dss = {
-      image = var.image
+      image     = var.image
+      resources = var.core_service_resources
 
       conf = {
         pubKeys = [
@@ -271,7 +275,8 @@ resource "local_file" "helm_chart_values" {
     }
 
     dss = {
-      image = var.image
+      image     = var.image
+      resources = var.core_service_resources
 
       conf = {
         pubKeys = [

deploy/infrastructure/dependencies/terraform-commons-dss/tanka.tf

@@ -20,8 +20,12 @@ resource "local_file" "tanka_config_main" {
     VAR_CRDB_EXTERNAL_NODES                  = join(",", [for a in var.crdb_external_nodes : "'${a}'"])
     VAR_STORAGE_CLASS                        = var.kubernetes_storage_class
     VAR_DOCKER_IMAGE_NAME                    = var.image
+    VAR_CORE_SERVICE_RESOURCES               = jsonencode(var.core_service_resources)
     VAR_CRDB_DOCKER_IMAGE_NAME               = "cockroachdb/cockroach:${var.crdb_image_tag}"
     VAR_CRDB_CLUSTER_NAME                    = var.crdb_cluster_name
+    VAR_CRDB_CACHE                           = var.crdb_cache
+    VAR_CRDB_MAX_SQL_MEMORY                  = var.crdb_max_sql_memory
+    VAR_CRDB_RESOURCES                       = jsonencode(var.crdb_resources)
     VAR_YUGABYTE_CLOUD                       = var.yugabyte_cloud
     VAR_YUGABYTE_REGION                      = var.yugabyte_region
     VAR_YUGABYTE_ZONE                        = var.yugabyte_zone

deploy/infrastructure/dependencies/terraform-commons-dss/templates/main.jsonnet.tmp

@@ -25,6 +25,9 @@ local metadata = metadataBase {
     JoinExisting: [${VAR_CRDB_EXTERNAL_NODES}],
     storageClass: '${VAR_STORAGE_CLASS}',
     clusterName: '${VAR_CRDB_CLUSTER_NAME}',
+    cache: '${VAR_CRDB_CACHE}',
+    maxSqlMemory: '${VAR_CRDB_MAX_SQL_MEMORY}',
+    resources: ${VAR_CRDB_RESOURCES},
   },
   yugabyte+: {
     image: '${VAR_YUGABYTE_DOCKER_IMAGE_NAME}',
@@ -62,7 +65,8 @@ local metadata = metadataBase {
     hostname: '${VAR_APP_HOSTNAME}',
     publicEndpoint: 'https://${VAR_APP_HOSTNAME}',
     dumpRequests: false,
-    sslPolicy: '${VAR_SSL_POLICY}'
+    sslPolicy: '${VAR_SSL_POLICY}',
+    resources: ${VAR_CORE_SERVICE_RESOURCES},
   },
   schema_manager+: {
     enable: true,

deploy/infrastructure/dependencies/terraform-commons-dss/variables.gen.tf

@@ -284,6 +284,68 @@ variable "crdb_cluster_name" {
 }
 
 
+variable "crdb_cache" {
+  type        = string
+  description = <<-EOT
+  Size of the CockroachDB storage engine cache, passed to the `--cache` flag of `cockroach start`.
+  Either a percentage of the memory available to the container or an absolute size.
+
+  Example: `25%` or `2GiB`
+  EOT
+
+  default = "25%"
+}
+
+
+variable "crdb_max_sql_memory" {
+  type        = string
+  description = <<-EOT
+  Maximum memory CockroachDB can use for SQL queries, passed to the `--max-sql-memory` flag of `cockroach start`.
+  Either a percentage of the memory available to the container or an absolute size.
+
+  Example: `25%` or `2GiB`
+  EOT
+
+  default = "25%"
+}
+
+
+variable "crdb_resources" {
+  type        = map(map(string))
+  description = <<-EOT
+  Kubernetes CPU and memory requests and limits of the CockroachDB containers.
+  Leave empty to not set any.
+
+  Example: `{ requests = { cpu = "3.5", memory = "14Gi" }, limits = { cpu = "3.5", memory = "14Gi" } }`
+  EOT
+
+  default = {}
+
+  validation {
+    condition     = alltrue([for k in keys(var.crdb_resources) : contains(["requests", "limits"], k)])
+    error_message = "crdb_resources keys must be `requests` or `limits`."
+  }
+}
+
+
+variable "core_service_resources" {
+  type        = map(map(string))
+  description = <<-EOT
+  Kubernetes CPU and memory requests and limits of the core-service containers.
+  Leave empty to not set any.
+
+  Example: `{ requests = { cpu = "0.5", memory = "2Gi" }, limits = { memory = "2Gi" } }`
+  EOT
+
+  default = {}
+
+  validation {
+    condition     = alltrue([for k in keys(var.core_service_resources) : contains(["requests", "limits"], k)])
+    error_message = "core_service_resources keys must be `requests` or `limits`."
+  }
+}
+
+
 variable "locality" {
   type        = string
   description = <<-EOT

deploy/infrastructure/modules/terraform-aws-dss/TFVARS.gen.md

@@ -119,6 +119,18 @@ Leave empty to disable record creation.</p>
                 <td><p>Name of the kubernetes cluster that will host this DSS instance (should generally describe the DSS instance being hosted)</p>
 <p>Example: <code>dss-che-1</code></p>
 </td>
+            </tr><tr>
+                <td>core_service_resources (<code>map(map(string))</code>)</td>
+                <td><p>Kubernetes CPU and memory requests and limits of the core-service containers.
+Leave empty to not set any.</p>
+<p>Example: <code>{ requests = { cpu = &quot;0.5&quot;, memory = &quot;2Gi&quot; }, limits = { memory = &quot;2Gi&quot; } }</code></p>
+<br/>Default value: <code>{}</code></td>
+            </tr><tr>
+                <td>crdb_cache (<code>string</code>)</td>
+                <td><p>Size of the CockroachDB storage engine cache, passed to the <code>--cache</code> flag of <code>cockroach start</code>.
+Either a percentage of the memory available to the container or an absolute size.</p>
+<p>Example: <code>25%</code> or <code>2GiB</code></p>
+<br/>Default value: <code>"25%"</code></td>
             </tr><tr>
                 <td>crdb_cluster_name (<code>string</code>)</td>
                 <td><p>A string that specifies a CRDB cluster name. This is used together to ensure that all newly created
@@ -148,6 +160,18 @@ From v.17, the recommended CockroachDB version is v24.1.3.</p>
                 <td>crdb_locality (<code>string</code>)</td>
                 <td><p>This variable has been renamed to locality and is left to warn users about migration.</p>
 <br/>Default value: <code>""</code></td>
+            </tr><tr>
+                <td>crdb_max_sql_memory (<code>string</code>)</td>
+                <td><p>Maximum memory CockroachDB can use for SQL queries, passed to the <code>--max-sql-memory</code> flag of <code>cockroach start</code>.
+Either a percentage of the memory available to the container or an absolute size.</p>
+<p>Example: <code>25%</code> or <code>2GiB</code></p>
+<br/>Default value: <code>"25%"</code></td>
+            </tr><tr>
+                <td>crdb_resources (<code>map(map(string))</code>)</td>
+                <td><p>Kubernetes CPU and memory requests and limits of the CockroachDB containers.
+Leave empty to not set any.</p>
+<p>Example: <code>{ requests = { cpu = &quot;3.5&quot;, memory = &quot;14Gi&quot; }, limits = { cpu = &quot;3.5&quot;, memory = &quot;14Gi&quot; } }</code></p>
+<br/>Default value: <code>{}</code></td>
             </tr><tr>
                 <td>datastore_max_open_conns (<code>number</code>)</td>
                 <td><p>Maximum number of open connections to the datastore.</p>

deploy/infrastructure/modules/terraform-aws-dss/main.tf

@@ -25,6 +25,10 @@ module "terraform-commons-dss" {
   app_hostname                         = var.app_hostname
   crdb_image_tag                       = var.crdb_image_tag
   crdb_cluster_name                    = var.crdb_cluster_name
+  crdb_cache                           = var.crdb_cache
+  crdb_max_sql_memory                  = var.crdb_max_sql_memory
+  crdb_resources                       = var.crdb_resources
+  core_service_resources               = var.core_service_resources
   db_hostname_suffix                   = var.db_hostname_suffix
   datastore_type                       = var.datastore_type
   datastore_max_open_conns             = var.datastore_max_open_conns

deploy/infrastructure/modules/terraform-aws-dss/variables.gen.tf

@@ -411,6 +411,68 @@ variable "crdb_cluster_name" {
 }
 
 
+variable "crdb_cache" {
+  type        = string
+  description = <<-EOT
+  Size of the CockroachDB storage engine cache, passed to the `--cache` flag of `cockroach start`.
+  Either a percentage of the memory available to the container or an absolute size.
+
+  Example: `25%` or `2GiB`
+  EOT
+
+  default = "25%"
+}
+
+
+variable "crdb_max_sql_memory" {
+  type        = string
+  description = <<-EOT
+  Maximum memory CockroachDB can use for SQL queries, passed to the `--max-sql-memory` flag of `cockroach start`.
+  Either a percentage of the memory available to the container or an absolute size.
+
+  Example: `25%` or `2GiB`
+  EOT
+
+  default = "25%"
+}
+
+
+variable "crdb_resources" {
+  type        = map(map(string))
+  description = <<-EOT
+  Kubernetes CPU and memory requests and limits of the CockroachDB containers.
+  Leave empty to not set any.
+
+  Example: `{ requests = { cpu = "3.5", memory = "14Gi" }, limits = { cpu = "3.5", memory = "14Gi" } }`
+  EOT
+
+  default = {}
+
+  validation {
+    condition     = alltrue([for k in keys(var.crdb_resources) : contains(["requests", "limits"], k)])
+    error_message = "crdb_resources keys must be `requests` or `limits`."
+  }
+}
+
+
+variable "core_service_resources" {
+  type        = map(map(string))
+  description = <<-EOT
+  Kubernetes CPU and memory requests and limits of the core-service containers.
+  Leave empty to not set any.
+
+  Example: `{ requests = { cpu = "0.5", memory = "2Gi" }, limits = { memory = "2Gi" } }`
+  EOT
+
+  default = {}
+
+  validation {
+    condition     = alltrue([for k in keys(var.core_service_resources) : contains(["requests", "limits"], k)])
+    error_message = "core_service_resources keys must be `requests` or `limits`."
+  }
+}
+
+
 variable "locality" {
   type        = string
   description = <<-EOT

deploy/infrastructure/modules/terraform-google-dss/TFVARS.gen.md

@@ -76,6 +76,18 @@ Example:</li>
                 <td><p>Name of the kubernetes cluster that will host this DSS instance (should generally describe the DSS instance being hosted)</p>
 <p>Example: <code>dss-che-1</code></p>
 </td>
+            </tr><tr>
+                <td>core_service_resources (<code>map(map(string))</code>)</td>
+                <td><p>Kubernetes CPU and memory requests and limits of the core-service containers.
+Leave empty to not set any.</p>
+<p>Example: <code>{ requests = { cpu = &quot;0.5&quot;, memory = &quot;2Gi&quot; }, limits = { memory = &quot;2Gi&quot; } }</code></p>
+<br/>Default value: <code>{}</code></td>
+            </tr><tr>
+                <td>crdb_cache (<code>string</code>)</td>
+                <td><p>Size of the CockroachDB storage engine cache, passed to the <code>--cache</code> flag of <code>cockroach start</code>.
+Either a percentage of the memory available to the container or an absolute size.</p>
+<p>Example: <code>25%</code> or <code>2GiB</code></p>
+<br/>Default value: <code>"25%"</code></td>
             </tr><tr>
                 <td>crdb_cluster_name (<code>string</code>)</td>
                 <td><p>A string that specifies a CRDB cluster name. This is used together to ensure that all newly created
@@ -105,6 +117,18 @@ From v.17, the recommended CockroachDB version is v24.1.3.</p>
                 <td>crdb_locality (<code>string</code>)</td>
                 <td><p>This variable has been renamed to locality and is left to warn users about migration.</p>
 <br/>Default value: <code>""</code></td>
+            </tr><tr>
+                <td>crdb_max_sql_memory (<code>string</code>)</td>
+                <td><p>Maximum memory CockroachDB can use for SQL queries, passed to the <code>--max-sql-memory</code> flag of <code>cockroach start</code>.
+Either a percentage of the memory available to the container or an absolute size.</p>
+<p>Example: <code>25%</code> or <code>2GiB</code></p>
+<br/>Default value: <code>"25%"</code></td>
+            </tr><tr>
+                <td>crdb_resources (<code>map(map(string))</code>)</td>
+                <td><p>Kubernetes CPU and memory requests and limits of the CockroachDB containers.
+Leave empty to not set any.</p>
+<p>Example: <code>{ requests = { cpu = &quot;3.5&quot;, memory = &quot;14Gi&quot; }, limits = { cpu = &quot;3.5&quot;, memory = &quot;14Gi&quot; } }</code></p>
+<br/>Default value: <code>{}</code></td>
             </tr><tr>
                 <td>datastore_max_open_conns (<code>number</code>)</td>
                 <td><p>Maximum number of open connections to the datastore.</p>

deploy/infrastructure/modules/terraform-google-dss/main.tf

@@ -25,6 +25,10 @@ module "terraform-commons-dss" {
   app_hostname                         = var.app_hostname
   crdb_image_tag                       = var.crdb_image_tag
   crdb_cluster_name                    = var.crdb_cluster_name
+  crdb_cache                           = var.crdb_cache
+  crdb_max_sql_memory                  = var.crdb_max_sql_memory
+  crdb_resources                       = var.crdb_resources
+  core_service_resources               = var.core_service_resources
   db_hostname_suffix                   = var.db_hostname_suffix
   datastore_type                       = var.datastore_type
   datastore_max_open_conns             = var.datastore_max_open_conns

deploy/infrastructure/modules/terraform-google-dss/variables.gen.tf

@@ -400,6 +400,68 @@ variable "crdb_cluster_name" {
 }
 
 
+variable "crdb_cache" {
+  type        = string
+  description = <<-EOT
+  Size of the CockroachDB storage engine cache, passed to the `--cache` flag of `cockroach start`.
+  Either a percentage of the memory available to the container or an absolute size.
+
+  Example: `25%` or `2GiB`
+  EOT
+
+  default = "25%"
+}
+
+
+variable "crdb_max_sql_memory" {
+  type        = string
+  description = <<-EOT
+  Maximum memory CockroachDB can use for SQL queries, passed to the `--max-sql-memory` flag of `cockroach start`.
+  Either a percentage of the memory available to the container or an absolute size.
+
+  Example: `25%` or `2GiB`
+  EOT
+
+  default = "25%"
+}
+
+
+variable "crdb_resources" {
+  type        = map(map(string))
+  description = <<-EOT
+  Kubernetes CPU and memory requests and limits of the CockroachDB containers.
+  Leave empty to not set any.
+
+  Example: `{ requests = { cpu = "3.5", memory = "14Gi" }, limits = { cpu = "3.5", memory = "14Gi" } }`
+  EOT
+
+  default = {}
+
+  validation {
+    condition     = alltrue([for k in keys(var.crdb_resources) : contains(["requests", "limits"], k)])
+    error_message = "crdb_resources keys must be `requests` or `limits`."
+  }
+}
+
+
+variable "core_service_resources" {
+  type        = map(map(string))
+  description = <<-EOT
+  Kubernetes CPU and memory requests and limits of the core-service containers.
+  Leave empty to not set any.
+
+  Example: `{ requests = { cpu = "0.5", memory = "2Gi" }, limits = { memory = "2Gi" } }`
+  EOT
+
+  default = {}
+
+  validation {
+    condition     = alltrue([for k in keys(var.core_service_resources) : contains(["requests", "limits"], k)])
+    error_message = "core_service_resources keys must be `requests` or `limits`."
+  }
+}
+
+
 variable "locality" {
   type        = string
   description = <<-EOT