feat: [CPCAP-12620] Default partition size monitoring

operator/charts/patroni-services/monitoring/grafana-dashboard.json

@@ -58,7 +58,7 @@
   ],
   "panels": [
     {
-      "collapsed": true,
+      "collapsed": false,
       "gridPos": {
         "h": 1,
         "w": 24,
@@ -1825,7 +1825,7 @@
       "type": "row"
     },
     {
-      "collapsed": false,
+      "collapsed": true,
       "gridPos": {
         "h": 1,
         "w": 24,
@@ -8898,6 +8898,207 @@
             }
           ],
           "type": "table"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "$datasource"
+          },
+          "description": "",
+          "fieldConfig": {
+            "defaults": {
+              "color": {
+                "fixedColor": "green",
+                "mode": "palette-classic"
+              },
+              "custom": {
+                "axisBorderShow": false,
+                "axisCenteredZero": false,
+                "axisColorMode": "text",
+                "axisLabel": "",
+                "axisPlacement": "auto",
+                "barAlignment": 0,
+                "barWidthFactor": 0.6,
+                "drawStyle": "line",
+                "fillOpacity": 25,
+                "gradientMode": "hue",
+                "hideFrom": {
+                  "legend": false,
+                  "tooltip": false,
+                  "viz": false
+                },
+                "insertNulls": false,
+                "lineInterpolation": "linear",
+                "lineStyle": {
+                  "fill": "solid"
+                },
+                "lineWidth": 1,
+                "pointSize": 5,
+                "scaleDistribution": {
+                  "type": "linear"
+                },
+                "showPoints": "auto",
+                "showValues": false,
+                "spanNulls": false,
+                "stacking": {
+                  "group": "A",
+                  "mode": "none"
+                },
+                "thresholdsStyle": {
+                  "mode": "off"
+                }
+              },
+              "thresholds": {
+                "mode": "percentage",
+                "steps": [
+                  {
+                    "color": "green",
+                    "value": 0
+                  },
+                  {
+                    "color": "red",
+                    "value": 80
+                  }
+                ]
+              },
+              "unit": "decbytes"
+            },
+            "overrides": []
+          },
+          "gridPos": {
+            "h": 8,
+            "w": 12,
+            "x": 0,
+            "y": 131
+          },
+          "id": 1103,
+          "options": {
+            "legend": {
+              "calcs": [],
+              "displayMode": "table",
+              "placement": "bottom",
+              "showLegend": true
+            },
+            "tooltip": {
+              "mode": "multi",
+              "sort": "none"
+            }
+          },
+          "targets": [
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "$datasource"
+              },
+              "editorMode": "builder",
+              "expr": "sum by(relname, datname) (ma_pg_default_partition_size_bytes{namespace=~\"$namespace\", datname=~\"$datname\", relname=~\"$relname\", cluster=~\"$cluster\"})",
+              "legendFormat": "{{datname}}/{{ relname }}",
+              "range": true,
+              "refId": "B"
+            }
+          ],
+          "title": "Default Partition Size (bytes)",
+          "type": "timeseries"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "$datasource"
+          },
+          "description": "",
+          "fieldConfig": {
+            "defaults": {
+              "color": {
+                "fixedColor": "green",
+                "mode": "palette-classic"
+              },
+              "custom": {
+                "axisBorderShow": false,
+                "axisCenteredZero": false,
+                "axisColorMode": "text",
+                "axisLabel": "",
+                "axisPlacement": "auto",
+                "barAlignment": 0,
+                "barWidthFactor": 0.6,
+                "drawStyle": "line",
+                "fillOpacity": 100,
+                "gradientMode": "opacity",
+                "hideFrom": {
+                  "legend": false,
+                  "tooltip": false,
+                  "viz": false
+                },
+                "insertNulls": false,
+                "lineInterpolation": "linear",
+                "lineStyle": {
+                  "fill": "solid"
+                },
+                "lineWidth": 1,
+                "pointSize": 1,
+                "scaleDistribution": {
+                  "type": "linear"
+                },
+                "showPoints": "auto",
+                "showValues": false,
+                "spanNulls": false,
+                "stacking": {
+                  "group": "A",
+                  "mode": "none"
+                },
+                "thresholdsStyle": {
+                  "mode": "off"
+                }
+              },
+              "thresholds": {
+                "mode": "absolute",
+                "steps": [
+                  {
+                    "color": "green",
+                    "value": 0
+                  },
+                  {
+                    "color": "red",
+                    "value": 80
+                  }
+                ]
+              }
+            },
+            "overrides": []
+          },
+          "gridPos": {
+            "h": 8,
+            "w": 12,
+            "x": 12,
+            "y": 131
+          },
+          "id": 1104,
+          "options": {
+            "legend": {
+              "calcs": [],
+              "displayMode": "table",
+              "placement": "bottom",
+              "showLegend": true
+            },
+            "tooltip": {
+              "mode": "single",
+              "sort": "none"
+            }
+          },
+          "targets": [
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "$datasource"
+              },
+              "editorMode": "builder",
+              "expr": "sum by(relname, datname) (ma_pg_default_partition_row_count{relname=~\"$relname\", datname=~\"$datname\", cluster=~\"$cluster\", namespace=\"$namespace\"})",
+              "legendFormat": "{{datname}}/{{ relname }}",
+              "range": true,
+              "refId": "A"
+            }
+          ],
+          "title": "Default Partition Row count",
+          "type": "timeseries"
         }
       ],
       "title": "Tables Statistics",
@@ -9229,6 +9430,33 @@
         "sort": 1,
         "type": "query"
       },
+      {
+        "allValue": ".*",
+        "current": {
+          "text": "All",
+          "value": [
+            "$__all"
+          ]
+        },
+        "datasource": {
+          "type": "prometheus",
+          "uid": "${datasource}"
+        },
+        "definition": "label_values(ma_pg_default_partition_size_bytes{namespace=~\"$namespace\", datname=~\"$datname\", cluster=~\"$cluster\"}, relname)",
+        "includeAll": true,
+        "label": "Partition Name",
+        "multi": true,
+        "name": "relname",
+        "options": [],
+        "query": {
+          "query": "label_values(ma_pg_default_partition_size_bytes{namespace=~\"$namespace\", datname=~\"$datname\", cluster=~\"$cluster\"}, relname)",
+          "refId": "PrometheusVariableQueryEditor-VariableQuery"
+        },
+        "refresh": 2,
+        "regex": "",
+        "sort": 1,
+        "type": "query"
+      },
       {
         "current": {
           "text": "10",
@@ -9421,4 +9649,4 @@
   "uid": "bc9f739dee3fe919d1a280c6e2bbea5255ec26ff",
   "version": 12,
   "weekStart": ""
-}
+}

operator/charts/patroni-services/templates/monitoring-templates/prometheus-rule.yml

@@ -522,4 +522,15 @@ spec:
         namespace: {{ .Release.Namespace }}
         service: {{ .Release.Name }}
 {{- end }}
+    - alert: PostgreSQL default partition size high
+      annotations:
+        description: PostgreSQL default partition size high
+        summary: >-
+          "Default partition \{\{ \$labels.relname \}\} of table \{\{ \$labels.parent_relname \}\} has exceeded the size threshold in namespace {{ .Release.Namespace }}."
+      expr: ma_pg_default_partition_size_bytes{namespace="{{ .Release.Namespace }}"} / ma_pg_default_partition_parent_size_bytes{namespace="{{ .Release.Namespace }}"} * 100 > {{ default "20" .Values.metricCollector.prometheusRules.defaultPartitionSizePercentThreshold }}
+      for: {{ default "3m" .Values.metricCollector.prometheusRules.alertDelay }}
+      labels:
+        severity: average
+        namespace: {{ .Release.Namespace }}
+        service: {{ .Release.Name }}
 {{ end }}

operator/charts/patroni-services/values.schema.json

@@ -488,6 +488,10 @@
               "type": "integer",
               "default": 104857600
             },
+            "defaultPartitionSizePercentThreshold": {
+              "type": "integer",
+              "default": 20
+            },
             "maxConnectionExceedPercentageThreshold": {
               "type": "integer",
               "default": 90

operator/charts/patroni-services/values.yaml

@@ -153,6 +153,7 @@ metricCollector:
     # should be specified in bytes, default is set to 32MB
     replicationLagValue: 33554432
     largeObjectSizeThreshold: 104857600  # 100MB
+    defaultPartitionSizePercentThreshold: 20  # 20% of table size
     maxConnectionExceedPercentageThreshold: 90
     maxConnectionReachedPercentageThreshold: 80
     warnWaitEventTreshold: 5

services/monitoring-agent/collector/pkg/metrics/performance_metrics.go

@@ -105,6 +105,9 @@ var (
 	(SELECT setting::int max_connections FROM pg_settings
 	WHERE name=$$max_connections$$) t3;
 `
+	defaultPartitionQuery = `SELECT n.nspname AS schemaname, c.relname AS relname, p.relname AS parent_relname, pg_total_relation_size(c.oid) AS size_bytes, COALESCE(s.n_live_tup, 0) AS row_count FROM pg_class c JOIN pg_namespace n ON n.oid = c.relnamespace JOIN pg_inherits i ON i.inhrelid = c.oid JOIN pg_class p ON p.oid = i.inhparent LEFT JOIN pg_stat_user_tables s ON s.relid = c.oid WHERE c.relispartition = true AND pg_get_expr(c.relpartbound, c.oid) = 'DEFAULT'`
+
+	parentTableQuery = `SELECT n.nspname AS schemaname, p.relname AS parent_relname, COALESCE(sum(pg_total_relation_size(i.inhrelid)), 0) AS parent_size_bytes FROM pg_inherits i JOIN pg_class p ON p.oid = i.inhparent JOIN pg_namespace n ON n.oid = p.relnamespace WHERE p.relkind = 'p' GROUP BY n.nspname, p.relname`
 )
 
 func (s *Scraper) CollectPerformanceMetrics() {
@@ -276,6 +279,7 @@ func (s *Scraper) collectMetricsPerDB(ctx context.Context, pg *postgres.Postgres
 		}
 		s.collectTableMetrics(ctx, pg)
 		s.collectDBaaSData(ctx, pg)
+		s.collectPartitionMetrics(ctx, pg)
 	}
 	logger.Info("Tables statistic metrics collection finished")
 }
@@ -359,6 +363,37 @@ func (s *Scraper) collectCountStatMetrics(ctx context.Context, pc *postgres.Post
 	}
 }
 
+func (s *Scraper) collectPartitionMetrics(ctx context.Context, pg *postgres.PostgresConnector) {
+	_, parentRows := getData(ctx, pg, parentTableQuery)
+	parentSizes := map[string]string{}
+	for _, row := range parentRows {
+		key := fmt.Sprintf("%v|%v", row["schemaname"], row["parent_relname"])
+		parentSizes[key] = fmt.Sprintf("%v", row["parent_size_bytes"])
+	}
+
+	partColumns, partRows := getData(ctx, pg, defaultPartitionQuery)
+	for _, row := range partRows {
+		labels := gauges.DefaultLabels()
+		labels["datname"] = pg.GetDatabase()
+		var sizeValue, rowCountValue string
+		for _, column := range partColumns {
+			rValue := fmt.Sprintf("%v", row[column])
+			switch column {
+			case "size_bytes":
+				sizeValue = rValue
+			case "row_count":
+				rowCountValue = rValue
+			default:
+				labels[column] = rValue
+			}
+		}
+		key := fmt.Sprintf("%v|%v", row["schemaname"], row["parent_relname"])
+		s.metrics = append(s.metrics, NewMetric("ma_pg_default_partition_size_bytes").withLabels(labels).setValue(sizeValue))
+		s.metrics = append(s.metrics, NewMetric("ma_pg_default_partition_row_count").withLabels(labels).setValue(rowCountValue))
+		s.metrics = append(s.metrics, NewMetric("ma_pg_default_partition_parent_size_bytes").withLabels(labels).setValue(parentSizes[key]))
+	}
+}
+
 func getDatabasesList(ctx context.Context, pg *postgres.PostgresConnector) ([]string, error) {
 	databases := []string{}
 	rows, err := pg.Query(ctx, dbQuery)