From 578d2c74b5ed945c5599065509570a7ede93f71b Mon Sep 17 00:00:00 2001 From: Benedikt Labrenz Date: Thu, 24 Sep 2026 15:07:32 +0200 Subject: [PATCH 01/93] Add agent-binary as workspace member --- Cargo.lock | 12 +++++ Cargo.nix | 70 +++++++++++++++++++++----- Cargo.toml | 2 +- rust/agent-binary/Cargo.toml | 24 +++++++++ rust/agent-binary/build.rs | 3 ++ rust/agent-binary/src/framework/mod.rs | 12 +++++ rust/agent-binary/src/main.rs | 47 +++++++++++++++++ 7 files changed, 157 insertions(+), 13 deletions(-) create mode 100644 rust/agent-binary/Cargo.toml create mode 100644 rust/agent-binary/build.rs create mode 100644 rust/agent-binary/src/framework/mod.rs create mode 100644 rust/agent-binary/src/main.rs diff --git a/Cargo.lock b/Cargo.lock index 75704686b..b08d28809 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3173,6 +3173,18 @@ dependencies = [ "zeroize", ] +[[package]] +name = "stackable-kafka-agent" +version = "0.0.0-dev" +dependencies = [ + "anyhow", + "built", + "clap", + "stackable-operator", + "tokio", + "tracing", +] + [[package]] name = "stackable-kafka-operator" version = "0.0.0-dev" diff --git a/Cargo.nix b/Cargo.nix index 1b1ba51b5..292f308c7 100644 --- a/Cargo.nix +++ b/Cargo.nix @@ -38,23 +38,21 @@ rec { # "public" attributes that we attempt to keep stable with new versions of crate2nix. # - rootCrate = rec { - packageId = "stackable-kafka-operator"; - # Use this attribute to refer to the derivation building your root crate package. - # You can override the features with rootCrate.build.override { features = [ "default" "feature1" ... ]; }. - build = internal.buildRustCrateWithFeatures { - inherit packageId; - }; - - # Debug support which might change between releases. - # File a bug if you depend on any for non-debug work! - debug = internal.debugCrate { inherit packageId; }; - }; # Refer your crate build derivation by name here. # You can override the features with # workspaceMembers."${crateName}".build.override { features = [ "default" "feature1" ... ]; }. workspaceMembers = { + "stackable-kafka-agent" = rec { + packageId = "stackable-kafka-agent"; + build = internal.buildRustCrateWithFeatures { + packageId = "stackable-kafka-agent"; + }; + + # Debug support which might change between releases. + # File a bug if you depend on any for non-debug work! + debug = internal.debugCrate { inherit packageId; }; + }; "stackable-kafka-operator" = rec { packageId = "stackable-kafka-operator"; build = internal.buildRustCrateWithFeatures { @@ -10423,6 +10421,54 @@ rec { }; resolvedDefaultFeatures = [ "default" "rustls" ]; }; + "stackable-kafka-agent" = rec { + crateName = "stackable-kafka-agent"; + version = "0.0.0-dev"; + edition = "2024"; + crateBin = [ + { + name = "stackable-kafka-agent"; + path = "src/main.rs"; + requiredFeatures = [ ]; + } + ]; + src = lib.cleanSourceWith { filter = sourceFilter; src = ./rust/agent-binary; }; + authors = [ + "Stackable GmbH " + ]; + dependencies = [ + { + name = "anyhow"; + packageId = "anyhow"; + } + { + name = "clap"; + packageId = "clap"; + } + { + name = "stackable-operator"; + packageId = "stackable-operator"; + features = [ "crds" "webhook" ]; + } + { + name = "tokio"; + packageId = "tokio"; + features = [ "full" ]; + } + { + name = "tracing"; + packageId = "tracing"; + } + ]; + buildDependencies = [ + { + name = "built"; + packageId = "built"; + features = [ "chrono" "git2" ]; + } + ]; + + }; "stackable-kafka-operator" = rec { crateName = "stackable-kafka-operator"; version = "0.0.0-dev"; diff --git a/Cargo.toml b/Cargo.toml index 2eac6bf58..5e9c33132 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,5 +1,5 @@ [workspace] -members = ["rust/operator-binary"] +members = ["rust/operator-binary", "rust/agent-binary"] resolver = "2" [workspace.package] diff --git a/rust/agent-binary/Cargo.toml b/rust/agent-binary/Cargo.toml new file mode 100644 index 000000000..c138e675b --- /dev/null +++ b/rust/agent-binary/Cargo.toml @@ -0,0 +1,24 @@ +[package] +name = "stackable-kafka-agent" +description = "Stackable Agent for Apache Kafka" +version.workspace = true +authors.workspace = true +license.workspace = true +edition.workspace = true +repository.workspace = true +publish = false + +[dependencies] +stackable-operator.workspace = true + +anyhow.workspace = true +clap.workspace = true +tokio.workspace = true +tracing.workspace = true + +[build-dependencies] +built.workspace = true + +[[bin]] +name = "stackable-kafka-agent" +path = "src/main.rs" diff --git a/rust/agent-binary/build.rs b/rust/agent-binary/build.rs new file mode 100644 index 000000000..fa809bfd1 --- /dev/null +++ b/rust/agent-binary/build.rs @@ -0,0 +1,3 @@ +fn main() { + built::write_built_file().unwrap(); +} diff --git a/rust/agent-binary/src/framework/mod.rs b/rust/agent-binary/src/framework/mod.rs new file mode 100644 index 000000000..c26b5e711 --- /dev/null +++ b/rust/agent-binary/src/framework/mod.rs @@ -0,0 +1,12 @@ +//! Reusable, product-agnostic agent building blocks. Staged here until they move to operator-rs. + +use clap::Args; +use stackable_operator::cli::CommonOptions; + +/// Agent subcommands. Like [`stackable_operator::cli::Command`], but without `crd` as CRDs are only +/// deployed by operators. +#[derive(Debug, clap::Parser)] +pub enum AgentCommand { + /// Run the agent. + Run(Run), +} diff --git a/rust/agent-binary/src/main.rs b/rust/agent-binary/src/main.rs new file mode 100644 index 000000000..64e2cad91 --- /dev/null +++ b/rust/agent-binary/src/main.rs @@ -0,0 +1,47 @@ +//! The Stackable Kafka agent: a per-cluster controller that reconciles in-cluster resources like `KafkaTopic`s. + +use clap::Parser; +use stackable_operator::{ + cli::CommonOptions, + telemetry::Tracing, + utils::signal::SignalWatcher, +}; + +use crate::framework::AgentCommand; + +mod framework; + +mod built_info { + include!(concat!(env!("OUT_DIR"), "/built.rs")); +} + +#[derive(clap::Parser)] +#[clap(about, author)] +struct Opts { + #[clap(subcommand)] + cmd: AgentCommand, +} + +#[tokio::main] +async fn main() -> anyhow::Result<()> { + match Opts::parse().cmd { + AgentCommand::Run(CommonOptions { telemetry, .. }) => { + let _tracing_guard = Tracing::pre_configured(built_info::PKG_NAME, telemetry).init()?; + + tracing::info!( + built_info.pkg_version = built_info::PKG_VERSION, + built_info.git_version = built_info::GIT_VERSION, + built_info.target = built_info::TARGET, + built_info.built_time_utc = built_info::BUILT_TIME_UTC, + built_info.rustc_version = built_info::RUSTC_VERSION, + "Starting {description}", + description = built_info::PKG_DESCRIPTION + ); + + let sigterm_watcher = SignalWatcher::sigterm()?; + sigterm_watcher.handle().await; + } + } + + Ok(()) +} From 2a1fae09ec06ce67308b1fccdad5350d96f4d23a Mon Sep 17 00:00:00 2001 From: Benedikt Labrenz Date: Thu, 24 Sep 2026 16:03:50 +0200 Subject: [PATCH 02/93] deploy the agent if spec.platformAcces.enabled is configured --- extra/crds.yaml | 41 +++ rust/agent-binary/src/main.rs | 6 +- rust/operator-binary/src/controller.rs | 28 +- rust/operator-binary/src/controller/apply.rs | 3 + .../src/controller/build/mod.rs | 34 +- .../src/controller/build/resource/agent.rs | 303 ++++++++++++++++++ .../src/controller/build/resource/mod.rs | 1 + .../src/controller/validate.rs | 16 +- rust/operator-binary/src/crd/mod.rs | 44 ++- .../src/framework/commons/mod.rs | 1 + .../framework/commons/platform_access/mod.rs | 3 + .../framework/commons/platform_access/tls.rs | 87 +++++ .../src/framework/constants/mod.rs | 1 + .../src/framework/constants/secret.rs | 2 + .../src/framework/kvp/label.rs | 45 +++ rust/operator-binary/src/framework/kvp/mod.rs | 1 + rust/operator-binary/src/framework/mod.rs | 6 + rust/operator-binary/src/main.rs | 18 +- 18 files changed, 619 insertions(+), 21 deletions(-) create mode 100644 rust/operator-binary/src/controller/build/resource/agent.rs create mode 100644 rust/operator-binary/src/framework/commons/mod.rs create mode 100644 rust/operator-binary/src/framework/commons/platform_access/mod.rs create mode 100644 rust/operator-binary/src/framework/commons/platform_access/tls.rs create mode 100644 rust/operator-binary/src/framework/constants/mod.rs create mode 100644 rust/operator-binary/src/framework/constants/secret.rs create mode 100644 rust/operator-binary/src/framework/kvp/label.rs create mode 100644 rust/operator-binary/src/framework/kvp/mod.rs create mode 100644 rust/operator-binary/src/framework/mod.rs diff --git a/extra/crds.yaml b/extra/crds.yaml index 9283cc4ff..7c5f5161f 100644 --- a/extra/crds.yaml +++ b/extra/crds.yaml @@ -2136,6 +2136,47 @@ spec: type: object x-kubernetes-preserve-unknown-fields: true type: array + platformAccess: + description: Access of the Stackable Data Platform to this cluster. + nullable: true + properties: + authentication: + description: The credential the agent authenticates to Kafka with. + oneOf: + - required: + - tls + properties: + tls: + description: 'Source of a TLS client certificate: a secret-operator SecretClass or a static Secret.' + oneOf: + - required: + - secretClass + - required: + - secret + properties: + secret: + description: |- + A static Secret holding the certificate in the keys `tls.crt` and `tls.key` (PEM), such as a + Secret of type `kubernetes.io/tls`. + maxLength: 253 + minLength: 1 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ + type: string + secretClass: + description: An AutoTLS SecretClass used to provision the certificate. + maxLength: 253 + minLength: 1 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ + type: string + type: object + type: object + enabled: + default: false + description: Whether the operator deploys the agent. False by default. + type: boolean + required: + - authentication + type: object required: - brokers - image diff --git a/rust/agent-binary/src/main.rs b/rust/agent-binary/src/main.rs index 64e2cad91..6ab5c58f7 100644 --- a/rust/agent-binary/src/main.rs +++ b/rust/agent-binary/src/main.rs @@ -1,11 +1,7 @@ //! The Stackable Kafka agent: a per-cluster controller that reconciles in-cluster resources like `KafkaTopic`s. use clap::Parser; -use stackable_operator::{ - cli::CommonOptions, - telemetry::Tracing, - utils::signal::SignalWatcher, -}; +use stackable_operator::{cli::CommonOptions, telemetry::Tracing, utils::signal::SignalWatcher}; use crate::framework::AgentCommand; diff --git a/rust/operator-binary/src/controller.rs b/rust/operator-binary/src/controller.rs index 21e1d8de4..1dc977ea1 100644 --- a/rust/operator-binary/src/controller.rs +++ b/rust/operator-binary/src/controller.rs @@ -22,7 +22,7 @@ use stackable_operator::{ constant, crd::listener, k8s_openapi::api::{ - apps::v1::StatefulSet, + apps::v1::{Deployment, StatefulSet}, core::v1::{ConfigMap, Service, ServiceAccount}, policy::v1::PodDisruptionBudget, rbac::v1::RoleBinding, @@ -65,7 +65,8 @@ use crate::{ update_status::update_status, }, crd::{ - APP_NAME, KAFKA_OPERATOR_NAME, KafkaPodDescriptor, MetadataManager, + APP_NAME, KAFKA_OPERATOR_NAME, KafkaPlatformAccessAuthentication, KafkaPodDescriptor, + MetadataManager, authorization::KafkaAuthorizationConfig, role::{AnyConfig, AnyConfigOverrides, KafkaRole}, v1alpha1, @@ -112,6 +113,7 @@ pub struct Applied; /// bootstrap [`Listener`](listener)s. pub struct KubernetesResources { pub stateful_sets: Vec, + pub deployments: Vec, pub services: Vec, pub listeners: Vec, pub config_maps: Vec, @@ -151,6 +153,7 @@ pub struct ValidatedCluster { /// address-less around the first reconcile runs; the listener-operator populates the ingress /// addresses and the `Listener` watch triggers a new run once it does. pub bootstrap_listeners: Vec, + pub agent_config: Option, } impl ValidatedCluster { @@ -165,6 +168,7 @@ impl ValidatedCluster { role_configs: BTreeMap, role_group_configs: BTreeMap>, bootstrap_listeners: Vec, + agent_config: Option, ) -> Self { // `app_version_label_value` is constructed to be a valid label value, so it is also a // valid `ProductVersion`. @@ -187,6 +191,7 @@ impl ValidatedCluster { role_configs, role_group_configs, bootstrap_listeners, + agent_config, } } @@ -339,6 +344,12 @@ impl ValidatedClusterConfig { } } +/// The configuration of the agent deployed with the cluster. +pub struct ValidatedAgentConfig { + pub image: String, + pub authentication: KafkaPlatformAccessAuthentication, +} + /// Per-role configuration extracted during validation. /// /// Resolved from the raw [`v1alpha1::KafkaCluster`] spec during validation so the reconcile loop @@ -397,6 +408,7 @@ pub type ValidatedRoleGroupConfig = stackable_operator::v2::role_utils::RoleGrou pub struct Ctx { pub client: stackable_operator::client::Client, pub operator_environment: OperatorEnvironmentOptions, + pub agent_image: String, } #[derive(Snafu, Debug, EnumDiscriminants)] @@ -466,9 +478,13 @@ pub async fn reconcile_kafka( .context(DereferenceSnafu)?; // validate (no client required) - let validated_cluster = - validate::validate(kafka, dereferenced_objects, &ctx.operator_environment) - .context(ValidateClusterSnafu)?; + let validated_cluster = validate::validate( + kafka, + dereferenced_objects, + &ctx.operator_environment, + &ctx.agent_image, + ) + .context(ValidateClusterSnafu)?; tracing::debug!( kerberos_enabled = validated_cluster.cluster_config.kafka_security.has_kerberos_enabled(), @@ -628,6 +644,7 @@ pub(crate) mod test_support { bootstrap_listeners: Vec::new(), }, &operator_environment(), + &crate::crd::default_agent_image(&operator_environment().image_repository), ) } } @@ -780,6 +797,7 @@ spec: {} operator_service_name: "kafka-operator".to_owned(), image_repository: "oci.stackable.tech/sdp".to_owned(), }, + agent_image: crate::crd::default_agent_image("oci.stackable.tech/sdp"), }); reconcile_kafka(Arc::new(kafka), ctx).await diff --git a/rust/operator-binary/src/controller/apply.rs b/rust/operator-binary/src/controller/apply.rs index 1ce080f87..5f740ecc4 100644 --- a/rust/operator-binary/src/controller/apply.rs +++ b/rust/operator-binary/src/controller/apply.rs @@ -78,6 +78,7 @@ impl<'a> Applier<'a> { // compile here instead of silently never being applied. let KubernetesResources { stateful_sets, + deployments, services, listeners, config_maps, @@ -97,6 +98,7 @@ impl<'a> Applier<'a> { let config_maps = self.add_resources(config_maps).await?; let pod_disruption_budgets = self.add_resources(pod_disruption_budgets).await?; let stateful_sets = self.add_resources(stateful_sets).await?; + let deployments = self.add_resources(deployments).await?; self.cluster_resources .delete_orphaned_resources(self.client) @@ -105,6 +107,7 @@ impl<'a> Applier<'a> { Ok(KubernetesResources { stateful_sets, + deployments, services, listeners, config_maps, diff --git a/rust/operator-binary/src/controller/build/mod.rs b/rust/operator-binary/src/controller/build/mod.rs index 1be5313e1..2283c4448 100644 --- a/rust/operator-binary/src/controller/build/mod.rs +++ b/rust/operator-binary/src/controller/build/mod.rs @@ -14,6 +14,9 @@ use crate::{ listener::get_kafka_listener_config, product_logging::vector_config_file_content, }, resource::{ + agent::{ + build_agent_deployment, build_agent_role_binding, build_agent_service_account, + }, config_map::build_rolegroup_config_map, discovery::build_discovery_configmap, listener::build_broker_rolegroup_bootstrap_listener, @@ -27,6 +30,7 @@ use crate::{ }, }, crd::role::{AnyConfig, KafkaRole}, + framework::kvp::label as framework_label, }; pub mod command; @@ -149,14 +153,24 @@ pub fn build(cluster: &ValidatedCluster) -> Result config_maps.push(build_discovery_configmap(cluster).context(DiscoveryConfigMapSnafu)?); + let mut deployments = vec![]; + let mut service_accounts = vec![build_service_account(cluster)]; + let mut role_bindings = vec![build_role_binding(cluster)]; + if let Some(agent_config) = &cluster.agent_config { + deployments.push(build_agent_deployment(cluster, agent_config)); + service_accounts.push(build_agent_service_account(cluster)); + role_bindings.push(build_agent_role_binding(cluster)); + } + Ok(KubernetesResources { stateful_sets, + deployments, services, listeners, config_maps, pod_disruption_budgets, - service_accounts: vec![build_service_account(cluster)], - role_bindings: vec![build_role_binding(cluster)], + service_accounts, + role_bindings, status: PhantomData, }) } @@ -171,6 +185,16 @@ pub(crate) fn recommended_labels_for_cluster_resources(cluster: &ValidatedCluste ) } +pub(crate) fn recommended_labels_for_agent_resources(cluster: &ValidatedCluster) -> Labels { + framework_label::recommended_labels_for_agent_resources( + &cluster.name, + &PRODUCT_NAME, + &cluster.product_version, + &OPERATOR_NAME, + &CONTROLLER_NAME, + ) +} + pub(crate) fn recommended_labels_for_role_resources( cluster: &ValidatedCluster, role_name: &RoleName, @@ -217,6 +241,10 @@ pub(crate) fn recommended_labels_for_unversioned_role_group_resources( } /// Selector labels matching the pods of a role group. +pub(crate) fn agent_selector(cluster: &ValidatedCluster) -> Labels { + framework_label::agent_selector(&cluster.name, &PRODUCT_NAME) +} + pub(crate) fn role_group_selector( cluster: &ValidatedCluster, role_name: &RoleName, @@ -363,6 +391,8 @@ mod tests { sorted_names(&resources.role_bindings), ["simple-kafka-rolebinding"] ); + // No agent without platform access. + assert!(resources.deployments.is_empty()); } #[test] diff --git a/rust/operator-binary/src/controller/build/resource/agent.rs b/rust/operator-binary/src/controller/build/resource/agent.rs new file mode 100644 index 000000000..e466512fd --- /dev/null +++ b/rust/operator-binary/src/controller/build/resource/agent.rs @@ -0,0 +1,303 @@ +//! Builds the resources of the platform-access agent deployed alongside the cluster. + +use std::str::FromStr; + +use stackable_operator::{ + builder::{ + meta::ObjectMetaBuilder, + pod::{ + PodBuilder, resources::ResourceRequirementsBuilder, security::PodSecurityContextBuilder, + }, + }, + constant, + k8s_openapi::{ + api::{ + apps::v1::{Deployment, DeploymentSpec}, + core::v1::{EnvVarSource, ObjectFieldSelector, ServiceAccount}, + rbac::v1::{RoleBinding, RoleRef, Subject}, + }, + apimachinery::pkg::apis::meta::v1::LabelSelector, + }, + kube::api::ObjectMeta, + v2::{ + builder::{meta::ownerreference_from_resource, pod::container::new_container_builder}, + types::kubernetes::ContainerName, + }, +}; + +use crate::{ + controller::{ + ValidatedAgentConfig, ValidatedCluster, + build::{agent_selector, recommended_labels_for_agent_resources}, + }, + crd::KafkaPlatformAccessAuthentication, +}; + +constant!(AGENT_CONTAINER_NAME: ContainerName = "agent"); +pub const AGENT_CLUSTER_ROLE_NAME: &str = "kafka-agent-clusterrole"; + +pub fn build_agent_deployment( + cluster: &ValidatedCluster, + agent_config: &ValidatedAgentConfig, +) -> Deployment { + let mut cb_agent = new_container_builder(&AGENT_CONTAINER_NAME); + let mut pod_builder = PodBuilder::new(); + + match &agent_config.authentication { + KafkaPlatformAccessAuthentication::Tls(credential) => { + credential.add_volumes_and_mounts(&mut pod_builder, vec![&mut cb_agent]) + } + } + + cb_agent + .image(&agent_config.image) + .args(vec!["run".to_owned()]) + .add_env_var_from_source( + "KUBERNETES_NODE_NAME", + EnvVarSource { + field_ref: Some(ObjectFieldSelector { + field_path: "spec.nodeName".to_owned(), + ..ObjectFieldSelector::default() + }), + ..EnvVarSource::default() + }, + ) + // Saves the agent from looking the domain up via the kubelet, which needs extra RBAC. + .add_env_var( + "KUBERNETES_CLUSTER_DOMAIN", + cluster.cluster_domain.to_string(), + ) + .resources( + ResourceRequirementsBuilder::new() + .with_cpu_request("100m") + .with_cpu_limit("500m") + .with_memory_request("128Mi") + .with_memory_limit("128Mi") + .build(), + ); + + pod_builder + .metadata( + ObjectMetaBuilder::new() + .with_labels(recommended_labels_for_agent_resources(cluster)) + .build(), + ) + .add_container(cb_agent.build()) + .service_account_name(agent_name(cluster)) + .security_context(PodSecurityContextBuilder::with_stackable_defaults().build()); + + Deployment { + metadata: agent_metadata(cluster), + spec: Some(DeploymentSpec { + replicas: Some(1), + selector: LabelSelector { + match_labels: Some(agent_selector(cluster).into()), + ..LabelSelector::default() + }, + template: pod_builder.build_template(), + ..DeploymentSpec::default() + }), + status: None, + } +} + +pub fn build_agent_service_account(cluster: &ValidatedCluster) -> ServiceAccount { + ServiceAccount { + metadata: agent_metadata(cluster), + ..ServiceAccount::default() + } +} + +/// Binds the agent ServiceAccount to the agent ClusterRole deployed by the Helm chart. +pub fn build_agent_role_binding(cluster: &ValidatedCluster) -> RoleBinding { + RoleBinding { + metadata: agent_metadata(cluster), + role_ref: RoleRef { + api_group: Some("rbac.authorization.k8s.io".to_owned()), + kind: "ClusterRole".to_owned(), + name: AGENT_CLUSTER_ROLE_NAME.to_owned(), + }, + subjects: Some(vec![Subject { + kind: "ServiceAccount".to_owned(), + name: agent_name(cluster), + namespace: Some(cluster.namespace.to_string()), + ..Subject::default() + }]), + } +} + +/// The name of all agent resources. Kinds don't share a namespace, so unlike a suffix per kind, this +/// cannot collide with the resources of a cluster named `-agent`. +fn agent_name(cluster: &ValidatedCluster) -> String { + format!("{}-agent", cluster.name) +} + +fn agent_metadata(cluster: &ValidatedCluster) -> ObjectMeta { + ObjectMetaBuilder::new() + .name_and_namespace(cluster) + .name(agent_name(cluster)) + .ownerreference(ownerreference_from_resource(cluster, None, Some(true))) + .with_labels(recommended_labels_for_agent_resources(cluster)) + .build() +} + +#[cfg(test)] +mod tests { + use serde_json::{Value, json}; + + use super::*; + use crate::{ + controller::test_support::{app_version_label, minimal_kafka, validated_cluster}, + crd::default_agent_image, + }; + + fn cluster(platform_access: &str) -> ValidatedCluster { + let kafka = minimal_kafka(&format!( + r#" + apiVersion: kafka.stackable.tech/v1alpha1 + kind: KafkaCluster + metadata: + name: simple-kafka + namespace: default + uid: 12345678-1234-1234-1234-123456789012 + spec: + image: + productVersion: 3.9.2 + clusterConfig: + zookeeperConfigMapName: xyz + platformAccess: {platform_access} + brokers: + roleGroups: + default: + replicas: 1 + "# + )); + validated_cluster(&kafka) + } + + fn agent_deployment(platform_access: &str) -> Value { + let cluster = cluster(platform_access); + let agent_config = cluster + .agent_config + .as_ref() + .expect("platform access is enabled"); + serde_json::to_value(build_agent_deployment(&cluster, agent_config)) + .expect("must be serializable") + } + + #[test] + fn agent_config_is_only_resolved_if_platform_access_is_enabled() { + assert!( + cluster("{enabled: false, authentication: {tls: {secretClass: tls}}}") + .agent_config + .is_none() + ); + + let cluster = cluster("{enabled: true, authentication: {tls: {secretClass: tls}}}"); + let agent_config = cluster.agent_config.expect("platform access is enabled"); + assert_eq!(agent_config.image, default_agent_image("oci.example.org")); + } + + #[test] + fn test_deployment() { + let deployment = + agent_deployment("{enabled: true, authentication: {tls: {secretClass: tls}}}"); + + assert_eq!(deployment["metadata"]["name"], "simple-kafka-agent"); + assert_eq!(deployment["spec"]["replicas"], 1); + // The selector must not match the broker or controller Pods. + assert_eq!( + deployment["spec"]["selector"]["matchLabels"], + json!({ + "app.kubernetes.io/component": "agent", + "app.kubernetes.io/instance": "simple-kafka", + "app.kubernetes.io/name": "kafka" + }) + ); + assert_eq!( + deployment["spec"]["template"]["metadata"]["labels"], + json!({ + "app.kubernetes.io/component": "agent", + "app.kubernetes.io/instance": "simple-kafka", + "app.kubernetes.io/managed-by": "kafka.stackable.tech_kafkacluster", + "app.kubernetes.io/name": "kafka", + "app.kubernetes.io/version": app_version_label("3.9.2"), + "stackable.tech/vendor": "Stackable" + }) + ); + + let pod_spec = &deployment["spec"]["template"]["spec"]; + assert_eq!(pod_spec["serviceAccountName"], "simple-kafka-agent"); + assert_eq!( + pod_spec["containers"][0]["image"], + default_agent_image("oci.example.org") + ); + assert_eq!(pod_spec["containers"][0]["args"], json!(["run"])); + assert_eq!( + pod_spec["containers"][0]["volumeMounts"], + json!([{"mountPath": "/stackable/secrets/tls-tls-cert", "name": "tls-tls-cert"}]) + ); + assert_eq!(pod_spec["volumes"][0]["name"], "tls-tls-cert"); + assert_eq!( + pod_spec["volumes"][0]["ephemeral"]["volumeClaimTemplate"]["metadata"]["annotations"]["secrets.stackable.tech/class"], + "tls" + ); + } + + #[test] + fn test_deployment_with_static_secret() { + let deployment = + agent_deployment("{enabled: true, authentication: {tls: {secret: my-cert}}}"); + + assert_eq!( + deployment["spec"]["template"]["spec"]["volumes"], + json!([{"name": "my-cert-tls-cert", "secret": {"secretName": "my-cert"}}]) + ); + } + + #[test] + fn test_role_binding() { + let cluster = cluster("{enabled: true, authentication: {tls: {secretClass: tls}}}"); + + assert_eq!( + json!({ + "apiVersion": "rbac.authorization.k8s.io/v1", + "kind": "RoleBinding", + "metadata": { + "labels": { + "app.kubernetes.io/component": "agent", + "app.kubernetes.io/instance": "simple-kafka", + "app.kubernetes.io/managed-by": "kafka.stackable.tech_kafkacluster", + "app.kubernetes.io/name": "kafka", + "app.kubernetes.io/version": app_version_label("3.9.2"), + "stackable.tech/vendor": "Stackable" + }, + "name": "simple-kafka-agent", + "namespace": "default", + "ownerReferences": [ + { + "apiVersion": "kafka.stackable.tech/v1alpha1", + "controller": true, + "kind": "KafkaCluster", + "name": "simple-kafka", + "uid": "12345678-1234-1234-1234-123456789012" + } + ] + }, + "roleRef": { + "apiGroup": "rbac.authorization.k8s.io", + "kind": "ClusterRole", + "name": "kafka-agent-clusterrole" + }, + "subjects": [ + { + "kind": "ServiceAccount", + "name": "simple-kafka-agent", + "namespace": "default" + } + ] + }), + serde_json::to_value(build_agent_role_binding(&cluster)).expect("must be serializable") + ); + } +} diff --git a/rust/operator-binary/src/controller/build/resource/mod.rs b/rust/operator-binary/src/controller/build/resource/mod.rs index 7f458714f..4acba6265 100644 --- a/rust/operator-binary/src/controller/build/resource/mod.rs +++ b/rust/operator-binary/src/controller/build/resource/mod.rs @@ -1,5 +1,6 @@ //! Builders that assemble Kubernetes resources for kafka rolegroups. +pub mod agent; pub mod config_map; pub mod discovery; pub mod listener; diff --git a/rust/operator-binary/src/controller/validate.rs b/rust/operator-binary/src/controller/validate.rs index a10c879f6..cc562eeee 100644 --- a/rust/operator-binary/src/controller/validate.rs +++ b/rust/operator-binary/src/controller/validate.rs @@ -30,8 +30,8 @@ use stackable_operator::{ use crate::{ controller::{ - RoleGroupName, ValidatedCluster, ValidatedClusterConfig, ValidatedKafkaConfig, - ValidatedRoleConfig, ValidatedRoleGroupConfig, + RoleGroupName, ValidatedAgentConfig, ValidatedCluster, ValidatedClusterConfig, + ValidatedKafkaConfig, ValidatedRoleConfig, ValidatedRoleGroupConfig, dereference::DereferencedObjects, security::{self, ValidatedKafkaSecurity}, }, @@ -199,6 +199,7 @@ pub fn validate( kafka: &v1alpha1::KafkaCluster, dereferenced_objects: DereferencedObjects, operator_environment: &OperatorEnvironmentOptions, + agent_image: &str, ) -> Result { let image = kafka .spec @@ -333,6 +334,16 @@ pub fn validate( .cluster_domain .clone(); + let agent_config = kafka + .spec + .platform_access + .as_ref() + .filter(|platform_access| platform_access.enabled) + .map(|platform_access| ValidatedAgentConfig { + image: agent_image.to_owned(), + authentication: platform_access.authentication.clone(), + }); + Ok(ValidatedCluster::new( name, namespace, @@ -353,6 +364,7 @@ pub fn validate( role_configs, role_group_configs, dereferenced_objects.bootstrap_listeners, + agent_config, )) } diff --git a/rust/operator-binary/src/crd/mod.rs b/rust/operator-binary/src/crd/mod.rs index 3402151fa..34081db27 100644 --- a/rust/operator-binary/src/crd/mod.rs +++ b/rust/operator-binary/src/crd/mod.rs @@ -38,13 +38,25 @@ use stackable_operator::{ }; use strum::{Display, EnumIter, EnumString}; -use crate::crd::{ - authorization::KafkaAuthorization, - role::{KafkaRole, broker::BrokerConfigFragment, controller::ControllerConfigFragment}, - tls::KafkaTls, +use crate::{ + crd::{ + authorization::KafkaAuthorization, + role::{KafkaRole, broker::BrokerConfigFragment, controller::ControllerConfigFragment}, + tls::KafkaTls, + }, + framework::commons::platform_access::tls::TlsClientCredential, }; pub const CONTAINER_IMAGE_BASE_NAME: &str = "kafka"; +pub const AGENT_IMAGE_BASE_NAME: &str = "kafka-agent"; + +/// The agent image of the same release as the operator. +pub fn default_agent_image(image_repository: &str) -> String { + format!( + "{image_repository}/{AGENT_IMAGE_BASE_NAME}:{}", + crate::built_info::PKG_VERSION + ) +} pub const APP_NAME: &str = "kafka"; pub const KAFKA_OPERATOR_NAME: &str = "kafka.stackable.tech"; pub const FIELD_MANAGER: &str = "kafka-operator"; @@ -98,6 +110,26 @@ pub type ControllerRole = Role< JavaCommonConfig, >; +/// Access of the Stackable Data Platform to this Kafka cluster, used by an agent to manage resources +/// like `KafkaTopic`s. The credential must be authorized in Kafka by the user. +#[derive(Clone, Debug, Deserialize, Eq, JsonSchema, PartialEq, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct KafkaPlatformAccess { + /// Whether the operator deploys the agent. False by default. + #[serde(default)] + pub enabled: bool, + + /// The credential the agent authenticates to Kafka with. + pub authentication: KafkaPlatformAccessAuthentication, +} + +/// How the agent authenticates to Kafka. +#[derive(Clone, Debug, Deserialize, Eq, JsonSchema, PartialEq, Serialize)] +#[serde(rename_all = "camelCase")] +pub enum KafkaPlatformAccessAuthentication { + Tls(TlsClientCredential), +} + #[versioned( version(name = "v1alpha1"), crates( @@ -138,6 +170,10 @@ pub mod versioned { #[serde(default)] pub cluster_config: v1alpha1::KafkaClusterConfig, + /// Access of the Stackable Data Platform to this cluster. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub platform_access: Option, + // no doc - docs in ClusterOperation struct. #[serde(default)] pub cluster_operation: ClusterOperation, diff --git a/rust/operator-binary/src/framework/commons/mod.rs b/rust/operator-binary/src/framework/commons/mod.rs new file mode 100644 index 000000000..ad8d7e6bd --- /dev/null +++ b/rust/operator-binary/src/framework/commons/mod.rs @@ -0,0 +1 @@ +pub mod platform_access; diff --git a/rust/operator-binary/src/framework/commons/platform_access/mod.rs b/rust/operator-binary/src/framework/commons/platform_access/mod.rs new file mode 100644 index 000000000..7cedc76bf --- /dev/null +++ b/rust/operator-binary/src/framework/commons/platform_access/mod.rs @@ -0,0 +1,3 @@ +//! Credentials a product cluster grants the platform-access agent, and how to mount them. + +pub mod tls; diff --git a/rust/operator-binary/src/framework/commons/platform_access/tls.rs b/rust/operator-binary/src/framework/commons/platform_access/tls.rs new file mode 100644 index 000000000..592d1e51b --- /dev/null +++ b/rust/operator-binary/src/framework/commons/platform_access/tls.rs @@ -0,0 +1,87 @@ +use serde::{Deserialize, Serialize}; +use stackable_operator::{ + builder::pod::{ + PodBuilder, + container::ContainerBuilder, + volume::{ + SecretFormat, SecretOperatorVolumeSourceBuilder, VolumeBuilder, VolumeMountBuilder, + }, + }, + commons::secret_class::SecretClassVolumeProvisionParts, + k8s_openapi::api::core::v1::{SecretVolumeSource, Volume, VolumeMount}, + schemars::{self, JsonSchema}, + v2::types::kubernetes::{SecretClassName, SecretName}, +}; + +use crate::framework::constants::secret::SECRET_BASE_PATH; + +/// Source of a TLS client certificate: a secret-operator SecretClass or a static Secret. +#[derive(Clone, Debug, Deserialize, Eq, JsonSchema, PartialEq, Serialize)] +#[serde(rename_all = "camelCase")] +pub enum TlsClientCredential { + /// An AutoTLS SecretClass used to provision the certificate. + SecretClass(SecretClassName), + + /// A static Secret holding the certificate in the keys `tls.crt` and `tls.key` (PEM), such as a + /// Secret of type `kubernetes.io/tls`. + Secret(SecretName), +} + +impl TlsClientCredential { + /// Adds the certificate volume to the Pod and mounts it into all given containers. + pub fn add_volumes_and_mounts( + &self, + pod_builder: &mut PodBuilder, + container_builders: Vec<&mut ContainerBuilder>, + ) { + let (volumes, mounts) = self.volumes_and_mounts(); + pod_builder + .add_volumes(volumes) + .expect("The volume name is derived from the credential and should not collide."); + for container_builder in container_builders { + container_builder + .add_volume_mounts(mounts.clone()) + .expect("The mount path is derived from the credential and should not collide."); + } + } + + fn volumes_and_mounts(&self) -> (Vec, Vec) { + let volume_name = self.volume_name(); + let volume = match self { + Self::SecretClass(secret_class) => VolumeBuilder::new(&volume_name) + .ephemeral( + SecretOperatorVolumeSourceBuilder::new( + secret_class, + SecretClassVolumeProvisionParts::PublicPrivate, + ) + .with_pod_scope() + .with_format(SecretFormat::TlsPem) + .build() + .expect("the annotations are built from a valid SecretClass name"), + ) + .build(), + Self::Secret(secret) => Volume { + name: volume_name.clone(), + secret: Some(SecretVolumeSource { + secret_name: Some(secret.to_string()), + ..SecretVolumeSource::default() + }), + ..Volume::default() + }, + }; + let mount = VolumeMountBuilder::new(&volume_name, self.mount_path()).build(); + (vec![volume], vec![mount]) + } + + /// The directory containing `tls.crt`, `tls.key` and `ca.crt` (PEM). + pub fn mount_path(&self) -> String { + format!("{SECRET_BASE_PATH}/{}", self.volume_name()) + } + + fn volume_name(&self) -> String { + match self { + Self::SecretClass(secret_class) => format!("{secret_class}-tls-cert"), + Self::Secret(secret) => format!("{secret}-tls-cert"), + } + } +} diff --git a/rust/operator-binary/src/framework/constants/mod.rs b/rust/operator-binary/src/framework/constants/mod.rs new file mode 100644 index 000000000..73b12dbbd --- /dev/null +++ b/rust/operator-binary/src/framework/constants/mod.rs @@ -0,0 +1 @@ +pub mod secret; diff --git a/rust/operator-binary/src/framework/constants/secret.rs b/rust/operator-binary/src/framework/constants/secret.rs new file mode 100644 index 000000000..320faf979 --- /dev/null +++ b/rust/operator-binary/src/framework/constants/secret.rs @@ -0,0 +1,2 @@ +// Mirrors the crate-private `SECRET_BASE_PATH` of operator-rs. +pub(crate) const SECRET_BASE_PATH: &str = "/stackable/secrets"; diff --git a/rust/operator-binary/src/framework/kvp/label.rs b/rust/operator-binary/src/framework/kvp/label.rs new file mode 100644 index 000000000..f3695b8c7 --- /dev/null +++ b/rust/operator-binary/src/framework/kvp/label.rs @@ -0,0 +1,45 @@ +use stackable_operator::{ + kvp::{Label, Labels}, + v2::{ + kvp::label::{ + label_app_kubernetes_io_instance, label_app_kubernetes_io_managed_by, + label_app_kubernetes_io_name, label_app_kubernetes_io_version, + label_stackable_tech_vendor, + }, + types::operator::{ClusterName, ControllerName, OperatorName, ProductName, ProductVersion}, + }, +}; + +/// Creates the recommended labels for agent resources, like the agent Deployment. +pub fn recommended_labels_for_agent_resources( + cluster_name: &ClusterName, + product_name: &ProductName, + product_version: &ProductVersion, + operator_name: &OperatorName, + controller_name: &ControllerName, +) -> Labels { + Labels::from_iter([ + label_app_kubernetes_io_instance(cluster_name), + label_app_kubernetes_io_name(product_name), + label_app_kubernetes_io_version(product_version), + label_app_kubernetes_io_component_agent(), + label_app_kubernetes_io_managed_by(operator_name, controller_name), + label_stackable_tech_vendor(), + ]) +} + +/// Creates the agent selector. +/// +/// The returned labels are a subset of the recommended labels for agent resources. +pub fn agent_selector(cluster_name: &ClusterName, product_name: &ProductName) -> Labels { + Labels::from_iter([ + label_app_kubernetes_io_instance(cluster_name), + label_app_kubernetes_io_name(product_name), + label_app_kubernetes_io_component_agent(), + ]) +} + +/// Creates the `app.kubernetes.io/component` label with the value `agent`. +pub fn label_app_kubernetes_io_component_agent() -> Label { + Label::component("agent").expect("\"agent\" is a valid label value") +} diff --git a/rust/operator-binary/src/framework/kvp/mod.rs b/rust/operator-binary/src/framework/kvp/mod.rs new file mode 100644 index 000000000..0006163a0 --- /dev/null +++ b/rust/operator-binary/src/framework/kvp/mod.rs @@ -0,0 +1 @@ +pub mod label; diff --git a/rust/operator-binary/src/framework/mod.rs b/rust/operator-binary/src/framework/mod.rs new file mode 100644 index 000000000..842c16dde --- /dev/null +++ b/rust/operator-binary/src/framework/mod.rs @@ -0,0 +1,6 @@ +//! Product-agnostic building blocks, staged here until they move to operator-rs. The module paths +//! mirror operator-rs, so moving code there only changes imports. + +pub mod commons; +pub mod constants; +pub mod kvp; diff --git a/rust/operator-binary/src/main.rs b/rust/operator-binary/src/main.rs index 075b48e11..941f4aa1e 100644 --- a/rust/operator-binary/src/main.rs +++ b/rust/operator-binary/src/main.rs @@ -14,7 +14,7 @@ use stackable_operator::{ crd::listener, eos::EndOfSupportChecker, k8s_openapi::api::{ - apps::v1::StatefulSet, + apps::v1::{Deployment, StatefulSet}, core::v1::{ConfigMap, Service, ServiceAccount}, policy::v1::PodDisruptionBudget, rbac::v1::RoleBinding, @@ -37,12 +37,13 @@ use stackable_operator::{ use crate::{ controller::KAFKA_FULL_CONTROLLER_NAME, - crd::{KAFKA_OPERATOR_NAME, KafkaCluster, KafkaClusterVersion, v1alpha1}, + crd::{KAFKA_OPERATOR_NAME, KafkaCluster, KafkaClusterVersion, default_agent_image, v1alpha1}, webhooks::conversion::create_webhook_server, }; mod controller; mod crd; +mod framework; mod webhooks; mod built_info { @@ -67,6 +68,10 @@ struct Opts { struct KafkaRun { #[clap(flatten)] common: RunArguments, + + /// The agent image. Defaults to the agent image belonging to this operator release. + #[arg(long, env)] + agent_image: Option, } #[tokio::main] @@ -83,7 +88,7 @@ async fn main() -> anyhow::Result<()> { maintenance, common, }, - .. + agent_image, }) => { // NOTE (@NickLarsenNZ): Before stackable-telemetry was used: // - The console log level was set by `KAFKA_OPERATOR_LOG`, and is now `CONSOLE_LOG` (when using Tracing::pre_configured). @@ -171,6 +176,10 @@ async fn main() -> anyhow::Result<()> { watch_namespace.get_api::>(&client), watcher::Config::default(), ) + .owns( + watch_namespace.get_api::>(&client), + watcher::Config::default(), + ) .watches( watch_namespace.get_api::>(&client), watcher::Config::default(), @@ -188,6 +197,9 @@ async fn main() -> anyhow::Result<()> { controller::error_policy, Arc::new(controller::Ctx { client: client.clone(), + agent_image: agent_image.unwrap_or_else(|| { + default_agent_image(&operator_environment.image_repository) + }), operator_environment, }), ) From f15777c1e5bef9a0b90fbd2f44ce8555b91faa22 Mon Sep 17 00:00:00 2001 From: Benedikt Labrenz Date: Thu, 24 Sep 2026 16:39:54 +0200 Subject: [PATCH 03/93] add agent image and cluster role to Helm chart --- .../kafka-operator/templates/_helpers.tpl | 7 +++++++ .../templates/clusterrole-agent.yaml | 21 +++++++++++++++++++ .../templates/clusterrole-operator.yaml | 15 ++++++++++++- .../kafka-operator/templates/deployment.yaml | 8 +++++++ 4 files changed, 50 insertions(+), 1 deletion(-) create mode 100644 deploy/helm/kafka-operator/templates/clusterrole-agent.yaml diff --git a/deploy/helm/kafka-operator/templates/_helpers.tpl b/deploy/helm/kafka-operator/templates/_helpers.tpl index 9c61cd511..27536afa2 100644 --- a/deploy/helm/kafka-operator/templates/_helpers.tpl +++ b/deploy/helm/kafka-operator/templates/_helpers.tpl @@ -84,3 +84,10 @@ Build the full operator container image reference. {{- define "kafka-operator.image" -}} {{- printf "%s/%s:%s" .Values.image.repository .Chart.Name (.Values.image.tag | default .Chart.AppVersion) -}} {{- end }} + +{{/* +Build the full agent container image reference. The agent is released together with the operator. +*/}} +{{- define "kafka-operator.agentImage" -}} +{{- printf "%s/%s-agent:%s" .Values.image.repository (include "kafka-operator.name" .) (.Values.image.tag | default .Chart.AppVersion) -}} +{{- end }} diff --git a/deploy/helm/kafka-operator/templates/clusterrole-agent.yaml b/deploy/helm/kafka-operator/templates/clusterrole-agent.yaml new file mode 100644 index 000000000..8288ab33a --- /dev/null +++ b/deploy/helm/kafka-operator/templates/clusterrole-agent.yaml @@ -0,0 +1,21 @@ +--- +# Agent ClusterRole: bound (via per KafkaCluster RoleBinding) to the ServiceAccount that the +# platform-access agent pod runs as. +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ include "kafka-operator.name" . }}-agent-clusterrole + labels: + {{- include "kafka-operator.labels" . | nindent 4 }} +rules: +{{ if .Capabilities.APIVersions.Has "security.openshift.io/v1" }} + # On OpenShift, the agent pod must be allowed to use the nonroot-v2 SCC + - apiGroups: + - security.openshift.io + resources: + - securitycontextconstraints + resourceNames: + - nonroot-v2 + verbs: + - use +{{ end }} diff --git a/deploy/helm/kafka-operator/templates/clusterrole-operator.yaml b/deploy/helm/kafka-operator/templates/clusterrole-operator.yaml index 59a34e6ce..96045367c 100644 --- a/deploy/helm/kafka-operator/templates/clusterrole-operator.yaml +++ b/deploy/helm/kafka-operator/templates/clusterrole-operator.yaml @@ -41,7 +41,7 @@ rules: - list - patch - watch - # Required to bind the product ClusterRole to the per-cluster ServiceAccount. + # Required to bind the product and agent ClusterRoles to the per-cluster ServiceAccounts. - apiGroups: - rbac.authorization.k8s.io resources: @@ -50,6 +50,7 @@ rules: - bind resourceNames: - {{ include "kafka-operator.name" . }}-clusterrole + - {{ include "kafka-operator.name" . }}-agent-clusterrole # StatefulSet created per role group (broker, KRaft controller). Applied via # SSA, tracked for orphan cleanup, and owned by the controller. - apiGroups: @@ -63,6 +64,18 @@ rules: - list - patch - watch + # Deployment of the platform-access agent. + - apiGroups: + - apps + resources: + - deployments + verbs: + - create + - delete + - get + - list + - patch + - watch # PodDisruptionBudget created per role group. Applied via SSA and tracked for orphan cleanup, and owned by the controller. - apiGroups: - policy diff --git a/deploy/helm/kafka-operator/templates/deployment.yaml b/deploy/helm/kafka-operator/templates/deployment.yaml index 6f7163dcd..9fac4f33b 100644 --- a/deploy/helm/kafka-operator/templates/deployment.yaml +++ b/deploy/helm/kafka-operator/templates/deployment.yaml @@ -16,6 +16,7 @@ spec: metadata: annotations: internal.stackable.tech/image: {{ include "kafka-operator.image" . }} + internal.stackable.tech/agent-image: {{ include "kafka-operator.agentImage" . }} {{- with .Values.podAnnotations }} {{- toYaml . | nindent 8 }} {{- end }} @@ -70,6 +71,13 @@ spec: - name: IMAGE_REPOSITORY value: {{ .Values.image.productRepository | default .Values.image.repository }} + # The image of the agent the operator deploys alongside a cluster. + - name: AGENT_IMAGE + # Tilt can use annotations as image paths, but not env variables + valueFrom: + fieldRef: + fieldPath: metadata.annotations['internal.stackable.tech/agent-image'] + # Operators need to know the node name they are running on, to e.g. discover the # Kubernetes domain name from the kubelet API. - name: KUBERNETES_NODE_NAME From ed51b0e91be5a2e028c95edf3faa171e10a5f705 Mon Sep 17 00:00:00 2001 From: Benedikt Labrenz Date: Thu, 24 Sep 2026 16:44:26 +0200 Subject: [PATCH 04/93] build a separate agent image --- .gitignore | 1 + Tiltfile | 17 ++++- default.nix | 36 ++++++++- docker/Dockerfile | 2 +- docker/Dockerfile.agent | 158 ++++++++++++++++++++++++++++++++++++++++ nix/meta.json | 2 +- 6 files changed, 210 insertions(+), 6 deletions(-) create mode 100644 docker/Dockerfile.agent diff --git a/.gitignore b/.gitignore index 411cc85da..cc2e5c12e 100644 --- a/.gitignore +++ b/.gitignore @@ -13,6 +13,7 @@ target/ *.tgz result +result-agent image.tar tilt_options.json diff --git a/Tiltfile b/Tiltfile index 1769d19a5..fba0c44f2 100644 --- a/Tiltfile +++ b/Tiltfile @@ -34,7 +34,22 @@ custom_build( # We need to set the correct image annotation on the operator Deployment to use e.g. # oci.stackable.tech/sandbox/opa-operator:7y19m3d8clwxlv34v5q2x4p7v536s00g instead of # oci.stackable.tech/sandbox/opa-operator:0.0.0-dev (which does not exist) -k8s_kind('Deployment', image_json_path='{.spec.template.metadata.annotations.internal\\.stackable\\.tech/image}') +deployment_image_json_paths = ['{.spec.template.metadata.annotations.internal\\.stackable\\.tech/image}'] + +# Operators that ship an agent (see `agent` in nix/meta.json) also build the agent image and pass +# it to the operator via the `internal.stackable.tech/agent-image` annotation. +if 'agent' in meta: + agent_image_name = operator_repository + '/' + meta['agent']['name'] + custom_build( + agent_image_name, + 'nix-build . -A dockerAgent --argstr dockerNameAgent "' + agent_image_name + '" -o result-agent && ./result-agent/load-image | docker load', + deps=['rust', 'Cargo.toml', 'Cargo.lock', 'default.nix', "nix", 'build.rs', 'vendor'], + ignore=['*.~undo-tree~'], + outputs_image_ref_to='result-agent/ref', + ) + deployment_image_json_paths.append('{.spec.template.metadata.annotations.internal\\.stackable\\.tech/agent-image}') + +k8s_kind('Deployment', image_json_path=deployment_image_json_paths) k8s_kind('DaemonSet', image_json_path='{.spec.template.metadata.annotations.internal\\.stackable\\.tech/image}') # Optionally specify a custom Helm values file to be passed to the Helm deployment below. diff --git a/default.nix b/default.nix index 6feb190a3..abaa026b6 100644 --- a/default.nix +++ b/default.nix @@ -101,6 +101,8 @@ } , meta ? pkgsLocal.lib.importJSON ./nix/meta.json , dockerName ? "oci.stackable.tech/sandbox/${meta.operator.name}" +# Only used by operators that ship an agent (see `agent` in nix/meta.json) +, dockerNameAgent ? "oci.stackable.tech/sandbox/${meta.agent.name}" , dockerTag ? null # Controls the amount of debug information included in the built operator binaries, # see https://doc.rust-lang.org/rustc/codegen-options/index.html#debuginfo @@ -125,7 +127,11 @@ rec { inherit cargo sources pkgsLocal pkgsTarget meta; inherit (pkgsLocal) lib; pkgs = lib.warn "pkgs is not cross-compilation-aware, explicitly use either pkgsLocal or pkgsTarget" pkgsLocal; - build = cargo.allWorkspaceMembers; + # Operators that ship an agent (see `agent` in nix/meta.json) keep the agent binary out of the + # operator image, as it has its own image (`dockerAgent`). + build = if meta ? agent + then cargo.workspaceMembers."stackable-${meta.operator.name}".build + else cargo.allWorkspaceMembers; entrypoint = build+"/bin/stackable-${meta.operator.name}"; # Run crds in the target environment, to avoid compiling everything twice crds = pkgsTarget.runCommand "${meta.operator.name}-crds.yaml" {} @@ -143,13 +149,13 @@ rec { # build it in the local environment so that the generated load-image # can run locally. # That's still fine, as long as we only refer to pkgsTarget *inside* of the image. - dockerImage = pkgsLocal.dockerTools.streamLayeredImage { + dockerImageArgs = { name = dockerName; tag = dockerTag; contents = [ # Kerberos 5 must be installed globally to load plugins correctly pkgsTarget.krb5 - # Make the whole cargo workspace available on $PATH + # Make the operator binaries available on $PATH build ] ++ lib.optional includeShell [ pkgsTarget.bashInteractive @@ -189,6 +195,7 @@ rec { User = toString stackableUserUid; }; }; + dockerImage = pkgsLocal.dockerTools.streamLayeredImage dockerImageArgs; docker = pkgsLocal.linkFarm "${dockerImage.name}-docker" [ { name = "load-image"; @@ -212,6 +219,29 @@ rec { } ]; + # The image of the agent, only built for operators that ship one. + dockerAgent = if meta ? agent then + let + agentBuild = cargo.workspaceMembers."stackable-${meta.agent.name}".build; + agentImage = pkgsLocal.dockerTools.streamLayeredImage (dockerImageArgs // { + name = dockerNameAgent; + contents = [ agentBuild ] ++ lib.optional includeShell [ + pkgsTarget.bashInteractive + pkgsTarget.coreutils + pkgsTarget.util-linuxMinimal + ]; + config = dockerImageArgs.config // { + Entrypoint = [ "${agentBuild}/bin/stackable-${meta.agent.name}" ]; + }; + }); + in pkgsLocal.linkFarm "${agentImage.name}-docker" [ + { name = "load-image"; path = agentImage; } + { name = "ref"; path = pkgsLocal.writeText "${agentImage.name}-image-tag" "${agentImage.imageName}:${agentImage.imageTag}"; } + { name = "image-repo"; path = pkgsLocal.writeText "${agentImage.name}-repo" agentImage.imageName; } + { name = "image-tag"; path = pkgsLocal.writeText "${agentImage.name}-tag" agentImage.imageTag; } + ] + else null; + # need to use vendored crate2nix because of https://github.com/kolloch/crate2nix/issues/264 crate2nix = import sources.crate2nix { pkgs = pkgsLocal; }; tilt = pkgsLocal.tilt; diff --git a/docker/Dockerfile b/docker/Dockerfile index 84458c3a2..a96fc6111 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -190,7 +190,7 @@ EOF COPY LICENSE /licenses/LICENSE -COPY --from=builder --chown=${STACKABLE_USER_UID}:0 /app/* /usr/local/bin/ +COPY --from=builder --chown=${STACKABLE_USER_UID}:0 /app/stackable-kafka-operator* /app/stackable-src.tar.gz /usr/local/bin/ USER ${STACKABLE_USER_UID} diff --git a/docker/Dockerfile.agent b/docker/Dockerfile.agent new file mode 100644 index 000000000..04f55b38d --- /dev/null +++ b/docker/Dockerfile.agent @@ -0,0 +1,158 @@ +# syntax=docker/dockerfile:1.16.0@sha256:e2dd261f92e4b763d789984f6eab84be66ab4f5f08052316d8eb8f173593acf7 +# NOTE: The syntax directive needs to be the first line in a Dockerfile +# Find the latest versions here: https://hub.docker.com/r/docker/dockerfile/tags +# And the changelogs: https://docs.docker.com/build/buildkit/dockerfile-release-notes/ or https://github.com/moby/buildkit/releases + +# https://docs.docker.com/build/checks/#fail-build-on-check-violations +# check=error=true + +# We want to automatically use the latest. We also don't tag our images with a version. +# hadolint ignore=DL3007 +FROM oci.stackable.tech/sdp/ubi10-rust-builder:latest AS builder + + +# We want to automatically use the latest. +# hadolint ignore=DL3007 +FROM registry.access.redhat.com/ubi10/ubi-minimal:latest AS agent + +ARG VERSION +# NOTE (@Techassi): This is required for OpenShift/Red Hat certification +# Keeping this as "1" seems to be fine since a couple of years /shrug +ARG RELEASE="1" + +# These are chosen at random and are this high on purpose to have very little chance to clash with an existing user or group on the host system +# NOTE: Please also update default.nix accordingly! +ARG STACKABLE_USER_GID="574654813" +ARG STACKABLE_USER_UID="782252253" +ARG STACKABLE_USER_NAME="stackable" + +# Sets the default shell to Bash with strict error handling and robust pipeline processing. +# "-e": Exits immediately if a command exits with a non-zero status +# "-u": Treats unset variables as an error, preventing unexpected behavior from undefined variables. +# "-o pipefail": Causes a pipeline to return the exit status of the last command in the pipe that failed, ensuring errors in any part of a pipeline are not ignored. +# "-c": Allows the execution of commands passed as a string +SHELL ["/bin/bash", "-euo", "pipefail", "-c"] + +# These labels have mostly been superseded by the OpenContainer spec annotations below but it doesn't hurt to include them +# http://label-schema.org/rc1/ +LABEL name="Stackable Agent for Apache Kafka" +LABEL maintainer="info@stackable.tech" +LABEL vendor="Stackable GmbH" +LABEL version="${VERSION}" +LABEL release="${RELEASE}" +LABEL summary="Manage resources inside Apache Kafka clusters." +LABEL description="Manage resources inside Apache Kafka clusters." + +# Overwriting/Pinning UBI labels +# https://github.com/projectatomic/ContainerApplicationGenericLabels +LABEL vcs-ref="" +LABEL distribution-scope="public" +LABEL url="https://stackable.tech" +ARG TARGETARCH +LABEL architecture="${TARGETARCH}" +LABEL com.redhat.component="" +# It complains about it being an invalid label but RedHat uses it and we want to override it and it works.... +# hadolint ignore=DL3048 +LABEL com.redhat.license_terms="" +LABEL io.buildah.version="" +LABEL io.openshift.expose-services="" + +# https://github.com/opencontainers/image-spec/blob/64294bd7a2bf2537e1a6a34d687caae70300b0c4/annotations.md#annotations +LABEL org.opencontainers.image.authors="info@stackable.tech" +LABEL org.opencontainers.image.url="https://stackable.tech" +LABEL org.opencontainers.image.vendor="Stackable GmbH" +LABEL org.opencontainers.image.licenses="OSL-3.0" +LABEL org.opencontainers.image.documentation="https://docs.stackable.tech/home/stable/kafka/" +LABEL org.opencontainers.image.version="${VERSION}" +LABEL org.opencontainers.image.revision="${RELEASE}" +LABEL org.opencontainers.image.title="Stackable Agent for Apache Kafka" +LABEL org.opencontainers.image.description="Manage resources inside Apache Kafka clusters." + +# https://docs.openshift.com/container-platform/4.16/openshift_images/create-images.html#defining-image-metadata +# https://github.com/projectatomic/ContainerApplicationGenericLabels/blob/master/vendor/redhat/labels.md +LABEL io.openshift.tags="ubi10,stackable,sdp,kafka" +LABEL io.k8s.description="Manage resources inside Apache Kafka clusters." +LABEL io.k8s.display-name="Stackable Agent for Apache Kafka" + +COPY <> /stackable/.bashrc + +echo -e "if [ -f ~/.bashrc ]; then\n\tsource ~/.bashrc\nfi" >> /stackable/.profile + +chown ${STACKABLE_USER_UID}:0 /stackable/.bashrc +chown ${STACKABLE_USER_UID}:0 /stackable/.profile + +# All files and folders owned by root to support running as arbitrary users +# This is best practice as all container users will belong to the root group (0) +chown -R ${STACKABLE_USER_UID}:0 /stackable +chmod -R g=u /stackable +EOF + +COPY < Date: Thu, 24 Sep 2026 19:47:34 +0200 Subject: [PATCH 05/93] run ghs build for matrix of operator and agent --- .github/workflows/build.yaml | 42 ++++++++++++++++++++++++++---------- 1 file changed, 31 insertions(+), 11 deletions(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 7251f20b1..2c6cc0009 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -62,8 +62,22 @@ jobs: - '**/Cargo.toml' - 'Cargo.lock' - '**/*.rs' + - 'nix/meta.json' + + # Operators that ship an agent (see `agent` in nix/meta.json) also build and publish its image. + - name: Determine Container Images + id: images + shell: bash + run: | + set -euo pipefail + IMAGES=$(jq -c --arg operator "$OPERATOR_NAME" ' + [{component: "operator", name: $operator, "container-file": "docker/Dockerfile"}] + + if .agent then [{component: "agent", name: .agent.name, "container-file": "docker/Dockerfile.agent"}] else [] end + ' nix/meta.json) + echo "IMAGES=$IMAGES" | tee -a "$GITHUB_OUTPUT" outputs: detected: ${{ steps.check.outputs.detected }} + images: ${{ steps.images.outputs.IMAGES }} helm-lint: name: Lint Helm Chart @@ -131,7 +145,7 @@ jobs: run: cargo udeps --workspace --all-targets build-container-image: - name: Build/Publish ${{ matrix.runner.arch }} Image + name: Build/Publish ${{ matrix.image.component }} ${{ matrix.runner.arch }} Image if: | github.repository_owner == 'stackabletech' && (github.event_name != 'merge_group') @@ -146,6 +160,7 @@ jobs: runner: - { name: "ubuntu-latest", arch: "amd64" } - { name: "ubicloud-standard-8-arm", arch: "arm64" } + image: ${{ fromJSON(needs.detect-changes.outputs.images) }} runs-on: ${{ matrix.runner.name }} outputs: operator-version: ${{ steps.version.outputs.OPERATOR_VERSION }} @@ -208,10 +223,10 @@ jobs: id: build uses: stackabletech/actions/build-container-image@34a64229959b15db6c5f307db8809805ba7edc55 # v0.18.3 with: - image-name: ${{ env.OPERATOR_NAME }} + image-name: ${{ matrix.image.name }} image-index-manifest-tag: ${{ steps.version.outputs.OPERATOR_VERSION }} build-arguments: VERSION=${{ steps.version.outputs.OPERATOR_VERSION }} - container-file: docker/Dockerfile + container-file: ${{ matrix.image.container-file }} - name: Publish Container Image to oci.stackable.tech if: ${{ !github.event.pull_request.head.repo.fork }} @@ -220,7 +235,7 @@ jobs: image-registry-uri: oci.stackable.tech image-registry-username: robot$sdp+github-action-build image-registry-password: ${{ secrets.HARBOR_ROBOT_SDP_GITHUB_ACTION_BUILD_SECRET }} - image-repository: sdp/${{ env.OPERATOR_NAME }} + image-repository: sdp/${{ matrix.image.name }} canonical-image-manifest-tag: ${{ steps.build.outputs.image-manifest-tag }} canonical-source-image-uri: ${{ steps.build.outputs.image-manifest-uri }} @@ -231,12 +246,12 @@ jobs: image-registry-uri: quay.io image-registry-username: stackable+robot_sdp_github_action_build image-registry-password: ${{ secrets.QUAY_ROBOT_SDP_GITHUB_ACTION_BUILD_SECRET }} - image-repository: stackable/sdp/${{ env.OPERATOR_NAME }} + image-repository: stackable/sdp/${{ matrix.image.name }} canonical-image-manifest-tag: ${{ steps.build.outputs.image-manifest-tag }} canonical-source-image-uri: ${{ steps.build.outputs.image-manifest-uri }} publish-index-manifest: - name: Publish/Sign ${{ needs.build-container-image.outputs.operator-version }} Index + name: Publish/Sign ${{ matrix.image.component }} ${{ needs.build-container-image.outputs.operator-version }} Index if: | github.repository_owner == 'stackabletech' && (github.event_name != 'merge_group') @@ -248,6 +263,10 @@ jobs: permissions: contents: read id-token: write + strategy: + fail-fast: false + matrix: + image: ${{ fromJSON(needs.detect-changes.outputs.images) }} runs-on: ubuntu-latest steps: - name: Checkout Repository @@ -261,7 +280,7 @@ jobs: image-registry-uri: oci.stackable.tech image-registry-username: robot$sdp+github-action-build image-registry-password: ${{ secrets.HARBOR_ROBOT_SDP_GITHUB_ACTION_BUILD_SECRET }} - image-repository: sdp/${{ env.OPERATOR_NAME }} + image-repository: sdp/${{ matrix.image.name }} canonical-image-index-manifest-tag: ${{ needs.build-container-image.outputs.operator-version }} - name: Publish and Sign Image Index to quay.io @@ -270,7 +289,7 @@ jobs: image-registry-uri: quay.io image-registry-username: stackable+robot_sdp_github_action_build image-registry-password: ${{ secrets.QUAY_ROBOT_SDP_GITHUB_ACTION_BUILD_SECRET }} - image-repository: stackable/sdp/${{ env.OPERATOR_NAME }} + image-repository: stackable/sdp/${{ matrix.image.name }} canonical-image-index-manifest-tag: ${{ needs.build-container-image.outputs.operator-version }} publish-helm-chart: @@ -320,7 +339,7 @@ jobs: helm-version: v3.17.4 # This is currently the latest version which supports pushing to quay.io openshift-preflight-check: - name: Run OpenShift Preflight Check for ${{ needs.build-container-image.outputs.operator-version }}-${{ matrix.arch }} + name: Run OpenShift Preflight Check for ${{ matrix.image.component }} ${{ needs.build-container-image.outputs.operator-version }}-${{ matrix.arch }} if: | github.repository_owner == 'stackabletech' && (github.event_name != 'merge_group') @@ -336,18 +355,19 @@ jobs: arch: - amd64 - arm64 + image: ${{ fromJSON(needs.detect-changes.outputs.images) }} runs-on: ubuntu-latest steps: - name: Run OpenShift Preflight Check for oci.stackable.tech uses: stackabletech/actions/run-openshift-preflight@34a64229959b15db6c5f307db8809805ba7edc55 # v0.18.3 with: - image-index-uri: oci.stackable.tech/sdp/${{ env.OPERATOR_NAME }}:${{ needs.build-container-image.outputs.operator-version }} + image-index-uri: oci.stackable.tech/sdp/${{ matrix.image.name }}:${{ needs.build-container-image.outputs.operator-version }} image-architecture: ${{ matrix.arch }} - name: Run OpenShift Preflight Check for quay.io uses: stackabletech/actions/run-openshift-preflight@34a64229959b15db6c5f307db8809805ba7edc55 # v0.18.3 with: - image-index-uri: quay.io/stackable/sdp/${{ env.OPERATOR_NAME }}:${{ needs.build-container-image.outputs.operator-version }} + image-index-uri: quay.io/stackable/sdp/${{ matrix.image.name }}:${{ needs.build-container-image.outputs.operator-version }} image-architecture: ${{ matrix.arch }} # This job is a required check in GitHub Settings for this repository. From 177cfb18a49a1bcce842bc0b0d74ac84bd02f29d Mon Sep 17 00:00:00 2001 From: Benedikt Labrenz Date: Thu, 1 Oct 2026 14:16:33 +0200 Subject: [PATCH 06/93] move staged framework code to operator-rs --- Cargo.lock | 87 +++++----- Cargo.nix | 154 ++++++++---------- Cargo.toml | 2 +- crate-hashes.json | 18 +- extra/crds.yaml | 2 +- rust/agent-binary/src/framework/mod.rs | 12 -- rust/agent-binary/src/main.rs | 15 +- .../src/controller/build/mod.rs | 5 +- rust/operator-binary/src/crd/mod.rs | 13 +- .../src/framework/commons/mod.rs | 1 - .../framework/commons/platform_access/mod.rs | 3 - .../framework/commons/platform_access/tls.rs | 87 ---------- .../src/framework/constants/mod.rs | 1 - .../src/framework/constants/secret.rs | 2 - .../src/framework/kvp/label.rs | 45 ----- rust/operator-binary/src/framework/kvp/mod.rs | 1 - rust/operator-binary/src/framework/mod.rs | 6 - rust/operator-binary/src/main.rs | 3 +- .../src/webhooks/conversion.rs | 11 +- 19 files changed, 138 insertions(+), 330 deletions(-) delete mode 100644 rust/agent-binary/src/framework/mod.rs delete mode 100644 rust/operator-binary/src/framework/commons/mod.rs delete mode 100644 rust/operator-binary/src/framework/commons/platform_access/mod.rs delete mode 100644 rust/operator-binary/src/framework/commons/platform_access/tls.rs delete mode 100644 rust/operator-binary/src/framework/constants/mod.rs delete mode 100644 rust/operator-binary/src/framework/constants/secret.rs delete mode 100644 rust/operator-binary/src/framework/kvp/label.rs delete mode 100644 rust/operator-binary/src/framework/kvp/mod.rs delete mode 100644 rust/operator-binary/src/framework/mod.rs diff --git a/Cargo.lock b/Cargo.lock index b08d28809..77847b010 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -439,22 +439,13 @@ dependencies = [ [[package]] name = "convert_case" -version = "0.11.0" +version = "0.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "affbf0190ed2caf063e3def54ff444b449371d55c58e513a95ab98eca50adb49" +checksum = "1af709f1f33454bf52eadfc8c78b3b9ef9cb26fb54d16dc9cd9a7299f899fd1b" dependencies = [ "unicode-segmentation", ] -[[package]] -name = "convert_case_extras" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "589c70f0faf8aa9d17787557d5eae854d7755cac50f5c3d12c81d3d57661cebb" -dependencies = [ - "convert_case", -] - [[package]] name = "core-foundation" version = "0.10.1" @@ -773,9 +764,9 @@ dependencies = [ [[package]] name = "educe" -version = "0.7.6" +version = "0.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e451fac8dd8dece16234604bf1efce6e90fddd8ab6ad4d66eec0eca5160959dd" +checksum = "1ebdb1f8f8d2815bbe7b369a7da10343c82026a4503b68bd1279ba557611e96d" dependencies = [ "enum-ordinalize", "proc-macro2", @@ -1707,7 +1698,7 @@ dependencies = [ [[package]] name = "k8s-version" version = "0.1.3" -source = "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.118.0#bc6c84025c2dcc834b94bfb57ec72810ae5f5eb1" +source = "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#7cb677e73bd218bcafb4ed4d4b459dda319fcbac" dependencies = [ "darling 0.24.1", "regex", @@ -1978,7 +1969,7 @@ dependencies = [ "num-integer", "num-iter", "num-traits", - "rand 0.8.7", + "rand 0.8.8", "smallvec", "zeroize", ] @@ -2038,9 +2029,9 @@ checksum = "7c87def4c32ab89d880effc9e097653c8da5d6ef28e6b539d313baaacfbafcbe" [[package]] name = "opentelemetry" -version = "0.32.0" +version = "0.33.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b0142c63252a9e054e68a4c61a5778f7b14f576274d593f8ce883d191a099682" +checksum = "6cdb0b1b267eb9db3331b434ed9ddab10d50e280a9adf9d13e5233e2002b61b5" dependencies = [ "futures-core", "futures-sink", @@ -2052,9 +2043,9 @@ dependencies = [ [[package]] name = "opentelemetry-appender-tracing" -version = "0.32.0" +version = "0.33.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2c0080f0dc1d7c786f467cd85a4e395fcab11ee852004f39a29a18ab7c25d837" +checksum = "e2b304e8d8f11326ba366a99ca5c7aaf7cd397b67268a332437175aeb42a40db" dependencies = [ "opentelemetry", "tracing", @@ -2064,9 +2055,9 @@ dependencies = [ [[package]] name = "opentelemetry-http" -version = "0.32.0" +version = "0.33.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5683015d09e2df236ef005b17f6f196f0d5f6313c4fa43a7b6a53b52776e4331" +checksum = "ee2c3625b8aa04209f7e01e513bc8044da9687fa38a9eacf59628ca5f3b87300" dependencies = [ "async-trait", "bytes", @@ -2077,11 +2068,12 @@ dependencies = [ [[package]] name = "opentelemetry-otlp" -version = "0.32.0" +version = "0.33.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9966929966d17620d7c316c643ba62631826e10021409357772d5eea84f62c35" +checksum = "699a67345e21962a231955b9059a157e428b219fa5df8efe11f08346fb23a344" dependencies = [ "http", + "httpdate", "opentelemetry", "opentelemetry-http", "opentelemetry-proto", @@ -2096,9 +2088,9 @@ dependencies = [ [[package]] name = "opentelemetry-proto" -version = "0.32.0" +version = "0.33.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "56d658ba1faf63f7b9c492cfbe6e0ec365440a16132d3270c1065f7b33f1b638" +checksum = "25da1ac11a0aeccf38d7f77ee0348715adaf8340f65ad46c94a02c6b20e2f65d" dependencies = [ "opentelemetry", "opentelemetry_sdk", @@ -2109,15 +2101,15 @@ dependencies = [ [[package]] name = "opentelemetry-semantic-conventions" -version = "0.32.1" +version = "0.33.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c913ac17a6c451661ee255f4625d143e51647ae78ebd969b75e41c4442f4fe47" +checksum = "d288b1e1bc3590052fbb8f0f635ef248dd9022cf1fbaeb6582b7ad6081a17b87" [[package]] name = "opentelemetry_sdk" -version = "0.32.1" +version = "0.33.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9b59f80e1ac4d5ff7a2db8fb6c80badb7f0f3f858211fba08dd9aaec750894f9" +checksum = "cb39533d9d1c912123efd7d41d7e0c29d16917b60ce15b4c8d87cb1af7f67520" dependencies = [ "futures-channel", "futures-executor", @@ -2422,9 +2414,9 @@ checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" [[package]] name = "rand" -version = "0.8.7" +version = "0.8.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "22f6172bdec972074665ed81ed53b71da00bfc44b65a753cfde883ec4c702a1a" +checksum = "e058c7de0b26af77780c769414d6257830bb240f3c38477dbc2c16e5f54d6d4c" dependencies = [ "rand_chacha 0.3.1", "rand_core 0.6.4", @@ -3152,7 +3144,7 @@ checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596" [[package]] name = "stackable-certs" version = "0.4.1" -source = "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.118.0#bc6c84025c2dcc834b94bfb57ec72810ae5f5eb1" +source = "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#7cb677e73bd218bcafb4ed4d4b459dda319fcbac" dependencies = [ "const-oid", "ecdsa", @@ -3209,15 +3201,15 @@ dependencies = [ [[package]] name = "stackable-operator" -version = "0.118.0" -source = "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.118.0#bc6c84025c2dcc834b94bfb57ec72810ae5f5eb1" +version = "0.119.0" +source = "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#7cb677e73bd218bcafb4ed4d4b459dda319fcbac" dependencies = [ "base64 0.23.1", "clap", "const_format", "delegate", "dockerfile-parser", - "educe 0.7.6", + "educe 0.8.1", "either", "futures", "http", @@ -3254,7 +3246,7 @@ dependencies = [ [[package]] name = "stackable-operator-derive" version = "0.3.1" -source = "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.118.0#bc6c84025c2dcc834b94bfb57ec72810ae5f5eb1" +source = "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#7cb677e73bd218bcafb4ed4d4b459dda319fcbac" dependencies = [ "darling 0.24.1", "proc-macro2", @@ -3265,7 +3257,7 @@ dependencies = [ [[package]] name = "stackable-shared" version = "0.1.2" -source = "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.118.0#bc6c84025c2dcc834b94bfb57ec72810ae5f5eb1" +source = "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#7cb677e73bd218bcafb4ed4d4b459dda319fcbac" dependencies = [ "jiff", "k8s-openapi", @@ -3282,7 +3274,7 @@ dependencies = [ [[package]] name = "stackable-telemetry" version = "0.6.5" -source = "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.118.0#bc6c84025c2dcc834b94bfb57ec72810ae5f5eb1" +source = "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#7cb677e73bd218bcafb4ed4d4b459dda319fcbac" dependencies = [ "axum", "clap", @@ -3306,7 +3298,7 @@ dependencies = [ [[package]] name = "stackable-versioned" version = "0.11.1" -source = "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.118.0#bc6c84025c2dcc834b94bfb57ec72810ae5f5eb1" +source = "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#7cb677e73bd218bcafb4ed4d4b459dda319fcbac" dependencies = [ "kube", "schemars", @@ -3320,10 +3312,9 @@ dependencies = [ [[package]] name = "stackable-versioned-macros" version = "0.11.1" -source = "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.118.0#bc6c84025c2dcc834b94bfb57ec72810ae5f5eb1" +source = "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#7cb677e73bd218bcafb4ed4d4b459dda319fcbac" dependencies = [ "convert_case", - "convert_case_extras", "darling 0.24.1", "indoc", "itertools 0.15.0", @@ -3337,8 +3328,8 @@ dependencies = [ [[package]] name = "stackable-webhook" -version = "0.9.2" -source = "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.118.0#bc6c84025c2dcc834b94bfb57ec72810ae5f5eb1" +version = "0.10.0" +source = "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#7cb677e73bd218bcafb4ed4d4b459dda319fcbac" dependencies = [ "arc-swap", "async-trait", @@ -3361,7 +3352,7 @@ dependencies = [ "tokio", "tokio-rustls", "tower", - "tower-http 0.7.0", + "tower-http 0.7.1", "tracing", "tracing-opentelemetry", "x509-cert", @@ -3754,9 +3745,9 @@ dependencies = [ [[package]] name = "tower-http" -version = "0.7.0" +version = "0.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b11f75e912b0c2be01b63d8cf8057b8c3f97cf34abb3d431a3a4c8675498e233" +checksum = "08a05a66a4fdd61cbbe0a1d755ffe0ca6aba159dd4820936a0ff8a8278245b9c" dependencies = [ "bitflags 2.13.1", "bytes", @@ -3840,9 +3831,9 @@ dependencies = [ [[package]] name = "tracing-opentelemetry" -version = "0.33.0" +version = "0.34.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "adbc64cba7137545b8044cb1fe9814f7aacf3c6b5f9b45be8bb5db538befdb26" +checksum = "0a904802a1b902f43638b677ff2a650847e3b4404101b6c586d648e8c1e3e8fe" dependencies = [ "js-sys", "opentelemetry", diff --git a/Cargo.nix b/Cargo.nix index 292f308c7..0f1de0737 100644 --- a/Cargo.nix +++ b/Cargo.nix @@ -1376,9 +1376,9 @@ rec { }; "convert_case" = rec { crateName = "convert_case"; - version = "0.11.0"; + version = "0.12.0"; edition = "2021"; - sha256 = "0jfv1ajyr65bjlx533n5alfkfjdl8ks4zxfywdiz1jnj1qcz1yxg"; + sha256 = "06zxk7w9jwlsrp4nvlalzckcpycy7f5wgj6zx99bym1lygqhkxqs"; authors = [ "rutrum " ]; @@ -1390,25 +1390,6 @@ rec { ]; }; - "convert_case_extras" = rec { - crateName = "convert_case_extras"; - version = "0.2.0"; - edition = "2021"; - sha256 = "1fyfc5vdblw15k8w7xahmif7bmslx3mdamvmg0brvapqzbq7172q"; - authors = [ - "rutrum " - ]; - dependencies = [ - { - name = "convert_case"; - packageId = "convert_case"; - } - ]; - features = { - "rand" = [ "dep:rand" ]; - "random" = [ "rand" ]; - }; - }; "core-foundation" = rec { crateName = "core-foundation"; version = "0.10.1"; @@ -2378,11 +2359,11 @@ rec { }; resolvedDefaultFeatures = [ "Clone" "Debug" "Hash" "PartialEq" ]; }; - "educe 0.7.6" = rec { + "educe 0.8.1" = rec { crateName = "educe"; - version = "0.7.6"; + version = "0.8.1"; edition = "2024"; - sha256 = "1par14babv60xrk4vbdnibfzv43frvpz2jv06iif3v4dvp4gllg4"; + sha256 = "0vg925v5bfkr2aynhfshlhk21j230fhpv6ingfz5p0fjz3wb3g8y"; procMacro = true; authors = [ "Magic Len " @@ -2405,13 +2386,7 @@ rec { { name = "syn"; packageId = "syn 3.0.4"; - } - ]; - devDependencies = [ - { - name = "syn"; - packageId = "syn 3.0.4"; - features = [ "full" ]; + features = [ "visit" "visit-mut" ]; } ]; features = { @@ -5410,7 +5385,7 @@ rec { workspace_member = null; src = pkgs.fetchgit { url = "https://github.com/stackabletech/operator-rs.git"; - rev = "bc6c84025c2dcc834b94bfb57ec72810ae5f5eb1"; + rev = "7cb677e73bd218bcafb4ed4d4b459dda319fcbac"; sha256 = "0cgziqra8097hp05ynib1qpw4c95n972f4w2rk9l3llyp8r1vmci"; }; libName = "k8s_version"; @@ -6459,7 +6434,7 @@ rec { } { name = "rand"; - packageId = "rand 0.8.7"; + packageId = "rand 0.8.8"; optional = true; usesDefaultFeatures = false; } @@ -6478,7 +6453,7 @@ rec { devDependencies = [ { name = "rand"; - packageId = "rand 0.8.7"; + packageId = "rand 0.8.8"; features = [ "small_rng" ]; } ]; @@ -6625,9 +6600,9 @@ rec { }; "opentelemetry" = rec { crateName = "opentelemetry"; - version = "0.32.0"; + version = "0.33.0"; edition = "2021"; - sha256 = "10ln14d1jgc8rvw97mblc9blzcgpg1bimim4d170b7ia4mijq55h"; + sha256 = "1db15c0f4csj7v8zkbd9h3i503divafysd5l64rxpfby4qdhpnvc"; dependencies = [ { name = "futures-core"; @@ -6679,9 +6654,9 @@ rec { }; "opentelemetry-appender-tracing" = rec { crateName = "opentelemetry-appender-tracing"; - version = "0.32.0"; + version = "0.33.0"; edition = "2021"; - sha256 = "0dyq4myan64sl8wly02jx0gb3jjz7575mn3w8rpphz0xvkq8001c"; + sha256 = "1ns05asawxbi8cra6s3jnsbx6z5gg9fcm6ba6sx2c4ziv3l09cz2"; libName = "opentelemetry_appender_tracing"; dependencies = [ { @@ -6730,9 +6705,9 @@ rec { }; "opentelemetry-http" = rec { crateName = "opentelemetry-http"; - version = "0.32.0"; + version = "0.33.0"; edition = "2021"; - sha256 = "0ca3drvm4fx5nskl7yn42dimy3bg35ppzc85y1p27pz215fh30sn"; + sha256 = "003kp3rsb332b77yma9qza3rdnj4h2y17r81gsgj015ap0jkcb7f"; libName = "opentelemetry_http"; dependencies = [ { @@ -6768,16 +6743,15 @@ rec { "internal-logs" = [ "opentelemetry/internal-logs" ]; "reqwest" = [ "dep:reqwest" ]; "reqwest-blocking" = [ "dep:reqwest" "reqwest/blocking" ]; - "reqwest-rustls" = [ "dep:reqwest" "reqwest/default-tls" ]; - "reqwest-rustls-webpki-roots" = [ "dep:reqwest" "reqwest/default-tls" "reqwest/webpki-roots" ]; + "reqwest-rustls" = [ "dep:reqwest" "reqwest/rustls" ]; }; resolvedDefaultFeatures = [ "reqwest" "reqwest-blocking" ]; }; "opentelemetry-otlp" = rec { crateName = "opentelemetry-otlp"; - version = "0.32.0"; + version = "0.33.0"; edition = "2021"; - sha256 = "0d9cys2flpidfxbr6h1103hjc633cax47ihnqgbj0xnicscr4rlr"; + sha256 = "0i534gxld0zh27z8xpx5kwhqnhky2nd0bfam34ijm5i1bqs6g6k9"; libName = "opentelemetry_otlp"; dependencies = [ { @@ -6787,6 +6761,11 @@ rec { usesDefaultFeatures = false; features = [ "std" ]; } + { + name = "httpdate"; + packageId = "httpdate"; + optional = true; + } { name = "opentelemetry"; packageId = "opentelemetry"; @@ -6829,7 +6808,7 @@ rec { packageId = "tokio"; optional = true; usesDefaultFeatures = false; - features = [ "sync" "rt" ]; + features = [ "sync" "rt" "time" ]; } { name = "tonic"; @@ -6865,17 +6844,15 @@ rec { ]; features = { "default" = [ "http-proto" "reqwest-blocking-client" "trace" "metrics" "logs" "internal-logs" ]; - "experimental-grpc-retry" = [ "grpc-tonic" "opentelemetry_sdk/experimental_async_runtime" "opentelemetry_sdk/rt-tokio" ]; - "experimental-http-retry" = [ "opentelemetry_sdk/experimental_async_runtime" "opentelemetry_sdk/rt-tokio" "tokio" "httpdate" ]; "flate2" = [ "dep:flate2" ]; "grpc-tonic" = [ "tonic" "tonic-types" "prost" "http" "tokio" "opentelemetry-proto/gen-tonic" ]; "gzip-http" = [ "flate2" ]; "gzip-tonic" = [ "tonic/gzip" ]; "http" = [ "dep:http" ]; - "http-json" = [ "serde_json" "prost" "opentelemetry-http" "opentelemetry-proto/gen-tonic-messages" "opentelemetry-proto/with-serde" "http" "trace" "metrics" ]; - "http-proto" = [ "prost" "opentelemetry-http" "opentelemetry-proto/gen-tonic-messages" "http" "trace" "metrics" ]; + "http-json" = [ "serde_json" "prost" "opentelemetry-http" "opentelemetry-proto/gen-tonic-messages" "opentelemetry-proto/with-serde" "http" "httpdate" "trace" "metrics" ]; + "http-proto" = [ "prost" "opentelemetry-http" "opentelemetry-proto/gen-tonic-messages" "http" "httpdate" "trace" "metrics" ]; "httpdate" = [ "dep:httpdate" ]; - "hyper-client" = [ "opentelemetry-http/hyper" ]; + "hyper-client" = [ "opentelemetry-http/hyper" "tokio" ]; "integration-testing" = [ "tonic" "prost" "tokio/full" "trace" "logs" ]; "internal-logs" = [ "opentelemetry_sdk/internal-logs" "opentelemetry/internal-logs" ]; "logs" = [ "opentelemetry/logs" "opentelemetry_sdk/logs" "opentelemetry-proto/logs" ]; @@ -6884,13 +6861,9 @@ rec { "prost" = [ "dep:prost" ]; "reqwest" = [ "dep:reqwest" ]; "reqwest-blocking-client" = [ "reqwest/blocking" "opentelemetry-http/reqwest-blocking" ]; - "reqwest-client" = [ "reqwest" "opentelemetry-http/reqwest" ]; + "reqwest-client" = [ "reqwest" "opentelemetry-http/reqwest" "tokio" ]; "reqwest-rustls" = [ "reqwest" "opentelemetry-http/reqwest-rustls" ]; - "reqwest-rustls-webpki-roots" = [ "reqwest" "opentelemetry-http/reqwest-rustls-webpki-roots" ]; - "serde" = [ "dep:serde" ]; "serde_json" = [ "dep:serde_json" ]; - "serialize" = [ "serde" "serde_json" ]; - "tls" = [ "tls-ring" ]; "tls-aws-lc" = [ "tonic/tls-aws-lc" ]; "tls-provider-agnostic" = [ "tonic/_tls-any" ]; "tls-ring" = [ "tonic/tls-ring" ]; @@ -6904,13 +6877,13 @@ rec { "zstd-http" = [ "zstd" ]; "zstd-tonic" = [ "tonic/zstd" ]; }; - resolvedDefaultFeatures = [ "default" "grpc-tonic" "gzip-tonic" "http" "http-proto" "internal-logs" "logs" "metrics" "opentelemetry-http" "prost" "reqwest" "reqwest-blocking-client" "tokio" "tonic" "tonic-types" "trace" ]; + resolvedDefaultFeatures = [ "default" "grpc-tonic" "gzip-tonic" "http" "http-proto" "httpdate" "internal-logs" "logs" "metrics" "opentelemetry-http" "prost" "reqwest" "reqwest-blocking-client" "tokio" "tonic" "tonic-types" "trace" ]; }; "opentelemetry-proto" = rec { crateName = "opentelemetry-proto"; - version = "0.32.0"; + version = "0.33.0"; edition = "2021"; - sha256 = "0f5ny4rpnpq6q5q34b8k2q548rf31rpbxkwjqjwzfqxg3yx5imjn"; + sha256 = "0pgnw8h6nb50jind8npn821szb8mhwsf0zppswwczv0a3b0imni5"; libName = "opentelemetry_proto"; dependencies = [ { @@ -6974,9 +6947,9 @@ rec { }; "opentelemetry-semantic-conventions" = rec { crateName = "opentelemetry-semantic-conventions"; - version = "0.32.1"; + version = "0.33.0"; edition = "2021"; - sha256 = "0izyyi148774fndrdgcfwxx68l9y2ifn5x2mw8g6clf4lqbsq4y9"; + sha256 = "11vvl60n1bdph9jypfhzrwi91pa8y9g663wgpcphb41mpkhv326j"; libName = "opentelemetry_semantic_conventions"; features = { }; @@ -6984,21 +6957,24 @@ rec { }; "opentelemetry_sdk" = rec { crateName = "opentelemetry_sdk"; - version = "0.32.1"; + version = "0.33.0"; edition = "2021"; - sha256 = "1ycl11syranrinhgn4c2hlzhyzyvpa06ryxq5mxgzmf4387ghncv"; + sha256 = "083myvvimjw7im65pq8cnqbnkl991iz1vm6pxwij348wklym6ffb"; dependencies = [ { name = "futures-channel"; packageId = "futures-channel"; + optional = true; } { name = "futures-executor"; packageId = "futures-executor"; + optional = true; } { name = "futures-util"; packageId = "futures-util"; + optional = true; usesDefaultFeatures = false; features = [ "std" "sink" "async-await-macro" ]; } @@ -7029,6 +7005,7 @@ rec { { name = "thiserror"; packageId = "thiserror 2.0.20"; + optional = true; usesDefaultFeatures = false; } { @@ -7053,6 +7030,7 @@ rec { ]; features = { "default" = [ "trace" "metrics" "logs" "internal-logs" ]; + "experimental_async_runtime" = [ "dep:futures-channel" "dep:futures-executor" "dep:futures-util" "dep:thiserror" ]; "experimental_logs_batch_log_processor_with_async_runtime" = [ "logs" "experimental_async_runtime" ]; "experimental_metrics_bound_instruments" = [ "metrics" "opentelemetry/experimental_metrics_bound_instruments" ]; "experimental_metrics_custom_reader" = [ "metrics" ]; @@ -7062,8 +7040,8 @@ rec { "http" = [ "dep:http" ]; "internal-logs" = [ "opentelemetry/internal-logs" ]; "jaeger_remote_sampler" = [ "trace" "opentelemetry-http" "http" "serde" "serde_json" "url" "experimental_async_runtime" ]; - "logs" = [ "opentelemetry/logs" ]; - "metrics" = [ "opentelemetry/metrics" ]; + "logs" = [ "opentelemetry/logs" "dep:futures-channel" "dep:futures-executor" "dep:futures-util" ]; + "metrics" = [ "opentelemetry/metrics" "dep:futures-channel" "dep:futures-executor" "dep:futures-util" "dep:thiserror" ]; "opentelemetry-http" = [ "dep:opentelemetry-http" ]; "percent-encoding" = [ "dep:percent-encoding" ]; "rand" = [ "dep:rand" ]; @@ -7075,7 +7053,7 @@ rec { "testing" = [ "opentelemetry/testing" "trace" "metrics" "logs" "tokio/sync" ]; "tokio" = [ "dep:tokio" ]; "tokio-stream" = [ "dep:tokio-stream" ]; - "trace" = [ "opentelemetry/trace" "rand" "percent-encoding" ]; + "trace" = [ "opentelemetry/trace" "rand" "percent-encoding" "dep:futures-channel" "dep:futures-executor" "dep:futures-util" "dep:thiserror" ]; "url" = [ "dep:url" ]; }; resolvedDefaultFeatures = [ "default" "experimental_async_runtime" "internal-logs" "logs" "metrics" "percent-encoding" "rand" "rt-tokio" "tokio" "tokio-stream" "trace" ]; @@ -7901,11 +7879,11 @@ rec { }; resolvedDefaultFeatures = [ "alloc" "default" "std" "std_rng" "sys_rng" "thread_rng" ]; }; - "rand 0.8.7" = rec { + "rand 0.8.8" = rec { crateName = "rand"; - version = "0.8.7"; + version = "0.8.8"; edition = "2018"; - sha256 = "06iaf16fr0z8zly7anmn8ky0p80xnx9yv0gdcm30fwn9vqmigxi2"; + sha256 = "0k3d9psya5icpiylff1w1wjbnc3q4pb1953n1iw7gbr61ggcfn70"; authors = [ "The Rand Project Developers" "The Rust Project Developers" @@ -10325,7 +10303,7 @@ rec { workspace_member = null; src = pkgs.fetchgit { url = "https://github.com/stackabletech/operator-rs.git"; - rev = "bc6c84025c2dcc834b94bfb57ec72810ae5f5eb1"; + rev = "7cb677e73bd218bcafb4ed4d4b459dda319fcbac"; sha256 = "0cgziqra8097hp05ynib1qpw4c95n972f4w2rk9l3llyp8r1vmci"; }; libName = "stackable_certs"; @@ -10563,12 +10541,12 @@ rec { }; "stackable-operator" = rec { crateName = "stackable-operator"; - version = "0.118.0"; + version = "0.119.0"; edition = "2024"; workspace_member = null; src = pkgs.fetchgit { url = "https://github.com/stackabletech/operator-rs.git"; - rev = "bc6c84025c2dcc834b94bfb57ec72810ae5f5eb1"; + rev = "7cb677e73bd218bcafb4ed4d4b459dda319fcbac"; sha256 = "0cgziqra8097hp05ynib1qpw4c95n972f4w2rk9l3llyp8r1vmci"; }; libName = "stackable_operator"; @@ -10599,7 +10577,7 @@ rec { } { name = "educe"; - packageId = "educe 0.7.6"; + packageId = "educe 0.8.1"; usesDefaultFeatures = false; features = [ "Clone" "Debug" "Default" "PartialEq" "Eq" ]; } @@ -10767,7 +10745,7 @@ rec { workspace_member = null; src = pkgs.fetchgit { url = "https://github.com/stackabletech/operator-rs.git"; - rev = "bc6c84025c2dcc834b94bfb57ec72810ae5f5eb1"; + rev = "7cb677e73bd218bcafb4ed4d4b459dda319fcbac"; sha256 = "0cgziqra8097hp05ynib1qpw4c95n972f4w2rk9l3llyp8r1vmci"; }; procMacro = true; @@ -10802,7 +10780,7 @@ rec { workspace_member = null; src = pkgs.fetchgit { url = "https://github.com/stackabletech/operator-rs.git"; - rev = "bc6c84025c2dcc834b94bfb57ec72810ae5f5eb1"; + rev = "7cb677e73bd218bcafb4ed4d4b459dda319fcbac"; sha256 = "0cgziqra8097hp05ynib1qpw4c95n972f4w2rk9l3llyp8r1vmci"; }; libName = "stackable_shared"; @@ -10883,7 +10861,7 @@ rec { workspace_member = null; src = pkgs.fetchgit { url = "https://github.com/stackabletech/operator-rs.git"; - rev = "bc6c84025c2dcc834b94bfb57ec72810ae5f5eb1"; + rev = "7cb677e73bd218bcafb4ed4d4b459dda319fcbac"; sha256 = "0cgziqra8097hp05ynib1qpw4c95n972f4w2rk9l3llyp8r1vmci"; }; libName = "stackable_telemetry"; @@ -10993,7 +10971,7 @@ rec { workspace_member = null; src = pkgs.fetchgit { url = "https://github.com/stackabletech/operator-rs.git"; - rev = "bc6c84025c2dcc834b94bfb57ec72810ae5f5eb1"; + rev = "7cb677e73bd218bcafb4ed4d4b459dda319fcbac"; sha256 = "0cgziqra8097hp05ynib1qpw4c95n972f4w2rk9l3llyp8r1vmci"; }; libName = "stackable_versioned"; @@ -11043,7 +11021,7 @@ rec { workspace_member = null; src = pkgs.fetchgit { url = "https://github.com/stackabletech/operator-rs.git"; - rev = "bc6c84025c2dcc834b94bfb57ec72810ae5f5eb1"; + rev = "7cb677e73bd218bcafb4ed4d4b459dda319fcbac"; sha256 = "0cgziqra8097hp05ynib1qpw4c95n972f4w2rk9l3llyp8r1vmci"; }; procMacro = true; @@ -11056,10 +11034,6 @@ rec { name = "convert_case"; packageId = "convert_case"; } - { - name = "convert_case_extras"; - packageId = "convert_case_extras"; - } { name = "darling"; packageId = "darling 0.24.1"; @@ -11106,12 +11080,12 @@ rec { }; "stackable-webhook" = rec { crateName = "stackable-webhook"; - version = "0.9.2"; + version = "0.10.0"; edition = "2024"; workspace_member = null; src = pkgs.fetchgit { url = "https://github.com/stackabletech/operator-rs.git"; - rev = "bc6c84025c2dcc834b94bfb57ec72810ae5f5eb1"; + rev = "7cb677e73bd218bcafb4ed4d4b459dda319fcbac"; sha256 = "0cgziqra8097hp05ynib1qpw4c95n972f4w2rk9l3llyp8r1vmci"; }; libName = "stackable_webhook"; @@ -11217,7 +11191,7 @@ rec { } { name = "tower-http"; - packageId = "tower-http 0.7.0"; + packageId = "tower-http 0.7.1"; features = [ "trace" ]; } { @@ -11422,7 +11396,7 @@ rec { "proc-macro" = [ "proc-macro2/proc-macro" "quote?/proc-macro" ]; "test" = [ "syn-test-suite/all-features" ]; }; - resolvedDefaultFeatures = [ "clone-impls" "default" "derive" "extra-traits" "full" "parsing" "printing" "proc-macro" "visit-mut" ]; + resolvedDefaultFeatures = [ "clone-impls" "default" "derive" "extra-traits" "full" "parsing" "printing" "proc-macro" "visit" "visit-mut" ]; }; "sync_wrapper" = rec { crateName = "sync_wrapper"; @@ -12588,11 +12562,11 @@ rec { }; resolvedDefaultFeatures = [ "auth" "base64" "default" "follow-redirect" "futures-util" "map-response-body" "mime" "tower" "trace" "tracing" "util" "validate-request" ]; }; - "tower-http 0.7.0" = rec { + "tower-http 0.7.1" = rec { crateName = "tower-http"; - version = "0.7.0"; + version = "0.7.1"; edition = "2018"; - sha256 = "0cz2k1a6gj54lcqx9cxb6k7rfgwcgc2zi31xnq0vxhmh2blpa7xi"; + sha256 = "172v4iw852pzl0v0k0nlklavlsnaw3zmbmx1w2xirmpxlik5m808"; libName = "tower_http"; authors = [ "Tower Maintainers " @@ -12890,9 +12864,9 @@ rec { }; "tracing-opentelemetry" = rec { crateName = "tracing-opentelemetry"; - version = "0.33.0"; + version = "0.34.0"; edition = "2021"; - sha256 = "09nvxy5m7nxmifz4b6szdcyczapp2jcgxcac0jw4ax8klz5n9g5d"; + sha256 = "1zp8wg0yhj6nhv2vc0a182sf6iq8clmgyxxn70vg80mrl414i40a"; libName = "tracing_opentelemetry"; dependencies = [ { diff --git a/Cargo.toml b/Cargo.toml index 5e9c33132..8f9321a8c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -10,7 +10,7 @@ edition = "2024" repository = "https://github.com/stackabletech/kafka-operator" [workspace.dependencies] -stackable-operator = { git = "https://github.com/stackabletech/operator-rs.git", tag = "stackable-operator-0.118.0", features = ["crds", "webhook"] } +stackable-operator = { git = "https://github.com/stackabletech/operator-rs.git", branch = "feat/platform-access", features = ["crds", "webhook"] } anyhow = "1.0" built = { version = "0.8", features = ["chrono", "git2"] } diff --git a/crate-hashes.json b/crate-hashes.json index 8357dcf27..e4f5f7b1d 100644 --- a/crate-hashes.json +++ b/crate-hashes.json @@ -1,11 +1,11 @@ { - "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.118.0#k8s-version@0.1.3": "0cgziqra8097hp05ynib1qpw4c95n972f4w2rk9l3llyp8r1vmci", - "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.118.0#stackable-certs@0.4.1": "0cgziqra8097hp05ynib1qpw4c95n972f4w2rk9l3llyp8r1vmci", - "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.118.0#stackable-operator-derive@0.3.1": "0cgziqra8097hp05ynib1qpw4c95n972f4w2rk9l3llyp8r1vmci", - "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.118.0#stackable-operator@0.118.0": "0cgziqra8097hp05ynib1qpw4c95n972f4w2rk9l3llyp8r1vmci", - "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.118.0#stackable-shared@0.1.2": "0cgziqra8097hp05ynib1qpw4c95n972f4w2rk9l3llyp8r1vmci", - "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.118.0#stackable-telemetry@0.6.5": "0cgziqra8097hp05ynib1qpw4c95n972f4w2rk9l3llyp8r1vmci", - "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.118.0#stackable-versioned-macros@0.11.1": "0cgziqra8097hp05ynib1qpw4c95n972f4w2rk9l3llyp8r1vmci", - "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.118.0#stackable-versioned@0.11.1": "0cgziqra8097hp05ynib1qpw4c95n972f4w2rk9l3llyp8r1vmci", - "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.118.0#stackable-webhook@0.9.2": "0cgziqra8097hp05ynib1qpw4c95n972f4w2rk9l3llyp8r1vmci" + "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#k8s-version@0.1.3": "0cgziqra8097hp05ynib1qpw4c95n972f4w2rk9l3llyp8r1vmci", + "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#stackable-certs@0.4.1": "0cgziqra8097hp05ynib1qpw4c95n972f4w2rk9l3llyp8r1vmci", + "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#stackable-operator-derive@0.3.1": "0cgziqra8097hp05ynib1qpw4c95n972f4w2rk9l3llyp8r1vmci", + "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#stackable-operator@0.119.0": "0cgziqra8097hp05ynib1qpw4c95n972f4w2rk9l3llyp8r1vmci", + "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#stackable-shared@0.1.2": "0cgziqra8097hp05ynib1qpw4c95n972f4w2rk9l3llyp8r1vmci", + "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#stackable-telemetry@0.6.5": "0cgziqra8097hp05ynib1qpw4c95n972f4w2rk9l3llyp8r1vmci", + "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#stackable-versioned-macros@0.11.1": "0cgziqra8097hp05ynib1qpw4c95n972f4w2rk9l3llyp8r1vmci", + "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#stackable-versioned@0.11.1": "0cgziqra8097hp05ynib1qpw4c95n972f4w2rk9l3llyp8r1vmci", + "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#stackable-webhook@0.10.0": "0cgziqra8097hp05ynib1qpw4c95n972f4w2rk9l3llyp8r1vmci" } \ No newline at end of file diff --git a/extra/crds.yaml b/extra/crds.yaml index 7c5f5161f..eb762d076 100644 --- a/extra/crds.yaml +++ b/extra/crds.yaml @@ -2156,7 +2156,7 @@ spec: properties: secret: description: |- - A static Secret holding the certificate in the keys `tls.crt` and `tls.key` (PEM), such as a + A static Secret holding the certificate in the keys `tls.crt` and `tls.key` (PEM), e.g. as a Secret of type `kubernetes.io/tls`. maxLength: 253 minLength: 1 diff --git a/rust/agent-binary/src/framework/mod.rs b/rust/agent-binary/src/framework/mod.rs deleted file mode 100644 index c26b5e711..000000000 --- a/rust/agent-binary/src/framework/mod.rs +++ /dev/null @@ -1,12 +0,0 @@ -//! Reusable, product-agnostic agent building blocks. Staged here until they move to operator-rs. - -use clap::Args; -use stackable_operator::cli::CommonOptions; - -/// Agent subcommands. Like [`stackable_operator::cli::Command`], but without `crd` as CRDs are only -/// deployed by operators. -#[derive(Debug, clap::Parser)] -pub enum AgentCommand { - /// Run the agent. - Run(Run), -} diff --git a/rust/agent-binary/src/main.rs b/rust/agent-binary/src/main.rs index 6ab5c58f7..0934f803f 100644 --- a/rust/agent-binary/src/main.rs +++ b/rust/agent-binary/src/main.rs @@ -1,11 +1,11 @@ //! The Stackable Kafka agent: a per-cluster controller that reconciles in-cluster resources like `KafkaTopic`s. use clap::Parser; -use stackable_operator::{cli::CommonOptions, telemetry::Tracing, utils::signal::SignalWatcher}; - -use crate::framework::AgentCommand; - -mod framework; +use stackable_operator::{ + cli::{Command, CommonOptions}, + telemetry::Tracing, + utils::signal::SignalWatcher, +}; mod built_info { include!(concat!(env!("OUT_DIR"), "/built.rs")); @@ -15,13 +15,14 @@ mod built_info { #[clap(about, author)] struct Opts { #[clap(subcommand)] - cmd: AgentCommand, + cmd: Command, } #[tokio::main] async fn main() -> anyhow::Result<()> { match Opts::parse().cmd { - AgentCommand::Run(CommonOptions { telemetry, .. }) => { + Command::Crd => anyhow::bail!("this agent has no CRDs, they are owned by the operator"), + Command::Run(CommonOptions { telemetry, .. }) => { let _tracing_guard = Tracing::pre_configured(built_info::PKG_NAME, telemetry).init()?; tracing::info!( diff --git a/rust/operator-binary/src/controller/build/mod.rs b/rust/operator-binary/src/controller/build/mod.rs index 2283c4448..18e220a57 100644 --- a/rust/operator-binary/src/controller/build/mod.rs +++ b/rust/operator-binary/src/controller/build/mod.rs @@ -30,7 +30,6 @@ use crate::{ }, }, crd::role::{AnyConfig, KafkaRole}, - framework::kvp::label as framework_label, }; pub mod command; @@ -186,7 +185,7 @@ pub(crate) fn recommended_labels_for_cluster_resources(cluster: &ValidatedCluste } pub(crate) fn recommended_labels_for_agent_resources(cluster: &ValidatedCluster) -> Labels { - framework_label::recommended_labels_for_agent_resources( + label::recommended_labels_for_agent_resources( &cluster.name, &PRODUCT_NAME, &cluster.product_version, @@ -242,7 +241,7 @@ pub(crate) fn recommended_labels_for_unversioned_role_group_resources( /// Selector labels matching the pods of a role group. pub(crate) fn agent_selector(cluster: &ValidatedCluster) -> Labels { - framework_label::agent_selector(&cluster.name, &PRODUCT_NAME) + label::agent_selector(&cluster.name, &PRODUCT_NAME) } pub(crate) fn role_group_selector( diff --git a/rust/operator-binary/src/crd/mod.rs b/rust/operator-binary/src/crd/mod.rs index 5af1e5428..7f486d93b 100644 --- a/rust/operator-binary/src/crd/mod.rs +++ b/rust/operator-binary/src/crd/mod.rs @@ -13,7 +13,7 @@ use snafu::Snafu; use stackable_operator::{ commons::{ cluster_operation::ClusterOperation, networking::DomainName, - product_image_selection::ProductImage, + platform_access::tls::TlsClientCredential, product_image_selection::ProductImage, }, config::merge::Merge, constant, @@ -38,13 +38,10 @@ use stackable_operator::{ }; use strum::{Display, EnumIter, EnumString}; -use crate::{ - crd::{ - authorization::KafkaAuthorization, - role::{KafkaRole, broker::BrokerConfigFragment, controller::ControllerConfigFragment}, - tls::KafkaTls, - }, - framework::commons::platform_access::tls::TlsClientCredential, +use crate::crd::{ + authorization::KafkaAuthorization, + role::{KafkaRole, broker::BrokerConfigFragment, controller::ControllerConfigFragment}, + tls::KafkaTls, }; pub const CONTAINER_IMAGE_BASE_NAME: &str = "kafka"; diff --git a/rust/operator-binary/src/framework/commons/mod.rs b/rust/operator-binary/src/framework/commons/mod.rs deleted file mode 100644 index ad8d7e6bd..000000000 --- a/rust/operator-binary/src/framework/commons/mod.rs +++ /dev/null @@ -1 +0,0 @@ -pub mod platform_access; diff --git a/rust/operator-binary/src/framework/commons/platform_access/mod.rs b/rust/operator-binary/src/framework/commons/platform_access/mod.rs deleted file mode 100644 index 7cedc76bf..000000000 --- a/rust/operator-binary/src/framework/commons/platform_access/mod.rs +++ /dev/null @@ -1,3 +0,0 @@ -//! Credentials a product cluster grants the platform-access agent, and how to mount them. - -pub mod tls; diff --git a/rust/operator-binary/src/framework/commons/platform_access/tls.rs b/rust/operator-binary/src/framework/commons/platform_access/tls.rs deleted file mode 100644 index 592d1e51b..000000000 --- a/rust/operator-binary/src/framework/commons/platform_access/tls.rs +++ /dev/null @@ -1,87 +0,0 @@ -use serde::{Deserialize, Serialize}; -use stackable_operator::{ - builder::pod::{ - PodBuilder, - container::ContainerBuilder, - volume::{ - SecretFormat, SecretOperatorVolumeSourceBuilder, VolumeBuilder, VolumeMountBuilder, - }, - }, - commons::secret_class::SecretClassVolumeProvisionParts, - k8s_openapi::api::core::v1::{SecretVolumeSource, Volume, VolumeMount}, - schemars::{self, JsonSchema}, - v2::types::kubernetes::{SecretClassName, SecretName}, -}; - -use crate::framework::constants::secret::SECRET_BASE_PATH; - -/// Source of a TLS client certificate: a secret-operator SecretClass or a static Secret. -#[derive(Clone, Debug, Deserialize, Eq, JsonSchema, PartialEq, Serialize)] -#[serde(rename_all = "camelCase")] -pub enum TlsClientCredential { - /// An AutoTLS SecretClass used to provision the certificate. - SecretClass(SecretClassName), - - /// A static Secret holding the certificate in the keys `tls.crt` and `tls.key` (PEM), such as a - /// Secret of type `kubernetes.io/tls`. - Secret(SecretName), -} - -impl TlsClientCredential { - /// Adds the certificate volume to the Pod and mounts it into all given containers. - pub fn add_volumes_and_mounts( - &self, - pod_builder: &mut PodBuilder, - container_builders: Vec<&mut ContainerBuilder>, - ) { - let (volumes, mounts) = self.volumes_and_mounts(); - pod_builder - .add_volumes(volumes) - .expect("The volume name is derived from the credential and should not collide."); - for container_builder in container_builders { - container_builder - .add_volume_mounts(mounts.clone()) - .expect("The mount path is derived from the credential and should not collide."); - } - } - - fn volumes_and_mounts(&self) -> (Vec, Vec) { - let volume_name = self.volume_name(); - let volume = match self { - Self::SecretClass(secret_class) => VolumeBuilder::new(&volume_name) - .ephemeral( - SecretOperatorVolumeSourceBuilder::new( - secret_class, - SecretClassVolumeProvisionParts::PublicPrivate, - ) - .with_pod_scope() - .with_format(SecretFormat::TlsPem) - .build() - .expect("the annotations are built from a valid SecretClass name"), - ) - .build(), - Self::Secret(secret) => Volume { - name: volume_name.clone(), - secret: Some(SecretVolumeSource { - secret_name: Some(secret.to_string()), - ..SecretVolumeSource::default() - }), - ..Volume::default() - }, - }; - let mount = VolumeMountBuilder::new(&volume_name, self.mount_path()).build(); - (vec![volume], vec![mount]) - } - - /// The directory containing `tls.crt`, `tls.key` and `ca.crt` (PEM). - pub fn mount_path(&self) -> String { - format!("{SECRET_BASE_PATH}/{}", self.volume_name()) - } - - fn volume_name(&self) -> String { - match self { - Self::SecretClass(secret_class) => format!("{secret_class}-tls-cert"), - Self::Secret(secret) => format!("{secret}-tls-cert"), - } - } -} diff --git a/rust/operator-binary/src/framework/constants/mod.rs b/rust/operator-binary/src/framework/constants/mod.rs deleted file mode 100644 index 73b12dbbd..000000000 --- a/rust/operator-binary/src/framework/constants/mod.rs +++ /dev/null @@ -1 +0,0 @@ -pub mod secret; diff --git a/rust/operator-binary/src/framework/constants/secret.rs b/rust/operator-binary/src/framework/constants/secret.rs deleted file mode 100644 index 320faf979..000000000 --- a/rust/operator-binary/src/framework/constants/secret.rs +++ /dev/null @@ -1,2 +0,0 @@ -// Mirrors the crate-private `SECRET_BASE_PATH` of operator-rs. -pub(crate) const SECRET_BASE_PATH: &str = "/stackable/secrets"; diff --git a/rust/operator-binary/src/framework/kvp/label.rs b/rust/operator-binary/src/framework/kvp/label.rs deleted file mode 100644 index f3695b8c7..000000000 --- a/rust/operator-binary/src/framework/kvp/label.rs +++ /dev/null @@ -1,45 +0,0 @@ -use stackable_operator::{ - kvp::{Label, Labels}, - v2::{ - kvp::label::{ - label_app_kubernetes_io_instance, label_app_kubernetes_io_managed_by, - label_app_kubernetes_io_name, label_app_kubernetes_io_version, - label_stackable_tech_vendor, - }, - types::operator::{ClusterName, ControllerName, OperatorName, ProductName, ProductVersion}, - }, -}; - -/// Creates the recommended labels for agent resources, like the agent Deployment. -pub fn recommended_labels_for_agent_resources( - cluster_name: &ClusterName, - product_name: &ProductName, - product_version: &ProductVersion, - operator_name: &OperatorName, - controller_name: &ControllerName, -) -> Labels { - Labels::from_iter([ - label_app_kubernetes_io_instance(cluster_name), - label_app_kubernetes_io_name(product_name), - label_app_kubernetes_io_version(product_version), - label_app_kubernetes_io_component_agent(), - label_app_kubernetes_io_managed_by(operator_name, controller_name), - label_stackable_tech_vendor(), - ]) -} - -/// Creates the agent selector. -/// -/// The returned labels are a subset of the recommended labels for agent resources. -pub fn agent_selector(cluster_name: &ClusterName, product_name: &ProductName) -> Labels { - Labels::from_iter([ - label_app_kubernetes_io_instance(cluster_name), - label_app_kubernetes_io_name(product_name), - label_app_kubernetes_io_component_agent(), - ]) -} - -/// Creates the `app.kubernetes.io/component` label with the value `agent`. -pub fn label_app_kubernetes_io_component_agent() -> Label { - Label::component("agent").expect("\"agent\" is a valid label value") -} diff --git a/rust/operator-binary/src/framework/kvp/mod.rs b/rust/operator-binary/src/framework/kvp/mod.rs deleted file mode 100644 index 0006163a0..000000000 --- a/rust/operator-binary/src/framework/kvp/mod.rs +++ /dev/null @@ -1 +0,0 @@ -pub mod label; diff --git a/rust/operator-binary/src/framework/mod.rs b/rust/operator-binary/src/framework/mod.rs deleted file mode 100644 index 842c16dde..000000000 --- a/rust/operator-binary/src/framework/mod.rs +++ /dev/null @@ -1,6 +0,0 @@ -//! Product-agnostic building blocks, staged here until they move to operator-rs. The module paths -//! mirror operator-rs, so moving code there only changes imports. - -pub mod commons; -pub mod constants; -pub mod kvp; diff --git a/rust/operator-binary/src/main.rs b/rust/operator-binary/src/main.rs index 941f4aa1e..809bd5664 100644 --- a/rust/operator-binary/src/main.rs +++ b/rust/operator-binary/src/main.rs @@ -43,7 +43,6 @@ use crate::{ mod controller; mod crd; -mod framework; mod webhooks; mod built_info { @@ -223,7 +222,7 @@ async fn main() -> anyhow::Result<()> { .map(anyhow::Ok); let delayed_kafka_controller = async { - signal::crd_established(&client, v1alpha1::KafkaCluster::crd_name(), None).await?; + signal::crd_established(&client, v1alpha1::KafkaCluster::crd_name()).await?; kafka_controller.await }; diff --git a/rust/operator-binary/src/webhooks/conversion.rs b/rust/operator-binary/src/webhooks/conversion.rs index 912ea5936..232e4dac4 100644 --- a/rust/operator-binary/src/webhooks/conversion.rs +++ b/rust/operator-binary/src/webhooks/conversion.rs @@ -4,6 +4,7 @@ use stackable_operator::{ kube::{Client, core::crd::MergeError}, webhook::{ WebhookServer, WebhookServerError, WebhookServerOptions, + health::HealthCheckRegistry, webhooks::{ConversionWebhook, ConversionWebhookOptions}, }, }; @@ -46,7 +47,11 @@ pub async fn create_webhook_server( webhook_service_name: operator_environment.operator_service_name.to_owned(), }; - WebhookServer::new(vec![Box::new(conversion_webhook)], webhook_server_options) - .await - .context(CreateWebhookSnafu) + WebhookServer::new( + vec![Box::new(conversion_webhook)], + webhook_server_options, + HealthCheckRegistry::new(), + ) + .await + .context(CreateWebhookSnafu) } From 7f6d5a1d9b897b556b98c0bfdb9e7ca1c67415b0 Mon Sep 17 00:00:00 2001 From: Benedikt Labrenz Date: Fri, 2 Oct 2026 15:47:43 +0200 Subject: [PATCH 07/93] fix issues identified by Claude --- Cargo.lock | 18 +++++----- Cargo.nix | 18 +++++----- .../kafka-operator/templates/_helpers.tpl | 10 +++++- .../templates/clusterrole-agent.yaml | 2 +- .../templates/clusterrole-operator.yaml | 2 +- .../src/controller/build/resource/agent.rs | 33 ++++++++++++++++--- 6 files changed, 57 insertions(+), 26 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 77847b010..9842b5ab4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1698,7 +1698,7 @@ dependencies = [ [[package]] name = "k8s-version" version = "0.1.3" -source = "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#7cb677e73bd218bcafb4ed4d4b459dda319fcbac" +source = "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#f3d2da737aa49086cbdaba806aaf2c8645251cfb" dependencies = [ "darling 0.24.1", "regex", @@ -3144,7 +3144,7 @@ checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596" [[package]] name = "stackable-certs" version = "0.4.1" -source = "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#7cb677e73bd218bcafb4ed4d4b459dda319fcbac" +source = "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#f3d2da737aa49086cbdaba806aaf2c8645251cfb" dependencies = [ "const-oid", "ecdsa", @@ -3202,7 +3202,7 @@ dependencies = [ [[package]] name = "stackable-operator" version = "0.119.0" -source = "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#7cb677e73bd218bcafb4ed4d4b459dda319fcbac" +source = "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#f3d2da737aa49086cbdaba806aaf2c8645251cfb" dependencies = [ "base64 0.23.1", "clap", @@ -3246,7 +3246,7 @@ dependencies = [ [[package]] name = "stackable-operator-derive" version = "0.3.1" -source = "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#7cb677e73bd218bcafb4ed4d4b459dda319fcbac" +source = "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#f3d2da737aa49086cbdaba806aaf2c8645251cfb" dependencies = [ "darling 0.24.1", "proc-macro2", @@ -3257,7 +3257,7 @@ dependencies = [ [[package]] name = "stackable-shared" version = "0.1.2" -source = "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#7cb677e73bd218bcafb4ed4d4b459dda319fcbac" +source = "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#f3d2da737aa49086cbdaba806aaf2c8645251cfb" dependencies = [ "jiff", "k8s-openapi", @@ -3274,7 +3274,7 @@ dependencies = [ [[package]] name = "stackable-telemetry" version = "0.6.5" -source = "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#7cb677e73bd218bcafb4ed4d4b459dda319fcbac" +source = "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#f3d2da737aa49086cbdaba806aaf2c8645251cfb" dependencies = [ "axum", "clap", @@ -3298,7 +3298,7 @@ dependencies = [ [[package]] name = "stackable-versioned" version = "0.11.1" -source = "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#7cb677e73bd218bcafb4ed4d4b459dda319fcbac" +source = "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#f3d2da737aa49086cbdaba806aaf2c8645251cfb" dependencies = [ "kube", "schemars", @@ -3312,7 +3312,7 @@ dependencies = [ [[package]] name = "stackable-versioned-macros" version = "0.11.1" -source = "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#7cb677e73bd218bcafb4ed4d4b459dda319fcbac" +source = "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#f3d2da737aa49086cbdaba806aaf2c8645251cfb" dependencies = [ "convert_case", "darling 0.24.1", @@ -3329,7 +3329,7 @@ dependencies = [ [[package]] name = "stackable-webhook" version = "0.10.0" -source = "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#7cb677e73bd218bcafb4ed4d4b459dda319fcbac" +source = "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#f3d2da737aa49086cbdaba806aaf2c8645251cfb" dependencies = [ "arc-swap", "async-trait", diff --git a/Cargo.nix b/Cargo.nix index 0f1de0737..b76c6a054 100644 --- a/Cargo.nix +++ b/Cargo.nix @@ -5385,7 +5385,7 @@ rec { workspace_member = null; src = pkgs.fetchgit { url = "https://github.com/stackabletech/operator-rs.git"; - rev = "7cb677e73bd218bcafb4ed4d4b459dda319fcbac"; + rev = "f3d2da737aa49086cbdaba806aaf2c8645251cfb"; sha256 = "0cgziqra8097hp05ynib1qpw4c95n972f4w2rk9l3llyp8r1vmci"; }; libName = "k8s_version"; @@ -10303,7 +10303,7 @@ rec { workspace_member = null; src = pkgs.fetchgit { url = "https://github.com/stackabletech/operator-rs.git"; - rev = "7cb677e73bd218bcafb4ed4d4b459dda319fcbac"; + rev = "f3d2da737aa49086cbdaba806aaf2c8645251cfb"; sha256 = "0cgziqra8097hp05ynib1qpw4c95n972f4w2rk9l3llyp8r1vmci"; }; libName = "stackable_certs"; @@ -10546,7 +10546,7 @@ rec { workspace_member = null; src = pkgs.fetchgit { url = "https://github.com/stackabletech/operator-rs.git"; - rev = "7cb677e73bd218bcafb4ed4d4b459dda319fcbac"; + rev = "f3d2da737aa49086cbdaba806aaf2c8645251cfb"; sha256 = "0cgziqra8097hp05ynib1qpw4c95n972f4w2rk9l3llyp8r1vmci"; }; libName = "stackable_operator"; @@ -10745,7 +10745,7 @@ rec { workspace_member = null; src = pkgs.fetchgit { url = "https://github.com/stackabletech/operator-rs.git"; - rev = "7cb677e73bd218bcafb4ed4d4b459dda319fcbac"; + rev = "f3d2da737aa49086cbdaba806aaf2c8645251cfb"; sha256 = "0cgziqra8097hp05ynib1qpw4c95n972f4w2rk9l3llyp8r1vmci"; }; procMacro = true; @@ -10780,7 +10780,7 @@ rec { workspace_member = null; src = pkgs.fetchgit { url = "https://github.com/stackabletech/operator-rs.git"; - rev = "7cb677e73bd218bcafb4ed4d4b459dda319fcbac"; + rev = "f3d2da737aa49086cbdaba806aaf2c8645251cfb"; sha256 = "0cgziqra8097hp05ynib1qpw4c95n972f4w2rk9l3llyp8r1vmci"; }; libName = "stackable_shared"; @@ -10861,7 +10861,7 @@ rec { workspace_member = null; src = pkgs.fetchgit { url = "https://github.com/stackabletech/operator-rs.git"; - rev = "7cb677e73bd218bcafb4ed4d4b459dda319fcbac"; + rev = "f3d2da737aa49086cbdaba806aaf2c8645251cfb"; sha256 = "0cgziqra8097hp05ynib1qpw4c95n972f4w2rk9l3llyp8r1vmci"; }; libName = "stackable_telemetry"; @@ -10971,7 +10971,7 @@ rec { workspace_member = null; src = pkgs.fetchgit { url = "https://github.com/stackabletech/operator-rs.git"; - rev = "7cb677e73bd218bcafb4ed4d4b459dda319fcbac"; + rev = "f3d2da737aa49086cbdaba806aaf2c8645251cfb"; sha256 = "0cgziqra8097hp05ynib1qpw4c95n972f4w2rk9l3llyp8r1vmci"; }; libName = "stackable_versioned"; @@ -11021,7 +11021,7 @@ rec { workspace_member = null; src = pkgs.fetchgit { url = "https://github.com/stackabletech/operator-rs.git"; - rev = "7cb677e73bd218bcafb4ed4d4b459dda319fcbac"; + rev = "f3d2da737aa49086cbdaba806aaf2c8645251cfb"; sha256 = "0cgziqra8097hp05ynib1qpw4c95n972f4w2rk9l3llyp8r1vmci"; }; procMacro = true; @@ -11085,7 +11085,7 @@ rec { workspace_member = null; src = pkgs.fetchgit { url = "https://github.com/stackabletech/operator-rs.git"; - rev = "7cb677e73bd218bcafb4ed4d4b459dda319fcbac"; + rev = "f3d2da737aa49086cbdaba806aaf2c8645251cfb"; sha256 = "0cgziqra8097hp05ynib1qpw4c95n972f4w2rk9l3llyp8r1vmci"; }; libName = "stackable_webhook"; diff --git a/deploy/helm/kafka-operator/templates/_helpers.tpl b/deploy/helm/kafka-operator/templates/_helpers.tpl index 27536afa2..281aa5495 100644 --- a/deploy/helm/kafka-operator/templates/_helpers.tpl +++ b/deploy/helm/kafka-operator/templates/_helpers.tpl @@ -5,6 +5,14 @@ Expand the name of the chart. {{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-operator" }} {{- end }} +{{/* +Fixed product name. Unlike kafka-operator.name it ignores nameOverride, because it must match +names hardcoded in the operator and the published image names. +*/}} +{{- define "kafka-operator.productName" -}} +kafka +{{- end }} + {{/* Expand the name of the chart. */}} @@ -89,5 +97,5 @@ Build the full operator container image reference. Build the full agent container image reference. The agent is released together with the operator. */}} {{- define "kafka-operator.agentImage" -}} -{{- printf "%s/%s-agent:%s" .Values.image.repository (include "kafka-operator.name" .) (.Values.image.tag | default .Chart.AppVersion) -}} +{{- printf "%s/%s-agent:%s" .Values.image.repository (include "kafka-operator.productName" .) (.Values.image.tag | default .Chart.AppVersion) -}} {{- end }} diff --git a/deploy/helm/kafka-operator/templates/clusterrole-agent.yaml b/deploy/helm/kafka-operator/templates/clusterrole-agent.yaml index 8288ab33a..a17015062 100644 --- a/deploy/helm/kafka-operator/templates/clusterrole-agent.yaml +++ b/deploy/helm/kafka-operator/templates/clusterrole-agent.yaml @@ -4,7 +4,7 @@ apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: - name: {{ include "kafka-operator.name" . }}-agent-clusterrole + name: {{ include "kafka-operator.productName" . }}-agent-clusterrole labels: {{- include "kafka-operator.labels" . | nindent 4 }} rules: diff --git a/deploy/helm/kafka-operator/templates/clusterrole-operator.yaml b/deploy/helm/kafka-operator/templates/clusterrole-operator.yaml index 96045367c..cabc631e4 100644 --- a/deploy/helm/kafka-operator/templates/clusterrole-operator.yaml +++ b/deploy/helm/kafka-operator/templates/clusterrole-operator.yaml @@ -50,7 +50,7 @@ rules: - bind resourceNames: - {{ include "kafka-operator.name" . }}-clusterrole - - {{ include "kafka-operator.name" . }}-agent-clusterrole + - {{ include "kafka-operator.productName" . }}-agent-clusterrole # StatefulSet created per role group (broker, KRaft controller). Applied via # SSA, tracked for orphan cleanup, and owned by the controller. - apiGroups: diff --git a/rust/operator-binary/src/controller/build/resource/agent.rs b/rust/operator-binary/src/controller/build/resource/agent.rs index e466512fd..35db34806 100644 --- a/rust/operator-binary/src/controller/build/resource/agent.rs +++ b/rust/operator-binary/src/controller/build/resource/agent.rs @@ -82,6 +82,7 @@ pub fn build_agent_deployment( .with_labels(recommended_labels_for_agent_resources(cluster)) .build(), ) + .image_pull_secrets_from_product_image(&cluster.image) .add_container(cb_agent.build()) .service_account_name(agent_name(cluster)) .security_context(PodSecurityContextBuilder::with_stackable_defaults().build()); @@ -152,6 +153,10 @@ mod tests { }; fn cluster(platform_access: &str) -> ValidatedCluster { + cluster_with_image("{productVersion: 3.9.2}", platform_access) + } + + fn cluster_with_image(image: &str, platform_access: &str) -> ValidatedCluster { let kafka = minimal_kafka(&format!( r#" apiVersion: kafka.stackable.tech/v1alpha1 @@ -161,8 +166,7 @@ mod tests { namespace: default uid: 12345678-1234-1234-1234-123456789012 spec: - image: - productVersion: 3.9.2 + image: {image} clusterConfig: zookeeperConfigMapName: xyz platformAccess: {platform_access} @@ -235,9 +239,9 @@ mod tests { assert_eq!(pod_spec["containers"][0]["args"], json!(["run"])); assert_eq!( pod_spec["containers"][0]["volumeMounts"], - json!([{"mountPath": "/stackable/secrets/tls-tls-cert", "name": "tls-tls-cert"}]) + json!([{"mountPath": "/stackable/secrets/tls-client-cert", "name": "tls-client-cert"}]) ); - assert_eq!(pod_spec["volumes"][0]["name"], "tls-tls-cert"); + assert_eq!(pod_spec["volumes"][0]["name"], "tls-client-cert"); assert_eq!( pod_spec["volumes"][0]["ephemeral"]["volumeClaimTemplate"]["metadata"]["annotations"]["secrets.stackable.tech/class"], "tls" @@ -251,7 +255,26 @@ mod tests { assert_eq!( deployment["spec"]["template"]["spec"]["volumes"], - json!([{"name": "my-cert-tls-cert", "secret": {"secretName": "my-cert"}}]) + json!([{"name": "tls-client-cert", "secret": {"secretName": "my-cert"}}]) + ); + } + + #[test] + fn test_deployment_uses_product_image_pull_secrets() { + let cluster = cluster_with_image( + "{productVersion: 3.9.2, pullSecrets: [{name: regcred}]}", + "{enabled: true, authentication: {tls: {secretClass: tls}}}", + ); + let agent_config = cluster + .agent_config + .as_ref() + .expect("platform access is enabled"); + let deployment = serde_json::to_value(build_agent_deployment(&cluster, agent_config)) + .expect("must be serializable"); + + assert_eq!( + deployment["spec"]["template"]["spec"]["imagePullSecrets"], + json!([{"name": "regcred"}]) ); } From e5f431bea08f83527a1d082aec51ed5658823ddd Mon Sep 17 00:00:00 2001 From: Benedikt Labrenz Date: Tue, 6 Oct 2026 15:13:35 +0200 Subject: [PATCH 08/93] add support for anonymous access via operator-rs --- Cargo.lock | 373 +++++++------ Cargo.nix | 523 ++++++++++-------- extra/crds.yaml | 7 +- .../src/controller/build/resource/agent.rs | 10 + rust/operator-binary/src/crd/mod.rs | 9 +- 5 files changed, 528 insertions(+), 394 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 9842b5ab4..da8c1babd 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -169,7 +169,7 @@ checksum = "82f6aeea286b8eb4dd3431a1be1b59d290ace00f5bfd8e2a159bc2a05e2c1667" dependencies = [ "proc-macro2", "quote", - "syn 3.0.4", + "syn 3.0.6", ] [[package]] @@ -279,9 +279,9 @@ checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" [[package]] name = "bitflags" -version = "2.13.1" +version = "2.13.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b588b76d00fde79687d7646a9b5bdf3cc0f655e0bbd080335a95d7e96f3587da" +checksum = "3ded4057c258ba199e2d26386d3af3780957ecaee6c4ef4041c6b4b8b97c0b06" [[package]] name = "block-buffer" @@ -316,9 +316,9 @@ checksum = "fc652a48c352aef3ea3aed32080501cf3ef6ed5da78602a020c991775b0aff04" [[package]] name = "cc" -version = "1.4.4" +version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ad534f4357a5264cce5019c989cf66a4f0dc4e0d1b1d15f8aacec0ff7360273" +checksum = "f74872d07caf508b30a21f6836e7d7016a2eaf7d9ff4f48deaa58cd8a0407630" dependencies = [ "find-msvc-tools", "jobserver", @@ -328,9 +328,9 @@ dependencies = [ [[package]] name = "cfg-if" -version = "1.0.4" +version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" +checksum = "4e7648175b45a9a48536d676f68d918270699102aa8dab5496df06904c914600" [[package]] name = "chacha20" @@ -339,7 +339,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "65c35e4b699c7e15ccbe7ee35c005e4fc0a278d22238a2857e6ce2dadeda1b06" dependencies = [ "cfg-if", - "cpufeatures 0.3.0", + "cpufeatures 0.3.1", "rand_core 0.10.1", ] @@ -356,9 +356,9 @@ dependencies = [ [[package]] name = "clap" -version = "4.6.6" +version = "4.6.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "473c7e07f409a8d772161724aa8db6a765a2532a70f9667eeb7b49d3d02fbdca" +checksum = "aa8876b300ab35ba921adea3dfd70157a46249b33f95c9084ae5709785478946" dependencies = [ "clap_builder", "clap_derive", @@ -366,9 +366,9 @@ dependencies = [ [[package]] name = "clap_builder" -version = "4.6.6" +version = "4.6.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b48fea5a88e9ae728a2dcbedbfc0e730f7d60da42e1cb049a83c9fb8b789889" +checksum = "ec0797fb7aeb1406c84efac526901f7ec3ead2124f946b494e72879d4b54704d" dependencies = [ "anstream", "anstyle", @@ -378,21 +378,21 @@ dependencies = [ [[package]] name = "clap_derive" -version = "4.6.4" +version = "4.6.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d012d2b9d65aca7f18f4d9878a045bc17899bba951561ba5ec3c2ba1eed9a061" +checksum = "f9c751b79415d4e559e3d1fcf128e09e720eb673a06d26cf6f392d37d75b66e0" dependencies = [ "heck", "proc-macro2", "quote", - "syn 3.0.4", + "syn 3.0.6", ] [[package]] name = "clap_lex" -version = "1.1.0" +version = "1.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c8d4a3bb8b1e0c1050499d1815f5ab16d04f0959b233085fb31653fbfc9d98f9" +checksum = "1c133bc6a41be0d194c306b5506d15e6feeea7b1d6604bd3f8310dfb2ca96486" [[package]] name = "colorchoice" @@ -402,9 +402,9 @@ checksum = "1d07550c9036bf2ae0c684c4297d503f838287c83c53686d05370d0e139ae570" [[package]] name = "combine" -version = "4.6.7" +version = "4.6.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ba5a308b75df32fe02788e748662718f03fde005016435c444eea572398219fd" +checksum = "cfc320937d09e6de266b31b9afb480f197d7a861be86be7cb2ea7e5d1bfffc5e" dependencies = [ "bytes", "memchr", @@ -462,6 +462,12 @@ version = "0.8.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" +[[package]] +name = "core_detect" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f8f80099a98041a3d1622845c271458a2d73e688351bf3cb999266764b81d48" + [[package]] name = "cpufeatures" version = "0.2.17" @@ -473,36 +479,36 @@ dependencies = [ [[package]] name = "cpufeatures" -version = "0.3.0" +version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8b2a41393f66f16b0823bb79094d54ac5fbd34ab292ddafb9a0456ac9f87d201" +checksum = "5ca28b0ae3115b884660db4118d803791fd6756b6e88f39c0f3f7859060d7566" dependencies = [ "libc", ] [[package]] name = "crc32fast" -version = "1.5.1" +version = "1.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8498c871161e1742aaa9d52551b2d6ebdd4c3d45a3be423e3728f33b955be550" +checksum = "01a7799fd6b852db0e61728dde9a204c423b44d689dbd432522543614b490e78" dependencies = [ "cfg-if", ] [[package]] name = "crossbeam-channel" -version = "0.5.16" +version = "0.5.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d85363c37faeca707aef026efa9f3b34d077bce547e48f770770625c6013679e" +checksum = "98b0cc327b5bc766e7fda9c9260cc0fa81b43a8e240440422dff70788e3f9ef1" dependencies = [ "crossbeam-utils", ] [[package]] name = "crossbeam-utils" -version = "0.8.22" +version = "0.8.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "61803da095bee82a81bb1a452ecc25d3b2f1416d1897eb86430c6159ef717c17" +checksum = "a31eee39dddec8330830986fcd7625edb5a24ec90ea038215273bbc3adb08ac6" [[package]] name = "crypto-bigint" @@ -569,7 +575,7 @@ dependencies = [ "proc-macro2", "quote", "strsim", - "syn 3.0.4", + "syn 3.0.6", ] [[package]] @@ -591,7 +597,7 @@ checksum = "2ac7135c3ef02b2f7833bbeb1be5ba7f966dcde8a87c6b87f65a778d71a02785" dependencies = [ "darling_core 0.24.1", "quote", - "syn 3.0.4", + "syn 3.0.6", ] [[package]] @@ -622,7 +628,7 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "10d60334b3b2e7c9d91ef8150abfb6fa4c1c39ebbcf4a81c2e346aad939fee3e" dependencies = [ - "thiserror 2.0.20", + "thiserror 2.0.21", ] [[package]] @@ -707,7 +713,7 @@ checksum = "c6232dd377dcc64799954cbd3a9bb882e9cdc1308ccd87b1c098f1fb2eaf82a8" dependencies = [ "proc-macro2", "quote", - "syn 3.0.4", + "syn 3.0.6", ] [[package]] @@ -771,7 +777,7 @@ dependencies = [ "enum-ordinalize", "proc-macro2", "quote", - "syn 3.0.4", + "syn 3.0.6", ] [[package]] @@ -802,11 +808,16 @@ dependencies = [ [[package]] name = "encoding_rs" -version = "0.8.35" +version = "0.8.42" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "75030f3c4f45dafd7586dd6780965a8c7e8e285a5ecb86713e63a79c5b2766f3" +checksum = "8e985e0451871ad22fb8d2b6b076e2028a502a0d3950998c2c5c0a4f9b5d9679" dependencies = [ "cfg-if", + "core_detect", + "multiversion_no_op", + "rustversion", + "scopeguard", + "simdutf8", ] [[package]] @@ -844,7 +855,7 @@ checksum = "a65863d15a4ce2888bd2f0f543cc963d3879c3a022c8ee43f6141d479a3ac815" dependencies = [ "proc-macro2", "quote", - "syn 3.0.4", + "syn 3.0.6", ] [[package]] @@ -901,9 +912,9 @@ dependencies = [ [[package]] name = "find-msvc-tools" -version = "0.1.11" +version = "0.1.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d45db016d36b838f563236e9193d0ee6ce38f3f68b6c94e914b4929c96bbb890" +checksum = "aedcfb3409746eddb02b9e19ebda1c3394f759a152e48ee875a0844d1b955484" [[package]] name = "flagset" @@ -913,12 +924,13 @@ checksum = "b7ac824320a75a52197e8f2d787f6a38b6718bb6897a35142d749af3c0e8f4fe" [[package]] name = "flate2" -version = "1.1.9" +version = "1.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "843fba2746e448b37e26a819579957415c8cef339bf08564fe8b7ddbd959573c" +checksum = "6e634e2e0ebac1ee034020da1ca582e17ffe4e0f5e985823721e168928136dcb" dependencies = [ "crc32fast", "miniz_oxide", + "zlib-rs", ] [[package]] @@ -998,7 +1010,7 @@ checksum = "9fb9654ba8355388abeb8dcb4fc62f511300867002afc858860463bdd9fe0c44" dependencies = [ "proc-macro2", "quote", - "syn 3.0.4", + "syn 3.0.6", ] [[package]] @@ -1090,7 +1102,7 @@ version = "0.21.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ddddbf932745a6be37109b6112d3ee09696106f848449069d3a57bba937ab82e" dependencies = [ - "bitflags 2.13.1", + "bitflags 2.13.2", "libc", "libgit2-sys", "log", @@ -1137,9 +1149,9 @@ dependencies = [ [[package]] name = "h2" -version = "0.4.18" +version = "0.4.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "839c0e8a181239723652be9062bb56ca5bf5f64011f73b623f6f4fc59086a228" +checksum = "ef8e5e5a340588f4452631496976cf8636d4a7ecf600239fdc27615d2530bc16" dependencies = [ "atomic-waker", "bytes", @@ -1250,9 +1262,9 @@ checksum = "15cdd26707701c53297e2fa6afb323d55fbc1d0810c3aec078ae3ef0424c3c15" [[package]] name = "hyper" -version = "1.11.0" +version = "1.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d22053281f852e11534f5198498373cbb59295120a20771d90f7ed1897490a72" +checksum = "27b501faa50e7a26c3d3560ca625132f4078a17771f4810baf70475ae48cbe43" dependencies = [ "atomic-waker", "bytes", @@ -1272,9 +1284,9 @@ dependencies = [ [[package]] name = "hyper-rustls" -version = "0.27.9" +version = "0.27.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "33ca68d021ef39cf6463ab54c1d0f5daf03377b70561305bb89a8f83aab66e0f" +checksum = "dfa8e654703247911e29c23fbeaa261834bd9bb74efba2f9acddc37bfb127f53" dependencies = [ "http", "hyper", @@ -1302,16 +1314,17 @@ dependencies = [ [[package]] name = "hyper-util" -version = "0.1.20" +version = "0.1.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "96547c2556ec9d12fb1578c4eaf448b04993e7fb79cbaad930a656880a6bdfa0" +checksum = "ddc03d96684f9226b8a787cdb71488417b53ab5ea8fdb1dac946cb9431cc8bff" dependencies = [ - "base64 0.22.1", + "base64 0.23.1", "bytes", "futures-channel", "futures-util", "http", "http-body", + "httparse", "hyper", "ipnet", "libc", @@ -1459,9 +1472,9 @@ dependencies = [ [[package]] name = "indexmap" -version = "2.14.0" +version = "2.14.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d466e9454f08e4a911e14806c24e16fba1b4c121d1ea474396f396069cf949d9" +checksum = "cc4e190f5d26ca7051642629da2c52fc03bde85a03197c99408dcd291734c855" dependencies = [ "equivalent", "hashbrown 0.17.1", @@ -1478,9 +1491,9 @@ dependencies = [ [[package]] name = "ipnet" -version = "2.12.1" +version = "2.12.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6a756c3fac73139e83f14c2d742155dd2b78d3ee56597b419a0579b7bdd6dd78" +checksum = "791930b43c0d5973160d90a8f3894509f2b273430f5c5c73b668636d0287c5c0" [[package]] name = "is_terminal_polyfill" @@ -1525,9 +1538,9 @@ dependencies = [ [[package]] name = "jiff" -version = "0.2.35" +version = "0.2.37" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "668b7183bd07af9a4885f5c35b0cc5c83c4607a913c16b7e17291832910d2dcc" +checksum = "0ab1baf72f08796de0260609515130699b890ac25f30e610ad894bc5856cafdb" dependencies = [ "defmt", "jiff-core", @@ -1542,18 +1555,19 @@ dependencies = [ [[package]] name = "jiff-core" -version = "0.1.0" +version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7feca88439efe53da3754500c1851dedf3cb36c524dd5cf8225cc0794de95d09" +checksum = "5e52fe76043ccecc9005d2305ebaadf7d7fc0cc89ca6baa10a94d6bc68c7128c" dependencies = [ "defmt", + "log", ] [[package]] name = "jiff-static" -version = "0.2.35" +version = "0.2.37" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3a69dcb3a21cfb32ce1cd056169337ca284af0766dd766e7878819b251a49204" +checksum = "378268a1116ad67ae6228701118ac9f491d78fda38a40a1f1a9e1348de6f7212" dependencies = [ "jiff-core", "proc-macro2", @@ -1588,7 +1602,7 @@ dependencies = [ "jni-sys", "log", "simd_cesu8", - "thiserror 2.0.20", + "thiserror 2.0.21", "walkdir", "windows-link", ] @@ -1637,9 +1651,9 @@ dependencies = [ [[package]] name = "js-sys" -version = "0.3.104" +version = "0.3.106" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0e0c1080212aad755ea003d18543e8768dd432c48819efd73a7bf1e39b7a5a3a" +checksum = "7883d941dae510fb2d978fc3fe018c71c9e2892fd38854de3e8b92c2e5ad9cc5" dependencies = [ "cfg-if", "futures-util", @@ -1656,20 +1670,20 @@ dependencies = [ "schemars", "serde", "serde_json", - "thiserror 2.0.20", + "thiserror 2.0.21", ] [[package]] name = "jsonpath-rust" -version = "1.0.8" +version = "1.0.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2e07021eefc09f897611b067ba7897b5e9266edfc902768418968772e5bb06a7" +checksum = "d23892d4caa103325a558250e9f4e406d81cf64f7c01679ae7a64e0413321e51" dependencies = [ "pest", "pest_derive", "regex", "serde_json", - "thiserror 2.0.20", + "thiserror 2.0.21", ] [[package]] @@ -1698,7 +1712,7 @@ dependencies = [ [[package]] name = "k8s-version" version = "0.1.3" -source = "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#f3d2da737aa49086cbdaba806aaf2c8645251cfb" +source = "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#3b91d73a3e66639654597d6304780b5ccc75d9c0" dependencies = [ "darling 0.24.1", "regex", @@ -1761,7 +1775,7 @@ dependencies = [ "serde", "serde-saphyr", "serde_json", - "thiserror 2.0.20", + "thiserror 2.0.21", "tokio", "tokio-util", "tower", @@ -1785,7 +1799,7 @@ dependencies = [ "serde", "serde-value", "serde_json", - "thiserror 2.0.20", + "thiserror 2.0.21", ] [[package]] @@ -1823,7 +1837,7 @@ dependencies = [ "pin-project", "serde", "serde_json", - "thiserror 2.0.20", + "thiserror 2.0.21", "tokio", "tokio-util", "tracing", @@ -1831,18 +1845,18 @@ dependencies = [ [[package]] name = "lazy_static" -version = "1.5.0" +version = "1.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" +checksum = "20870f649af7073d53e38067b2a84312175d56ea15217e1b15bc83506ec50afb" dependencies = [ "spin", ] [[package]] name = "libc" -version = "0.2.189" +version = "0.2.190" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3eaf3ede3fee6db1a4c2ee091bf8a8b4dccdc6d17f656fb07896ee72867612f2" +checksum = "ce5d3ddc6d3fa000eb1536d85e147bfe31aacaba692ed6a876f95cb7c855be78" [[package]] name = "libgit2-sys" @@ -1924,9 +1938,9 @@ checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" [[package]] name = "miniz_oxide" -version = "0.8.9" +version = "0.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1fa76a2c86f704bdb222d66965fb3d63269ce38518b83cb0575fca855ebb6316" +checksum = "b63fbc4a50860e98e7b2aa7804ded1db5cbc3aff9193adaff57a6931bf7c4b4c" dependencies = [ "adler2", "simd-adler32", @@ -1934,15 +1948,21 @@ dependencies = [ [[package]] name = "mio" -version = "1.2.2" +version = "1.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "30d65c71f1ce40ab09135ce117d742b9f8a19ff91a41a8b57ed50bc2de59c427" +checksum = "1788edb87fdc09c7e26304471e2f5be8cdefb1b6930d6e3985fc02ff53bf86ee" dependencies = [ "libc", "wasi", "windows-sys 0.61.2", ] +[[package]] +name = "multiversion_no_op" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "743fb55ba31b18fb1ecef6bdc9aa2743314978ac084044301a7eee33fb99a20d" + [[package]] name = "nohash-hasher" version = "0.2.0" @@ -2037,7 +2057,7 @@ dependencies = [ "futures-sink", "js-sys", "pin-project-lite", - "thiserror 2.0.20", + "thiserror 2.0.21", "tracing", ] @@ -2080,7 +2100,7 @@ dependencies = [ "opentelemetry_sdk", "prost", "reqwest", - "thiserror 2.0.20", + "thiserror 2.0.21", "tokio", "tonic", "tonic-types", @@ -2118,7 +2138,7 @@ dependencies = [ "percent-encoding", "portable-atomic", "rand 0.9.5", - "thiserror 2.0.20", + "thiserror 2.0.21", "tokio", "tokio-stream", ] @@ -2200,9 +2220,9 @@ checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" [[package]] name = "pest" -version = "2.9.0" +version = "2.9.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5a07a60cc7a4d00c91f95c685609d1d2f79050e6804b70ebedd7650f0b839bcf" +checksum = "45d3aca230fad2e6f6317ca0a72724338c4960cb97168a85cdee66df4a9a21a8" dependencies = [ "memchr", "ucd-trie", @@ -2210,9 +2230,9 @@ dependencies = [ [[package]] name = "pest_derive" -version = "2.9.0" +version = "2.9.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b3a83744a5c8455b8b3e0dc5031362780a347c878bdd11584d1a8984228cc88d" +checksum = "284b60557f2c4a2e72ad3f2d34d42685a2fa4a6a61d0d2a10c0ae2a5e916c2cf" dependencies = [ "pest", "pest_generator", @@ -2220,9 +2240,9 @@ dependencies = [ [[package]] name = "pest_generator" -version = "2.9.0" +version = "2.9.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e0cd3451aa3de60d4b9a1e736885e4dea6b31617598026f12256ad566d63304a" +checksum = "1d9d1f08a115309ee99268cf85e5228e0e56aa9caf8841ec12866b6be07c3109" dependencies = [ "pest", "pest_meta", @@ -2233,9 +2253,9 @@ dependencies = [ [[package]] name = "pest_meta" -version = "2.9.0" +version = "2.9.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e04d3a0849e241d7dfce834c83b1c5edc8622009e8dd51a12ba1927c32f05496" +checksum = "ed93ba1a9ffcca32130a5188701c81c0c49cf00d4b7c5007d5148951d743adcb" dependencies = [ "pest", ] @@ -2301,9 +2321,9 @@ checksum = "05c8b63e8d9609db387f0324918f81d68fe27748f084ef092fb35954d0539a85" [[package]] name = "portable-atomic-util" -version = "0.2.7" +version = "0.2.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c2a106d1259c23fac8e543272398ae0e3c0b8d33c88ed73d0cc71b0f1d902618" +checksum = "10ab3eb7f3becc3a1cbc4f2c6f20267996cfc1a6467a873763411b136a122715" dependencies = [ "portable-atomic", ] @@ -2319,9 +2339,9 @@ dependencies = [ [[package]] name = "powerfmt" -version = "0.2.0" +version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391" +checksum = "4a6394b9e965e73d0a289ee54f589087e2c676aedf60885baf52c76b771e4958" [[package]] name = "ppv-lite86" @@ -2434,9 +2454,9 @@ dependencies = [ [[package]] name = "rand" -version = "0.10.2" +version = "0.10.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c7f5fa3a058cd35567ef9bfa5e75732bee0f9e4c55fa90477bef2dfcdbc4be80" +checksum = "65c9fb96cbc91e3478eaae79a69fcd3f1ae4ad052e471fe6732fff548984b4af" dependencies = [ "chacha20", "getrandom 0.4.3", @@ -2493,7 +2513,7 @@ version = "0.5.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ed2bf2547551a7053d6fdfafda3f938979645c44812fbfcda098faae3f1a362d" dependencies = [ - "bitflags 2.13.1", + "bitflags 2.13.2", ] [[package]] @@ -2513,7 +2533,7 @@ checksum = "92ecd8964f8453721699a1ed72037b0db49ce2f5a5138486ee89bed6f67cdf3a" dependencies = [ "proc-macro2", "quote", - "syn 3.0.4", + "syn 3.0.6", ] [[package]] @@ -2553,11 +2573,11 @@ checksum = "ba39f3699c378cd8970968dcbff9c43159ea4cfbd88d43c00b22f2ef10a435d2" [[package]] name = "reqwest" -version = "0.13.4" +version = "0.13.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "219c5811de6525e5416c7d5d53bb656d3afdbc6c5af816e0802bcfa42dbdc1c3" +checksum = "16a1cfa75cc186dd73d5818e510e042e40927bccc9c236b061cea97e1eb08029" dependencies = [ - "base64 0.22.1", + "base64 0.23.1", "bytes", "futures-channel", "futures-core", @@ -2703,9 +2723,9 @@ dependencies = [ [[package]] name = "rustls-platform-verifier" -version = "0.7.0" +version = "0.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "26d1e2536ce4f35f4846aa13bff16bd0ff40157cdb14cc056c7b14ba41233ba0" +checksum = "1167586491e2b18b8bfbb293e8180ec17c201c4f076d7cb3070ca964e7598f98" dependencies = [ "core-foundation", "core-foundation-sys", @@ -2724,9 +2744,9 @@ dependencies = [ [[package]] name = "rustls-platform-verifier-android" -version = "0.1.1" +version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f87165f0995f63a9fbeea62b64d10b4d9d8e78ec6d7d51fb2125fda7bb36788f" +checksum = "eec689c0bc40ff2458a5977b6619cb718087084a18e02a131c599b62d05e1a5f" [[package]] name = "rustls-webpki" @@ -2792,7 +2812,7 @@ dependencies = [ "proc-macro2", "quote", "serde_derive_internals", - "syn 3.0.4", + "syn 3.0.6", ] [[package]] @@ -2830,7 +2850,7 @@ version = "3.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b7f4bc775c73d9a02cde8bf7b2ec4c9d12743edf609006c7facc23998404cd1d" dependencies = [ - "bitflags 2.13.1", + "bitflags 2.13.2", "core-foundation", "core-foundation-sys", "libc", @@ -2913,7 +2933,7 @@ checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348" dependencies = [ "proc-macro2", "quote", - "syn 3.0.4", + "syn 3.0.6", ] [[package]] @@ -2924,7 +2944,7 @@ checksum = "f852137cce035d6a4df67ccce505ff6b3e9fd3a10e3e52b24dc71e650bb1a9bd" dependencies = [ "proc-macro2", "quote", - "syn 3.0.4", + "syn 3.0.6", ] [[package]] @@ -3144,14 +3164,14 @@ checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596" [[package]] name = "stackable-certs" version = "0.4.1" -source = "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#f3d2da737aa49086cbdaba806aaf2c8645251cfb" +source = "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#3b91d73a3e66639654597d6304780b5ccc75d9c0" dependencies = [ "const-oid", "ecdsa", "k8s-openapi", "kube", "p256", - "rand 0.10.2", + "rand 0.10.3", "rand_core 0.6.4", "rsa", "sha2", @@ -3202,7 +3222,7 @@ dependencies = [ [[package]] name = "stackable-operator" version = "0.119.0" -source = "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#f3d2da737aa49086cbdaba806aaf2c8645251cfb" +source = "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#3b91d73a3e66639654597d6304780b5ccc75d9c0" dependencies = [ "base64 0.23.1", "clap", @@ -3219,7 +3239,7 @@ dependencies = [ "json-patch", "k8s-openapi", "kube", - "rand 0.10.2", + "rand 0.10.3", "regex", "schemars", "semver", @@ -3246,18 +3266,18 @@ dependencies = [ [[package]] name = "stackable-operator-derive" version = "0.3.1" -source = "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#f3d2da737aa49086cbdaba806aaf2c8645251cfb" +source = "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#3b91d73a3e66639654597d6304780b5ccc75d9c0" dependencies = [ "darling 0.24.1", "proc-macro2", "quote", - "syn 3.0.4", + "syn 3.0.6", ] [[package]] name = "stackable-shared" version = "0.1.2" -source = "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#f3d2da737aa49086cbdaba806aaf2c8645251cfb" +source = "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#3b91d73a3e66639654597d6304780b5ccc75d9c0" dependencies = [ "jiff", "k8s-openapi", @@ -3274,7 +3294,7 @@ dependencies = [ [[package]] name = "stackable-telemetry" version = "0.6.5" -source = "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#f3d2da737aa49086cbdaba806aaf2c8645251cfb" +source = "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#3b91d73a3e66639654597d6304780b5ccc75d9c0" dependencies = [ "axum", "clap", @@ -3298,7 +3318,7 @@ dependencies = [ [[package]] name = "stackable-versioned" version = "0.11.1" -source = "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#f3d2da737aa49086cbdaba806aaf2c8645251cfb" +source = "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#3b91d73a3e66639654597d6304780b5ccc75d9c0" dependencies = [ "kube", "schemars", @@ -3312,7 +3332,7 @@ dependencies = [ [[package]] name = "stackable-versioned-macros" version = "0.11.1" -source = "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#f3d2da737aa49086cbdaba806aaf2c8645251cfb" +source = "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#3b91d73a3e66639654597d6304780b5ccc75d9c0" dependencies = [ "convert_case", "darling 0.24.1", @@ -3323,13 +3343,13 @@ dependencies = [ "kube", "proc-macro2", "quote", - "syn 3.0.4", + "syn 3.0.6", ] [[package]] name = "stackable-webhook" version = "0.10.0" -source = "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#f3d2da737aa49086cbdaba806aaf2c8645251cfb" +source = "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#3b91d73a3e66639654597d6304780b5ccc75d9c0" dependencies = [ "arc-swap", "async-trait", @@ -3342,7 +3362,7 @@ dependencies = [ "kube", "opentelemetry", "opentelemetry-semantic-conventions", - "rand 0.10.2", + "rand 0.10.3", "serde", "serde_json", "snafu 0.9.2", @@ -3421,9 +3441,9 @@ dependencies = [ [[package]] name = "syn" -version = "3.0.4" +version = "3.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e6275cddf4610d1775e6d1fe9469b2e77d0f39fd98fb7450901b821e0c53649f" +checksum = "8593e8e72159ed2257d083c7a454a85cbf854f37a0966d8d483aff8c8a3ebcee" dependencies = [ "proc-macro2", "quote", @@ -3441,13 +3461,13 @@ dependencies = [ [[package]] name = "synstructure" -version = "0.13.2" +version = "0.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "728a70f3dbaf5bab7f0c4b1ac8d7ae5ea60a4b5549c8a5914361c99147a709d2" +checksum = "901704edd0dfe137f1987838ee4f259e4e063c31371bdb423f7ae38ec6f77f02" dependencies = [ "proc-macro2", "quote", - "syn 2.0.119", + "syn 3.0.6", ] [[package]] @@ -3461,11 +3481,11 @@ dependencies = [ [[package]] name = "thiserror" -version = "2.0.20" +version = "2.0.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ec86235f5fcc2a73650310756d2ac5b138a5780bbbdfae3eeccec992c435ba4f" +checksum = "09e52cb86a36cede5cb101bf8908837b3e4c6e5e59fe7fd85c23fb56200d189e" dependencies = [ - "thiserror-impl 2.0.20", + "thiserror-impl 2.0.21", ] [[package]] @@ -3481,13 +3501,13 @@ dependencies = [ [[package]] name = "thiserror-impl" -version = "2.0.20" +version = "2.0.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bc04cd3e1236dd4a98afca4569f2deb3f120e5422a4023be2cb683f8486292af" +checksum = "fe5197923287db20a58125f0bc85c062f7f2c892de97b18c356f9efb14b28524" dependencies = [ "proc-macro2", "quote", - "syn 3.0.4", + "syn 3.0.6", ] [[package]] @@ -3562,9 +3582,9 @@ dependencies = [ [[package]] name = "tokio" -version = "1.53.1" +version = "1.53.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "202caea871b69668250d242070849eb495be178ed697a3e98aebce5bc81a0bed" +checksum = "e95f91fcc7a621e8b030f6aa23c71fe9838ae2fb4d8118b75602a328f5144044" dependencies = [ "bytes", "libc", @@ -3585,14 +3605,14 @@ checksum = "78773a2a397f451582ce068015985c33193cf6dea8b74d2a639fe457b2f07b0e" dependencies = [ "proc-macro2", "quote", - "syn 3.0.4", + "syn 3.0.6", ] [[package]] name = "tokio-rustls" -version = "0.26.4" +version = "0.26.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1729aa945f29d91ba541258c8df89027d5792d85a8841fb65e8bf0f4ede4ef61" +checksum = "c9cc2678c2cdd569ef8215e2afd7954ada2ae20b4fdd2c5fe6139a3b02d105db" dependencies = [ "rustls", "tokio", @@ -3635,9 +3655,9 @@ dependencies = [ [[package]] name = "toml_edit" -version = "0.25.13+spec-1.1.0" +version = "0.25.15+spec-1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6975367e4d2ef766d86af01ffad14b622fecc8d4357a998fbc4deb6e9bacaf9b" +checksum = "1340ea94a5856333492c9064b02c778b191dd2c853778d9609debdcdfea3a614" dependencies = [ "indexmap", "toml_datetime", @@ -3729,7 +3749,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4cfcf7e2740e6fc6d4d688b4ef00650406bb94adf4731e43c096c3a19fe40840" dependencies = [ "base64 0.22.1", - "bitflags 2.13.1", + "bitflags 2.13.2", "bytes", "futures-util", "http", @@ -3749,7 +3769,7 @@ version = "0.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "08a05a66a4fdd61cbbe0a1d755ffe0ca6aba159dd4820936a0ff8a8278245b9c" dependencies = [ - "bitflags 2.13.1", + "bitflags 2.13.2", "bytes", "http", "http-body", @@ -3792,7 +3812,7 @@ checksum = "050686193eb999b4bb3bc2acfa891a13da00f79734704c4b8b4ef1a10b368a3c" dependencies = [ "crossbeam-channel", "symlink", - "thiserror 2.0.20", + "thiserror 2.0.21", "time", "tracing-subscriber", ] @@ -3896,9 +3916,9 @@ checksum = "2896d95c02a80c6d6a5d6e953d479f5ddf2dfdb6a244441010e373ac0fb88971" [[package]] name = "unicode-ident" -version = "1.0.24" +version = "1.0.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" +checksum = "d245f478577f809a851594d02313b640fb437e0bb33866753cff937863096954" [[package]] name = "unicode-segmentation" @@ -3957,9 +3977,9 @@ checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" [[package]] name = "uuid" -version = "1.25.0" +version = "1.27.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f053576934f05a761a402421fbbe3d425d9366f75f978806a037b3ca481abecc" +checksum = "97277d36b9c3ace13e58fa6e753f8b0bbbf302a18dd193240d70f9e29681059a" dependencies = [ "js-sys", "wasm-bindgen", @@ -3995,9 +4015,9 @@ dependencies = [ [[package]] name = "want" -version = "0.3.1" +version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bfa7760aed19e106de2c7c0b581b509f2f25d3dacaf737cb82ac61bc6d760b0e" +checksum = "ec4cdd0dd910afe868b7ef477227d8d538b46b3075031afee8a9f2acb0a2ed0b" dependencies = [ "try-lock", ] @@ -4019,9 +4039,9 @@ dependencies = [ [[package]] name = "wasm-bindgen" -version = "0.2.127" +version = "0.2.129" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1b70935747edd64d89de3efa29d73789b806c15798f8e7dca4d8ac356b50ce70" +checksum = "9bb54f33acc68fd454578d9820b0bde1a1a3d17aa17bb7b6595806d02886d409" dependencies = [ "cfg-if", "once_cell", @@ -4032,19 +4052,20 @@ dependencies = [ [[package]] name = "wasm-bindgen-futures" -version = "0.4.77" +version = "0.4.79" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6b7777d5cc23d0e91404e53ce2d5e8ec7acae3026b16233dba62cd3246457950" +checksum = "3cbab34de2d982e9b48e18d216d04c4a6f641066ff19ffb699980f591ee3610e" dependencies = [ "js-sys", + "tokio", "wasm-bindgen", ] [[package]] name = "wasm-bindgen-macro" -version = "0.2.127" +version = "0.2.129" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "77775f8f3f7217702089053b94958f8f54061a3f663417df76e19cbdcca29bc1" +checksum = "2e29d0c35b16e224a7eeb5cd2d25e3e1968fbd65604117b44d3b789d00ee8535" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -4052,31 +4073,31 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.127" +version = "0.2.129" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e11d33f857dc2fb11b8bc75aee111aa9cbeb12cd9f25efd3d4c2a3dd4e235284" +checksum = "6f501a8bc3719dba86ef8ae4728879c08001bea749eb1333ac5b91e040e2a6b7" dependencies = [ "bumpalo", "proc-macro2", "quote", - "syn 2.0.119", + "syn 3.0.6", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-shared" -version = "0.2.127" +version = "0.2.129" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7ef64dbcc55df09c7e5a46182d181c2cfa3e925f3da937ea764728b4bbb9dcbf" +checksum = "23f0c9c52aa7cd7d77769a4cfe2a9adb1b331f489a41d912ce14513d5ab995c6" dependencies = [ "unicode-ident", ] [[package]] name = "web-sys" -version = "0.3.104" +version = "0.3.106" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c435338968042f4f59a557f690a253676d47ce13ceb55d70100e7facf6620a30" +checksum = "88261b9deccee56594c11a3460c462c41f58d148598fe70ad77070126a68aba4" dependencies = [ "js-sys", "wasm-bindgen", @@ -4305,30 +4326,30 @@ dependencies = [ [[package]] name = "yoke-derive" -version = "0.8.2" +version = "0.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "de844c262c8848816172cef550288e7dc6c7b7814b4ee56b3e1553f275f1858e" +checksum = "ec8ebde2db3681e8c9980cc27822030e68752690ddfa9473e739aeb4dbde6d71" dependencies = [ "proc-macro2", "quote", - "syn 2.0.119", + "syn 3.0.6", "synstructure", ] [[package]] name = "zerocopy" -version = "0.8.56" +version = "0.8.60" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "556764e583adb45a9f8d413c2a147fa7e8d821e48e12b14fd560b607998b75eb" +checksum = "6400fc4bb7426f6faac3fb07566122bb18622e3022bd2c36426a92ed489b1dbe" dependencies = [ "zerocopy-derive", ] [[package]] name = "zerocopy-derive" -version = "0.8.56" +version = "0.8.60" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f2ab42fc20575779bd240faa45f94a74256f755c0fa9e89f0ede20d91d0cdfc1" +checksum = "99c27cc7525bad4df59a45e073c45b2c2aa820efb588234cc23a547ec69a34db" dependencies = [ "proc-macro2", "quote", @@ -4346,13 +4367,13 @@ dependencies = [ [[package]] name = "zerofrom-derive" -version = "0.1.7" +version = "0.1.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "11532158c46691caf0f2593ea8358fed6bbf68a0315e80aae9bd41fbade684a1" +checksum = "f75b4683f6c7f45248d4d64056a24298c6281e0993356d7d1b4a1a962ef10d4a" dependencies = [ "proc-macro2", "quote", - "syn 2.0.119", + "syn 3.0.6", "synstructure", ] @@ -4406,9 +4427,15 @@ checksum = "34df6fc39dbd26ddc9c10e6a2984476e13acce22e64e4487636ef494369225da" dependencies = [ "proc-macro2", "quote", - "syn 3.0.4", + "syn 3.0.6", ] +[[package]] +name = "zlib-rs" +version = "0.6.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b268e58e7c693d7c271f93ffc4ba3b380412554231c85bf61ca7af91042a4112" + [[package]] name = "zmij" version = "1.0.23" diff --git a/Cargo.nix b/Cargo.nix index b76c6a054..17baabbeb 100644 --- a/Cargo.nix +++ b/Cargo.nix @@ -506,7 +506,7 @@ rec { } { name = "syn"; - packageId = "syn 3.0.4"; + packageId = "syn 3.0.6"; usesDefaultFeatures = false; features = [ "clone-impls" "full" "parsing" "printing" "proc-macro" "visit-mut" ]; } @@ -889,11 +889,11 @@ rec { }; resolvedDefaultFeatures = [ "default" ]; }; - "bitflags 2.13.1" = rec { + "bitflags 2.13.2" = rec { crateName = "bitflags"; - version = "2.13.1"; + version = "2.13.2"; edition = "2021"; - sha256 = "1nl76mpykmwmb8rq1l5vw1azdh1wvxdrnsk4sy3rdrzx01nvg25m"; + sha256 = "01hbgjwvid66850fzi76mvn5f2bqycx6sf165ng1kfjqq9bl1v9x"; authors = [ "The Rust Project Developers" ]; @@ -987,9 +987,9 @@ rec { }; "cc" = rec { crateName = "cc"; - version = "1.4.4"; + version = "1.6.0"; edition = "2021"; - sha256 = "0wq26vvhzv5ci9gx3cfiw320skvaysf9i701wp668lks6ps39m8a"; + sha256 = "0c3n82hdi355xa6z9x4zgnpjwsh1szkkcs0zl8q8nl5ggk874j7p"; dependencies = [ { name = "find-msvc-tools"; @@ -1020,9 +1020,9 @@ rec { }; "cfg-if" = rec { crateName = "cfg-if"; - version = "1.0.4"; + version = "1.0.5"; edition = "2018"; - sha256 = "008q28ajc546z5p2hcwdnckmg0hia7rnx52fni04bwqkzyrghc4k"; + sha256 = "0026j56901nzjraap3da0a8njw42j66zcxnn6s2s9aa5bcblhxjf"; libName = "cfg_if"; authors = [ "Alex Crichton " @@ -1047,7 +1047,7 @@ rec { } { name = "cpufeatures"; - packageId = "cpufeatures 0.3.0"; + packageId = "cpufeatures 0.3.1"; target = { target, features }: (("x86_64" == target."arch" or null) || ("x86" == target."arch" or null)); } { @@ -1118,10 +1118,10 @@ rec { }; "clap" = rec { crateName = "clap"; - version = "4.6.6"; + version = "4.6.7"; edition = "2024"; crateBin = []; - sha256 = "1jmx5z8d6jbvxdz6dybh599s4rd7ns6sl90p2rrdga09yh3pwg27"; + sha256 = "0il98y2rfw75984ck59znd4n592p07bxz8yy3a9blddb02rpd25a"; dependencies = [ { name = "clap_builder"; @@ -1160,9 +1160,9 @@ rec { }; "clap_builder" = rec { crateName = "clap_builder"; - version = "4.6.6"; + version = "4.6.7"; edition = "2024"; - sha256 = "12cqg25zpjc3k82cpqa2v9h7s3vk1vydpgnwl8lfg6lfm2jzwj3v"; + sha256 = "0kbhai5rv1vj9r4np52g2b9fmhvy3y82digs9v40c57bgbxrf1zc"; dependencies = [ { name = "anstream"; @@ -1199,9 +1199,9 @@ rec { }; "clap_derive" = rec { crateName = "clap_derive"; - version = "4.6.4"; + version = "4.6.7"; edition = "2024"; - sha256 = "0qd0v7pa2arwxjjinmjim6xrjy61bc28m1yryhc7zjjssswx44nh"; + sha256 = "1q36bgbkfb9rdz7jcvd0ffv0wwlyw0lg3z6iwdcybm0mjjvm3izr"; procMacro = true; dependencies = [ { @@ -1218,7 +1218,7 @@ rec { } { name = "syn"; - packageId = "syn 3.0.4"; + packageId = "syn 3.0.6"; features = [ "full" ]; } ]; @@ -1231,9 +1231,9 @@ rec { }; "clap_lex" = rec { crateName = "clap_lex"; - version = "1.1.0"; + version = "1.1.1"; edition = "2024"; - sha256 = "1ycqkpygnlqnndghhcxjb44lzl0nmgsia64x9581030yifxs7m68"; + sha256 = "11k4m4ngn39iz39lnq6nn6kyxzp62mnm1d86qfad3q0vlk33n4qw"; }; "colorchoice" = rec { @@ -1245,9 +1245,9 @@ rec { }; "combine" = rec { crateName = "combine"; - version = "4.6.7"; + version = "4.6.8"; edition = "2018"; - sha256 = "1z8rh8wp59gf8k23ar010phgs0wgf5i8cx4fg01gwcnzfn5k0nms"; + sha256 = "0ppwzwdmszpan9ybx1myc6ldg5zih2sazf9idckdxrh9gn9j1hyg"; authors = [ "Markus Westerlind " ]; @@ -1433,6 +1433,18 @@ rec { }; resolvedDefaultFeatures = [ "default" "link" ]; }; + "core_detect" = rec { + crateName = "core_detect"; + version = "1.0.0"; + edition = "2018"; + sha256 = "0j0xp1j6f9lrp4ybylc3d0zdg8jq2hkmr1122qyil14qk84q13vz"; + authors = [ + "Thom Chiovoloni " + ]; + features = { + }; + resolvedDefaultFeatures = [ "default" ]; + }; "cpufeatures 0.2.17" = rec { crateName = "cpufeatures"; version = "0.2.17"; @@ -1469,11 +1481,11 @@ rec { ]; }; - "cpufeatures 0.3.0" = rec { + "cpufeatures 0.3.1" = rec { crateName = "cpufeatures"; - version = "0.3.0"; + version = "0.3.1"; edition = "2024"; - sha256 = "00fjhygsqmh4kbxxlb99mcsbspxcai6hjydv4c46pwb67wwl2alb"; + sha256 = "0rkm1l35jy1z1yfg723fddsxc7vr0gc1hhfvc138hnqiwc58p8jw"; authors = [ "RustCrypto Developers" ]; @@ -1507,9 +1519,9 @@ rec { }; "crc32fast" = rec { crateName = "crc32fast"; - version = "1.5.1"; + version = "1.5.2"; edition = "2021"; - sha256 = "0l75bfakpwr86wz45gm38lylrpgbssr529fmm6m445qy2rqwi644"; + sha256 = "0y0f955n2hr5a8rd9nw9sr23nhjc42ddx3bjc47dnlmqssgpk9q1"; authors = [ "Sam Rijs " "Alex Crichton " @@ -1523,13 +1535,13 @@ rec { features = { "default" = [ "std" ]; }; - resolvedDefaultFeatures = [ "default" "std" ]; + resolvedDefaultFeatures = [ "std" ]; }; "crossbeam-channel" = rec { crateName = "crossbeam-channel"; - version = "0.5.16"; + version = "0.5.17"; edition = "2021"; - sha256 = "17k72dh5qqkh0xvqzr27wny7gl1l7fgzlvh2xxx71jmfgz1n6lyq"; + sha256 = "1wcy7y77hw7z5m140114iqxb90gsq062djd9zpkndisvgcrcrc4q"; libName = "crossbeam_channel"; dependencies = [ { @@ -1546,9 +1558,9 @@ rec { }; "crossbeam-utils" = rec { crateName = "crossbeam-utils"; - version = "0.8.22"; + version = "0.8.23"; edition = "2021"; - sha256 = "05vwf7pmjq8c8f3fp5qqdm0z3cnk4p62wi8spf0jms5yjnh3v031"; + sha256 = "1ilan2nw7fvka8hki80fr57a5dgd4mvcsvwq60437j6yvlwyw7m3"; libName = "crossbeam_utils"; features = { "default" = [ "std" ]; @@ -1753,7 +1765,7 @@ rec { } { name = "syn"; - packageId = "syn 3.0.4"; + packageId = "syn 3.0.6"; features = [ "full" "extra-traits" ]; } ]; @@ -1809,7 +1821,7 @@ rec { } { name = "syn"; - packageId = "syn 3.0.4"; + packageId = "syn 3.0.6"; } ]; @@ -1882,7 +1894,7 @@ rec { dependencies = [ { name = "thiserror"; - packageId = "thiserror 2.0.20"; + packageId = "thiserror 2.0.21"; } ]; features = { @@ -2177,7 +2189,7 @@ rec { } { name = "syn"; - packageId = "syn 3.0.4"; + packageId = "syn 3.0.6"; } ]; features = { @@ -2385,7 +2397,7 @@ rec { } { name = "syn"; - packageId = "syn 3.0.4"; + packageId = "syn 3.0.6"; features = [ "visit" "visit-mut" ]; } ]; @@ -2507,9 +2519,9 @@ rec { }; "encoding_rs" = rec { crateName = "encoding_rs"; - version = "0.8.35"; - edition = "2018"; - sha256 = "1wv64xdrr9v37rqqdjsyb8l8wzlcbab80ryxhrszvnj59wy0y0vm"; + version = "0.8.42"; + edition = "2024"; + sha256 = "0ycnbndly2jw5j69jl1r1lm512h2w9vb1dnjp0px46l7a425x64f"; authors = [ "Henri Sivonen " ]; @@ -2518,13 +2530,47 @@ rec { name = "cfg-if"; packageId = "cfg-if"; } + { + name = "core_detect"; + packageId = "core_detect"; + target = { target, features }: (("x86_64" == target."arch" or null) || ("x86" == target."arch" or null)); + } + { + name = "multiversion_no_op"; + packageId = "multiversion_no_op"; + } + { + name = "rustversion"; + packageId = "rustversion"; + optional = true; + } + { + name = "scopeguard"; + packageId = "scopeguard"; + usesDefaultFeatures = false; + } + { + name = "simdutf8"; + packageId = "simdutf8"; + usesDefaultFeatures = false; + target = { target, features }: (("x86_64" == target."arch" or null) || ("x86" == target."arch" or null) || ("aarch64" == target."arch" or null) || (("wasm32" == target."arch" or null) && (builtins.elem "simd128" targetFeatures))); + features = [ "aarch64_neon" "public_imp" ]; + } + ]; + buildDependencies = [ + { + name = "rustversion"; + packageId = "rustversion"; + } ]; features = { "any_all_workaround" = [ "dep:any_all_workaround" ]; "default" = [ "alloc" ]; "fast-legacy-encode" = [ "fast-hangul-encode" "fast-hanja-encode" "fast-kanji-encode" "fast-gb-hanzi-encode" "fast-big5-hanzi-encode" ]; + "rustversion" = [ "dep:rustversion" ]; "serde" = [ "dep:serde" ]; - "simd-accel" = [ "any_all_workaround" ]; + "simd-accel" = [ "any_all_workaround" "rustversion" ]; + "std" = [ "dep:multiversion" ]; }; resolvedDefaultFeatures = [ "alloc" "default" ]; }; @@ -2599,7 +2645,7 @@ rec { } { name = "syn"; - packageId = "syn 3.0.4"; + packageId = "syn 3.0.6"; } ]; features = { @@ -2767,9 +2813,9 @@ rec { }; "find-msvc-tools" = rec { crateName = "find-msvc-tools"; - version = "0.1.11"; + version = "0.1.14"; edition = "2021"; - sha256 = "145qpfb9r4ml2klr8v4byvrkikp61qyiks9n69b8z0vbscbb0pfl"; + sha256 = "112ljldlv150fpl8xr2jl5czg51k3kdfn6cy5fqdsvkl14sgpp5f"; libName = "find_msvc_tools"; }; @@ -2787,9 +2833,9 @@ rec { }; "flate2" = rec { crateName = "flate2"; - version = "1.1.9"; + version = "1.1.10"; edition = "2018"; - sha256 = "0g2pb7cxnzcbzrj8bw4v6gpqqp21aycmf6d84rzb6j748qkvlgw4"; + sha256 = "1jvd2cl8j5hyf8imi62y1x7gwzz1hajirni0801yxhds1qp4wqvf"; authors = [ "Alex Crichton " "Josh Triplett " @@ -2799,33 +2845,33 @@ rec { name = "crc32fast"; packageId = "crc32fast"; optional = true; + usesDefaultFeatures = false; } { name = "miniz_oxide"; packageId = "miniz_oxide"; optional = true; - usesDefaultFeatures = false; - features = [ "with-alloc" "simd" ]; + features = [ "simd" ]; } { - name = "miniz_oxide"; - packageId = "miniz_oxide"; + name = "zlib-rs"; + packageId = "zlib-rs"; + optional = true; usesDefaultFeatures = false; - target = { target, features }: (("wasm32" == target."arch" or null) && (!("emscripten" == target."os" or null))); - features = [ "with-alloc" "simd" ]; + features = [ "rust-allocator" ]; } ]; features = { "any_c_zlib" = [ "any_zlib" ]; "any_zlib" = [ "any_impl" ]; - "cloudflare-zlib-sys" = [ "dep:cloudflare-zlib-sys" ]; - "cloudflare_zlib" = [ "any_c_zlib" "cloudflare-zlib-sys" "dep:crc32fast" ]; - "default" = [ "rust_backend" ]; + "cloudflare_zlib" = [ "zlib" ]; + "default" = [ "rust_backend" "runtime_detection" ]; "document-features" = [ "dep:document-features" ]; "libz-ng-sys" = [ "dep:libz-ng-sys" ]; "libz-sys" = [ "dep:libz-sys" ]; "miniz-sys" = [ "rust_backend" ]; "miniz_oxide" = [ "any_impl" "dep:miniz_oxide" "dep:crc32fast" ]; + "runtime_detection" = [ "zlib-rs?/std" "crc32fast?/std" ]; "rust_backend" = [ "miniz_oxide" "any_impl" ]; "zlib" = [ "any_c_zlib" "libz-sys" "dep:crc32fast" ]; "zlib-default" = [ "any_c_zlib" "libz-sys/default" "dep:crc32fast" ]; @@ -2833,7 +2879,7 @@ rec { "zlib-ng-compat" = [ "zlib" "libz-sys/zlib-ng" "dep:crc32fast" ]; "zlib-rs" = [ "any_zlib" "dep:zlib-rs" ]; }; - resolvedDefaultFeatures = [ "any_impl" "default" "miniz_oxide" "rust_backend" ]; + resolvedDefaultFeatures = [ "any_impl" "default" "miniz_oxide" "runtime_detection" "rust_backend" ]; }; "fnv" = rec { crateName = "fnv"; @@ -3045,7 +3091,7 @@ rec { } { name = "syn"; - packageId = "syn 3.0.4"; + packageId = "syn 3.0.6"; features = [ "full" ]; } ]; @@ -3432,7 +3478,7 @@ rec { dependencies = [ { name = "bitflags"; - packageId = "bitflags 2.13.1"; + packageId = "bitflags 2.13.2"; } { name = "libc"; @@ -3570,9 +3616,9 @@ rec { }; "h2" = rec { crateName = "h2"; - version = "0.4.18"; + version = "0.4.19"; edition = "2021"; - sha256 = "0a52hs8cakvg7xi3pxqi83vganyaasxn545ya8v74f8j3250x743"; + sha256 = "05mw60jmsq97vjgj607nxjkx8dl6rxv6jj9i4r2z92056id5x3pg"; authors = [ "Carl Lerche " "Sean McArthur " @@ -3881,9 +3927,9 @@ rec { }; "hyper" = rec { crateName = "hyper"; - version = "1.11.0"; + version = "1.11.1"; edition = "2021"; - sha256 = "0wha96biivgpj0fpf80a2aar5dfbff1lk62i9x9i2bl53wl5686j"; + sha256 = "0hxyikj5livhmw5q3x3ifyhphh1g2cjsc32nsg1jcyhflpx03d97"; authors = [ "Sean McArthur " ]; @@ -3986,9 +4032,9 @@ rec { }; "hyper-rustls" = rec { crateName = "hyper-rustls"; - version = "0.27.9"; + version = "0.27.10"; edition = "2021"; - sha256 = "03vfnsm873wsp1dk0q85nxvk7w6syp8c2m5bcdjcyfgg4786ijik"; + sha256 = "0lvz2bxpphyxmkws5ysfnydvsd0q4smbwgy254g92irjf1afda6z"; libName = "hyper_rustls"; dependencies = [ { @@ -4125,9 +4171,9 @@ rec { }; "hyper-util" = rec { crateName = "hyper-util"; - version = "0.1.20"; - edition = "2021"; - sha256 = "186zdc58hmm663csmjvrzgkr6jdh93sfmi3q2pxi57gcaqjpqm4n"; + version = "0.1.21"; + edition = "2024"; + sha256 = "1zwbrhqr9js6r7db3zd8bsmm6ys1i0abgkc7lyw2d4jgd2b3vh6x"; libName = "hyper_util"; authors = [ "Sean McArthur " @@ -4135,7 +4181,7 @@ rec { dependencies = [ { name = "base64"; - packageId = "base64 0.22.1"; + packageId = "base64 0.23.1"; optional = true; } { @@ -4161,6 +4207,11 @@ rec { name = "http-body"; packageId = "http-body"; } + { + name = "httparse"; + packageId = "httparse"; + optional = true; + } { name = "hyper"; packageId = "hyper"; @@ -4228,18 +4279,19 @@ rec { { name = "tokio"; packageId = "tokio"; - features = [ "macros" "test-util" "signal" ]; + features = [ "macros" "test-util" "signal" "net" "io-util" ]; } ]; features = { - "client" = [ "hyper/client" "tokio/net" "dep:tracing" "dep:futures-channel" "dep:tower-service" ]; - "client-legacy" = [ "client" "dep:socket2" "tokio/sync" "dep:libc" "dep:futures-util" ]; - "client-pool" = [ "client" "dep:futures-util" "dep:tower-layer" ]; + "client" = [ "hyper/client" "dep:tracing" "dep:futures-channel" "dep:tower-service" ]; + "client-legacy" = [ "client" "tokio/net" "dep:socket2" "tokio/sync" "dep:libc" "dep:futures-util" "dep:httparse" ]; + "client-pool" = [ "client" "dep:futures-util" "dep:tower-layer" "tokio/sync" ]; "client-proxy" = [ "client" "dep:base64" "dep:ipnet" "dep:percent-encoding" ]; "client-proxy-system" = [ "dep:system-configuration" "dep:windows-registry" ]; "full" = [ "client" "client-legacy" "client-pool" "client-proxy" "client-proxy-system" "server" "server-auto" "server-graceful" "service" "http1" "http2" "tokio" "tracing" ]; "http1" = [ "hyper/http1" ]; "http2" = [ "hyper/http2" ]; + "rt-tracing-exec-force" = [ "tokio" "tracing" ]; "server" = [ "hyper/server" ]; "server-auto" = [ "server" "http1" "http2" ]; "server-graceful" = [ "server" "tokio/sync" ]; @@ -4685,9 +4737,9 @@ rec { }; "indexmap" = rec { crateName = "indexmap"; - version = "2.14.0"; + version = "2.14.2"; edition = "2024"; - sha256 = "1na9z6f0d5pkjr1lgsni470v98gv2r7c41j8w48skr089x2yjrnl"; + sha256 = "0mf86hbjkkcd82cpq683bblbs0zwa8ndla96ci8p1ji6bl7ijknc"; dependencies = [ { name = "equivalent"; @@ -4736,9 +4788,9 @@ rec { }; "ipnet" = rec { crateName = "ipnet"; - version = "2.12.1"; + version = "2.12.2"; edition = "2018"; - sha256 = "0y6xssyvfy85k90pnnanxv9phayxalhp8bacy61rw4vkmhznqxba"; + sha256 = "1h65hw16sqv8nrrmqp0g8drv5wh98n4z7a4h1lb76n8d7js306br"; authors = [ "Kris Price " ]; @@ -4844,9 +4896,9 @@ rec { }; "jiff" = rec { crateName = "jiff"; - version = "0.2.35"; + version = "0.2.37"; edition = "2021"; - sha256 = "1k1d1n8k46192xz6ph8km43lcg68ql65phzmhm49mbq7pn1p32v6"; + sha256 = "1nxgdj2wajw9ml8fcc2zq858k6v9618m22864vh6sy885zvvmc8a"; authors = [ "Andrew Gallant " ]; @@ -4917,6 +4969,7 @@ rec { ]; features = { "alloc" = [ "jcore/alloc" "serde_core?/alloc" "portable-atomic-util/alloc" "defmt?/alloc" ]; + "arbitrary" = [ "dep:arbitrary" "jcore/arbitrary" "std" ]; "default" = [ "std" "tz-system" "tz-fat" "tzdb-bundle-platform" "tzdb-zoneinfo" "tzdb-concatenated" "perf-inline" ]; "defmt" = [ "dep:defmt" "jcore/defmt" ]; "js" = [ "dep:wasm-bindgen" "dep:js-sys" ]; @@ -4936,9 +4989,9 @@ rec { }; "jiff-core" = rec { crateName = "jiff-core"; - version = "0.1.0"; + version = "0.1.1"; edition = "2021"; - sha256 = "02axx56pkh2w4bw5rp94qlvcpwzd3n2w2025fnikvrgg762aiv3z"; + sha256 = "130jqxlbrmll1ahvm9lwr06grmzpmnx5wc6j0n8crkiw0ivgwljy"; libName = "jiff_core"; authors = [ "Andrew Gallant " @@ -4949,21 +5002,28 @@ rec { packageId = "defmt"; optional = true; } + { + name = "log"; + packageId = "log"; + optional = true; + usesDefaultFeatures = false; + } ]; features = { "alloc" = [ "defmt?/alloc" ]; + "arbitrary" = [ "dep:arbitrary" "std" ]; "default" = [ "std" "tz-fat" ]; "defmt" = [ "dep:defmt" ]; "logging" = [ "dep:log" ]; - "std" = [ "alloc" ]; + "std" = [ "alloc" "log?/std" ]; }; resolvedDefaultFeatures = [ "alloc" "default" "std" "tz-fat" ]; }; "jiff-static" = rec { crateName = "jiff-static"; - version = "0.2.35"; + version = "0.2.37"; edition = "2021"; - sha256 = "014jli8v46c8hzkndmvdfvq4la6a6y9icmnh3k735yqwlarxqs9s"; + sha256 = "04kjdzg4h4wy38ghm91qva7xg4glr65120c74bk7mmka26hni0ip"; procMacro = true; libName = "jiff_static"; authors = [ @@ -5058,7 +5118,7 @@ rec { } { name = "thiserror"; - packageId = "thiserror 2.0.20"; + packageId = "thiserror 2.0.21"; } { name = "windows-link"; @@ -5176,9 +5236,9 @@ rec { }; "js-sys" = rec { crateName = "js-sys"; - version = "0.3.104"; + version = "0.3.106"; edition = "2021"; - sha256 = "0fjsgady7wbv7bbyy6c8qhrd93bnx11qbl83l1g7bb9a4601030f"; + sha256 = "1icwmpjw54lb7vg5926k5y4y5jbiih0zxhwgjwnzn475v90xk0vq"; libName = "js_sys"; authors = [ "The wasm-bindgen Developers" @@ -5193,7 +5253,6 @@ rec { packageId = "futures-util"; optional = true; usesDefaultFeatures = false; - features = [ "std" ]; } { name = "wasm-bindgen"; @@ -5204,7 +5263,7 @@ rec { features = { "default" = [ "std" "unsafe-eval" ]; "futures-core-03-stream" = [ "dep:futures-util" "dep:futures-core" ]; - "std" = [ "wasm-bindgen/std" "dep:futures-util" ]; + "std" = [ "wasm-bindgen/std" "futures-util/std" ]; }; resolvedDefaultFeatures = [ "default" "std" "unsafe-eval" ]; }; @@ -5238,7 +5297,7 @@ rec { } { name = "thiserror"; - packageId = "thiserror 2.0.20"; + packageId = "thiserror 2.0.21"; } ]; devDependencies = [ @@ -5261,9 +5320,9 @@ rec { }; "jsonpath-rust" = rec { crateName = "jsonpath-rust"; - version = "1.0.8"; + version = "1.0.11"; edition = "2021"; - sha256 = "19q6pgjp51wn3227c0n9vxp2dsdmjxwblrxh25v8k7y0xwg041rf"; + sha256 = "0l8y689h8km6wyd6f0bw9zv1rn06wksfjl42amd340x1rba94f6j"; libName = "jsonpath_rust"; authors = [ "BorisZhguchev " @@ -5287,7 +5346,7 @@ rec { } { name = "thiserror"; - packageId = "thiserror 2.0.20"; + packageId = "thiserror 2.0.21"; } ]; @@ -5385,7 +5444,7 @@ rec { workspace_member = null; src = pkgs.fetchgit { url = "https://github.com/stackabletech/operator-rs.git"; - rev = "f3d2da737aa49086cbdaba806aaf2c8645251cfb"; + rev = "3b91d73a3e66639654597d6304780b5ccc75d9c0"; sha256 = "0cgziqra8097hp05ynib1qpw4c95n972f4w2rk9l3llyp8r1vmci"; }; libName = "k8s_version"; @@ -5664,7 +5723,7 @@ rec { } { name = "thiserror"; - packageId = "thiserror 2.0.20"; + packageId = "thiserror 2.0.21"; } { name = "tokio"; @@ -5830,7 +5889,7 @@ rec { } { name = "thiserror"; - packageId = "thiserror 2.0.20"; + packageId = "thiserror 2.0.21"; } ]; devDependencies = [ @@ -5982,7 +6041,7 @@ rec { } { name = "thiserror"; - packageId = "thiserror 2.0.20"; + packageId = "thiserror 2.0.21"; } { name = "tokio"; @@ -6022,9 +6081,9 @@ rec { }; "lazy_static" = rec { crateName = "lazy_static"; - version = "1.5.0"; + version = "1.5.1"; edition = "2015"; - sha256 = "1zk6dqqni0193xg6iijh7i3i44sryglwgvx20spdvwk3r6sbrlmv"; + sha256 = "1yqaqmp510xw2ldpw88mx9b5s5qj8flb4rw0wd9ks1zpk9j0z1r0"; authors = [ "Marvin Löbel " ]; @@ -6045,9 +6104,9 @@ rec { }; "libc" = rec { crateName = "libc"; - version = "0.2.189"; + version = "0.2.190"; edition = "2021"; - sha256 = "1whjfs375vlng2q6yrbzs73cvp5lm3w1n2gfqajb2vgf7zg3xbry"; + sha256 = "0y5yap4bfp7rfsldcbk9pb5alcgygca5xn1n2pmh181zdpf3spff"; features = { "default" = [ "std" ]; "rustc-dep-of-std" = [ "align" "rustc-std-workspace-core" ]; @@ -6285,9 +6344,9 @@ rec { }; "miniz_oxide" = rec { crateName = "miniz_oxide"; - version = "0.8.9"; + version = "0.9.1"; edition = "2021"; - sha256 = "05k3pdg8bjjzayq3rf0qhpirq9k37pxnasfn4arbs17phqn6m9qz"; + sha256 = "0k2bgjzk2sbsynpsv4wizwxbqp6vs7g08y5anbkrh3l6a15bqgxn"; authors = [ "Frommi " "oyvindln " @@ -6314,14 +6373,15 @@ rec { "serde" = [ "dep:serde" ]; "simd" = [ "simd-adler32" ]; "simd-adler32" = [ "dep:simd-adler32" ]; + "std" = [ "serde?/std" ]; }; - resolvedDefaultFeatures = [ "simd" "simd-adler32" "with-alloc" ]; + resolvedDefaultFeatures = [ "default" "simd" "simd-adler32" "with-alloc" ]; }; "mio" = rec { crateName = "mio"; - version = "1.2.2"; + version = "1.2.4"; edition = "2021"; - sha256 = "09y4b7gc42ymgssshh8sz6gs3y5r8bbigqaw2c4snh6fy5qmrmih"; + sha256 = "1vl6px9zy0pwhlwnw3cknsqyzkg8bcpiwiq4cgicf2fwgywfv20p"; authors = [ "Carl Lerche " "Thomas de Zeeuw " @@ -6352,6 +6412,17 @@ rec { }; resolvedDefaultFeatures = [ "net" "os-ext" "os-poll" ]; }; + "multiversion_no_op" = rec { + crateName = "multiversion_no_op"; + version = "1.0.0"; + edition = "2021"; + sha256 = "03d2k7xk7vky38q48h08miw4jca34ymckggnrqggn60vlddvagvl"; + procMacro = true; + authors = [ + "Henri Sivonen " + ]; + + }; "nohash-hasher" = rec { crateName = "nohash-hasher"; version = "0.2.0"; @@ -6626,7 +6697,7 @@ rec { } { name = "thiserror"; - packageId = "thiserror 2.0.20"; + packageId = "thiserror 2.0.21"; optional = true; usesDefaultFeatures = false; } @@ -6800,7 +6871,7 @@ rec { } { name = "thiserror"; - packageId = "thiserror 2.0.20"; + packageId = "thiserror 2.0.21"; usesDefaultFeatures = false; } { @@ -7004,7 +7075,7 @@ rec { } { name = "thiserror"; - packageId = "thiserror 2.0.20"; + packageId = "thiserror 2.0.21"; optional = true; usesDefaultFeatures = false; } @@ -7307,9 +7378,9 @@ rec { }; "pest" = rec { crateName = "pest"; - version = "2.9.0"; + version = "2.9.2"; edition = "2021"; - sha256 = "1kwvhc5hyrfpxpmp0jw0wr891xyjs44mcs2wz68hrl54qw6ac1ss"; + sha256 = "1a11k95dyrpfrn2ql5lprdh4k31k4hksg83w67vfdlps62iarls5"; authors = [ "Dragoș Tiselice " ]; @@ -7336,9 +7407,9 @@ rec { }; "pest_derive" = rec { crateName = "pest_derive"; - version = "2.9.0"; + version = "2.9.2"; edition = "2021"; - sha256 = "13f8ihi8928s9mc13pcbhxy382kqc89h7i8d7s5mnif8lm23ga5k"; + sha256 = "1ky22vlsbqha1jhx5l31d95gm8l54va38b9zmmr2wjicgxan0jr8"; procMacro = true; authors = [ "Dragoș Tiselice " @@ -7364,9 +7435,9 @@ rec { }; "pest_generator" = rec { crateName = "pest_generator"; - version = "2.9.0"; + version = "2.9.2"; edition = "2021"; - sha256 = "0jihcdnmdban4bqjd02r2wbb79nywj2nhwqyk95hvrixm98k9kg0"; + sha256 = "029igkh6nsw62bn4325gkjm5c3lf4bjqbkv8jblrwc0ml441z78x"; authors = [ "Dragoș Tiselice " ]; @@ -7402,9 +7473,9 @@ rec { }; "pest_meta" = rec { crateName = "pest_meta"; - version = "2.9.0"; + version = "2.9.2"; edition = "2021"; - sha256 = "15jly0r7r4m15fhm3pg814h65j7dqnqq6k43rvgxfhg29443lkg0"; + sha256 = "1jxd8gbm328lsl3m0z2b1pq9ri60h4f7122i189k5jpwkwdbm4zd"; authors = [ "Dragoș Tiselice " ]; @@ -7559,9 +7630,9 @@ rec { }; "portable-atomic-util" = rec { crateName = "portable-atomic-util"; - version = "0.2.7"; + version = "0.2.8"; edition = "2018"; - sha256 = "0616j0fhy6y71hyxg3n86f6hng0fmsc269s3wp4gl8ww4p8hd8f2"; + sha256 = "059729m166s1ccvqfyj6lv0wz5kr4qh6yb2gphf3mk5yyfvkxaqh"; libName = "portable_atomic_util"; dependencies = [ { @@ -7605,9 +7676,9 @@ rec { }; "powerfmt" = rec { crateName = "powerfmt"; - version = "0.2.0"; + version = "0.2.1"; edition = "2021"; - sha256 = "14ckj2xdpkhv3h6l5sdmb9f1d57z8hbfpdldjc2vl5givq2y77j3"; + sha256 = "0n293rvnpisjmxdqhq6zmrvcdql7j1c4zrcy5053vrv5x6wr8qsa"; authors = [ "Jacob Pratt " ]; @@ -7840,11 +7911,11 @@ rec { "rustc-dep-of-std" = [ "core" ]; }; }; - "rand 0.10.2" = rec { + "rand 0.10.3" = rec { crateName = "rand"; - version = "0.10.2"; + version = "0.10.3"; edition = "2024"; - sha256 = "105yqkdzqbgggd3r1yjm9jg0zvibfdsmxylvxxkmblwc0lxgmxf7"; + sha256 = "1bxlhj4m9zrgfgk1yirf0nny86izrngscydfx9w387n9rfbgpjb5"; authors = [ "The Rand Project Developers" "The Rust Project Developers" @@ -8080,7 +8151,7 @@ rec { dependencies = [ { name = "bitflags"; - packageId = "bitflags 2.13.1"; + packageId = "bitflags 2.13.2"; } ]; features = { @@ -8128,7 +8199,7 @@ rec { } { name = "syn"; - packageId = "syn 3.0.4"; + packageId = "syn 3.0.6"; } ]; @@ -8285,16 +8356,16 @@ rec { }; "reqwest" = rec { crateName = "reqwest"; - version = "0.13.4"; + version = "0.13.5"; edition = "2021"; - sha256 = "1hy1plns9krbh3h1dy2sdjygsfkdcnxm6pbxdi0ya9b5vq8mi711"; + sha256 = "0ac0n0g7xaffc6q3dhn9rixr4h1f0h7533l1smrxv1n1bjkwz88n"; authors = [ "Sean McArthur " ]; dependencies = [ { name = "base64"; - packageId = "base64 0.22.1"; + packageId = "base64 0.23.1"; } { name = "bytes"; @@ -8412,7 +8483,7 @@ rec { name = "web-sys"; packageId = "web-sys"; target = { target, features }: (("wasm32" == target."arch" or null) && (("unknown" == target."os" or null) || ("none" == target."os" or null))); - features = [ "AbortController" "AbortSignal" "Headers" "Request" "RequestInit" "RequestMode" "Response" "Window" "FormData" "Blob" "BlobPropertyBag" "ServiceWorkerGlobalScope" "RequestCredentials" "File" "ReadableStream" "RequestCache" ]; + features = [ "AbortController" "AbortSignal" "Headers" "Request" "RequestInit" "RequestMode" "Response" "Window" "FormData" "Blob" "BlobPropertyBag" "ServiceWorkerGlobalScope" "RequestCredentials" "File" "ReadableStream" "ReferrerPolicy" "RequestCache" ]; } ]; devDependencies = [ @@ -8916,9 +8987,9 @@ rec { }; "rustls-platform-verifier" = rec { crateName = "rustls-platform-verifier"; - version = "0.7.0"; + version = "0.7.1"; edition = "2021"; - sha256 = "181v4d0vl53vdh2wq56vghal1zyhdgqvy4xa8r45zwz4di9y5l96"; + sha256 = "164gb7kn9a8c0yrpqv879wf20z611qcfi4xjzf5qpcg2j5j5hrqi"; libName = "rustls_platform_verifier"; dependencies = [ { @@ -9022,7 +9093,7 @@ rec { name = "rustls"; packageId = "rustls"; usesDefaultFeatures = false; - features = [ "ring" ]; + features = [ "aws-lc-rs" ]; } { name = "webpki-root-certs"; @@ -9034,16 +9105,16 @@ rec { "base64" = [ "dep:base64" ]; "cert-logging" = [ "base64" ]; "docsrs" = [ "jni" "once_cell" ]; - "ffi-testing" = [ "android_logger" "rustls/ring" ]; + "ffi-testing" = [ "android_logger" "rustls/aws-lc-rs" ]; "jni" = [ "dep:jni" ]; "once_cell" = [ "dep:once_cell" ]; }; }; "rustls-platform-verifier-android" = rec { crateName = "rustls-platform-verifier-android"; - version = "0.1.1"; + version = "0.2.0"; edition = "2021"; - sha256 = "13vq6sxsgz9547xm2zbdxiw8x7ad1g8n8ax6xvxsjqszk7q6awgq"; + sha256 = "0pqsbv8656sr3h9jmq0q9848g03irccncywplmc29zs0pk08kipf"; libName = "rustls_platform_verifier_android"; }; @@ -9252,13 +9323,13 @@ rec { } { name = "syn"; - packageId = "syn 3.0.4"; + packageId = "syn 3.0.6"; } ]; devDependencies = [ { name = "syn"; - packageId = "syn 3.0.4"; + packageId = "syn 3.0.6"; features = [ "extra-traits" ]; } ]; @@ -9369,7 +9440,7 @@ rec { dependencies = [ { name = "bitflags"; - packageId = "bitflags 2.13.1"; + packageId = "bitflags 2.13.2"; } { name = "core-foundation"; @@ -9640,7 +9711,7 @@ rec { } { name = "syn"; - packageId = "syn 3.0.4"; + packageId = "syn 3.0.6"; usesDefaultFeatures = false; features = [ "clone-impls" "derive" "parsing" "printing" "proc-macro" ]; } @@ -9672,7 +9743,7 @@ rec { } { name = "syn"; - packageId = "syn 3.0.4"; + packageId = "syn 3.0.6"; usesDefaultFeatures = false; features = [ "clone-impls" "derive" "parsing" "printing" ]; } @@ -10041,7 +10112,7 @@ rec { features = { "default" = [ "std" ]; }; - resolvedDefaultFeatures = [ "std" ]; + resolvedDefaultFeatures = [ "aarch64_neon" "public_imp" "std" ]; }; "slab" = rec { crateName = "slab"; @@ -10303,7 +10374,7 @@ rec { workspace_member = null; src = pkgs.fetchgit { url = "https://github.com/stackabletech/operator-rs.git"; - rev = "f3d2da737aa49086cbdaba806aaf2c8645251cfb"; + rev = "3b91d73a3e66639654597d6304780b5ccc75d9c0"; sha256 = "0cgziqra8097hp05ynib1qpw4c95n972f4w2rk9l3llyp8r1vmci"; }; libName = "stackable_certs"; @@ -10340,7 +10411,7 @@ rec { } { name = "rand"; - packageId = "rand 0.10.2"; + packageId = "rand 0.10.3"; } { name = "rand_core"; @@ -10546,7 +10617,7 @@ rec { workspace_member = null; src = pkgs.fetchgit { url = "https://github.com/stackabletech/operator-rs.git"; - rev = "f3d2da737aa49086cbdaba806aaf2c8645251cfb"; + rev = "3b91d73a3e66639654597d6304780b5ccc75d9c0"; sha256 = "0cgziqra8097hp05ynib1qpw4c95n972f4w2rk9l3llyp8r1vmci"; }; libName = "stackable_operator"; @@ -10624,7 +10695,7 @@ rec { } { name = "rand"; - packageId = "rand 0.10.2"; + packageId = "rand 0.10.3"; } { name = "regex"; @@ -10745,7 +10816,7 @@ rec { workspace_member = null; src = pkgs.fetchgit { url = "https://github.com/stackabletech/operator-rs.git"; - rev = "f3d2da737aa49086cbdaba806aaf2c8645251cfb"; + rev = "3b91d73a3e66639654597d6304780b5ccc75d9c0"; sha256 = "0cgziqra8097hp05ynib1qpw4c95n972f4w2rk9l3llyp8r1vmci"; }; procMacro = true; @@ -10768,7 +10839,7 @@ rec { } { name = "syn"; - packageId = "syn 3.0.4"; + packageId = "syn 3.0.6"; } ]; @@ -10780,7 +10851,7 @@ rec { workspace_member = null; src = pkgs.fetchgit { url = "https://github.com/stackabletech/operator-rs.git"; - rev = "f3d2da737aa49086cbdaba806aaf2c8645251cfb"; + rev = "3b91d73a3e66639654597d6304780b5ccc75d9c0"; sha256 = "0cgziqra8097hp05ynib1qpw4c95n972f4w2rk9l3llyp8r1vmci"; }; libName = "stackable_shared"; @@ -10861,7 +10932,7 @@ rec { workspace_member = null; src = pkgs.fetchgit { url = "https://github.com/stackabletech/operator-rs.git"; - rev = "f3d2da737aa49086cbdaba806aaf2c8645251cfb"; + rev = "3b91d73a3e66639654597d6304780b5ccc75d9c0"; sha256 = "0cgziqra8097hp05ynib1qpw4c95n972f4w2rk9l3llyp8r1vmci"; }; libName = "stackable_telemetry"; @@ -10971,7 +11042,7 @@ rec { workspace_member = null; src = pkgs.fetchgit { url = "https://github.com/stackabletech/operator-rs.git"; - rev = "f3d2da737aa49086cbdaba806aaf2c8645251cfb"; + rev = "3b91d73a3e66639654597d6304780b5ccc75d9c0"; sha256 = "0cgziqra8097hp05ynib1qpw4c95n972f4w2rk9l3llyp8r1vmci"; }; libName = "stackable_versioned"; @@ -11021,7 +11092,7 @@ rec { workspace_member = null; src = pkgs.fetchgit { url = "https://github.com/stackabletech/operator-rs.git"; - rev = "f3d2da737aa49086cbdaba806aaf2c8645251cfb"; + rev = "3b91d73a3e66639654597d6304780b5ccc75d9c0"; sha256 = "0cgziqra8097hp05ynib1qpw4c95n972f4w2rk9l3llyp8r1vmci"; }; procMacro = true; @@ -11073,7 +11144,7 @@ rec { } { name = "syn"; - packageId = "syn 3.0.4"; + packageId = "syn 3.0.6"; } ]; @@ -11085,7 +11156,7 @@ rec { workspace_member = null; src = pkgs.fetchgit { url = "https://github.com/stackabletech/operator-rs.git"; - rev = "f3d2da737aa49086cbdaba806aaf2c8645251cfb"; + rev = "3b91d73a3e66639654597d6304780b5ccc75d9c0"; sha256 = "0cgziqra8097hp05ynib1qpw4c95n972f4w2rk9l3llyp8r1vmci"; }; libName = "stackable_webhook"; @@ -11145,7 +11216,7 @@ rec { } { name = "rand"; - packageId = "rand 0.10.2"; + packageId = "rand 0.10.3"; } { name = "serde"; @@ -11363,13 +11434,13 @@ rec { "proc-macro" = [ "proc-macro2/proc-macro" "quote?/proc-macro" ]; "test" = [ "syn-test-suite/all-features" ]; }; - resolvedDefaultFeatures = [ "clone-impls" "default" "derive" "extra-traits" "fold" "full" "parsing" "printing" "proc-macro" "visit" "visit-mut" ]; + resolvedDefaultFeatures = [ "clone-impls" "default" "derive" "extra-traits" "full" "parsing" "printing" "proc-macro" "visit" "visit-mut" ]; }; - "syn 3.0.4" = rec { + "syn 3.0.6" = rec { crateName = "syn"; - version = "3.0.4"; + version = "3.0.6"; edition = "2021"; - sha256 = "17v4ac61x0hvj1879ywqzlwhyzg7n9lr9zniwrsif3b1ykfmq9z6"; + sha256 = "1vmw7s58rzrs926nv5m06x7qbgswm1aa9iw3s1bj5var47kyi4w5"; authors = [ "David Tolnay " ]; @@ -11396,7 +11467,7 @@ rec { "proc-macro" = [ "proc-macro2/proc-macro" "quote?/proc-macro" ]; "test" = [ "syn-test-suite/all-features" ]; }; - resolvedDefaultFeatures = [ "clone-impls" "default" "derive" "extra-traits" "full" "parsing" "printing" "proc-macro" "visit" "visit-mut" ]; + resolvedDefaultFeatures = [ "clone-impls" "default" "derive" "extra-traits" "fold" "full" "parsing" "printing" "proc-macro" "visit" "visit-mut" ]; }; "sync_wrapper" = rec { crateName = "sync_wrapper"; @@ -11422,9 +11493,9 @@ rec { }; "synstructure" = rec { crateName = "synstructure"; - version = "0.13.2"; + version = "0.14.0"; edition = "2018"; - sha256 = "1lh9lx3r3jb18f8sbj29am5hm9jymvbwh6jb1izsnnxgvgrp12kj"; + sha256 = "00kzyz38xqvs7x1dn6rp64y0ckly4m7ywf3qk3qkgqfzs3nh85wh"; authors = [ "Nika Layzell " ]; @@ -11441,7 +11512,7 @@ rec { } { name = "syn"; - packageId = "syn 2.0.119"; + packageId = "syn 3.0.6"; usesDefaultFeatures = false; features = [ "derive" "parsing" "printing" "clone-impls" "visit" "extra-traits" ]; } @@ -11468,18 +11539,18 @@ rec { ]; }; - "thiserror 2.0.20" = rec { + "thiserror 2.0.21" = rec { crateName = "thiserror"; - version = "2.0.20"; + version = "2.0.21"; edition = "2021"; - sha256 = "0kxs6p295jffxhzaxpxv1dwaaf5iqlm6sx8h0djp6ancbxgj71pc"; + sha256 = "17hq1lh5dyr3bkc7zzjrbrp4qgkvhc48kgq1n5fdxkindaw2rr89"; authors = [ "David Tolnay " ]; dependencies = [ { name = "thiserror-impl"; - packageId = "thiserror-impl 2.0.20"; + packageId = "thiserror-impl 2.0.21"; } ]; features = { @@ -11513,11 +11584,11 @@ rec { ]; }; - "thiserror-impl 2.0.20" = rec { + "thiserror-impl 2.0.21" = rec { crateName = "thiserror-impl"; - version = "2.0.20"; + version = "2.0.21"; edition = "2021"; - sha256 = "1bwjc94gi0xn5jz26h1a8bjj1wdkvvr6jifamyc4mp9n28zcs15w"; + sha256 = "0945n8agp7kg6n6b35yyjb4g5xv2q22vrw15h6jj1nw76a99flgy"; procMacro = true; libName = "thiserror_impl"; authors = [ @@ -11534,7 +11605,7 @@ rec { } { name = "syn"; - packageId = "syn 3.0.4"; + packageId = "syn 3.0.6"; } ]; @@ -11758,9 +11829,9 @@ rec { }; "tokio" = rec { crateName = "tokio"; - version = "1.53.1"; + version = "1.53.2"; edition = "2021"; - sha256 = "1v8b3b45pkpbibls75yniqbvx5dlks2708141ljni5mnf6lawb10"; + sha256 = "0i202ksji8q2asvii0adzgi8m0z93z3j7apn62qfh8d6qzy92pz9"; authors = [ "Tokio Contributors " ]; @@ -11898,7 +11969,7 @@ rec { } { name = "syn"; - packageId = "syn 3.0.4"; + packageId = "syn 3.0.6"; features = [ "full" ]; } ]; @@ -11906,9 +11977,9 @@ rec { }; "tokio-rustls" = rec { crateName = "tokio-rustls"; - version = "0.26.4"; + version = "0.26.6"; edition = "2021"; - sha256 = "0qggwknz9w4bbsv1z158hlnpkm97j3w8v31586jipn99byaala8p"; + sha256 = "1nq5s413p6hkwrgjrpag1gi2mnjajpbszqhmhbpnkmfdq9w2dk69"; libName = "tokio_rustls"; dependencies = [ { @@ -12079,9 +12150,9 @@ rec { }; "toml_edit" = rec { crateName = "toml_edit"; - version = "0.25.13+spec-1.1.0"; + version = "0.25.15+spec-1.1.0"; edition = "2024"; - sha256 = "16xgmjdnxssdpj7rjyimsk4fqbv29g8zl7zhdbc6dxrf9mz3cxb9"; + sha256 = "0556lgzcvgfy16b8sxskr391s6cbfwnb0r4h5i4k6qw5lnaflh0k"; dependencies = [ { name = "indexmap"; @@ -12449,7 +12520,7 @@ rec { } { name = "bitflags"; - packageId = "bitflags 2.13.1"; + packageId = "bitflags 2.13.2"; } { name = "bytes"; @@ -12574,7 +12645,7 @@ rec { dependencies = [ { name = "bitflags"; - packageId = "bitflags 2.13.1"; + packageId = "bitflags 2.13.2"; } { name = "bytes"; @@ -12749,7 +12820,7 @@ rec { } { name = "thiserror"; - packageId = "thiserror 2.0.20"; + packageId = "thiserror 2.0.21"; } { name = "time"; @@ -13125,9 +13196,9 @@ rec { }; "unicode-ident" = rec { crateName = "unicode-ident"; - version = "1.0.24"; + version = "1.0.26"; edition = "2021"; - sha256 = "0xfs8y1g7syl2iykji8zk5hgfi5jw819f5zsrbaxmlzwsly33r76"; + sha256 = "0m3915ipi4zz7isncf5k1dz47ys0nq9j7l4l2n2rm03zaxwg8ifj"; libName = "unicode_ident"; authors = [ "David Tolnay " @@ -13284,9 +13355,9 @@ rec { }; "uuid" = rec { crateName = "uuid"; - version = "1.25.0"; + version = "1.27.0"; edition = "2021"; - sha256 = "1k5y394cmcrpl038i5szyxk96pa27nzgn89480d7cnph6ilmflzh"; + sha256 = "16h5h6bf5ybh1lj97lcdl41g7fqbiczpavpsb0zf3b63p4v7s9wp"; authors = [ "Ashley Mannix" "Dylan DPC" @@ -13398,9 +13469,9 @@ rec { }; "want" = rec { crateName = "want"; - version = "0.3.1"; + version = "0.3.2"; edition = "2018"; - sha256 = "03hbfrnvqqdchb5kgxyavb9jabwza0dmh2vw5kg0dq8rxl57d9xz"; + sha256 = "02zdlaqarwm9x3z1l0vm61mv8f6mv0kp4izgnxlfibqhv46xsk7c"; authors = [ "Sean McArthur " ]; @@ -13450,9 +13521,9 @@ rec { }; "wasm-bindgen" = rec { crateName = "wasm-bindgen"; - version = "0.2.127"; + version = "0.2.129"; edition = "2021"; - sha256 = "0w6fa1mkbb6qlkffgy4qaz0hdf496zbjkyiyvs4lvmpd8xbr6w0v"; + sha256 = "02flhqld01jqb6vbfyx1gb8s78g1pnq2164daxad93y6mhrlzdcv"; libName = "wasm_bindgen"; authors = [ "The wasm-bindgen Developers" @@ -13483,12 +13554,6 @@ rec { rename = "rustversion-compat"; } ]; - devDependencies = [ - { - name = "once_cell"; - packageId = "once_cell"; - } - ]; features = { "default" = [ "std" ]; "enable-interning" = [ "std" ]; @@ -13501,9 +13566,9 @@ rec { }; "wasm-bindgen-futures" = rec { crateName = "wasm-bindgen-futures"; - version = "0.4.77"; + version = "0.4.79"; edition = "2021"; - sha256 = "0l3r8m335kb2p8yj65kb0biwlypcx3ay4g750hafkl13rkapfxvb"; + sha256 = "03k1wcg5j3wqk6vgy6gzcq868vsa9k81dlhqissfk0nrw96v7fiw"; libName = "wasm_bindgen_futures"; authors = [ "The wasm-bindgen Developers" @@ -13514,6 +13579,13 @@ rec { packageId = "js-sys"; usesDefaultFeatures = false; } + { + name = "tokio"; + packageId = "tokio"; + usesDefaultFeatures = false; + target = { target, features }: (("emscripten" == target."os" or null) && (target."wasm_bindgen_unstable_tokio" or false)); + features = [ "rt" "time" ]; + } { name = "wasm-bindgen"; packageId = "wasm-bindgen"; @@ -13529,9 +13601,9 @@ rec { }; "wasm-bindgen-macro" = rec { crateName = "wasm-bindgen-macro"; - version = "0.2.127"; + version = "0.2.129"; edition = "2021"; - sha256 = "1hcvlb6bv771fvgifd367wd0cm4giyar8fq5i4h705vj7y7myxvp"; + sha256 = "0dc5xq09sy1v9ns1fhb0cnyqz5p1wcjjvkdmxskj9qhnbg1x0a9f"; procMacro = true; libName = "wasm_bindgen_macro"; authors = [ @@ -13553,9 +13625,9 @@ rec { }; "wasm-bindgen-macro-support" = rec { crateName = "wasm-bindgen-macro-support"; - version = "0.2.127"; + version = "0.2.129"; edition = "2021"; - sha256 = "112j4d7dv8y2sk9yy9czrl9fpjx9388ywnn7icdv2bywazw367g1"; + sha256 = "1dx6w90f14avmhri7ss9lyz03060g6475r4axy3bm7biqf5ill3g"; libName = "wasm_bindgen_macro_support"; authors = [ "The wasm-bindgen Developers" @@ -13575,7 +13647,7 @@ rec { } { name = "syn"; - packageId = "syn 2.0.119"; + packageId = "syn 3.0.6"; features = [ "visit" "visit-mut" "full" "extra-traits" ]; } { @@ -13589,10 +13661,10 @@ rec { }; "wasm-bindgen-shared" = rec { crateName = "wasm-bindgen-shared"; - version = "0.2.127"; + version = "0.2.129"; edition = "2021"; links = "wasm_bindgen"; - sha256 = "1gywp6xv8a27fvm3ga9xby93xyic3hc2s626b9z9rw2xqny4vxky"; + sha256 = "1ilmp5d3sl8lrq9djhcs90gk66yvk8mgwk4sfrvpvkd75b2wkw13"; libName = "wasm_bindgen_shared"; authors = [ "The wasm-bindgen Developers" @@ -13607,9 +13679,9 @@ rec { }; "web-sys" = rec { crateName = "web-sys"; - version = "0.3.104"; + version = "0.3.106"; edition = "2021"; - sha256 = "0c0acbvaqzqf21q5vdff2g74fvb7afi91xjplmclybq4d24k6df4"; + sha256 = "195bd1m14w3hsw5fg3sr938mh7y4cb260d0sq6a6brffxjfin9l8"; libName = "web_sys"; authors = [ "The wasm-bindgen Developers" @@ -14104,7 +14176,7 @@ rec { "default" = [ "std" ]; "std" = [ "wasm-bindgen/std" "js-sys/std" ]; }; - resolvedDefaultFeatures = [ "AbortController" "AbortSignal" "Blob" "BlobPropertyBag" "EventTarget" "File" "FormData" "Headers" "ReadableStream" "Request" "RequestCache" "RequestCredentials" "RequestInit" "RequestMode" "Response" "ServiceWorkerGlobalScope" "Window" "WorkerGlobalScope" "default" "std" ]; + resolvedDefaultFeatures = [ "AbortController" "AbortSignal" "Blob" "BlobPropertyBag" "EventTarget" "File" "FormData" "Headers" "ReadableStream" "ReferrerPolicy" "Request" "RequestCache" "RequestCredentials" "RequestInit" "RequestMode" "Response" "ServiceWorkerGlobalScope" "Window" "WorkerGlobalScope" "default" "std" ]; }; "web-time" = rec { crateName = "web-time"; @@ -15106,9 +15178,9 @@ rec { }; "yoke-derive" = rec { crateName = "yoke-derive"; - version = "0.8.2"; + version = "0.8.4"; edition = "2021"; - sha256 = "13l5y5sz4lqm7rmyakjbh6vwgikxiql51xfff9hq2j485hk4r16y"; + sha256 = "0wbdvvdv9birwxrr9ynxj0k7as0f0ci7ihhck34yi09nvgibv3pc"; procMacro = true; libName = "yoke_derive"; authors = [ @@ -15125,8 +15197,8 @@ rec { } { name = "syn"; - packageId = "syn 2.0.119"; - features = [ "fold" ]; + packageId = "syn 3.0.6"; + features = [ "fold" "visit" ]; } { name = "synstructure"; @@ -15137,9 +15209,9 @@ rec { }; "zerocopy" = rec { crateName = "zerocopy"; - version = "0.8.56"; + version = "0.8.60"; edition = "2021"; - sha256 = "1svmifchgdk0sm7v24lfwhhxis57gwa2lg21ingmmd5dhgjn8rsm"; + sha256 = "1ghxkd4fv4ka88v2rg9260p6465v49hmc1zvqfm6yvs2nx5zq034"; dependencies = [ { name = "zerocopy-derive"; @@ -15169,9 +15241,9 @@ rec { }; "zerocopy-derive" = rec { crateName = "zerocopy-derive"; - version = "0.8.56"; + version = "0.8.60"; edition = "2021"; - sha256 = "1hfz1hfxj86y1sgyia8gbisny9bl9bwlbahg4jypjmsp43y45azj"; + sha256 = "1nrlkb37wm1sq962725mxwhahaicbg277q25kbslvbavab3prhlr"; procMacro = true; libName = "zerocopy_derive"; dependencies = [ @@ -15222,9 +15294,9 @@ rec { }; "zerofrom-derive" = rec { crateName = "zerofrom-derive"; - version = "0.1.7"; + version = "0.1.8"; edition = "2021"; - sha256 = "18c4wsnznhdxx6m80piil1lbyszdiwsshgjrybqcm4b6qic22lqi"; + sha256 = "0jhdy4p9c6ja3dynsdck14g2iilq8ai5ch6nsi455x67ys1lcnzp"; procMacro = true; libName = "zerofrom_derive"; authors = [ @@ -15241,8 +15313,8 @@ rec { } { name = "syn"; - packageId = "syn 2.0.119"; - features = [ "fold" ]; + packageId = "syn 3.0.6"; + features = [ "fold" "visit" ]; } { name = "synstructure"; @@ -15408,12 +15480,29 @@ rec { } { name = "syn"; - packageId = "syn 3.0.4"; + packageId = "syn 3.0.6"; features = [ "extra-traits" ]; } ]; }; + "zlib-rs" = rec { + crateName = "zlib-rs"; + version = "0.6.8"; + edition = "2021"; + sha256 = "04j158293bx73kv5pj1i89ai411q7fxc9zwk3wkpqgb9gj7fas5j"; + libName = "zlib_rs"; + features = { + "__internal-fuzz" = [ "arbitrary" ]; + "__internal-test" = [ "quickcheck" ]; + "arbitrary" = [ "dep:arbitrary" ]; + "avx512" = [ "vpclmulqdq" ]; + "default" = [ "std" "c-allocator" ]; + "quickcheck" = [ "dep:quickcheck" ]; + "std" = [ "rust-allocator" ]; + }; + resolvedDefaultFeatures = [ "rust-allocator" "std" ]; + }; "zmij" = rec { crateName = "zmij"; version = "1.0.23"; diff --git a/extra/crds.yaml b/extra/crds.yaml index eb762d076..fe7d18a42 100644 --- a/extra/crds.yaml +++ b/extra/crds.yaml @@ -2141,11 +2141,16 @@ spec: nullable: true properties: authentication: - description: The credential the agent authenticates to Kafka with. + description: How the agent authenticates to Kafka. oneOf: - required: - tls + - required: + - anonymous properties: + anonymous: + description: 'No credential: the product does not authenticate the agent.' + type: object tls: description: 'Source of a TLS client certificate: a secret-operator SecretClass or a static Secret.' oneOf: diff --git a/rust/operator-binary/src/controller/build/resource/agent.rs b/rust/operator-binary/src/controller/build/resource/agent.rs index 35db34806..e9dc87ca9 100644 --- a/rust/operator-binary/src/controller/build/resource/agent.rs +++ b/rust/operator-binary/src/controller/build/resource/agent.rs @@ -47,6 +47,7 @@ pub fn build_agent_deployment( KafkaPlatformAccessAuthentication::Tls(credential) => { credential.add_volumes_and_mounts(&mut pod_builder, vec![&mut cb_agent]) } + KafkaPlatformAccessAuthentication::Anonymous(_) => {} } cb_agent @@ -259,6 +260,15 @@ mod tests { ); } + #[test] + fn deployment_with_anonymous_authentication_mounts_no_credential() { + let deployment = agent_deployment("{enabled: true, authentication: {anonymous: {}}}"); + + let pod_spec = &deployment["spec"]["template"]["spec"]; + assert_eq!(pod_spec["volumes"], Value::Null); + assert_eq!(pod_spec["containers"][0]["volumeMounts"], Value::Null); + } + #[test] fn test_deployment_uses_product_image_pull_secrets() { let cluster = cluster_with_image( diff --git a/rust/operator-binary/src/crd/mod.rs b/rust/operator-binary/src/crd/mod.rs index 7f486d93b..b09942d88 100644 --- a/rust/operator-binary/src/crd/mod.rs +++ b/rust/operator-binary/src/crd/mod.rs @@ -12,8 +12,10 @@ use serde::{Deserialize, Serialize}; use snafu::Snafu; use stackable_operator::{ commons::{ - cluster_operation::ClusterOperation, networking::DomainName, - platform_access::tls::TlsClientCredential, product_image_selection::ProductImage, + cluster_operation::ClusterOperation, + networking::DomainName, + platform_access::{anonymous::Anonymous, tls::TlsClientCredential}, + product_image_selection::ProductImage, }, config::merge::Merge, constant, @@ -116,7 +118,7 @@ pub struct KafkaPlatformAccess { #[serde(default)] pub enabled: bool, - /// The credential the agent authenticates to Kafka with. + /// How the agent authenticates to Kafka. pub authentication: KafkaPlatformAccessAuthentication, } @@ -125,6 +127,7 @@ pub struct KafkaPlatformAccess { #[serde(rename_all = "camelCase")] pub enum KafkaPlatformAccessAuthentication { Tls(TlsClientCredential), + Anonymous(Anonymous), } #[versioned( From 18a4ff2f610a9f6e2ff8e8881a04f50203417d24 Mon Sep 17 00:00:00 2001 From: Benedikt Labrenz Date: Tue, 13 Oct 2026 11:23:09 +0200 Subject: [PATCH 09/93] use versioned for platform access CRD fragments --- Cargo.lock | 222 ++++++++------ Cargo.nix | 283 +++++++++++------- extra/crds.yaml | 4 +- rust/operator-binary/src/controller.rs | 5 +- .../src/controller/build/resource/agent.rs | 2 +- rust/operator-binary/src/crd/mod.rs | 49 ++- 6 files changed, 326 insertions(+), 239 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index da8c1babd..bbeb84616 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -169,7 +169,7 @@ checksum = "82f6aeea286b8eb4dd3431a1be1b59d290ace00f5bfd8e2a159bc2a05e2c1667" dependencies = [ "proc-macro2", "quote", - "syn 3.0.6", + "syn 3.0.9", ] [[package]] @@ -316,9 +316,9 @@ checksum = "fc652a48c352aef3ea3aed32080501cf3ef6ed5da78602a020c991775b0aff04" [[package]] name = "cc" -version = "1.6.0" +version = "1.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f74872d07caf508b30a21f6836e7d7016a2eaf7d9ff4f48deaa58cd8a0407630" +checksum = "6651c9ed80effdc7db0ff72512157f901af5e3549e341e24b1dd4887d836d838" dependencies = [ "find-msvc-tools", "jobserver", @@ -385,7 +385,7 @@ dependencies = [ "heck", "proc-macro2", "quote", - "syn 3.0.6", + "syn 3.0.9", ] [[package]] @@ -575,7 +575,7 @@ dependencies = [ "proc-macro2", "quote", "strsim", - "syn 3.0.6", + "syn 3.0.9", ] [[package]] @@ -597,7 +597,7 @@ checksum = "2ac7135c3ef02b2f7833bbeb1be5ba7f966dcde8a87c6b87f65a778d71a02785" dependencies = [ "darling_core 0.24.1", "quote", - "syn 3.0.6", + "syn 3.0.9", ] [[package]] @@ -668,9 +668,9 @@ dependencies = [ [[package]] name = "deranged" -version = "0.5.8" +version = "0.5.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7cd812cc2bc1d69d4764bd80df88b4317eaef9e773c75226407d9bc0876b211c" +checksum = "9e9de72ce2ad1f90dc62fa25f0f430ef85eb4b0d8fa0be4f30373bc40a21d28e" [[package]] name = "derive_more" @@ -713,7 +713,7 @@ checksum = "c6232dd377dcc64799954cbd3a9bb882e9cdc1308ccd87b1c098f1fb2eaf82a8" dependencies = [ "proc-macro2", "quote", - "syn 3.0.6", + "syn 3.0.9", ] [[package]] @@ -777,14 +777,14 @@ dependencies = [ "enum-ordinalize", "proc-macro2", "quote", - "syn 3.0.6", + "syn 3.0.9", ] [[package]] name = "either" -version = "1.18.0" +version = "1.19.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "252afb9ae5eaa683babdc6a068b3f5726eb19e05070c731f9b2a23a7c3e8ed34" +checksum = "0e9c71c2167ca323c882b99918929403426e2373ea17242ff5653e0d5e1058be" [[package]] name = "elliptic-curve" @@ -855,7 +855,7 @@ checksum = "a65863d15a4ce2888bd2f0f543cc963d3879c3a022c8ee43f6141d479a3ac815" dependencies = [ "proc-macro2", "quote", - "syn 3.0.6", + "syn 3.0.9", ] [[package]] @@ -1010,7 +1010,7 @@ checksum = "9fb9654ba8355388abeb8dcb4fc62f511300867002afc858860463bdd9fe0c44" dependencies = [ "proc-macro2", "quote", - "syn 3.0.6", + "syn 3.0.9", ] [[package]] @@ -1149,9 +1149,9 @@ dependencies = [ [[package]] name = "h2" -version = "0.4.19" +version = "0.4.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ef8e5e5a340588f4452631496976cf8636d4a7ecf600239fdc27615d2530bc16" +checksum = "7d29020232d6aa3fb1daca64c1127cf662cf97f254ae16c18c05b8ab635fc118" dependencies = [ "atomic-waker", "bytes", @@ -1262,9 +1262,9 @@ checksum = "15cdd26707701c53297e2fa6afb323d55fbc1d0810c3aec078ae3ef0424c3c15" [[package]] name = "hyper" -version = "1.11.1" +version = "1.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "27b501faa50e7a26c3d3560ca625132f4078a17771f4810baf70475ae48cbe43" +checksum = "2c3e324da4c95177d6291d4c8730197c0d1822f8a9766814a4a44fa5ab797c9c" dependencies = [ "atomic-waker", "bytes", @@ -1482,9 +1482,9 @@ dependencies = [ [[package]] name = "indoc" -version = "2.0.7" +version = "2.0.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "79cf5c93f93228cf8efb3ba362535fb11199ac548a09ce117c9b1adc3030d706" +checksum = "a37b2691796cffeb8a8cd305ac66e65841559f147f4e63231d0eafa4db5384d1" dependencies = [ "rustversion", ] @@ -1538,9 +1538,9 @@ dependencies = [ [[package]] name = "jiff" -version = "0.2.37" +version = "0.2.38" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ab1baf72f08796de0260609515130699b890ac25f30e610ad894bc5856cafdb" +checksum = "b2b005715dcbeb0089a3c0dab99f2ff1cc3b2525323552703d648585d342a383" dependencies = [ "defmt", "jiff-core", @@ -1565,9 +1565,9 @@ dependencies = [ [[package]] name = "jiff-static" -version = "0.2.37" +version = "0.2.38" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "378268a1116ad67ae6228701118ac9f491d78fda38a40a1f1a9e1348de6f7212" +checksum = "2cc9817253cf7c7ee4684451bd327e88d6f3658014e54a29198625590650695c" dependencies = [ "jiff-core", "proc-macro2", @@ -1577,9 +1577,9 @@ dependencies = [ [[package]] name = "jiff-tzdb" -version = "0.1.8" +version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "142bd39932ad231f10513df9ab62661fead8719872150b7ad02a2df79f4e141e" +checksum = "fa8377070c6bae868759445e5a77f66d84f0b72f3a054bfb00e6d038b8282da7" [[package]] name = "jiff-tzdb-platform" @@ -1712,7 +1712,7 @@ dependencies = [ [[package]] name = "k8s-version" version = "0.1.3" -source = "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#3b91d73a3e66639654597d6304780b5ccc75d9c0" +source = "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#754e444b8cc3daf6a59fb69d7eaf632249ccc3be" dependencies = [ "darling 0.24.1", "regex", @@ -1878,9 +1878,9 @@ checksum = "b6d2cec3eae94f9f509c767b45932f1ada8350c4bdb85af2fcab4a3c14807981" [[package]] name = "libz-sys" -version = "1.1.29" +version = "1.1.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "85bc9657773828b90eeb625adff10eeac83cc21bbfd8e23a03eaa8a33c9e28d9" +checksum = "f710a23e6dbf193214fd46ca56a9d6864e550abe86202184532ae7275e46de19" dependencies = [ "cc", "libc", @@ -2049,9 +2049,9 @@ checksum = "7c87def4c32ab89d880effc9e097653c8da5d6ef28e6b539d313baaacfbafcbe" [[package]] name = "opentelemetry" -version = "0.33.0" +version = "0.33.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6cdb0b1b267eb9db3331b434ed9ddab10d50e280a9adf9d13e5233e2002b61b5" +checksum = "8c1f1a6ac9c875a4941e4c7c085b61e8f4daa326de03b4701768c3491a6ad9ee" dependencies = [ "futures-core", "futures-sink", @@ -2063,9 +2063,9 @@ dependencies = [ [[package]] name = "opentelemetry-appender-tracing" -version = "0.33.0" +version = "0.33.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e2b304e8d8f11326ba366a99ca5c7aaf7cd397b67268a332437175aeb42a40db" +checksum = "1f01f1c53c46ce80e5fd8cb7a646f13766fd8f315a8a5b75b9bad59a08d2c5c7" dependencies = [ "opentelemetry", "tracing", @@ -2075,9 +2075,9 @@ dependencies = [ [[package]] name = "opentelemetry-http" -version = "0.33.0" +version = "0.33.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ee2c3625b8aa04209f7e01e513bc8044da9687fa38a9eacf59628ca5f3b87300" +checksum = "302249bd8ee804789faaa7b5b2cbf79bc61cb864714079c644e3828af5b7e811" dependencies = [ "async-trait", "bytes", @@ -2088,9 +2088,9 @@ dependencies = [ [[package]] name = "opentelemetry-otlp" -version = "0.33.0" +version = "0.33.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "699a67345e21962a231955b9059a157e428b219fa5df8efe11f08346fb23a344" +checksum = "0756af24841e367128ef31c831999937b8b5c5197d47914590d8f4e2e88c2e1a" dependencies = [ "http", "httpdate", @@ -2108,9 +2108,9 @@ dependencies = [ [[package]] name = "opentelemetry-proto" -version = "0.33.0" +version = "0.33.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "25da1ac11a0aeccf38d7f77ee0348715adaf8340f65ad46c94a02c6b20e2f65d" +checksum = "d576f5dddc3299341bcd85a04be17b2564844c0c9563214f02d85f2524d1afae" dependencies = [ "opentelemetry", "opentelemetry_sdk", @@ -2121,15 +2121,15 @@ dependencies = [ [[package]] name = "opentelemetry-semantic-conventions" -version = "0.33.0" +version = "0.33.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d288b1e1bc3590052fbb8f0f635ef248dd9022cf1fbaeb6582b7ad6081a17b87" +checksum = "7a3cf639fefd3e852869114f9925b578da6dd8e99281a2089bb0e773ead46962" [[package]] name = "opentelemetry_sdk" -version = "0.33.0" +version = "0.33.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cb39533d9d1c912123efd7d41d7e0c29d16917b60ce15b4c8d87cb1af7f67520" +checksum = "372a912db2974d1990207006fd0e3aa0ef3fd4ad3be43c65cfaadf2206866061" dependencies = [ "futures-channel", "futures-executor", @@ -2220,19 +2220,21 @@ checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" [[package]] name = "pest" -version = "2.9.2" +version = "2.9.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "45d3aca230fad2e6f6317ca0a72724338c4960cb97168a85cdee66df4a9a21a8" +checksum = "b568374ba38b33a6c627141f891faf16902b08d2db26b8ede1bcb0a15b1919fa" dependencies = [ "memchr", + "psm", + "stacker", "ucd-trie", ] [[package]] name = "pest_derive" -version = "2.9.2" +version = "2.9.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "284b60557f2c4a2e72ad3f2d34d42685a2fa4a6a61d0d2a10c0ae2a5e916c2cf" +checksum = "b66e184b924cebaaff20ab2256ca52f12332d528a39aa76553b5d96f92aacf7f" dependencies = [ "pest", "pest_generator", @@ -2240,22 +2242,22 @@ dependencies = [ [[package]] name = "pest_generator" -version = "2.9.2" +version = "2.9.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1d9d1f08a115309ee99268cf85e5228e0e56aa9caf8841ec12866b6be07c3109" +checksum = "a87478d267e4de54a626af9754f2f0f58e927aac6ed0575fe89bc05ad6851694" dependencies = [ "pest", "pest_meta", "proc-macro2", "quote", - "syn 2.0.119", + "syn 3.0.9", ] [[package]] name = "pest_meta" -version = "2.9.2" +version = "2.9.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ed93ba1a9ffcca32130a5188701c81c0c49cf00d4b7c5007d5148951d743adcb" +checksum = "4f986f248b4241ac359b831f6139aaa34e03b08a37b6caf7e201a33f95c869e1" dependencies = [ "pest", ] @@ -2411,6 +2413,15 @@ dependencies = [ "prost", ] +[[package]] +name = "psm" +version = "0.1.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "200b9ff220857e53e184257720a14553b2f4aa02577d2ed9842d45d4b9654810" +dependencies = [ + "cc", +] + [[package]] name = "quote" version = "1.0.47" @@ -2533,7 +2544,7 @@ checksum = "92ecd8964f8453721699a1ed72037b0db49ce2f5a5138486ee89bed6f67cdf3a" dependencies = [ "proc-macro2", "quote", - "syn 3.0.6", + "syn 3.0.9", ] [[package]] @@ -2791,9 +2802,9 @@ dependencies = [ [[package]] name = "schemars" -version = "1.2.2" +version = "1.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "687274d293b6cdc6e73e0fee520bf2049650090d7164f87672d212a3c530cf4a" +checksum = "ab508826f74a77ca9d5aba6ff19b522583ee3eaf28a19384ff3d0e5835fadf6e" dependencies = [ "dyn-clone", "ref-cast", @@ -2805,14 +2816,14 @@ dependencies = [ [[package]] name = "schemars_derive" -version = "1.2.2" +version = "1.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d98c67716b46af2f0b8cf752abc930f6f9aecfbf671ecfb531db8a31dbe4e2ba" +checksum = "e1c3a92094fa7d61aa124645844facb6b554dfc797136d0f5fd1f890e2bffc69" dependencies = [ "proc-macro2", "quote", "serde_derive_internals", - "syn 3.0.6", + "syn 3.0.9", ] [[package]] @@ -2933,7 +2944,7 @@ checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348" dependencies = [ "proc-macro2", "quote", - "syn 3.0.6", + "syn 3.0.9", ] [[package]] @@ -2944,14 +2955,14 @@ checksum = "f852137cce035d6a4df67ccce505ff6b3e9fd3a10e3e52b24dc71e650bb1a9bd" dependencies = [ "proc-macro2", "quote", - "syn 3.0.6", + "syn 3.0.9", ] [[package]] name = "serde_json" -version = "1.0.151" +version = "1.0.154" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c841b55ecdae098c80dcae9cf767f6f8a0c2cdb3416bbef72181df4d0fe73f14" +checksum = "e7e9cc8b1b85264074fbcc02a88680c4096b1e47df8f739dceb03bf482f04bd6" dependencies = [ "itoa", "memchr", @@ -3164,7 +3175,7 @@ checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596" [[package]] name = "stackable-certs" version = "0.4.1" -source = "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#3b91d73a3e66639654597d6304780b5ccc75d9c0" +source = "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#754e444b8cc3daf6a59fb69d7eaf632249ccc3be" dependencies = [ "const-oid", "ecdsa", @@ -3222,7 +3233,7 @@ dependencies = [ [[package]] name = "stackable-operator" version = "0.119.0" -source = "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#3b91d73a3e66639654597d6304780b5ccc75d9c0" +source = "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#754e444b8cc3daf6a59fb69d7eaf632249ccc3be" dependencies = [ "base64 0.23.1", "clap", @@ -3266,18 +3277,18 @@ dependencies = [ [[package]] name = "stackable-operator-derive" version = "0.3.1" -source = "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#3b91d73a3e66639654597d6304780b5ccc75d9c0" +source = "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#754e444b8cc3daf6a59fb69d7eaf632249ccc3be" dependencies = [ "darling 0.24.1", "proc-macro2", "quote", - "syn 3.0.6", + "syn 3.0.9", ] [[package]] name = "stackable-shared" version = "0.1.2" -source = "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#3b91d73a3e66639654597d6304780b5ccc75d9c0" +source = "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#754e444b8cc3daf6a59fb69d7eaf632249ccc3be" dependencies = [ "jiff", "k8s-openapi", @@ -3294,7 +3305,7 @@ dependencies = [ [[package]] name = "stackable-telemetry" version = "0.6.5" -source = "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#3b91d73a3e66639654597d6304780b5ccc75d9c0" +source = "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#754e444b8cc3daf6a59fb69d7eaf632249ccc3be" dependencies = [ "axum", "clap", @@ -3318,7 +3329,7 @@ dependencies = [ [[package]] name = "stackable-versioned" version = "0.11.1" -source = "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#3b91d73a3e66639654597d6304780b5ccc75d9c0" +source = "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#754e444b8cc3daf6a59fb69d7eaf632249ccc3be" dependencies = [ "kube", "schemars", @@ -3332,7 +3343,7 @@ dependencies = [ [[package]] name = "stackable-versioned-macros" version = "0.11.1" -source = "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#3b91d73a3e66639654597d6304780b5ccc75d9c0" +source = "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#754e444b8cc3daf6a59fb69d7eaf632249ccc3be" dependencies = [ "convert_case", "darling 0.24.1", @@ -3343,13 +3354,13 @@ dependencies = [ "kube", "proc-macro2", "quote", - "syn 3.0.6", + "syn 3.0.9", ] [[package]] name = "stackable-webhook" version = "0.10.0" -source = "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#3b91d73a3e66639654597d6304780b5ccc75d9c0" +source = "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#754e444b8cc3daf6a59fb69d7eaf632249ccc3be" dependencies = [ "arc-swap", "async-trait", @@ -3378,6 +3389,19 @@ dependencies = [ "x509-cert", ] +[[package]] +name = "stacker" +version = "0.1.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "707f49d46706bacf8a2b00d51dace3f9de527c13eec3778f570c411f89e69967" +dependencies = [ + "cc", + "cfg-if", + "libc", + "psm", + "windows-sys 0.61.2", +] + [[package]] name = "strsim" version = "0.11.1" @@ -3441,9 +3465,9 @@ dependencies = [ [[package]] name = "syn" -version = "3.0.6" +version = "3.0.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8593e8e72159ed2257d083c7a454a85cbf854f37a0966d8d483aff8c8a3ebcee" +checksum = "d78c8dee4c7bf0e14673097256fed6142ce9d3b85a408189d07482442145823b" dependencies = [ "proc-macro2", "quote", @@ -3467,7 +3491,7 @@ checksum = "901704edd0dfe137f1987838ee4f259e4e063c31371bdb423f7ae38ec6f77f02" dependencies = [ "proc-macro2", "quote", - "syn 3.0.6", + "syn 3.0.9", ] [[package]] @@ -3507,7 +3531,7 @@ checksum = "fe5197923287db20a58125f0bc85c062f7f2c892de97b18c356f9efb14b28524" dependencies = [ "proc-macro2", "quote", - "syn 3.0.6", + "syn 3.0.9", ] [[package]] @@ -3605,7 +3629,7 @@ checksum = "78773a2a397f451582ce068015985c33193cf6dea8b74d2a639fe457b2f07b0e" dependencies = [ "proc-macro2", "quote", - "syn 3.0.6", + "syn 3.0.9", ] [[package]] @@ -3631,9 +3655,9 @@ dependencies = [ [[package]] name = "tokio-util" -version = "0.7.19" +version = "0.7.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "494815d09bf52b5548659851081238f0ca39ff638363907596da739561c62c52" +checksum = "e464cf451ba96ebfc6f9b6542f17ee8b8956e33f1e40d9690624e59d7a7f8a4b" dependencies = [ "bytes", "futures-core", @@ -3646,18 +3670,18 @@ dependencies = [ [[package]] name = "toml_datetime" -version = "1.1.1+spec-1.1.0" +version = "1.1.2+spec-1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3165f65f62e28e0115a00b2ebdd37eb6f3b641855f9d636d3cd4103767159ad7" +checksum = "2b86d767906c6c42421dcba507eb9d203e779497710a47782a224bb871653053" dependencies = [ "serde_core", ] [[package]] name = "toml_edit" -version = "0.25.15+spec-1.1.0" +version = "0.25.17+spec-1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1340ea94a5856333492c9064b02c778b191dd2c853778d9609debdcdfea3a614" +checksum = "e3641d5bbb5349a79e1020a242d251efbc546ad8048d133958323ce9c40a9c9c" dependencies = [ "indexmap", "toml_datetime", @@ -3667,9 +3691,9 @@ dependencies = [ [[package]] name = "toml_parser" -version = "1.1.3+spec-1.1.0" +version = "1.1.5+spec-1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1d38ac1cf9b95face32296c0a3ede1fdc270627c9d9c02a7274dd6d960dc4d56" +checksum = "baa693a8032d7e1cada7d0041e96126df243179ff061456783ac7f12bda4744c" dependencies = [ "winnow", ] @@ -3916,9 +3940,9 @@ checksum = "2896d95c02a80c6d6a5d6e953d479f5ddf2dfdb6a244441010e373ac0fb88971" [[package]] name = "unicode-ident" -version = "1.0.26" +version = "1.0.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d245f478577f809a851594d02313b640fb437e0bb33866753cff937863096954" +checksum = "a2c754d6c33795a1c324727428e5a7dedb5b06195f9890bdbcba760d3e246563" [[package]] name = "unicode-segmentation" @@ -3977,9 +4001,9 @@ checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" [[package]] name = "uuid" -version = "1.27.0" +version = "1.28.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "97277d36b9c3ace13e58fa6e753f8b0bbbf302a18dd193240d70f9e29681059a" +checksum = "7cc1186384beb7dd8eedea376413fd654937285ea6c9cfbb928dc3043ea4b606" dependencies = [ "js-sys", "wasm-bindgen", @@ -4080,7 +4104,7 @@ dependencies = [ "bumpalo", "proc-macro2", "quote", - "syn 3.0.6", + "syn 3.0.9", "wasm-bindgen-shared", ] @@ -4332,24 +4356,24 @@ checksum = "ec8ebde2db3681e8c9980cc27822030e68752690ddfa9473e739aeb4dbde6d71" dependencies = [ "proc-macro2", "quote", - "syn 3.0.6", + "syn 3.0.9", "synstructure", ] [[package]] name = "zerocopy" -version = "0.8.60" +version = "0.8.62" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6400fc4bb7426f6faac3fb07566122bb18622e3022bd2c36426a92ed489b1dbe" +checksum = "86502bf56ac7c77571a32e2647bb2a15894565e981fb2a48d7bde2d91c965a9d" dependencies = [ "zerocopy-derive", ] [[package]] name = "zerocopy-derive" -version = "0.8.60" +version = "0.8.62" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "99c27cc7525bad4df59a45e073c45b2c2aa820efb588234cc23a547ec69a34db" +checksum = "5457206954b06561e2608c7e19cf58b1926586d999c246eebe4502f7e2039d1a" dependencies = [ "proc-macro2", "quote", @@ -4373,15 +4397,15 @@ checksum = "f75b4683f6c7f45248d4d64056a24298c6281e0993356d7d1b4a1a962ef10d4a" dependencies = [ "proc-macro2", "quote", - "syn 3.0.6", + "syn 3.0.9", "synstructure", ] [[package]] name = "zeroize" -version = "1.9.0" +version = "1.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e13c156562582aa81c60cb29407084cdb54c4164760106ab78e6c5b0858cf64e" +checksum = "e13084392c5e4bc371903e2935a5eaeed24905a7511356b883835e18a78f6879" dependencies = [ "zeroize_derive", ] @@ -4427,7 +4451,7 @@ checksum = "34df6fc39dbd26ddc9c10e6a2984476e13acce22e64e4487636ef494369225da" dependencies = [ "proc-macro2", "quote", - "syn 3.0.6", + "syn 3.0.9", ] [[package]] diff --git a/Cargo.nix b/Cargo.nix index 17baabbeb..116feeff6 100644 --- a/Cargo.nix +++ b/Cargo.nix @@ -506,7 +506,7 @@ rec { } { name = "syn"; - packageId = "syn 3.0.6"; + packageId = "syn 3.0.9"; usesDefaultFeatures = false; features = [ "clone-impls" "full" "parsing" "printing" "proc-macro" "visit-mut" ]; } @@ -987,9 +987,9 @@ rec { }; "cc" = rec { crateName = "cc"; - version = "1.6.0"; + version = "1.8.0"; edition = "2021"; - sha256 = "0c3n82hdi355xa6z9x4zgnpjwsh1szkkcs0zl8q8nl5ggk874j7p"; + sha256 = "0f6q6vc8fj6xn4j1wd4yakiza6lhgwai49gp1zdwgzggh3nwjlb6"; dependencies = [ { name = "find-msvc-tools"; @@ -1218,7 +1218,7 @@ rec { } { name = "syn"; - packageId = "syn 3.0.6"; + packageId = "syn 3.0.9"; features = [ "full" ]; } ]; @@ -1765,7 +1765,7 @@ rec { } { name = "syn"; - packageId = "syn 3.0.6"; + packageId = "syn 3.0.9"; features = [ "full" "extra-traits" ]; } ]; @@ -1821,7 +1821,7 @@ rec { } { name = "syn"; - packageId = "syn 3.0.6"; + packageId = "syn 3.0.9"; } ]; @@ -2006,9 +2006,9 @@ rec { }; "deranged" = rec { crateName = "deranged"; - version = "0.5.8"; - edition = "2021"; - sha256 = "0711df3w16vx80k55ivkwzwswziinj4dz05xci3rvmn15g615n3w"; + version = "0.5.9"; + edition = "2024"; + sha256 = "13nj445c8frp617vx84g1m5yp1gg63sg09gscbf907xdw8nfg7cy"; authors = [ "Jacob Pratt " ]; @@ -2189,7 +2189,7 @@ rec { } { name = "syn"; - packageId = "syn 3.0.6"; + packageId = "syn 3.0.9"; } ]; features = { @@ -2397,7 +2397,7 @@ rec { } { name = "syn"; - packageId = "syn 3.0.6"; + packageId = "syn 3.0.9"; features = [ "visit" "visit-mut" ]; } ]; @@ -2409,9 +2409,9 @@ rec { }; "either" = rec { crateName = "either"; - version = "1.18.0"; + version = "1.19.0"; edition = "2021"; - sha256 = "0d7dx31sf8rakcgp63070ngb2vkjynrni866pnx879pawndgnai5"; + sha256 = "1gjq21g0sgk5ylpj85zafcinwhh3jj91i6drhb4278vw2v17370f"; features = { "default" = [ "std" ]; "serde" = [ "dep:serde" ]; @@ -2645,7 +2645,7 @@ rec { } { name = "syn"; - packageId = "syn 3.0.6"; + packageId = "syn 3.0.9"; } ]; features = { @@ -3091,7 +3091,7 @@ rec { } { name = "syn"; - packageId = "syn 3.0.6"; + packageId = "syn 3.0.9"; features = [ "full" ]; } ]; @@ -3616,9 +3616,9 @@ rec { }; "h2" = rec { crateName = "h2"; - version = "0.4.19"; + version = "0.4.20"; edition = "2021"; - sha256 = "05mw60jmsq97vjgj607nxjkx8dl6rxv6jj9i4r2z92056id5x3pg"; + sha256 = "0661bxispf05ik0idbjlyabwyqpngh9c2r6avaqkzann68104abx"; authors = [ "Carl Lerche " "Sean McArthur " @@ -3927,9 +3927,9 @@ rec { }; "hyper" = rec { crateName = "hyper"; - version = "1.11.1"; + version = "1.12.0"; edition = "2021"; - sha256 = "0hxyikj5livhmw5q3x3ifyhphh1g2cjsc32nsg1jcyhflpx03d97"; + sha256 = "173wg6msakx4lha6hxm9z0i1h3bw34q8fk0x57b7flf9li6k4gic"; authors = [ "Sean McArthur " ]; @@ -4023,7 +4023,7 @@ rec { "client" = [ "dep:want" "dep:pin-project-lite" "dep:smallvec" ]; "ffi" = [ "dep:http-body-util" "dep:futures-util" ]; "full" = [ "client" "http1" "http2" "server" ]; - "http1" = [ "dep:atomic-waker" "dep:futures-channel" "dep:futures-core" "dep:httparse" "dep:itoa" ]; + "http1" = [ "dep:atomic-waker" "dep:futures-core" "dep:httparse" "dep:itoa" ]; "http2" = [ "dep:atomic-waker" "dep:futures-channel" "dep:futures-core" "dep:h2" ]; "server" = [ "dep:httpdate" "dep:pin-project-lite" "dep:smallvec" ]; "tracing" = [ "dep:tracing" ]; @@ -4765,9 +4765,9 @@ rec { }; "indoc" = rec { crateName = "indoc"; - version = "2.0.7"; + version = "2.0.8"; edition = "2021"; - sha256 = "01np60qdq6lvgh8ww2caajn9j4dibx9n58rvzf7cya1jz69mrkvr"; + sha256 = "1lc4agds9bqf3lin6kkz2jgmahaqwrkaq1fkij5fpzvcg68jcyx3"; procMacro = true; authors = [ "David Tolnay " @@ -4896,9 +4896,9 @@ rec { }; "jiff" = rec { crateName = "jiff"; - version = "0.2.37"; + version = "0.2.38"; edition = "2021"; - sha256 = "1nxgdj2wajw9ml8fcc2zq858k6v9618m22864vh6sy885zvvmc8a"; + sha256 = "10x38b9qb1b47mq54d9j4ljkpk7i5ygvknn0lf4h1sybbmqhbc5j"; authors = [ "Andrew Gallant " ]; @@ -5021,9 +5021,9 @@ rec { }; "jiff-static" = rec { crateName = "jiff-static"; - version = "0.2.37"; + version = "0.2.38"; edition = "2021"; - sha256 = "04kjdzg4h4wy38ghm91qva7xg4glr65120c74bk7mmka26hni0ip"; + sha256 = "0p39a035j9c634llmr8lh1jz7ml8gqrbsla4d3j7wz6gadr83j9c"; procMacro = true; libName = "jiff_static"; authors = [ @@ -5055,9 +5055,9 @@ rec { }; "jiff-tzdb" = rec { crateName = "jiff-tzdb"; - version = "0.1.8"; + version = "0.1.9"; edition = "2021"; - sha256 = "07hl9sgzfb9as1x0n5bjk1qxishzcriapy9xa481y8xd6acx6aql"; + sha256 = "19rd52w3il7603xln19s5yvz113dyrvmlpj4b63qdbkb1h3pg0zs"; libName = "jiff_tzdb"; libPath = "lib.rs"; authors = [ @@ -5444,7 +5444,7 @@ rec { workspace_member = null; src = pkgs.fetchgit { url = "https://github.com/stackabletech/operator-rs.git"; - rev = "3b91d73a3e66639654597d6304780b5ccc75d9c0"; + rev = "754e444b8cc3daf6a59fb69d7eaf632249ccc3be"; sha256 = "0cgziqra8097hp05ynib1qpw4c95n972f4w2rk9l3llyp8r1vmci"; }; libName = "k8s_version"; @@ -6178,10 +6178,10 @@ rec { }; "libz-sys" = rec { crateName = "libz-sys"; - version = "1.1.29"; + version = "1.1.30"; edition = "2018"; links = "z"; - sha256 = "1n98kqya7a7a0cxf5n5z3g13rj7a1vqxynk2xc7bja1qfxbrdg45"; + sha256 = "06fy8rg2grraaf222846pq55akl6sslmdjj6zla346dzdlza447p"; libName = "libz_sys"; authors = [ "Alex Crichton " @@ -6671,9 +6671,9 @@ rec { }; "opentelemetry" = rec { crateName = "opentelemetry"; - version = "0.33.0"; + version = "0.33.1"; edition = "2021"; - sha256 = "1db15c0f4csj7v8zkbd9h3i503divafysd5l64rxpfby4qdhpnvc"; + sha256 = "1vnrd8d4khv82xqb80yy4sixmx78c5dhhz2c3saa8xf8r5m1l7wc"; dependencies = [ { name = "futures-core"; @@ -6725,9 +6725,9 @@ rec { }; "opentelemetry-appender-tracing" = rec { crateName = "opentelemetry-appender-tracing"; - version = "0.33.0"; + version = "0.33.1"; edition = "2021"; - sha256 = "1ns05asawxbi8cra6s3jnsbx6z5gg9fcm6ba6sx2c4ziv3l09cz2"; + sha256 = "1iy5s849mmdsp5smp2js667zsripy53addwczpjq1kj67k2z208z"; libName = "opentelemetry_appender_tracing"; dependencies = [ { @@ -6776,9 +6776,9 @@ rec { }; "opentelemetry-http" = rec { crateName = "opentelemetry-http"; - version = "0.33.0"; + version = "0.33.1"; edition = "2021"; - sha256 = "003kp3rsb332b77yma9qza3rdnj4h2y17r81gsgj015ap0jkcb7f"; + sha256 = "04g8nzsqm0p38k37jh3icjw1rilvyz5v5dd7magph178isylj8ih"; libName = "opentelemetry_http"; dependencies = [ { @@ -6820,9 +6820,9 @@ rec { }; "opentelemetry-otlp" = rec { crateName = "opentelemetry-otlp"; - version = "0.33.0"; + version = "0.33.1"; edition = "2021"; - sha256 = "0i534gxld0zh27z8xpx5kwhqnhky2nd0bfam34ijm5i1bqs6g6k9"; + sha256 = "06ifiklf5x6qj12r2ivx372vbf1pk6ck3j1ixwl72dhyhhjaymh7"; libName = "opentelemetry_otlp"; dependencies = [ { @@ -6952,9 +6952,9 @@ rec { }; "opentelemetry-proto" = rec { crateName = "opentelemetry-proto"; - version = "0.33.0"; + version = "0.33.1"; edition = "2021"; - sha256 = "0pgnw8h6nb50jind8npn821szb8mhwsf0zppswwczv0a3b0imni5"; + sha256 = "1bmgs4j2apyq097j2qwm1i688r15gghlp845rldk969jvkfzaxnm"; libName = "opentelemetry_proto"; dependencies = [ { @@ -7018,9 +7018,9 @@ rec { }; "opentelemetry-semantic-conventions" = rec { crateName = "opentelemetry-semantic-conventions"; - version = "0.33.0"; + version = "0.33.1"; edition = "2021"; - sha256 = "11vvl60n1bdph9jypfhzrwi91pa8y9g663wgpcphb41mpkhv326j"; + sha256 = "0qk9skm77rxhkc4a50cjx7c6vnkqnljrjkqid4l8agpxzqwzcg3s"; libName = "opentelemetry_semantic_conventions"; features = { }; @@ -7028,9 +7028,9 @@ rec { }; "opentelemetry_sdk" = rec { crateName = "opentelemetry_sdk"; - version = "0.33.0"; + version = "0.33.1"; edition = "2021"; - sha256 = "083myvvimjw7im65pq8cnqbnkl991iz1vm6pxwij348wklym6ffb"; + sha256 = "0qb0hq325pxarxjkrr1vmpa3zvx0787gs1kh4281jkcpn8nr2aip"; dependencies = [ { name = "futures-channel"; @@ -7378,9 +7378,9 @@ rec { }; "pest" = rec { crateName = "pest"; - version = "2.9.2"; + version = "2.9.3"; edition = "2021"; - sha256 = "1a11k95dyrpfrn2ql5lprdh4k31k4hksg83w67vfdlps62iarls5"; + sha256 = "1yhr35ds3c5ww7nvh9nvs842p40nmwgqj7ql4z3accwbld5kfs5m"; authors = [ "Dragoș Tiselice " ]; @@ -7390,6 +7390,16 @@ rec { packageId = "memchr"; optional = true; } + { + name = "psm"; + packageId = "psm"; + optional = true; + } + { + name = "stacker"; + packageId = "stacker"; + optional = true; + } { name = "ucd-trie"; packageId = "ucd-trie"; @@ -7401,15 +7411,15 @@ rec { "memchr" = [ "dep:memchr" ]; "miette-error" = [ "std" "pretty-print" "dep:miette" ]; "pretty-print" = [ "dep:serde" "dep:serde_json" ]; - "std" = [ "ucd-trie/std" ]; + "std" = [ "ucd-trie/std" "dep:stacker" "dep:psm" ]; }; resolvedDefaultFeatures = [ "default" "memchr" "std" ]; }; "pest_derive" = rec { crateName = "pest_derive"; - version = "2.9.2"; + version = "2.9.3"; edition = "2021"; - sha256 = "1ky22vlsbqha1jhx5l31d95gm8l54va38b9zmmr2wjicgxan0jr8"; + sha256 = "0zygma96zndmadjsg6m353ak48ziab55c8mb43zsmsscj95ihvmn"; procMacro = true; authors = [ "Dragoș Tiselice " @@ -7435,9 +7445,9 @@ rec { }; "pest_generator" = rec { crateName = "pest_generator"; - version = "2.9.2"; + version = "2.9.3"; edition = "2021"; - sha256 = "029igkh6nsw62bn4325gkjm5c3lf4bjqbkv8jblrwc0ml441z78x"; + sha256 = "150nhpb5mh4vx1gmgl3fmix953pmy3r595xg4sk59pp4cz97hx58"; authors = [ "Dragoș Tiselice " ]; @@ -7461,7 +7471,7 @@ rec { } { name = "syn"; - packageId = "syn 2.0.119"; + packageId = "syn 3.0.9"; } ]; features = { @@ -7473,9 +7483,9 @@ rec { }; "pest_meta" = rec { crateName = "pest_meta"; - version = "2.9.2"; + version = "2.9.3"; edition = "2021"; - sha256 = "1jxd8gbm328lsl3m0z2b1pq9ri60h4f7122i189k5jpwkwdbm4zd"; + sha256 = "1qb9r2akz8q1wbvwmdipiaq06km3m8wn27w3kcssqha2icj6z62g"; authors = [ "Dragoș Tiselice " ]; @@ -7867,6 +7877,22 @@ rec { }; resolvedDefaultFeatures = [ "default" "std" ]; }; + "psm" = rec { + crateName = "psm"; + version = "0.1.24"; + edition = "2015"; + sha256 = "0428cnwx8i9dhkcjwzap0amg9cjk8nhj0xr5hkhm6zl543r9y2r0"; + authors = [ + "Simonas Kazlauskas " + ]; + buildDependencies = [ + { + name = "cc"; + packageId = "cc"; + } + ]; + + }; "quote" = rec { crateName = "quote"; version = "1.0.47"; @@ -8199,7 +8225,7 @@ rec { } { name = "syn"; - packageId = "syn 3.0.6"; + packageId = "syn 3.0.9"; } ]; @@ -9222,9 +9248,9 @@ rec { }; "schemars" = rec { crateName = "schemars"; - version = "1.2.2"; + version = "1.2.3"; edition = "2021"; - sha256 = "0jng632s64njf9vghr3i1l4m15h4y85m5vhg7vkwdkdnjg978wk8"; + sha256 = "0vnzz8smh3ixzy297898mwzfx0r5aadz2vxsbafwlxsaywk8hl5b"; authors = [ "Graham Esau " ]; @@ -9301,9 +9327,9 @@ rec { }; "schemars_derive" = rec { crateName = "schemars_derive"; - version = "1.2.2"; + version = "1.2.3"; edition = "2021"; - sha256 = "1fp2wkdk32nv66swy7k7pz7sxygn634snlppih5jzbs6ddqng36r"; + sha256 = "0sgwpzi91y6ibw7ns4wpqzgm9ddnmi7q8ia62am62zgsjhhakhz1"; procMacro = true; authors = [ "Graham Esau " @@ -9323,13 +9349,13 @@ rec { } { name = "syn"; - packageId = "syn 3.0.6"; + packageId = "syn 3.0.9"; } ]; devDependencies = [ { name = "syn"; - packageId = "syn 3.0.6"; + packageId = "syn 3.0.9"; features = [ "extra-traits" ]; } ]; @@ -9711,7 +9737,7 @@ rec { } { name = "syn"; - packageId = "syn 3.0.6"; + packageId = "syn 3.0.9"; usesDefaultFeatures = false; features = [ "clone-impls" "derive" "parsing" "printing" "proc-macro" ]; } @@ -9743,7 +9769,7 @@ rec { } { name = "syn"; - packageId = "syn 3.0.6"; + packageId = "syn 3.0.9"; usesDefaultFeatures = false; features = [ "clone-impls" "derive" "parsing" "printing" ]; } @@ -9752,13 +9778,9 @@ rec { }; "serde_json" = rec { crateName = "serde_json"; - version = "1.0.151"; + version = "1.0.154"; edition = "2021"; - sha256 = "051zww7lvpw147vvwss1ng6w587qyrkzg75fvj08q2dfrmgbahf8"; - authors = [ - "Erick Tryzelaar " - "David Tolnay " - ]; + sha256 = "1mjby21g8fxhrsfp73yz8wg6n2f4h23ah0nczds409l53f5wrsg7"; dependencies = [ { name = "itoa"; @@ -9796,7 +9818,7 @@ rec { "alloc" = [ "serde_core/alloc" ]; "default" = [ "std" ]; "indexmap" = [ "dep:indexmap" ]; - "preserve_order" = [ "indexmap" "std" ]; + "preserve_order" = [ "indexmap" "alloc" "dep:foldhash" ]; "std" = [ "memchr/std" "serde_core/std" ]; }; resolvedDefaultFeatures = [ "alloc" "default" "raw_value" "std" ]; @@ -10374,7 +10396,7 @@ rec { workspace_member = null; src = pkgs.fetchgit { url = "https://github.com/stackabletech/operator-rs.git"; - rev = "3b91d73a3e66639654597d6304780b5ccc75d9c0"; + rev = "754e444b8cc3daf6a59fb69d7eaf632249ccc3be"; sha256 = "0cgziqra8097hp05ynib1qpw4c95n972f4w2rk9l3llyp8r1vmci"; }; libName = "stackable_certs"; @@ -10617,7 +10639,7 @@ rec { workspace_member = null; src = pkgs.fetchgit { url = "https://github.com/stackabletech/operator-rs.git"; - rev = "3b91d73a3e66639654597d6304780b5ccc75d9c0"; + rev = "754e444b8cc3daf6a59fb69d7eaf632249ccc3be"; sha256 = "0cgziqra8097hp05ynib1qpw4c95n972f4w2rk9l3llyp8r1vmci"; }; libName = "stackable_operator"; @@ -10816,7 +10838,7 @@ rec { workspace_member = null; src = pkgs.fetchgit { url = "https://github.com/stackabletech/operator-rs.git"; - rev = "3b91d73a3e66639654597d6304780b5ccc75d9c0"; + rev = "754e444b8cc3daf6a59fb69d7eaf632249ccc3be"; sha256 = "0cgziqra8097hp05ynib1qpw4c95n972f4w2rk9l3llyp8r1vmci"; }; procMacro = true; @@ -10839,7 +10861,7 @@ rec { } { name = "syn"; - packageId = "syn 3.0.6"; + packageId = "syn 3.0.9"; } ]; @@ -10851,7 +10873,7 @@ rec { workspace_member = null; src = pkgs.fetchgit { url = "https://github.com/stackabletech/operator-rs.git"; - rev = "3b91d73a3e66639654597d6304780b5ccc75d9c0"; + rev = "754e444b8cc3daf6a59fb69d7eaf632249ccc3be"; sha256 = "0cgziqra8097hp05ynib1qpw4c95n972f4w2rk9l3llyp8r1vmci"; }; libName = "stackable_shared"; @@ -10932,7 +10954,7 @@ rec { workspace_member = null; src = pkgs.fetchgit { url = "https://github.com/stackabletech/operator-rs.git"; - rev = "3b91d73a3e66639654597d6304780b5ccc75d9c0"; + rev = "754e444b8cc3daf6a59fb69d7eaf632249ccc3be"; sha256 = "0cgziqra8097hp05ynib1qpw4c95n972f4w2rk9l3llyp8r1vmci"; }; libName = "stackable_telemetry"; @@ -11042,7 +11064,7 @@ rec { workspace_member = null; src = pkgs.fetchgit { url = "https://github.com/stackabletech/operator-rs.git"; - rev = "3b91d73a3e66639654597d6304780b5ccc75d9c0"; + rev = "754e444b8cc3daf6a59fb69d7eaf632249ccc3be"; sha256 = "0cgziqra8097hp05ynib1qpw4c95n972f4w2rk9l3llyp8r1vmci"; }; libName = "stackable_versioned"; @@ -11092,7 +11114,7 @@ rec { workspace_member = null; src = pkgs.fetchgit { url = "https://github.com/stackabletech/operator-rs.git"; - rev = "3b91d73a3e66639654597d6304780b5ccc75d9c0"; + rev = "754e444b8cc3daf6a59fb69d7eaf632249ccc3be"; sha256 = "0cgziqra8097hp05ynib1qpw4c95n972f4w2rk9l3llyp8r1vmci"; }; procMacro = true; @@ -11144,7 +11166,7 @@ rec { } { name = "syn"; - packageId = "syn 3.0.6"; + packageId = "syn 3.0.9"; } ]; @@ -11156,7 +11178,7 @@ rec { workspace_member = null; src = pkgs.fetchgit { url = "https://github.com/stackabletech/operator-rs.git"; - rev = "3b91d73a3e66639654597d6304780b5ccc75d9c0"; + rev = "754e444b8cc3daf6a59fb69d7eaf632249ccc3be"; sha256 = "0cgziqra8097hp05ynib1qpw4c95n972f4w2rk9l3llyp8r1vmci"; }; libName = "stackable_webhook"; @@ -11280,6 +11302,49 @@ rec { } ]; + }; + "stacker" = rec { + crateName = "stacker"; + version = "0.1.25"; + edition = "2021"; + sha256 = "0rwrws4iyh8cay7pghzf2dy55pprwfn1vm805f5czfh6cza4jzvh"; + authors = [ + "Alex Crichton " + "Simonas Kazlauskas " + ]; + dependencies = [ + { + name = "cfg-if"; + packageId = "cfg-if"; + } + { + name = "libc"; + packageId = "libc"; + } + { + name = "psm"; + packageId = "psm"; + } + { + name = "windows-sys"; + packageId = "windows-sys 0.61.2"; + target = { target, features }: (target.name == "arm64ec-pc-windows-msvc"); + features = [ "Win32_System_Memory" "Win32_System_Threading" "Win32_Foundation" ]; + } + { + name = "windows-sys"; + packageId = "windows-sys 0.61.2"; + target = { target, features }: ((target."windows" or false) && (!("arm64ec" == target."arch" or null))); + features = [ "Win32_System_Memory" "Win32_System_Threading" "Win32_Foundation" ]; + } + ]; + buildDependencies = [ + { + name = "cc"; + packageId = "cc"; + } + ]; + }; "strsim" = rec { crateName = "strsim"; @@ -11436,11 +11501,11 @@ rec { }; resolvedDefaultFeatures = [ "clone-impls" "default" "derive" "extra-traits" "full" "parsing" "printing" "proc-macro" "visit" "visit-mut" ]; }; - "syn 3.0.6" = rec { + "syn 3.0.9" = rec { crateName = "syn"; - version = "3.0.6"; + version = "3.0.9"; edition = "2021"; - sha256 = "1vmw7s58rzrs926nv5m06x7qbgswm1aa9iw3s1bj5var47kyi4w5"; + sha256 = "0fw28lhl90kls24q2h2sp39yjb0lsvz5cwh9fd3f3w3v9kp8v36p"; authors = [ "David Tolnay " ]; @@ -11512,7 +11577,7 @@ rec { } { name = "syn"; - packageId = "syn 3.0.6"; + packageId = "syn 3.0.9"; usesDefaultFeatures = false; features = [ "derive" "parsing" "printing" "clone-impls" "visit" "extra-traits" ]; } @@ -11605,7 +11670,7 @@ rec { } { name = "syn"; - packageId = "syn 3.0.6"; + packageId = "syn 3.0.9"; } ]; @@ -11969,7 +12034,7 @@ rec { } { name = "syn"; - packageId = "syn 3.0.6"; + packageId = "syn 3.0.9"; features = [ "full" ]; } ]; @@ -12060,9 +12125,9 @@ rec { }; "tokio-util" = rec { crateName = "tokio-util"; - version = "0.7.19"; + version = "0.7.20"; edition = "2021"; - sha256 = "0licqrhrawysjrsr0qw3cgzkkjph7090hlcqcm45aazmkg81aj29"; + sha256 = "0jwagxx9vr940rlxjh0y7zimd2cbxqbjym5nz73byvm93d2wyr74"; libName = "tokio_util"; authors = [ "Tokio Contributors " @@ -12129,9 +12194,9 @@ rec { }; "toml_datetime" = rec { crateName = "toml_datetime"; - version = "1.1.1+spec-1.1.0"; + version = "1.1.2+spec-1.1.0"; edition = "2024"; - sha256 = "1mws2mkkf46l7inn77azhm0vdwxngv9vsbhbl0ah33p2c9gzcr9i"; + sha256 = "0lrhcmqvhjr259w4f2kijya7fgi0kpmhg9fb3m144v3cj1kxg1ib"; dependencies = [ { name = "serde_core"; @@ -12150,9 +12215,9 @@ rec { }; "toml_edit" = rec { crateName = "toml_edit"; - version = "0.25.15+spec-1.1.0"; + version = "0.25.17+spec-1.1.0"; edition = "2024"; - sha256 = "0556lgzcvgfy16b8sxskr391s6cbfwnb0r4h5i4k6qw5lnaflh0k"; + sha256 = "174w1b2fjg1jb0wi7384v1m59g7ga79458i022gafjakpddisr73"; dependencies = [ { name = "indexmap"; @@ -12185,9 +12250,9 @@ rec { }; "toml_parser" = rec { crateName = "toml_parser"; - version = "1.1.3+spec-1.1.0"; + version = "1.1.5+spec-1.1.0"; edition = "2024"; - sha256 = "0mjdvihdkmjd4ykh574xgii71hpxw7ns7h4n4bisqpxrz4faqf0x"; + sha256 = "0k3lljyi4zxchdklaqghkwbl7wkd2ab1w16hlyniqzid0fl979ms"; dependencies = [ { name = "winnow"; @@ -13196,9 +13261,9 @@ rec { }; "unicode-ident" = rec { crateName = "unicode-ident"; - version = "1.0.26"; + version = "1.0.27"; edition = "2021"; - sha256 = "0m3915ipi4zz7isncf5k1dz47ys0nq9j7l4l2n2rm03zaxwg8ifj"; + sha256 = "0qv54hz0sxmspjyr162z3435pnyylzjjhx3j4k1s359pqgb59ix2"; libName = "unicode_ident"; authors = [ "David Tolnay " @@ -13355,9 +13420,9 @@ rec { }; "uuid" = rec { crateName = "uuid"; - version = "1.27.0"; + version = "1.28.0"; edition = "2021"; - sha256 = "16h5h6bf5ybh1lj97lcdl41g7fqbiczpavpsb0zf3b63p4v7s9wp"; + sha256 = "01mnlhz09hwdjaxwzjd6bql3fjb5zl9n8dzaxn7dvdxyhiiiihbw"; authors = [ "Ashley Mannix" "Dylan DPC" @@ -13647,7 +13712,7 @@ rec { } { name = "syn"; - packageId = "syn 3.0.6"; + packageId = "syn 3.0.9"; features = [ "visit" "visit-mut" "full" "extra-traits" ]; } { @@ -15197,7 +15262,7 @@ rec { } { name = "syn"; - packageId = "syn 3.0.6"; + packageId = "syn 3.0.9"; features = [ "fold" "visit" ]; } { @@ -15209,9 +15274,9 @@ rec { }; "zerocopy" = rec { crateName = "zerocopy"; - version = "0.8.60"; + version = "0.8.62"; edition = "2021"; - sha256 = "1ghxkd4fv4ka88v2rg9260p6465v49hmc1zvqfm6yvs2nx5zq034"; + sha256 = "17asjqfdkqmxsx42myw1x5jlb28m5axlf9ifldqpbiy7dbsjnl46"; dependencies = [ { name = "zerocopy-derive"; @@ -15241,9 +15306,9 @@ rec { }; "zerocopy-derive" = rec { crateName = "zerocopy-derive"; - version = "0.8.60"; + version = "0.8.62"; edition = "2021"; - sha256 = "1nrlkb37wm1sq962725mxwhahaicbg277q25kbslvbavab3prhlr"; + sha256 = "06lx0gigf0j5pvp4dhlrv636b4mib37ijzlcc3i62rdhailj0msl"; procMacro = true; libName = "zerocopy_derive"; dependencies = [ @@ -15313,7 +15378,7 @@ rec { } { name = "syn"; - packageId = "syn 3.0.6"; + packageId = "syn 3.0.9"; features = [ "fold" "visit" ]; } { @@ -15325,9 +15390,9 @@ rec { }; "zeroize" = rec { crateName = "zeroize"; - version = "1.9.0"; + version = "1.9.1"; edition = "2024"; - sha256 = "0kpnij2v1ig6g2mhc0bnci0lrdfdhiq40afbc0fahajqc9jiag71"; + sha256 = "0yb8iykihpl3hfw5c4silw2lklpfxajkaa9yj1qw6jsy5hwq8c71"; authors = [ "The RustCrypto Project Developers" ]; @@ -15480,7 +15545,7 @@ rec { } { name = "syn"; - packageId = "syn 3.0.6"; + packageId = "syn 3.0.9"; features = [ "extra-traits" ]; } ]; diff --git a/extra/crds.yaml b/extra/crds.yaml index fe7d18a42..83287899a 100644 --- a/extra/crds.yaml +++ b/extra/crds.yaml @@ -2161,8 +2161,8 @@ spec: properties: secret: description: |- - A static Secret holding the certificate in the keys `tls.crt` and `tls.key` (PEM), e.g. as a - Secret of type `kubernetes.io/tls`. + A static Secret holding the certificate in the keys `tls.crt` and `tls.key` (PEM), e.g. as + a Secret of type `kubernetes.io/tls`. maxLength: 253 minLength: 1 pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ diff --git a/rust/operator-binary/src/controller.rs b/rust/operator-binary/src/controller.rs index 1dc977ea1..98eb8d541 100644 --- a/rust/operator-binary/src/controller.rs +++ b/rust/operator-binary/src/controller.rs @@ -65,8 +65,7 @@ use crate::{ update_status::update_status, }, crd::{ - APP_NAME, KAFKA_OPERATOR_NAME, KafkaPlatformAccessAuthentication, KafkaPodDescriptor, - MetadataManager, + APP_NAME, KAFKA_OPERATOR_NAME, KafkaPodDescriptor, MetadataManager, authorization::KafkaAuthorizationConfig, role::{AnyConfig, AnyConfigOverrides, KafkaRole}, v1alpha1, @@ -347,7 +346,7 @@ impl ValidatedClusterConfig { /// The configuration of the agent deployed with the cluster. pub struct ValidatedAgentConfig { pub image: String, - pub authentication: KafkaPlatformAccessAuthentication, + pub authentication: v1alpha1::KafkaPlatformAccessAuthentication, } /// Per-role configuration extracted during validation. diff --git a/rust/operator-binary/src/controller/build/resource/agent.rs b/rust/operator-binary/src/controller/build/resource/agent.rs index e9dc87ca9..bad1fb2a7 100644 --- a/rust/operator-binary/src/controller/build/resource/agent.rs +++ b/rust/operator-binary/src/controller/build/resource/agent.rs @@ -30,7 +30,7 @@ use crate::{ ValidatedAgentConfig, ValidatedCluster, build::{agent_selector, recommended_labels_for_agent_resources}, }, - crd::KafkaPlatformAccessAuthentication, + crd::v1alpha1::KafkaPlatformAccessAuthentication, }; constant!(AGENT_CONTAINER_NAME: ContainerName = "agent"); diff --git a/rust/operator-binary/src/crd/mod.rs b/rust/operator-binary/src/crd/mod.rs index b09942d88..8274f1b20 100644 --- a/rust/operator-binary/src/crd/mod.rs +++ b/rust/operator-binary/src/crd/mod.rs @@ -12,13 +12,12 @@ use serde::{Deserialize, Serialize}; use snafu::Snafu; use stackable_operator::{ commons::{ - cluster_operation::ClusterOperation, - networking::DomainName, - platform_access::{anonymous::Anonymous, tls::TlsClientCredential}, + cluster_operation::ClusterOperation, networking::DomainName, product_image_selection::ProductImage, }, config::merge::Merge, constant, + crd::platform_access::v1alpha1::{Anonymous, TlsClientCredential}, deep_merger::ObjectOverrides, kube::{CustomResource, runtime::reflector::ObjectRef}, role_utils::GenericRoleConfig, @@ -109,27 +108,6 @@ pub type ControllerRole = Role< JavaCommonConfig, >; -/// Access of the Stackable Data Platform to this Kafka cluster, used by an agent to manage resources -/// like `KafkaTopic`s. The credential must be authorized in Kafka by the user. -#[derive(Clone, Debug, Deserialize, Eq, JsonSchema, PartialEq, Serialize)] -#[serde(rename_all = "camelCase")] -pub struct KafkaPlatformAccess { - /// Whether the operator deploys the agent. False by default. - #[serde(default)] - pub enabled: bool, - - /// How the agent authenticates to Kafka. - pub authentication: KafkaPlatformAccessAuthentication, -} - -/// How the agent authenticates to Kafka. -#[derive(Clone, Debug, Deserialize, Eq, JsonSchema, PartialEq, Serialize)] -#[serde(rename_all = "camelCase")] -pub enum KafkaPlatformAccessAuthentication { - Tls(TlsClientCredential), - Anonymous(Anonymous), -} - #[versioned( version(name = "v1alpha1"), crates( @@ -172,7 +150,7 @@ pub mod versioned { /// Access of the Stackable Data Platform to this cluster. #[serde(default, skip_serializing_if = "Option::is_none")] - pub platform_access: Option, + pub platform_access: Option, // no doc - docs in ClusterOperation struct. #[serde(default)] @@ -183,6 +161,27 @@ pub mod versioned { pub object_overrides: ObjectOverrides, } + /// Access of the Stackable Data Platform to this Kafka cluster, used by an agent to manage + /// resources like `KafkaTopic`s. The credential must be authorized in Kafka by the user. + #[derive(Clone, Debug, Deserialize, Eq, JsonSchema, PartialEq, Serialize)] + #[serde(rename_all = "camelCase")] + pub struct KafkaPlatformAccess { + /// Whether the operator deploys the agent. False by default. + #[serde(default)] + pub enabled: bool, + + /// How the agent authenticates to Kafka. + pub authentication: v1alpha1::KafkaPlatformAccessAuthentication, + } + + /// How the agent authenticates to Kafka. + #[derive(Clone, Debug, Deserialize, Eq, JsonSchema, PartialEq, Serialize)] + #[serde(rename_all = "camelCase")] + pub enum KafkaPlatformAccessAuthentication { + Tls(TlsClientCredential), + Anonymous(Anonymous), + } + #[derive(Clone, Deserialize, Debug, Eq, JsonSchema, PartialEq, Serialize)] #[serde(rename_all = "camelCase")] pub struct KafkaClusterConfig { From c26dc429af8e3af78acbd2999226a1dbb619d6de Mon Sep 17 00:00:00 2001 From: Benedikt Labrenz Date: Fri, 16 Oct 2026 10:38:03 +0200 Subject: [PATCH 10/93] add kafka admin client managing topics, topic config and partition count --- Cargo.lock | 55 +++ Cargo.nix | 174 ++++++- Cargo.toml | 8 +- rust/kafka-admin/Cargo.toml | 25 + rust/kafka-admin/src/bootstrap.rs | 106 +++++ rust/kafka-admin/src/client.rs | 712 +++++++++++++++++++++++++++++ rust/kafka-admin/src/codec.rs | 121 +++++ rust/kafka-admin/src/connection.rs | 598 ++++++++++++++++++++++++ rust/kafka-admin/src/lib.rs | 7 + rust/kafka-admin/src/tls.rs | 106 +++++ 10 files changed, 1910 insertions(+), 2 deletions(-) create mode 100644 rust/kafka-admin/Cargo.toml create mode 100644 rust/kafka-admin/src/bootstrap.rs create mode 100644 rust/kafka-admin/src/client.rs create mode 100644 rust/kafka-admin/src/codec.rs create mode 100644 rust/kafka-admin/src/connection.rs create mode 100644 rust/kafka-admin/src/lib.rs create mode 100644 rust/kafka-admin/src/tls.rs diff --git a/Cargo.lock b/Cargo.lock index bbeb84616..be7309270 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -486,6 +486,30 @@ dependencies = [ "libc", ] +[[package]] +name = "crc" +version = "3.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5eb8a2a1cd12ab0d987a5d5e825195d372001a4094a0376319d5a0ad71c1ba0d" +dependencies = [ + "crc-catalog", +] + +[[package]] +name = "crc-catalog" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "217698eaf96b4a3f0bc4f3662aaa55bdf913cd54d7204591faa790070c6d0853" + +[[package]] +name = "crc32c" +version = "0.6.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a47af21622d091a8f0fb295b88bc886ac74efcc613efc19f5d0b21de5c89e47" +dependencies = [ + "rustc_version", +] + [[package]] name = "crc32fast" version = "1.5.2" @@ -1719,6 +1743,20 @@ dependencies = [ "snafu 0.9.2", ] +[[package]] +name = "kafka-protocol" +version = "0.18.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "099d5c2f1b40cd830cbf18ca4d2a0805f2875b811ca18f0372b2433e68fe2dda" +dependencies = [ + "anyhow", + "bytes", + "crc", + "crc32c", + "indexmap", + "uuid", +] + [[package]] name = "konst" version = "0.2.20" @@ -3196,6 +3234,23 @@ dependencies = [ "zeroize", ] +[[package]] +name = "stackable-kafka-admin" +version = "0.0.0-dev" +dependencies = [ + "anyhow", + "base64 0.22.1", + "bytes", + "futures", + "kafka-protocol", + "rstest", + "rustls", + "snafu 0.9.2", + "tokio", + "tokio-rustls", + "tokio-util", +] + [[package]] name = "stackable-kafka-agent" version = "0.0.0-dev" diff --git a/Cargo.nix b/Cargo.nix index 116feeff6..7003815a1 100644 --- a/Cargo.nix +++ b/Cargo.nix @@ -43,6 +43,16 @@ rec { # You can override the features with # workspaceMembers."${crateName}".build.override { features = [ "default" "feature1" ... ]; }. workspaceMembers = { + "stackable-kafka-admin" = rec { + packageId = "stackable-kafka-admin"; + build = internal.buildRustCrateWithFeatures { + packageId = "stackable-kafka-admin"; + }; + + # Debug support which might change between releases. + # File a bug if you depend on any for non-debug work! + debug = internal.debugCrate { inherit packageId; }; + }; "stackable-kafka-agent" = rec { packageId = "stackable-kafka-agent"; build = internal.buildRustCrateWithFeatures { @@ -1516,6 +1526,50 @@ rec { } ]; + }; + "crc" = rec { + crateName = "crc"; + version = "3.4.0"; + edition = "2021"; + sha256 = "03dsq5qsv86m35ikg84l80d00wnkjm8q4pjxgac0vaqjrnhs5f2y"; + authors = [ + "Rui Hu " + "Akhil Velagapudi <4@4khil.com>" + ]; + dependencies = [ + { + name = "crc-catalog"; + packageId = "crc-catalog"; + } + ]; + + }; + "crc-catalog" = rec { + crateName = "crc-catalog"; + version = "2.5.0"; + edition = "2018"; + sha256 = "0lq8dl60g457za8la86pak6i7ydxanm2lrpkqh5kyjkbz7m9hxi1"; + libName = "crc_catalog"; + authors = [ + "Akhil Velagapudi " + ]; + + }; + "crc32c" = rec { + crateName = "crc32c"; + version = "0.6.8"; + edition = "2018"; + sha256 = "0iwyr3jivcnhylczqgk1rkpp9b46r25vi5dj1y7il29dc8hsyirs"; + authors = [ + "Zack Owens" + ]; + buildDependencies = [ + { + name = "rustc_version"; + packageId = "rustc_version"; + } + ]; + }; "crc32fast" = rec { crateName = "crc32fast"; @@ -5472,6 +5526,52 @@ rec { }; resolvedDefaultFeatures = [ "darling" ]; }; + "kafka-protocol" = rec { + crateName = "kafka-protocol"; + version = "0.18.0"; + edition = "2021"; + sha256 = "1nidzrl3whxjf81qz88wh5dqgwh510m4vjhqpw687ka03cpmr789"; + libName = "kafka_protocol"; + authors = [ + "Diggory Blake " + "Charlotte McElwain " + "belltoy " + ]; + dependencies = [ + { + name = "anyhow"; + packageId = "anyhow"; + } + { + name = "bytes"; + packageId = "bytes"; + } + { + name = "crc"; + packageId = "crc"; + } + { + name = "crc32c"; + packageId = "crc32c"; + } + { + name = "indexmap"; + packageId = "indexmap"; + } + { + name = "uuid"; + packageId = "uuid"; + } + ]; + features = { + "default" = [ "client" "broker" "gzip" "zstd" "snappy" "lz4" ]; + "gzip" = [ "dep:flate2" ]; + "lz4" = [ "dep:lz4" ]; + "snappy" = [ "dep:snap" ]; + "zstd" = [ "dep:zstd" ]; + }; + resolvedDefaultFeatures = [ "broker" "client" ]; + }; "konst" = rec { crateName = "konst"; version = "0.2.20"; @@ -10492,6 +10592,78 @@ rec { }; resolvedDefaultFeatures = [ "default" "rustls" ]; }; + "stackable-kafka-admin" = rec { + crateName = "stackable-kafka-admin"; + version = "0.0.0-dev"; + edition = "2024"; + src = lib.cleanSourceWith { filter = sourceFilter; src = ./rust/kafka-admin; }; + libName = "stackable_kafka_admin"; + authors = [ + "Stackable GmbH " + ]; + dependencies = [ + { + name = "anyhow"; + packageId = "anyhow"; + } + { + name = "base64"; + packageId = "base64 0.22.1"; + } + { + name = "bytes"; + packageId = "bytes"; + } + { + name = "futures"; + packageId = "futures"; + } + { + name = "kafka-protocol"; + packageId = "kafka-protocol"; + usesDefaultFeatures = false; + features = [ "broker" "client" ]; + } + { + name = "rustls"; + packageId = "rustls"; + usesDefaultFeatures = false; + features = [ "logging" "ring" "std" "tls12" ]; + } + { + name = "snafu"; + packageId = "snafu 0.9.2"; + } + { + name = "tokio"; + packageId = "tokio"; + features = [ "full" ]; + } + { + name = "tokio-rustls"; + packageId = "tokio-rustls"; + usesDefaultFeatures = false; + features = [ "logging" "ring" "tls12" ]; + } + { + name = "tokio-util"; + packageId = "tokio-util"; + features = [ "codec" ]; + } + ]; + devDependencies = [ + { + name = "rstest"; + packageId = "rstest"; + } + { + name = "tokio"; + packageId = "tokio"; + features = [ "full" "test-util" ]; + } + ]; + + }; "stackable-kafka-agent" = rec { crateName = "stackable-kafka-agent"; version = "0.0.0-dev"; @@ -12011,7 +12183,7 @@ rec { "tracing" = [ "dep:tracing" ]; "windows-sys" = [ "dep:windows-sys" ]; }; - resolvedDefaultFeatures = [ "bytes" "default" "fs" "full" "io-std" "io-util" "libc" "macros" "mio" "net" "parking_lot" "process" "rt" "rt-multi-thread" "signal" "signal-hook-registry" "socket2" "sync" "time" "tokio-macros" "windows-sys" ]; + resolvedDefaultFeatures = [ "bytes" "default" "fs" "full" "io-std" "io-util" "libc" "macros" "mio" "net" "parking_lot" "process" "rt" "rt-multi-thread" "signal" "signal-hook-registry" "socket2" "sync" "test-util" "time" "tokio-macros" "windows-sys" ]; }; "tokio-macros" = rec { crateName = "tokio-macros"; diff --git a/Cargo.toml b/Cargo.toml index 8f9321a8c..540a2aebc 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,5 +1,5 @@ [workspace] -members = ["rust/operator-binary", "rust/agent-binary"] +members = ["rust/operator-binary", "rust/agent-binary", "rust/kafka-admin"] resolver = "2" [workspace.package] @@ -13,12 +13,16 @@ repository = "https://github.com/stackabletech/kafka-operator" stackable-operator = { git = "https://github.com/stackabletech/operator-rs.git", branch = "feat/platform-access", features = ["crds", "webhook"] } anyhow = "1.0" +base64 = "0.22" built = { version = "0.8", features = ["chrono", "git2"] } +bytes = "1.12" clap = "4.6" const_format = "0.2" futures = "0.3" indoc = "2.0" +kafka-protocol = { version = "0.18", default-features = false, features = ["broker", "client"] } rstest = "0.26" +rustls = { version = "0.23", default-features = false, features = ["logging", "ring", "std", "tls12"] } semver = "1.0" serde = { version = "1.0", features = ["derive"] } serde_json = "1.0" @@ -26,6 +30,8 @@ serde_yaml = "0.9" snafu = "0.9" strum = { version = "0.28", features = ["derive"] } tokio = { version = "1.53", features = ["full"] } +tokio-rustls = { version = "0.26", default-features = false, features = ["logging", "ring", "tls12"] } +tokio-util = { version = "0.7", features = ["codec"] } tracing = "0.1" [patch."https://github.com/stackabletech/operator-rs.git"] diff --git a/rust/kafka-admin/Cargo.toml b/rust/kafka-admin/Cargo.toml new file mode 100644 index 000000000..2540fda7b --- /dev/null +++ b/rust/kafka-admin/Cargo.toml @@ -0,0 +1,25 @@ +[package] +name = "stackable-kafka-admin" +description = "Kafka admin client of the Stackable agent for Apache Kafka" +version.workspace = true +authors.workspace = true +license.workspace = true +edition.workspace = true +repository.workspace = true +publish = false + +[dependencies] +anyhow.workspace = true +base64.workspace = true +bytes.workspace = true +futures.workspace = true +kafka-protocol.workspace = true +rustls.workspace = true +snafu.workspace = true +tokio.workspace = true +tokio-rustls.workspace = true +tokio-util.workspace = true + +[dev-dependencies] +rstest.workspace = true +tokio = { workspace = true, features = ["test-util"] } diff --git a/rust/kafka-admin/src/bootstrap.rs b/rust/kafka-admin/src/bootstrap.rs new file mode 100644 index 000000000..1ab97bb93 --- /dev/null +++ b/rust/kafka-admin/src/bootstrap.rs @@ -0,0 +1,106 @@ +//! The bootstrap servers of a Kafka cluster. + +use std::fmt; + +use snafu::{OptionExt, Snafu, ensure}; + +#[derive(Debug, Snafu)] +pub enum Error { + #[snafu(display("no bootstrap servers given"))] + NoBootstrapServers, + + #[snafu(display("{server:?} is not a valid bootstrap server, expected host:port"))] + InvalidBootstrapServer { server: String }, +} + +type Result = std::result::Result; + +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct BrokerAddress { + pub host: String, + pub port: u16, +} + +impl fmt::Display for BrokerAddress { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + if self.host.contains(':') { + write!(f, "[{}]:{}", self.host, self.port) + } else { + write!(f, "{}:{}", self.host, self.port) + } + } +} + +/// Parses a comma-separated list of `host:port` entries. IPv6 hosts may be enclosed in brackets. +pub fn parse_bootstrap_servers(value: &str) -> Result> { + let servers = value + .split(',') + .map(str::trim) + .filter(|server| !server.is_empty()) + .map(|server| { + // Splits only at the ':' separating host and port, not inside IPv6 hosts + let (host, port) = server + .rsplit_once(':') + .context(InvalidBootstrapServerSnafu { server })?; + let host = host + .strip_prefix('[') + .and_then(|host| host.strip_suffix(']')) + .unwrap_or(host); + let port = port + .parse() + .ok() + .context(InvalidBootstrapServerSnafu { server })?; + ensure!( + !host.is_empty() && !host.contains(['[', ']']), + InvalidBootstrapServerSnafu { server } + ); + Ok(BrokerAddress { + host: host.to_owned(), + port, + }) + }) + .collect::>>()?; + ensure!(!servers.is_empty(), NoBootstrapServersSnafu); + Ok(servers) +} + +#[cfg(test)] +mod tests { + use rstest::rstest; + + use super::*; + + #[rstest] + #[case("kafka:9092", &[("kafka", 9092)])] + #[case("kafka-0:9092, kafka-1:9093", &[("kafka-0", 9092), ("kafka-1", 9093)])] + #[case("kafka:9092,", &[("kafka", 9092)])] + #[case("[fd00::1]:9092", &[("fd00::1", 9092)])] + #[case("fd00::1:9092", &[("fd00::1", 9092)])] + fn parse_valid_bootstrap_servers(#[case] value: &str, #[case] expected: &[(&str, u16)]) { + let expected = expected + .iter() + .map(|(host, port)| BrokerAddress { + host: host.to_string(), + port: *port, + }) + .collect::>(); + + assert_eq!( + expected, + parse_bootstrap_servers(value).expect("the bootstrap servers should be valid") + ); + } + + #[rstest] + #[case("")] + #[case(" , ")] + #[case("kafka")] + #[case("kafka:port")] + #[case("kafka:99999")] + #[case(":9092")] + #[case("[fd00::1:9092")] + #[case("fd00::1]:9092")] + fn reject_invalid_bootstrap_servers(#[case] value: &str) { + parse_bootstrap_servers(value).expect_err("the bootstrap servers should be invalid"); + } +} diff --git a/rust/kafka-admin/src/client.rs b/rust/kafka-admin/src/client.rs new file mode 100644 index 000000000..39e5c390f --- /dev/null +++ b/rust/kafka-admin/src/client.rs @@ -0,0 +1,712 @@ +//! An admin client for the topics of a Kafka cluster in KRaft mode. + +use std::{ + collections::{BTreeMap, BTreeSet}, + time::Duration, +}; + +use base64::{Engine, engine::general_purpose::URL_SAFE_NO_PAD}; +use kafka_protocol::{ + ResponseError, + error::ParseResponseErrorCode, + messages::{ + CreatePartitionsRequest, CreateTopicsRequest, DeleteTopicsRequest, DeleteTopicsResponse, + DescribeConfigsRequest, DescribeConfigsResponse, IncrementalAlterConfigsRequest, + MetadataRequest, MetadataResponse, TopicName, + create_partitions_request::CreatePartitionsTopic, + create_topics_request::{CreatableTopic, CreatableTopicConfig}, + delete_topics_request::DeleteTopicState, + describe_configs_request::DescribeConfigsResource, + incremental_alter_configs_request::{AlterConfigsResource, AlterableConfig}, + metadata_request::MetadataRequestTopic, + }, + protocol::{Request, StrBytes}, +}; +use snafu::{OptionExt, ResultExt, Snafu}; +use tokio::time::Instant; + +use crate::{ + bootstrap::BrokerAddress, + connection::{self, Connection, Security}, +}; + +const METADATA_VERSION: i16 = 12; +const CREATE_TOPICS_VERSION: i16 = 7; +const DELETE_TOPICS_VERSION: i16 = 6; +const CREATE_PARTITIONS_VERSION: i16 = 3; +const DESCRIBE_CONFIGS_VERSION: i16 = 4; +const INCREMENTAL_ALTER_CONFIGS_VERSION: i16 = 1; + +/// Must stay below the connection's request timeout. +const BROKER_TIMEOUT_MS: i32 = 25_000; + +/// Must stay below the broker's `connections.max.idle.ms` (10 minutes by default). +const CONNECTION_MAX_IDLE: Duration = Duration::from_secs(5 * 60); + +const RESOURCE_TYPE_TOPIC: i8 = 2; +const CONFIG_SOURCE_DYNAMIC_TOPIC: i8 = 1; +const CONFIG_OPERATION_SET: i8 = 0; +const CONFIG_OPERATION_DELETE: i8 = 1; + +#[derive(Debug, Snafu)] +pub enum Error { + #[snafu(display("no bootstrap servers given"))] + NoBootstrapServers, + + #[snafu(display("failed to connect to the bootstrap server {server}"))] + Connect { + source: connection::Error, + server: BrokerAddress, + }, + + #[snafu(display("failed to send the request to the broker"))] + Send { source: connection::Error }, + + #[snafu(display( + "the response to the request with API key {api_key} contains no result for the topic {topic:?}" + ))] + MissingResult { api_key: i16, topic: String }, + + #[snafu(display( + "the broker rejected the request with API key {api_key} for the topic {topic:?} with {error}{}", + message.as_deref().map(|message| format!(": {message}")).unwrap_or_default() + ))] + Kafka { + api_key: i16, + topic: String, + error: ResponseError, + message: Option, + }, +} + +type Result = std::result::Result; + +impl Error { + pub fn is_retriable(&self) -> bool { + match self { + Error::Connect { source, .. } | Error::Send { source } => source.is_retriable(), + Error::Kafka { error, .. } => error.is_retriable(), + _ => false, + } + } + + pub fn kafka_error(&self) -> Option { + match self { + Error::Kafka { error, .. } => Some(*error), + _ => None, + } + } +} + +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct Topic { + /// The topic id in Kafka's string format (URL-safe base64 without padding), as shown by + /// `kafka-topics.sh` + pub id: String, + + /// Sorted by the partition index + pub partitions: Vec, +} + +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct Partition { + pub index: i32, + /// Broker ids, including replicas that are added or removed by a reassignment + pub replicas: Vec, +} + +pub struct KafkaAdminClient { + bootstrap_servers: Vec, + security: Security, + client_id: String, + /// The connection and the time of its last successful request + connection: Option<(Connection, Instant)>, +} + +impl KafkaAdminClient { + /// Creates the client without connecting. + pub fn new(bootstrap_servers: Vec, security: Security, client_id: &str) -> Self { + Self { + bootstrap_servers, + security, + client_id: client_id.to_owned(), + connection: None, + } + } + + /// Returns `None` if the topic does not exist. + pub async fn describe_topic(&mut self, topic: &str) -> Result> { + let response = self + .send(describe_topic_request(topic), METADATA_VERSION) + .await?; + describe_topic_result(response, topic) + } + + /// Returns the configs set on the topic itself, without broker or default values. + pub async fn describe_topic_configs( + &mut self, + topic: &str, + ) -> Result> { + let response = self + .send( + describe_topic_configs_request(topic), + DESCRIBE_CONFIGS_VERSION, + ) + .await?; + describe_topic_configs_result(response, topic) + } + + pub async fn create_topic( + &mut self, + topic: &str, + partitions: i32, + replication_factor: i16, + configs: &BTreeMap, + ) -> Result<()> { + let response = self + .send( + create_topic_request(topic, partitions, replication_factor, configs), + CREATE_TOPICS_VERSION, + ) + .await?; + + let result = first(response.topics, CreateTopicsRequest::KEY, topic)?; + check( + CreateTopicsRequest::KEY, + topic, + result.error_code, + result.error_message, + ) + } + + /// Increases the number of partitions to `partitions`. + pub async fn create_partitions(&mut self, topic: &str, partitions: i32) -> Result<()> { + let response = self + .send( + create_partitions_request(topic, partitions), + CREATE_PARTITIONS_VERSION, + ) + .await?; + + let result = first(response.results, CreatePartitionsRequest::KEY, topic)?; + check( + CreatePartitionsRequest::KEY, + topic, + result.error_code, + result.error_message, + ) + } + + /// Sets the `set` configs and resets the `delete` configs to their defaults. Other configs of + /// the topic stay untouched. + pub async fn incremental_alter_configs( + &mut self, + topic: &str, + set: &BTreeMap, + delete: &BTreeSet, + ) -> Result<()> { + let response = self + .send( + incremental_alter_configs_request(topic, set, delete), + INCREMENTAL_ALTER_CONFIGS_VERSION, + ) + .await?; + + let result = first( + response.responses, + IncrementalAlterConfigsRequest::KEY, + topic, + )?; + check( + IncrementalAlterConfigsRequest::KEY, + topic, + result.error_code, + result.error_message, + ) + } + + /// Succeeds if the topic does not exist. + pub async fn delete_topic(&mut self, topic: &str) -> Result<()> { + let response = self + .send(delete_topic_request(topic), DELETE_TOPICS_VERSION) + .await?; + delete_topic_result(response, topic) + } + + /// Sends the request on the current connection or a new one. A failed or cancelled request + /// drops the connection, and a connection idle for longer than [`CONNECTION_MAX_IDLE`] is + /// replaced. + async fn send(&mut self, request: R, version: i16) -> Result { + let mut connection = match self.connection.take() { + Some((connection, last_used)) if last_used.elapsed() < CONNECTION_MAX_IDLE => { + connection + } + _ => self.connect().await?, + }; + let response = connection.send(request, version).await.context(SendSnafu)?; + self.connection = Some((connection, Instant::now())); + Ok(response) + } + + /// Connects to the first bootstrap server that answers. Any broker works, because brokers in + /// KRaft mode forward admin requests to the active controller. + async fn connect(&self) -> Result { + let mut last_error = Error::NoBootstrapServers; + for server in &self.bootstrap_servers { + match Connection::connect(&server.host, server.port, &self.security, &self.client_id) + .await + { + Ok(connection) => return Ok(connection), + Err(source) => { + last_error = Error::Connect { + source, + server: server.clone(), + }; + } + } + } + Err(last_error) + } +} + +fn describe_topic_request(topic: &str) -> MetadataRequest { + MetadataRequest::default() + .with_topics(Some(vec![ + MetadataRequestTopic::default().with_name(Some(topic_name(topic))), + ])) + .with_allow_auto_topic_creation(false) +} + +fn describe_topic_result(response: MetadataResponse, topic: &str) -> Result> { + let result = first(response.topics, MetadataRequest::KEY, topic)?; + if result.error_code.err() == Some(ResponseError::UnknownTopicOrPartition) { + return Ok(None); + } + check(MetadataRequest::KEY, topic, result.error_code, None)?; + + let mut partitions = result + .partitions + .into_iter() + .map(|partition| Partition { + index: partition.partition_index, + replicas: partition.replica_nodes.into_iter().map(|id| *id).collect(), + }) + .collect::>(); + partitions.sort_by_key(|partition| partition.index); + Ok(Some(Topic { + id: URL_SAFE_NO_PAD.encode(result.topic_id.as_bytes()), + partitions, + })) +} + +fn describe_topic_configs_request(topic: &str) -> DescribeConfigsRequest { + DescribeConfigsRequest::default().with_resources(vec![ + DescribeConfigsResource::default() + .with_resource_type(RESOURCE_TYPE_TOPIC) + .with_resource_name(StrBytes::from_string(topic.to_owned())), + ]) +} + +fn describe_topic_configs_result( + response: DescribeConfigsResponse, + topic: &str, +) -> Result> { + let result = first(response.results, DescribeConfigsRequest::KEY, topic)?; + check( + DescribeConfigsRequest::KEY, + topic, + result.error_code, + result.error_message, + )?; + + Ok(result + .configs + .into_iter() + .filter(|config| config.config_source == CONFIG_SOURCE_DYNAMIC_TOPIC) + .filter_map(|config| Some((config.name.to_string(), config.value?.to_string()))) + .collect()) +} + +fn create_topic_request( + topic: &str, + partitions: i32, + replication_factor: i16, + configs: &BTreeMap, +) -> CreateTopicsRequest { + let configs = configs + .iter() + .map(|(name, value)| { + CreatableTopicConfig::default() + .with_name(StrBytes::from_string(name.clone())) + .with_value(Some(StrBytes::from_string(value.clone()))) + }) + .collect(); + CreateTopicsRequest::default() + .with_topics(vec![ + CreatableTopic::default() + .with_name(topic_name(topic)) + .with_num_partitions(partitions) + .with_replication_factor(replication_factor) + .with_configs(configs), + ]) + .with_timeout_ms(BROKER_TIMEOUT_MS) +} + +fn create_partitions_request(topic: &str, partitions: i32) -> CreatePartitionsRequest { + CreatePartitionsRequest::default() + .with_topics(vec![ + CreatePartitionsTopic::default() + .with_name(topic_name(topic)) + .with_count(partitions) + .with_assignments(None), + ]) + .with_timeout_ms(BROKER_TIMEOUT_MS) +} + +fn incremental_alter_configs_request( + topic: &str, + set: &BTreeMap, + delete: &BTreeSet, +) -> IncrementalAlterConfigsRequest { + let set = set.iter().map(|(name, value)| { + AlterableConfig::default() + .with_name(StrBytes::from_string(name.clone())) + .with_config_operation(CONFIG_OPERATION_SET) + .with_value(Some(StrBytes::from_string(value.clone()))) + }); + let delete = delete.iter().map(|name| { + AlterableConfig::default() + .with_name(StrBytes::from_string(name.clone())) + .with_config_operation(CONFIG_OPERATION_DELETE) + .with_value(None) + }); + // We can send all config operations (set & delete) together as one list + IncrementalAlterConfigsRequest::default().with_resources(vec![ + AlterConfigsResource::default() + .with_resource_type(RESOURCE_TYPE_TOPIC) + .with_resource_name(StrBytes::from_string(topic.to_owned())) + .with_configs(set.chain(delete).collect()), + ]) +} + +fn delete_topic_request(topic: &str) -> DeleteTopicsRequest { + DeleteTopicsRequest::default() + .with_topics(vec![ + DeleteTopicState::default().with_name(Some(topic_name(topic))), + ]) + .with_timeout_ms(BROKER_TIMEOUT_MS) +} + +fn delete_topic_result(response: DeleteTopicsResponse, topic: &str) -> Result<()> { + let result = first(response.responses, DeleteTopicsRequest::KEY, topic)?; + // It's ok if topic or partition are already being deleted + if matches!( + result.error_code.err(), + Some(ResponseError::UnknownTopicOrPartition | ResponseError::UnknownTopicId) + ) { + return Ok(()); + } + check( + DeleteTopicsRequest::KEY, + topic, + result.error_code, + result.error_message, + ) +} + +fn topic_name(topic: &str) -> TopicName { + TopicName::from(StrBytes::from_string(topic.to_owned())) +} + +// Used instead of [0] to correctly handle empty results & take ownership +fn first(results: Vec, api_key: i16, topic: &str) -> Result { + results + .into_iter() + .next() + .context(MissingResultSnafu { api_key, topic }) +} + +fn check(api_key: i16, topic: &str, error_code: i16, message: Option) -> Result<()> { + match error_code.err() { + None => Ok(()), + Some(error) => KafkaSnafu { + api_key, + topic, + error, + message: message.map(|message| message.to_string()), + } + .fail(), + } +} + +#[cfg(test)] +mod tests { + use kafka_protocol::messages::{ + delete_topics_response::DeletableTopicResult, + describe_configs_response::{DescribeConfigsResourceResult, DescribeConfigsResult}, + metadata_response::{MetadataResponsePartition, MetadataResponseTopic}, + }; + use rstest::rstest; + use tokio::net::TcpStream; + + use super::*; + use crate::connection::test_support::{accept, listen, read_request_body, write_response}; + + const TOPIC: &str = "orders"; + const APIS: &[(i16, i16, i16)] = &[(DeleteTopicsRequest::KEY, 0, DELETE_TOPICS_VERSION)]; + + #[tokio::test] + async fn try_the_next_bootstrap_server() { + let (unreachable, unreachable_port) = listen().await; + drop(unreachable); + let (listener, port) = listen().await; + let mut client = kafka_admin_client(&[unreachable_port, port]); + + let broker_side = async { + let mut broker = accept(&listener, APIS).await; + answer_delete_topics(&mut broker).await; + }; + let (result, ()) = tokio::join!(client.delete_topic(TOPIC), broker_side); + + result.expect("the second bootstrap server should be used"); + } + + #[tokio::test] + async fn reconnect_after_a_failed_request() { + let (listener, port) = listen().await; + let mut client = kafka_admin_client(&[port]); + + let broker_side = async { + let mut broker = accept(&listener, APIS).await; + read_request_body::(&mut broker).await; + drop(broker); + }; + let (result, ()) = tokio::join!(client.delete_topic(TOPIC), broker_side); + assert!(matches!( + result.expect_err("the broker should have closed the connection"), + Error::Send { .. } + )); + + let broker_side = async { + let mut broker = accept(&listener, APIS).await; + answer_delete_topics(&mut broker).await; + }; + let (result, ()) = tokio::join!(client.delete_topic(TOPIC), broker_side); + + result.expect("the client should reconnect"); + } + + #[test] + fn describe_topic_by_name_without_auto_creation() { + let request = describe_topic_request(TOPIC); + + let topics = request.topics.expect("the topics should be requested"); + assert_eq!(Some(topic_name(TOPIC)), topics[0].name); + assert!(!request.allow_auto_topic_creation); + } + + #[test] + fn describe_existing_topic() { + let response = MetadataResponse::default().with_topics(vec![ + MetadataResponseTopic::default() + .with_name(Some(topic_name(TOPIC))) + .with_topic_id( + "517c94f6-2e40-467e-96d3-438ddad69e58" + .parse() + .expect("the topic id should be valid"), + ) + .with_partitions(vec![ + MetadataResponsePartition::default() + .with_partition_index(1) + .with_replica_nodes(vec![2.into(), 3.into()]), + MetadataResponsePartition::default() + .with_partition_index(0) + .with_replica_nodes(vec![1.into(), 2.into()]), + ]), + ]); + + assert_eq!( + Some(Topic { + id: "UXyU9i5ARn6W00ON2taeWA".to_owned(), + partitions: vec![ + Partition { + index: 0, + replicas: vec![1, 2] + }, + Partition { + index: 1, + replicas: vec![2, 3] + }, + ] + }), + describe_topic_result(response, TOPIC).expect("the topic should be described") + ); + } + + #[test] + fn describe_missing_topic() { + let response = MetadataResponse::default().with_topics(vec![ + MetadataResponseTopic::default() + .with_error_code(ResponseError::UnknownTopicOrPartition.code()), + ]); + + assert_eq!( + None, + describe_topic_result(response, TOPIC).expect("a missing topic should not be an error") + ); + } + + #[test] + fn describe_topic_configs_of_the_topic_resource() { + let request = describe_topic_configs_request(TOPIC); + + assert_eq!(RESOURCE_TYPE_TOPIC, request.resources[0].resource_type); + assert_eq!(TOPIC, request.resources[0].resource_name.as_str()); + } + + #[test] + fn describe_only_dynamic_topic_configs() { + let config = |name: &'static str, value: &'static str, source: i8| { + DescribeConfigsResourceResult::default() + .with_name(StrBytes::from_static_str(name)) + .with_value(Some(StrBytes::from_static_str(value))) + .with_config_source(source) + }; + let response = DescribeConfigsResponse::default().with_results(vec![ + DescribeConfigsResult::default().with_configs(vec![ + config("retention.ms", "1000", CONFIG_SOURCE_DYNAMIC_TOPIC), + config("cleanup.policy", "delete", 5), + ]), + ]); + + assert_eq!( + BTreeMap::from([("retention.ms".to_owned(), "1000".to_owned())]), + describe_topic_configs_result(response, TOPIC) + .expect("the configs should be described") + ); + } + + #[test] + fn create_topic_with_configs() { + let configs = BTreeMap::from([("retention.ms".to_owned(), "1000".to_owned())]); + + let request = create_topic_request(TOPIC, 3, 2, &configs); + + let topic = &request.topics[0]; + assert_eq!(topic_name(TOPIC), topic.name); + assert_eq!(3, topic.num_partitions); + assert_eq!(2, topic.replication_factor); + assert!(topic.assignments.is_empty()); + assert_eq!("retention.ms", topic.configs[0].name.as_str()); + assert_eq!( + Some("1000"), + topic.configs[0].value.as_ref().map(|value| value.as_str()) + ); + assert_eq!(BROKER_TIMEOUT_MS, request.timeout_ms); + } + + #[test] + fn create_partitions_without_assignments() { + let request = create_partitions_request(TOPIC, 6); + + assert_eq!(topic_name(TOPIC), request.topics[0].name); + assert_eq!(6, request.topics[0].count); + assert_eq!(None, request.topics[0].assignments); + assert_eq!(BROKER_TIMEOUT_MS, request.timeout_ms); + } + + #[test] + fn set_and_delete_configs_in_one_request() { + let set = BTreeMap::from([("retention.ms".to_owned(), "1000".to_owned())]); + let delete = BTreeSet::from(["cleanup.policy".to_owned()]); + + let request = incremental_alter_configs_request(TOPIC, &set, &delete); + + let resource = &request.resources[0]; + assert_eq!(RESOURCE_TYPE_TOPIC, resource.resource_type); + assert_eq!(TOPIC, resource.resource_name.as_str()); + let configs = resource + .configs + .iter() + .map(|config| { + ( + config.name.as_str(), + config.config_operation, + config.value.as_ref().map(|value| value.as_str()), + ) + }) + .collect::>(); + assert_eq!( + vec![ + ("retention.ms", CONFIG_OPERATION_SET, Some("1000")), + ("cleanup.policy", CONFIG_OPERATION_DELETE, None), + ], + configs + ); + } + + #[test] + fn delete_topic_by_name() { + let request = delete_topic_request(TOPIC); + + assert_eq!(Some(topic_name(TOPIC)), request.topics[0].name); + assert!(request.topics[0].topic_id.is_nil()); + assert_eq!(BROKER_TIMEOUT_MS, request.timeout_ms); + } + + #[rstest] + #[case(ResponseError::UnknownTopicOrPartition)] + #[case(ResponseError::UnknownTopicId)] + fn delete_missing_topic(#[case] error: ResponseError) { + let response = DeleteTopicsResponse::default().with_responses(vec![ + DeletableTopicResult::default().with_error_code(error.code()), + ]); + + delete_topic_result(response, TOPIC).expect("deleting a missing topic should succeed"); + } + + #[test] + fn report_rejected_request() { + let error = check( + CreateTopicsRequest::KEY, + TOPIC, + ResponseError::TopicAlreadyExists.code(), + Some(StrBytes::from_static_str("Topic 'orders' already exists.")), + ) + .expect_err("the request should be rejected"); + + assert_eq!(Some(ResponseError::TopicAlreadyExists), error.kafka_error()); + assert!(!error.is_retriable()); + assert!(error.to_string().contains("Topic 'orders' already exists.")); + } + + #[test] + fn reject_response_without_result() { + let error = first( + Vec::::new(), + DeleteTopicsRequest::KEY, + TOPIC, + ) + .expect_err("the result should be missing"); + + assert!(matches!(error, Error::MissingResult { .. })); + } + + fn kafka_admin_client(ports: &[u16]) -> KafkaAdminClient { + let bootstrap_servers = ports + .iter() + .map(|port| BrokerAddress { + host: "127.0.0.1".to_owned(), + port: *port, + }) + .collect(); + KafkaAdminClient::new(bootstrap_servers, Security::Plaintext, "test") + } + + async fn answer_delete_topics(broker: &mut TcpStream) { + let (header, _) = read_request_body::(broker).await; + let response = + DeleteTopicsResponse::default().with_responses(vec![DeletableTopicResult::default()]); + write_response(broker, &header, header.correlation_id, response).await; + } +} diff --git a/rust/kafka-admin/src/codec.rs b/rust/kafka-admin/src/codec.rs new file mode 100644 index 000000000..0c69347df --- /dev/null +++ b/rust/kafka-admin/src/codec.rs @@ -0,0 +1,121 @@ +//! Encoding of request and decoding of response frames: a header followed by a message body. The +//! length prefix is added and removed by the connection. + +use bytes::{Bytes, BytesMut}; +use kafka_protocol::{ + messages::{RequestHeader, ResponseHeader}, + protocol::{Decodable, Encodable, HeaderVersion, Request, StrBytes}, +}; +use snafu::{ResultExt, Snafu, ensure}; + +#[derive(Debug, Snafu)] +pub enum Error { + #[snafu(display("failed to encode the request with API key {api_key}"))] + EncodeRequest { source: anyhow::Error, api_key: i16 }, + + #[snafu(display("failed to decode the response with API key {api_key}"))] + DecodeResponse { source: anyhow::Error, api_key: i16 }, + + #[snafu(display("expected correlation id {expected}, received {received}"))] + CorrelationIdMismatch { expected: i32, received: i32 }, +} + +pub fn encode_request( + request: &R, + version: i16, + correlation_id: i32, + client_id: &str, +) -> Result { + let header = RequestHeader::default() + .with_request_api_key(R::KEY) + .with_request_api_version(version) + .with_correlation_id(correlation_id) + .with_client_id(Some(StrBytes::from_string(client_id.to_owned()))); + + let mut frame = BytesMut::new(); + header + .encode(&mut frame, R::header_version(version)) + .context(EncodeRequestSnafu { api_key: R::KEY })?; + request + .encode(&mut frame, version) + .context(EncodeRequestSnafu { api_key: R::KEY })?; + Ok(frame.freeze()) +} + +pub fn decode_response( + mut frame: Bytes, + version: i16, + correlation_id: i32, +) -> Result { + let header = ResponseHeader::decode(&mut frame, R::Response::header_version(version)) + .context(DecodeResponseSnafu { api_key: R::KEY })?; + ensure!( + header.correlation_id == correlation_id, + CorrelationIdMismatchSnafu { + expected: correlation_id, + received: header.correlation_id, + } + ); + R::Response::decode(&mut frame, version).context(DecodeResponseSnafu { api_key: R::KEY }) +} + +#[cfg(test)] +mod tests { + use kafka_protocol::{ + messages::{MetadataRequest, MetadataResponse}, + protocol::decode_request_header_from_buffer, + }; + + use super::*; + + #[test] + fn encode_request_with_header() { + let mut frame = encode_request(&MetadataRequest::default(), 12, 7, "agent") + .expect("the request should be encodable"); + + let header = + decode_request_header_from_buffer(&mut frame).expect("the header should be decodable"); + assert_eq!(3, header.request_api_key); + assert_eq!(12, header.request_api_version); + assert_eq!(7, header.correlation_id); + assert_eq!(Some("agent"), header.client_id.as_deref()); + } + + fn response_frame(correlation_id: i32, version: i16, response: &MetadataResponse) -> Bytes { + let mut frame = BytesMut::new(); + ResponseHeader::default() + .with_correlation_id(correlation_id) + .encode(&mut frame, MetadataResponse::header_version(version)) + .expect("the header should be encodable"); + response + .encode(&mut frame, version) + .expect("the response should be encodable"); + frame.freeze() + } + + #[test] + fn decode_response_with_matching_correlation_id() { + let response = MetadataResponse::default().with_controller_id(2.into()); + + let decoded = decode_response::(response_frame(7, 12, &response), 12, 7) + .expect("the response should be decodable"); + + assert_eq!(response, decoded); + } + + #[test] + fn reject_response_with_other_correlation_id() { + let frame = response_frame(8, 12, &MetadataResponse::default()); + + let error = decode_response::(frame, 12, 7) + .expect_err("the correlation id should not match"); + + assert!(matches!( + error, + Error::CorrelationIdMismatch { + expected: 7, + received: 8 + } + )); + } +} diff --git a/rust/kafka-admin/src/connection.rs b/rust/kafka-admin/src/connection.rs new file mode 100644 index 000000000..d4a29ef2e --- /dev/null +++ b/rust/kafka-admin/src/connection.rs @@ -0,0 +1,598 @@ +//! A connection to a single Kafka broker. + +use std::{collections::BTreeMap, io, path::PathBuf, sync::Arc, time::Duration}; + +use futures::{SinkExt, StreamExt}; +use kafka_protocol::{ + ResponseError, + error::ParseResponseErrorCode, + messages::{ApiVersionsRequest, ApiVersionsResponse}, + protocol::{Request, StrBytes, VersionRange}, +}; +use rustls::{ClientConfig, pki_types::ServerName}; +use snafu::{OptionExt, ResultExt, Snafu, ensure}; +use tokio::{ + net::TcpStream, + time::{error::Elapsed, timeout}, +}; +use tokio_rustls::{TlsConnector, client::TlsStream}; +use tokio_util::{ + codec::{Framed, LengthDelimitedCodec}, + either::Either, +}; + +use crate::{ + codec::{self, decode_response, encode_request}, + tls, +}; + +const CONNECT_TIMEOUT: Duration = Duration::from_secs(10); +const REQUEST_TIMEOUT: Duration = Duration::from_secs(30); +const MAX_FRAME_SIZE: usize = 64 * 1024 * 1024; +const API_VERSIONS_VERSION: i16 = 3; + +#[derive(Debug, Snafu)] +pub enum Error { + #[snafu(display("failed to connect to {address:?}"))] + Connect { source: io::Error, address: String }, + + #[snafu(display("timed out connecting to {address:?}"))] + ConnectTimeout { source: Elapsed, address: String }, + + #[snafu(display("failed to configure TLS"))] + Tls { source: tls::Error }, + + #[snafu(display("{host:?} is not a valid TLS server name"))] + InvalidServerName { host: String }, + + #[snafu(display("failed the TLS handshake with {address:?}"))] + TlsHandshake { source: io::Error, address: String }, + + #[snafu(display("failed to write to the broker"))] + Write { source: io::Error }, + + #[snafu(display("failed to read from the broker"))] + Read { source: io::Error }, + + #[snafu(display("the broker closed the connection"))] + ConnectionClosed, + + #[snafu(display("the connection is broken by an earlier failed or cancelled request"))] + ConnectionBroken, + + #[snafu(display("timed out waiting for the response to the request with API key {api_key}"))] + RequestTimeout { source: Elapsed, api_key: i16 }, + + #[snafu(display("failed to encode or decode a message"))] + Codec { source: codec::Error }, + + #[snafu(display("the broker returned {error:?} for the request with API key {api_key}"))] + Kafka { error: ResponseError, api_key: i16 }, + + #[snafu(display("the broker does not support the API key {api_key}"))] + UnsupportedApi { api_key: i16 }, + + #[snafu(display( + "the broker supports the versions {broker_versions} of the API key {api_key}, but not {version}" + ))] + UnsupportedVersion { + api_key: i16, + version: i16, + broker_versions: VersionRange, + }, +} + +type Result = std::result::Result; + +impl Error { + pub fn is_retriable(&self) -> bool { + match self { + Error::Connect { .. } + | Error::ConnectTimeout { .. } + | Error::TlsHandshake { .. } + | Error::Write { .. } + | Error::Read { .. } + | Error::ConnectionClosed + | Error::ConnectionBroken + | Error::RequestTimeout { .. } => true, + Error::Kafka { error, .. } => error.is_retriable(), + _ => false, + } + } +} + +/// How to connect to the broker. Paths point to PEM files. +#[derive(Clone, Debug, Eq, PartialEq)] +pub enum Security { + Plaintext, + Tls { + ca: PathBuf, + }, + MutualTls { + ca: PathBuf, + certificate: PathBuf, + key: PathBuf, + }, +} + +pub struct Connection { + stream: Framed>, LengthDelimitedCodec>, + client_id: String, + correlation_id: i32, + awaiting_response: bool, + broker_api_versions: BTreeMap, +} + +impl Connection { + /// Connects to the broker and fetches the API versions it supports. + pub async fn connect( + host: &str, + port: u16, + security: &Security, + client_id: &str, + ) -> Result { + let address = format!("{host}:{port}"); + let tcp_stream = connect_tcp(host, port, &address).await?; + let stream = match tls::client_config(security).context(TlsSnafu)? { + None => Either::Left(tcp_stream), + Some(config) => Either::Right(tls_handshake(tcp_stream, host, &address, config).await?), + }; + + let mut connection = Self::new(stream, client_id); + let response = connection + .send_with_version(api_versions_request(), API_VERSIONS_VERSION) + .await?; + connection.broker_api_versions = api_versions_result(response)?; + Ok(connection) + } + + fn new(stream: Either>, client_id: &str) -> Self { + let codec = LengthDelimitedCodec::builder() + .length_field_type::() + .max_frame_length(MAX_FRAME_SIZE) + .new_codec(); + Self { + stream: Framed::new(stream, codec), + client_id: client_id.to_owned(), + correlation_id: 0, + awaiting_response: false, + broker_api_versions: BTreeMap::new(), + } + } + + /// Sends the request with the given version, if the broker supports it. + pub async fn send(&mut self, request: R, version: i16) -> Result { + // Each API that is relevant for us has a version that is supported by all of our Kafka versions. + // We use these versions and pin them in client.rs. + // We will have to add API version negotiation at some point in the future. + check_version(&self.broker_api_versions, R::KEY, version)?; + self.send_with_version(request, version).await + } + + async fn send_with_version( + &mut self, + request: R, + version: i16, + ) -> Result { + ensure!(!self.awaiting_response, ConnectionBrokenSnafu); + self.awaiting_response = true; + + self.correlation_id = self.correlation_id.wrapping_add(1); + let frame = encode_request(&request, version, self.correlation_id, &self.client_id) + .context(CodecSnafu)?; + + let response = timeout(REQUEST_TIMEOUT, async { + self.stream.send(frame).await.context(WriteSnafu)?; + self.stream + .next() + .await + .context(ConnectionClosedSnafu)? + .context(ReadSnafu) + }) + .await + .context(RequestTimeoutSnafu { api_key: R::KEY })??; + + let response = decode_response::(response.freeze(), version, self.correlation_id) + .context(CodecSnafu)?; + + self.awaiting_response = false; + Ok(response) + } +} + +async fn connect_tcp(host: &str, port: u16, address: &str) -> Result { + timeout(CONNECT_TIMEOUT, TcpStream::connect((host, port))) + .await + .context(ConnectTimeoutSnafu { address })? + .context(ConnectSnafu { address }) +} + +fn api_versions_request() -> ApiVersionsRequest { + ApiVersionsRequest::default() + .with_client_software_name(StrBytes::from_static_str("stackable-kafka-admin")) + .with_client_software_version(StrBytes::from_static_str(env!("CARGO_PKG_VERSION"))) +} + +fn api_versions_result(response: ApiVersionsResponse) -> Result> { + if let Some(error) = response.error_code.err() { + return KafkaSnafu { + error, + api_key: ApiVersionsRequest::KEY, + } + .fail(); + } + + Ok(response + .api_keys + .iter() + .map(|api| { + ( + api.api_key, + VersionRange { + min: api.min_version, + max: api.max_version, + }, + ) + }) + .collect()) +} + +fn check_version( + broker_api_versions: &BTreeMap, + api_key: i16, + version: i16, +) -> Result<()> { + let broker_versions = broker_api_versions + .get(&api_key) + .context(UnsupportedApiSnafu { api_key })?; + ensure!( + (broker_versions.min..=broker_versions.max).contains(&version), + UnsupportedVersionSnafu { + api_key, + version, + broker_versions: *broker_versions, + } + ); + Ok(()) +} + +async fn tls_handshake( + tcp_stream: TcpStream, + host: &str, + address: &str, + config: ClientConfig, +) -> Result> { + let server_name = ServerName::try_from(host.to_owned()) + .ok() + .context(InvalidServerNameSnafu { host })?; + timeout( + CONNECT_TIMEOUT, + TlsConnector::from(Arc::new(config)).connect(server_name, tcp_stream), + ) + .await + .context(ConnectTimeoutSnafu { address })? + .context(TlsHandshakeSnafu { address }) +} + +/// A mock Kafka broker for tests, speaking the wire protocol over a local TCP socket. +#[cfg(test)] +pub(crate) mod test_support { + use bytes::{BufMut, Bytes, BytesMut}; + use kafka_protocol::{ + messages::{ + ApiVersionsRequest, ApiVersionsResponse, RequestHeader, ResponseHeader, + api_versions_response::ApiVersion, + }, + protocol::{Encodable, HeaderVersion, Request, decode_request_header_from_buffer}, + }; + use tokio::{ + io::{AsyncReadExt, AsyncWriteExt}, + net::{TcpListener, TcpStream}, + }; + + use super::API_VERSIONS_VERSION; + + pub async fn listen() -> (TcpListener, u16) { + let listener = TcpListener::bind("127.0.0.1:0") + .await + .expect("the mock broker should be able to listen"); + let port = listener + .local_addr() + .expect("the mock broker should have an address") + .port(); + (listener, port) + } + + /// Accepts a connection and answers its ApiVersions request with the given + /// `(api_key, min_version, max_version)` ranges. + pub async fn accept(listener: &TcpListener, apis: &[(i16, i16, i16)]) -> TcpStream { + let (mut broker, _) = listener + .accept() + .await + .expect("the mock broker should accept the connection"); + let (request, _) = read_request(&mut broker).await; + assert_eq!(ApiVersionsRequest::KEY, request.request_api_key); + assert_eq!(API_VERSIONS_VERSION, request.request_api_version); + write_response( + &mut broker, + &request, + request.correlation_id, + api_versions(apis), + ) + .await; + broker + } + + pub async fn read_request(broker: &mut TcpStream) -> (RequestHeader, Bytes) { + let size = broker + .read_i32() + .await + .expect("the request size should be readable"); + let mut frame = vec![0; size as usize]; + broker + .read_exact(&mut frame) + .await + .expect("the request should be readable"); + let mut frame = Bytes::from(frame); + let header = decode_request_header_from_buffer(&mut frame) + .expect("the request header should be decodable"); + (header, frame) + } + + pub async fn read_request_body(broker: &mut TcpStream) -> (RequestHeader, R) { + let (header, mut body) = read_request(broker).await; + assert_eq!(R::KEY, header.request_api_key); + let request = R::decode(&mut body, header.request_api_version) + .expect("the request should be decodable"); + (header, request) + } + + pub async fn write_response( + broker: &mut TcpStream, + request: &RequestHeader, + correlation_id: i32, + response: M, + ) { + let mut body = BytesMut::new(); + ResponseHeader::default() + .with_correlation_id(correlation_id) + .encode(&mut body, M::header_version(request.request_api_version)) + .expect("the response header should be encodable"); + response + .encode(&mut body, request.request_api_version) + .expect("the response should be encodable"); + let mut frame = BytesMut::new(); + frame.put_i32(body.len() as i32); + frame.extend_from_slice(&body); + broker + .write_all(&frame) + .await + .expect("the response should be writable"); + } + + fn api_versions(apis: &[(i16, i16, i16)]) -> ApiVersionsResponse { + ApiVersionsResponse::default().with_api_keys( + apis.iter() + .map(|(api_key, min_version, max_version)| { + ApiVersion::default() + .with_api_key(*api_key) + .with_min_version(*min_version) + .with_max_version(*max_version) + }) + .collect(), + ) + } +} + +#[cfg(test)] +mod tests { + use kafka_protocol::messages::{ + MetadataRequest, MetadataResponse, api_versions_response::ApiVersion, + }; + use tokio::io::AsyncWriteExt; + + use super::{ + test_support::{accept, listen, read_request, write_response}, + *, + }; + + /// Connects to a mock broker supporting Metadata versions 0 to 13. + async fn connect() -> (Connection, TcpStream) { + let (listener, port) = listen().await; + let (connection, broker) = tokio::join!( + Connection::connect("127.0.0.1", port, &Security::Plaintext, "test"), + accept(&listener, &[(MetadataRequest::KEY, 0, 13)]) + ); + (connection.expect("the connection should succeed"), broker) + } + + #[test] + fn read_api_versions() { + let response = ApiVersionsResponse::default().with_api_keys(vec![ + ApiVersion::default() + .with_api_key(MetadataRequest::KEY) + .with_min_version(0) + .with_max_version(13), + ]); + + assert_eq!( + BTreeMap::from([(MetadataRequest::KEY, VersionRange { min: 0, max: 13 })]), + api_versions_result(response).expect("the API versions should be valid") + ); + } + + #[test] + fn reject_api_versions_with_error() { + let response = ApiVersionsResponse::default() + .with_error_code(ResponseError::UnsupportedVersion.code()); + + let error = api_versions_result(response).expect_err("the response should be rejected"); + + assert!(matches!( + error, + Error::Kafka { + error: ResponseError::UnsupportedVersion, + .. + } + )); + } + + #[test] + fn accept_supported_version() { + let broker_api_versions = + BTreeMap::from([(MetadataRequest::KEY, VersionRange { min: 0, max: 13 })]); + + check_version(&broker_api_versions, MetadataRequest::KEY, 12) + .expect("the version should be supported"); + } + + #[test] + fn reject_api_unknown_to_the_broker() { + let error = check_version(&BTreeMap::new(), MetadataRequest::KEY, 12) + .expect_err("the API should be unsupported"); + + assert!(matches!(error, Error::UnsupportedApi { .. })); + } + + #[test] + fn reject_version_unsupported_by_the_broker() { + let broker_api_versions = + BTreeMap::from([(MetadataRequest::KEY, VersionRange { min: 0, max: 9 })]); + + let error = check_version(&broker_api_versions, MetadataRequest::KEY, 12) + .expect_err("the version should be unsupported"); + + assert!(matches!( + error, + Error::UnsupportedVersion { version: 12, .. } + )); + } + + #[tokio::test] + async fn send_request_and_receive_response() { + let (mut connection, mut broker) = connect().await; + + let broker_side = async { + let (request, _) = read_request(&mut broker).await; + assert_eq!(MetadataRequest::KEY, request.request_api_key); + assert_eq!(12, request.request_api_version); + write_response( + &mut broker, + &request, + request.correlation_id, + MetadataResponse::default().with_controller_id(2.into()), + ) + .await; + }; + let (response, ()) = + tokio::join!(connection.send(MetadataRequest::default(), 12), broker_side); + + assert_eq!( + 2, + *response.expect("the request should succeed").controller_id + ); + } + + #[tokio::test] + async fn reject_response_with_other_correlation_id() { + let (mut connection, mut broker) = connect().await; + + let broker_side = async { + let (request, _) = read_request(&mut broker).await; + write_response( + &mut broker, + &request, + request.correlation_id + 1, + MetadataResponse::default(), + ) + .await; + }; + let (response, ()) = + tokio::join!(connection.send(MetadataRequest::default(), 12), broker_side); + + assert!(matches!( + response.expect_err("the correlation id should not match"), + Error::Codec { + source: codec::Error::CorrelationIdMismatch { .. } + } + )); + } + + #[tokio::test] + async fn reject_invalid_frame_size() { + let (mut connection, mut broker) = connect().await; + + let broker_side = async { + read_request(&mut broker).await; + broker + .write_i32(-1) + .await + .expect("the frame size should be writable"); + }; + let (response, ()) = + tokio::join!(connection.send(MetadataRequest::default(), 12), broker_side); + + assert!(matches!( + response.expect_err("the frame size should be invalid"), + Error::Read { .. } + )); + } + + #[tokio::test] + async fn time_out_without_response() { + let (mut connection, mut broker) = connect().await; + tokio::time::pause(); + + let broker_side = async { + read_request(&mut broker).await; + }; + let (response, ()) = + tokio::join!(connection.send(MetadataRequest::default(), 12), broker_side); + + assert!(matches!( + response.expect_err("the request should time out"), + Error::RequestTimeout { .. } + )); + } + + #[tokio::test] + async fn reject_requests_after_a_failed_request() { + let (mut connection, mut broker) = connect().await; + + let broker_side = async { + read_request(&mut broker).await; + drop(broker); + }; + let (failed, ()) = + tokio::join!(connection.send(MetadataRequest::default(), 12), broker_side); + assert!(matches!( + failed.expect_err("the broker should have closed the connection"), + Error::ConnectionClosed + )); + + let error = connection + .send(MetadataRequest::default(), 12) + .await + .expect_err("the connection should be broken"); + + assert!(matches!(error, Error::ConnectionBroken)); + } + + #[tokio::test] + async fn reject_requests_after_a_cancelled_request() { + let (mut connection, _broker) = connect().await; + + { + let request = connection.send(MetadataRequest::default(), 12); + tokio::pin!(request); + let _ = futures::poll!(request.as_mut()); + } + + let error = connection + .send(MetadataRequest::default(), 12) + .await + .expect_err("the connection should be broken"); + + assert!(matches!(error, Error::ConnectionBroken)); + } +} diff --git a/rust/kafka-admin/src/lib.rs b/rust/kafka-admin/src/lib.rs new file mode 100644 index 000000000..cb1f00ea3 --- /dev/null +++ b/rust/kafka-admin/src/lib.rs @@ -0,0 +1,7 @@ +#![allow(clippy::result_large_err)] + +pub mod bootstrap; +pub mod client; +mod codec; +pub mod connection; +pub mod tls; diff --git a/rust/kafka-admin/src/tls.rs b/rust/kafka-admin/src/tls.rs new file mode 100644 index 000000000..6e23a5123 --- /dev/null +++ b/rust/kafka-admin/src/tls.rs @@ -0,0 +1,106 @@ +//! TLS client configuration from PEM files. + +use std::{ + path::{Path, PathBuf}, + sync::Arc, +}; + +use rustls::{ + ClientConfig, RootCertStore, + pki_types::{CertificateDer, PrivateKeyDer, pem::PemObject}, +}; +use snafu::{ResultExt, Snafu, ensure}; + +use crate::connection::Security; + +#[derive(Debug, Snafu)] +pub enum Error { + #[snafu(display("failed to read the PEM file {path:?}"))] + ReadPem { + source: rustls::pki_types::pem::Error, + path: PathBuf, + }, + + #[snafu(display("the PEM file {path:?} contains no certificate"))] + NoCertificate { path: PathBuf }, + + #[snafu(display("failed to add the CA certificate from {path:?}"))] + AddCaCertificate { + source: rustls::Error, + path: PathBuf, + }, + + #[snafu(display("failed to build the TLS configuration"))] + BuildConfig { source: rustls::Error }, +} + +type Result = std::result::Result; + +/// Returns `None` for plaintext connections. The PEM files are read on every call. +pub(crate) fn client_config(security: &Security) -> Result> { + let (ca, client_certificate) = match security { + Security::Plaintext => return Ok(None), + Security::Tls { ca } => (ca, None), + Security::MutualTls { + ca, + certificate, + key, + } => (ca, Some((certificate, key))), + }; + + let builder = + ClientConfig::builder_with_provider(Arc::new(rustls::crypto::ring::default_provider())) + .with_safe_default_protocol_versions() + .context(BuildConfigSnafu)? + .with_root_certificates(root_certificates(ca)?); + + let config = match client_certificate { + None => builder.with_no_client_auth(), + Some((certificate, key)) => { + let key = PrivateKeyDer::from_pem_file(key).context(ReadPemSnafu { path: key })?; + builder + .with_client_auth_cert(certificate_chain(certificate)?, key) + .context(BuildConfigSnafu)? + } + }; + Ok(Some(config)) +} + +fn root_certificates(ca: &Path) -> Result { + let mut root_certificates = RootCertStore::empty(); + for certificate in CertificateDer::pem_file_iter(ca).context(ReadPemSnafu { path: ca })? { + root_certificates + .add(certificate.context(ReadPemSnafu { path: ca })?) + .context(AddCaCertificateSnafu { path: ca })?; + } + ensure!( + !root_certificates.is_empty(), + NoCertificateSnafu { path: ca } + ); + Ok(root_certificates) +} + +fn certificate_chain(path: &Path) -> Result>> { + let certificate_chain = CertificateDer::pem_file_iter(path) + .context(ReadPemSnafu { path })? + .collect::, _>>() + .context(ReadPemSnafu { path })?; + ensure!(!certificate_chain.is_empty(), NoCertificateSnafu { path }); + Ok(certificate_chain) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn reject_ca_file_without_certificate() { + let ca = std::env::temp_dir().join(format!("empty-ca-{}.crt", std::process::id())); + std::fs::write(&ca, "").expect("the CA file should be writable"); + + let error = root_certificates(&ca).expect_err("the CA file should be rejected"); + std::fs::remove_file(&ca).expect("the CA file should be removable"); + + assert!(matches!(error, Error::NoCertificate { .. })); + } +} From c65a16ce8fcc79a4e0ae69a7bab6f5ff9c40a61d Mon Sep 17 00:00:00 2001 From: Benedikt Labrenz Date: Tue, 13 Oct 2026 14:37:51 +0200 Subject: [PATCH 11/93] add shared crate with shared constants and client security enum --- Cargo.lock | 8 ++++++++ Cargo.nix | 34 +++++++++++++++++++++++++++++++++ Cargo.toml | 3 ++- rust/shared/Cargo.toml | 15 +++++++++++++++ rust/shared/src/agent.rs | 41 ++++++++++++++++++++++++++++++++++++++++ rust/shared/src/lib.rs | 1 + 6 files changed, 101 insertions(+), 1 deletion(-) create mode 100644 rust/shared/Cargo.toml create mode 100644 rust/shared/src/agent.rs create mode 100644 rust/shared/src/lib.rs diff --git a/Cargo.lock b/Cargo.lock index be7309270..a87628861 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3285,6 +3285,14 @@ dependencies = [ "tracing", ] +[[package]] +name = "stackable-kafka-shared" +version = "0.0.0-dev" +dependencies = [ + "rstest", + "strum", +] + [[package]] name = "stackable-operator" version = "0.119.0" diff --git a/Cargo.nix b/Cargo.nix index 7003815a1..480c24eec 100644 --- a/Cargo.nix +++ b/Cargo.nix @@ -69,6 +69,16 @@ rec { packageId = "stackable-kafka-operator"; }; + # Debug support which might change between releases. + # File a bug if you depend on any for non-debug work! + debug = internal.debugCrate { inherit packageId; }; + }; + "stackable-kafka-shared" = rec { + packageId = "stackable-kafka-shared"; + build = internal.buildRustCrateWithFeatures { + packageId = "stackable-kafka-shared"; + }; + # Debug support which might change between releases. # File a bug if you depend on any for non-debug work! debug = internal.debugCrate { inherit packageId; }; @@ -10803,6 +10813,30 @@ rec { } ]; + }; + "stackable-kafka-shared" = rec { + crateName = "stackable-kafka-shared"; + version = "0.0.0-dev"; + edition = "2024"; + src = lib.cleanSourceWith { filter = sourceFilter; src = ./rust/shared; }; + libName = "stackable_kafka_shared"; + authors = [ + "Stackable GmbH " + ]; + dependencies = [ + { + name = "strum"; + packageId = "strum"; + features = [ "derive" ]; + } + ]; + devDependencies = [ + { + name = "rstest"; + packageId = "rstest"; + } + ]; + }; "stackable-operator" = rec { crateName = "stackable-operator"; diff --git a/Cargo.toml b/Cargo.toml index 540a2aebc..423ccc605 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,5 +1,5 @@ [workspace] -members = ["rust/operator-binary", "rust/agent-binary", "rust/kafka-admin"] +members = ["rust/operator-binary", "rust/agent-binary", "rust/kafka-admin", "rust/shared"] resolver = "2" [workspace.package] @@ -11,6 +11,7 @@ repository = "https://github.com/stackabletech/kafka-operator" [workspace.dependencies] stackable-operator = { git = "https://github.com/stackabletech/operator-rs.git", branch = "feat/platform-access", features = ["crds", "webhook"] } +stackable-kafka-shared = { path = "rust/shared" } anyhow = "1.0" base64 = "0.22" diff --git a/rust/shared/Cargo.toml b/rust/shared/Cargo.toml new file mode 100644 index 000000000..4164d809c --- /dev/null +++ b/rust/shared/Cargo.toml @@ -0,0 +1,15 @@ +[package] +name = "stackable-kafka-shared" +description = "Types and contracts shared by the Stackable operator and agent for Apache Kafka" +version.workspace = true +authors.workspace = true +license.workspace = true +edition.workspace = true +repository.workspace = true +publish = false + +[dependencies] +strum.workspace = true + +[dev-dependencies] +rstest.workspace = true diff --git a/rust/shared/src/agent.rs b/rust/shared/src/agent.rs new file mode 100644 index 000000000..4af72434c --- /dev/null +++ b/rust/shared/src/agent.rs @@ -0,0 +1,41 @@ +//! Paths and environment variables through which the operator configures the agent. + +use strum::{Display, EnumString}; + +pub const DISCOVERY_DIR: &str = "/stackable/discovery"; + +/// Directory of the CA certificate the Kafka brokers are verified with. +pub const SERVER_CA_DIR: &str = "/stackable/secrets/tls-server-ca"; + +pub const ENV_CLUSTER_NAME: &str = "KAFKA_CLUSTER_NAME"; + +pub const ENV_CLUSTER_NAMESPACE: &str = "KAFKA_CLUSTER_NAMESPACE"; + +pub const ENV_CLIENT_SECURITY: &str = "KAFKA_CLIENT_SECURITY"; + +/// How the agent connects to the brokers. +#[derive(Clone, Copy, Debug, Display, EnumString, Eq, PartialEq)] +#[strum(serialize_all = "kebab-case")] +pub enum ClientSecurity { + Plaintext, + Tls, + MutualTls, +} + +#[cfg(test)] +mod tests { + use std::str::FromStr; + + use rstest::rstest; + + use super::*; + + #[rstest] + #[case::plaintext(ClientSecurity::Plaintext, "plaintext")] + #[case::tls(ClientSecurity::Tls, "tls")] + #[case::mutual_tls(ClientSecurity::MutualTls, "mutual-tls")] + fn roundtrip(#[case] security: ClientSecurity, #[case] value: &str) { + assert_eq!(value, security.to_string()); + assert_eq!(Ok(security), ClientSecurity::from_str(value)); + } +} diff --git a/rust/shared/src/lib.rs b/rust/shared/src/lib.rs new file mode 100644 index 000000000..f17bc55db --- /dev/null +++ b/rust/shared/src/lib.rs @@ -0,0 +1 @@ +pub mod agent; From 41db2c70adf507669c9a81f14d943329d79f08dd Mon Sep 17 00:00:00 2001 From: Benedikt Labrenz Date: Tue, 13 Oct 2026 17:14:37 +0200 Subject: [PATCH 12/93] add KafkaTopic CRD --- Cargo.lock | 7 + Cargo.nix | 34 +++- Cargo.toml | 1 + deploy/helm/kafka-operator/Chart.yaml | 5 + extra/crds.yaml | 138 ++++++++++++++ rust/operator-binary/Cargo.toml | 1 + rust/operator-binary/src/main.rs | 9 +- .../src/webhooks/conversion.rs | 15 +- rust/shared/Cargo.toml | 10 ++ rust/shared/src/crd/mod.rs | 1 + rust/shared/src/crd/topic.rs | 168 ++++++++++++++++++ rust/shared/src/lib.rs | 1 + 12 files changed, 383 insertions(+), 7 deletions(-) create mode 100644 rust/shared/src/crd/mod.rs create mode 100644 rust/shared/src/crd/topic.rs diff --git a/Cargo.lock b/Cargo.lock index a87628861..a50ea695d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3279,6 +3279,7 @@ dependencies = [ "serde_json", "serde_yaml", "snafu 0.9.2", + "stackable-kafka-shared", "stackable-operator", "strum", "tokio", @@ -3289,7 +3290,13 @@ dependencies = [ name = "stackable-kafka-shared" version = "0.0.0-dev" dependencies = [ + "indoc", + "regex", "rstest", + "serde", + "serde_json", + "snafu 0.9.2", + "stackable-operator", "strum", ] diff --git a/Cargo.nix b/Cargo.nix index 480c24eec..95a0e6424 100644 --- a/Cargo.nix +++ b/Cargo.nix @@ -10775,6 +10775,10 @@ rec { name = "snafu"; packageId = "snafu 0.9.2"; } + { + name = "stackable-kafka-shared"; + packageId = "stackable-kafka-shared"; + } { name = "stackable-operator"; packageId = "stackable-operator"; @@ -10824,6 +10828,28 @@ rec { "Stackable GmbH " ]; dependencies = [ + { + name = "regex"; + packageId = "regex"; + } + { + name = "serde"; + packageId = "serde"; + features = [ "derive" ]; + } + { + name = "serde_json"; + packageId = "serde_json"; + } + { + name = "snafu"; + packageId = "snafu 0.9.2"; + } + { + name = "stackable-operator"; + packageId = "stackable-operator"; + features = [ "crds" "webhook" ]; + } { name = "strum"; packageId = "strum"; @@ -10831,12 +10857,18 @@ rec { } ]; devDependencies = [ + { + name = "indoc"; + packageId = "indoc"; + } { name = "rstest"; packageId = "rstest"; } ]; - + features = { + }; + resolvedDefaultFeatures = [ "test-support" ]; }; "stackable-operator" = rec { crateName = "stackable-operator"; diff --git a/Cargo.toml b/Cargo.toml index 423ccc605..dfe32d4b2 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -22,6 +22,7 @@ const_format = "0.2" futures = "0.3" indoc = "2.0" kafka-protocol = { version = "0.18", default-features = false, features = ["broker", "client"] } +regex = "1.13" rstest = "0.26" rustls = { version = "0.23", default-features = false, features = ["logging", "ring", "std", "tls12"] } semver = "1.0" diff --git a/deploy/helm/kafka-operator/Chart.yaml b/deploy/helm/kafka-operator/Chart.yaml index aca052b25..e8472c047 100644 --- a/deploy/helm/kafka-operator/Chart.yaml +++ b/deploy/helm/kafka-operator/Chart.yaml @@ -31,3 +31,8 @@ annotations: name: kafkaclusters.kafka.stackable.tech displayName: KafkaCluster description: A Kafka cluster stacklet. This resource is managed by the Stackable operator for Apache Kafka. + - kind: KafkaTopic + version: v1alpha1 + name: kafkatopics.kafka.stackable.tech + displayName: KafkaTopic + description: A Kafka topic, created and updated by the agent of the referenced Kafka cluster. diff --git a/extra/crds.yaml b/extra/crds.yaml index 83287899a..3c8248781 100644 --- a/extra/crds.yaml +++ b/extra/crds.yaml @@ -2236,3 +2236,141 @@ spec: storage: true subresources: status: {} +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: kafkatopics.kafka.stackable.tech +spec: + group: kafka.stackable.tech + names: + kind: KafkaTopic + plural: kafkatopics + singular: kafkatopic + scope: Namespaced + versions: + - name: v1alpha1 + schema: + openAPIV3Schema: + description: A Kafka topic, created and updated by the agent of the referenced Kafka cluster. + properties: + spec: + description: |- + A Kafka topic managed by the agent of the referenced Kafka cluster. + + The referenced `KafkaCluster` must have `platformAccess` enabled or this resource will + not be reconciled. + properties: + clusterName: + description: Name of a KafkaCluster in the namespace of this KafkaTopic resource. Immutable. + maxLength: 40 + minLength: 1 + type: string + x-kubernetes-validations: + - message: clusterName is immutable + rule: self == oldSelf + config: + additionalProperties: + type: string + default: {} + description: |- + Topic configs like `retention.ms`. Keys removed from this map are reset to the broker + default. Keys not listed here are not managed by the agent. + type: object + managementPolicy: + default: + adoption: Refuse + deletion: Retain + description: What an agent may do with a resource inside the product beyond applying its spec. + properties: + adoption: + default: Refuse + description: |- + Whether a resource that already exists in the product, but was not created by the + agent, is taken over. `Refuse` by default. + enum: + - Refuse + - Adopt + type: string + deletion: + default: Retain + description: |- + What happens to the resource in the product when this object is deleted. `Retain` by + default. + enum: + - Retain + - Delete + type: string + type: object + partitions: + description: Number of partitions. Can only be increased. + format: int32 + minimum: 1.0 + type: integer + replicationFactor: + description: Number of replicas of each partition. Changes are not applied to existing topics. + format: int16 + maximum: 32767.0 + minimum: 1.0 + type: integer + topicName: + description: Name of the topic in Kafka. Immutable. + maxLength: 249 + minLength: 1 + pattern: ^([a-zA-Z0-9-][a-zA-Z0-9._-]*|_([a-zA-Z0-9.-][a-zA-Z0-9._-]*)?|\.[a-zA-Z0-9_-][a-zA-Z0-9._-]*|\.\.[a-zA-Z0-9._-]+)$ + type: string + x-kubernetes-validations: + - message: topicName is immutable + rule: self == oldSelf + required: + - clusterName + - partitions + - replicationFactor + - topicName + type: object + status: + nullable: true + properties: + conditions: + default: [] + description: '`Ready` and `Synced`.' + items: + description: Condition contains details for one aspect of the current state of this API Resource. + properties: + lastTransitionTime: + description: lastTransitionTime is the last time the condition transitioned from one status to another. This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. + format: date-time + type: string + message: + description: message is a human readable message indicating details about the transition. This may be an empty string. + type: string + observedGeneration: + description: observedGeneration represents the .metadata.generation that the condition was set based upon. For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date with respect to the current state of the instance. + format: int64 + type: integer + reason: + description: reason contains a programmatic identifier indicating the reason for the condition's last transition. Producers of specific condition types may define expected values and meanings for this field, and whether the values are considered a guaranteed API. The value should be a CamelCase string. This field may not be empty. + type: string + status: + description: status of the condition, one of True, False, Unknown. + type: string + type: + description: type of condition in CamelCase or in foo.example.com/CamelCase. + type: string + required: + - lastTransitionTime + - message + - reason + - status + - type + type: object + type: array + type: object + required: + - spec + title: KafkaTopic + type: object + served: true + storage: true + subresources: + status: {} diff --git a/rust/operator-binary/Cargo.toml b/rust/operator-binary/Cargo.toml index 5c926bea0..4adf90866 100644 --- a/rust/operator-binary/Cargo.toml +++ b/rust/operator-binary/Cargo.toml @@ -9,6 +9,7 @@ repository.workspace = true publish = false [dependencies] +stackable-kafka-shared.workspace = true stackable-operator.workspace = true indoc.workspace = true diff --git a/rust/operator-binary/src/main.rs b/rust/operator-binary/src/main.rs index 809bd5664..37020491d 100644 --- a/rust/operator-binary/src/main.rs +++ b/rust/operator-binary/src/main.rs @@ -7,6 +7,7 @@ use std::sync::Arc; use anyhow::anyhow; use clap::Parser; use futures::{FutureExt, StreamExt, TryFutureExt}; +use stackable_kafka_shared::crd::topic::{KafkaTopic, KafkaTopicVersion}; use stackable_operator::{ YamlSchema, cli::{Command, RunArguments}, @@ -77,8 +78,12 @@ struct KafkaRun { async fn main() -> anyhow::Result<()> { let opts = Opts::parse(); match opts.cmd { - Command::Crd => KafkaCluster::merged_crd(KafkaClusterVersion::V1Alpha1)? - .print_yaml_schema(built_info::PKG_VERSION, &SerializeOptions::default())?, + Command::Crd => { + KafkaCluster::merged_crd(KafkaClusterVersion::V1Alpha1)? + .print_yaml_schema(built_info::PKG_VERSION, &SerializeOptions::default())?; + KafkaTopic::merged_crd(KafkaTopicVersion::V1Alpha1)? + .print_yaml_schema(built_info::PKG_VERSION, &SerializeOptions::default())?; + } Command::Run(KafkaRun { common: RunArguments { diff --git a/rust/operator-binary/src/webhooks/conversion.rs b/rust/operator-binary/src/webhooks/conversion.rs index 232e4dac4..7c2693653 100644 --- a/rust/operator-binary/src/webhooks/conversion.rs +++ b/rust/operator-binary/src/webhooks/conversion.rs @@ -1,4 +1,5 @@ use snafu::{ResultExt, Snafu}; +use stackable_kafka_shared::crd::topic::{KafkaTopic, KafkaTopicVersion}; use stackable_operator::{ cli::OperatorEnvironmentOptions, kube::{Client, core::crd::MergeError}, @@ -28,10 +29,16 @@ pub async fn create_webhook_server( disable_crd_maintenance: bool, client: Client, ) -> Result { - let crds_and_handlers = vec![( - KafkaCluster::merged_crd(KafkaClusterVersion::V1Alpha1).context(MergeCrdSnafu)?, - KafkaCluster::try_convert, - )]; + let crds_and_handlers = vec![ + ( + KafkaCluster::merged_crd(KafkaClusterVersion::V1Alpha1).context(MergeCrdSnafu)?, + KafkaCluster::try_convert as fn(_) -> _, + ), + ( + KafkaTopic::merged_crd(KafkaTopicVersion::V1Alpha1).context(MergeCrdSnafu)?, + KafkaTopic::try_convert as fn(_) -> _, + ), + ]; let conversion_webhook_options = ConversionWebhookOptions { disable_crd_maintenance, diff --git a/rust/shared/Cargo.toml b/rust/shared/Cargo.toml index 4164d809c..9eedd8c66 100644 --- a/rust/shared/Cargo.toml +++ b/rust/shared/Cargo.toml @@ -9,7 +9,17 @@ repository.workspace = true publish = false [dependencies] +stackable-operator.workspace = true + +regex.workspace = true +serde.workspace = true +serde_json.workspace = true +snafu.workspace = true strum.workspace = true [dev-dependencies] +indoc.workspace = true rstest.workspace = true + +[features] +test-support = [] diff --git a/rust/shared/src/crd/mod.rs b/rust/shared/src/crd/mod.rs new file mode 100644 index 000000000..8a7c2fb7c --- /dev/null +++ b/rust/shared/src/crd/mod.rs @@ -0,0 +1 @@ +pub mod topic; diff --git a/rust/shared/src/crd/topic.rs b/rust/shared/src/crd/topic.rs new file mode 100644 index 000000000..7b855a8e4 --- /dev/null +++ b/rust/shared/src/crd/topic.rs @@ -0,0 +1,168 @@ +use std::collections::BTreeMap; + +use serde::{Deserialize, Serialize}; +use stackable_operator::{ + attributed_string_type, + crd::platform_access::v1alpha1::ManagementPolicy, + k8s_openapi::apimachinery::pkg::apis::meta::v1::Condition, + kube::CustomResource, + schemars::{self, JsonSchema}, + v2::types::operator::ClusterName, + versioned::versioned, +}; + +attributed_string_type! { + TopicName, + "The name of a Kafka topic, not `.` or `..` and not starting with `__`", + "shop.orders", + (min_length = 1), + (max_length = 249), + (regex = r"^([a-zA-Z0-9-][a-zA-Z0-9._-]*|_([a-zA-Z0-9.-][a-zA-Z0-9._-]*)?|\.[a-zA-Z0-9_-][a-zA-Z0-9._-]*|\.\.[a-zA-Z0-9._-]+)$") +} + +#[versioned( + version(name = "v1alpha1"), + crates( + kube_core = "stackable_operator::kube::core", + kube_client = "stackable_operator::kube::client", + k8s_openapi = "stackable_operator::k8s_openapi", + schemars = "stackable_operator::schemars", + versioned = "stackable_operator::versioned" + ) +)] +pub mod versioned { + /// A Kafka topic managed by the agent of the referenced Kafka cluster. + /// + /// The referenced `KafkaCluster` must have `platformAccess` enabled or this resource will + /// not be reconciled. + #[versioned(crd( + doc = "A Kafka topic, created and updated by the agent of the referenced Kafka cluster.", + group = "kafka.stackable.tech", + plural = "kafkatopics", + status = "KafkaTopicStatus", + namespaced + ))] + #[derive(Clone, CustomResource, Debug, Deserialize, JsonSchema, PartialEq, Serialize)] + #[serde(rename_all = "camelCase")] + pub struct KafkaTopicSpec { + /// Name of a KafkaCluster in the namespace of this KafkaTopic resource. Immutable. + #[schemars(extend("x-kubernetes-validations" = [{ + "rule": "self == oldSelf", + "message": "clusterName is immutable" + }]))] + pub cluster_name: ClusterName, + + #[serde(default)] + pub management_policy: ManagementPolicy, + + /// Name of the topic in Kafka. Immutable. + #[schemars(extend("x-kubernetes-validations" = [{ + "rule": "self == oldSelf", + "message": "topicName is immutable" + }]))] + pub topic_name: TopicName, + + /// Number of partitions. Can only be increased. + #[schemars(range(min = 1))] + pub partitions: i32, + + /// Number of replicas of each partition. Changes are not applied to existing topics. + #[schemars(range(min = 1))] + pub replication_factor: i16, + + /// Topic configs like `retention.ms`. Keys removed from this map are reset to the broker + /// default. Keys not listed here are not managed by the agent. + #[serde(default)] + pub config: BTreeMap, + } + + #[derive(Clone, Debug, Default, Deserialize, JsonSchema, PartialEq, Serialize)] + #[serde(rename_all = "camelCase")] + pub struct KafkaTopicStatus { + /// `Ready` and `Synced`. + #[serde(default)] + pub conditions: Vec, + } +} + +#[cfg(test)] +mod tests { + use std::str::FromStr; + + use rstest::rstest; + use stackable_operator::versioned::test_utils::RoundtripTestData; + + use super::*; + + impl RoundtripTestData for v1alpha1::KafkaTopicSpec { + fn roundtrip_test_data() -> Vec { + stackable_operator::utils::yaml_from_str_singleton_map(indoc::indoc! {r#" + - clusterName: simple-kafka + managementPolicy: + adoption: Adopt + deletion: Delete + topicName: shop.orders + partitions: 6 + replicationFactor: 3 + config: + retention.ms: "604800000" + - clusterName: simple-kafka + topicName: minimal + partitions: 1 + replicationFactor: 1 + "#}) + .expect("the roundtrip test data should be valid YAML") + } + } + + #[rstest] + #[case::dotted("shop.orders")] + #[case::uppercase_and_underscore("Shop_Orders-1")] + #[case::single_underscore("_internal")] + #[case::underscore_only("_")] + #[case::leading_dot(".hidden")] + #[case::three_dots("...")] + fn accept_topic_name(#[case] name: &str) { + assert!( + TopicName::from_str(name).is_ok(), + "{name:?} should be valid" + ); + } + + #[rstest] + #[case::empty("")] + #[case::dot(".")] + #[case::two_dots("..")] + #[case::double_underscore_prefix("__consumer_offsets")] + #[case::invalid_character("shop/orders")] + #[case::too_long(&"a".repeat(250))] + fn reject_topic_name(#[case] name: &str) { + assert!( + TopicName::from_str(name).is_err(), + "{name:?} should be invalid" + ); + } + + #[test] + fn schema_contains_validations() { + let crd = serde_json::to_value( + KafkaTopic::merged_crd(KafkaTopicVersion::V1Alpha1) + .expect("the CRD should be mergeable"), + ) + .expect("the CRD should be serializable"); + let spec = &crd["spec"]["versions"][0]["schema"]["openAPIV3Schema"]["properties"]["spec"]["properties"]; + + assert_eq!( + "self == oldSelf", + spec["clusterName"]["x-kubernetes-validations"][0]["rule"] + ); + assert_eq!( + "self == oldSelf", + spec["topicName"]["x-kubernetes-validations"][0]["rule"] + ); + assert_eq!(249, spec["topicName"]["maxLength"]); + assert!(spec["topicName"]["pattern"].is_string()); + assert_eq!(Some(1.0), spec["partitions"]["minimum"].as_f64()); + assert_eq!(Some(1.0), spec["replicationFactor"]["minimum"].as_f64()); + } +} diff --git a/rust/shared/src/lib.rs b/rust/shared/src/lib.rs index f17bc55db..0e0beca04 100644 --- a/rust/shared/src/lib.rs +++ b/rust/shared/src/lib.rs @@ -1 +1,2 @@ pub mod agent; +pub mod crd; From fe5203f4075e733f95823ace251a460b79e0355a Mon Sep 17 00:00:00 2001 From: Benedikt Labrenz Date: Wed, 14 Oct 2026 14:02:20 +0200 Subject: [PATCH 13/93] add necessary envs and volumes to agent pod --- Cargo.lock | 18 +- Cargo.nix | 18 +- rust/operator-binary/src/controller.rs | 133 +++++++++++- .../src/controller/build/resource/agent.rs | 146 +++++++++++-- .../src/controller/build/security.rs | 120 +---------- .../src/controller/security.rs | 12 ++ .../src/controller/validate.rs | 203 ++++++++++++++++-- rust/shared/src/agent.rs | 13 +- 8 files changed, 482 insertions(+), 181 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index a50ea695d..f646c56ea 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1736,7 +1736,7 @@ dependencies = [ [[package]] name = "k8s-version" version = "0.1.3" -source = "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#754e444b8cc3daf6a59fb69d7eaf632249ccc3be" +source = "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#92e1a3655432e3899da735a9d86ded2e38030c0e" dependencies = [ "darling 0.24.1", "regex", @@ -3213,7 +3213,7 @@ checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596" [[package]] name = "stackable-certs" version = "0.4.1" -source = "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#754e444b8cc3daf6a59fb69d7eaf632249ccc3be" +source = "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#92e1a3655432e3899da735a9d86ded2e38030c0e" dependencies = [ "const-oid", "ecdsa", @@ -3303,7 +3303,7 @@ dependencies = [ [[package]] name = "stackable-operator" version = "0.119.0" -source = "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#754e444b8cc3daf6a59fb69d7eaf632249ccc3be" +source = "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#92e1a3655432e3899da735a9d86ded2e38030c0e" dependencies = [ "base64 0.23.1", "clap", @@ -3347,7 +3347,7 @@ dependencies = [ [[package]] name = "stackable-operator-derive" version = "0.3.1" -source = "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#754e444b8cc3daf6a59fb69d7eaf632249ccc3be" +source = "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#92e1a3655432e3899da735a9d86ded2e38030c0e" dependencies = [ "darling 0.24.1", "proc-macro2", @@ -3358,7 +3358,7 @@ dependencies = [ [[package]] name = "stackable-shared" version = "0.1.2" -source = "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#754e444b8cc3daf6a59fb69d7eaf632249ccc3be" +source = "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#92e1a3655432e3899da735a9d86ded2e38030c0e" dependencies = [ "jiff", "k8s-openapi", @@ -3375,7 +3375,7 @@ dependencies = [ [[package]] name = "stackable-telemetry" version = "0.6.5" -source = "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#754e444b8cc3daf6a59fb69d7eaf632249ccc3be" +source = "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#92e1a3655432e3899da735a9d86ded2e38030c0e" dependencies = [ "axum", "clap", @@ -3399,7 +3399,7 @@ dependencies = [ [[package]] name = "stackable-versioned" version = "0.11.1" -source = "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#754e444b8cc3daf6a59fb69d7eaf632249ccc3be" +source = "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#92e1a3655432e3899da735a9d86ded2e38030c0e" dependencies = [ "kube", "schemars", @@ -3413,7 +3413,7 @@ dependencies = [ [[package]] name = "stackable-versioned-macros" version = "0.11.1" -source = "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#754e444b8cc3daf6a59fb69d7eaf632249ccc3be" +source = "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#92e1a3655432e3899da735a9d86ded2e38030c0e" dependencies = [ "convert_case", "darling 0.24.1", @@ -3430,7 +3430,7 @@ dependencies = [ [[package]] name = "stackable-webhook" version = "0.10.0" -source = "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#754e444b8cc3daf6a59fb69d7eaf632249ccc3be" +source = "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#92e1a3655432e3899da735a9d86ded2e38030c0e" dependencies = [ "arc-swap", "async-trait", diff --git a/Cargo.nix b/Cargo.nix index 95a0e6424..27002f0f9 100644 --- a/Cargo.nix +++ b/Cargo.nix @@ -5508,7 +5508,7 @@ rec { workspace_member = null; src = pkgs.fetchgit { url = "https://github.com/stackabletech/operator-rs.git"; - rev = "754e444b8cc3daf6a59fb69d7eaf632249ccc3be"; + rev = "92e1a3655432e3899da735a9d86ded2e38030c0e"; sha256 = "0cgziqra8097hp05ynib1qpw4c95n972f4w2rk9l3llyp8r1vmci"; }; libName = "k8s_version"; @@ -10506,7 +10506,7 @@ rec { workspace_member = null; src = pkgs.fetchgit { url = "https://github.com/stackabletech/operator-rs.git"; - rev = "754e444b8cc3daf6a59fb69d7eaf632249ccc3be"; + rev = "92e1a3655432e3899da735a9d86ded2e38030c0e"; sha256 = "0cgziqra8097hp05ynib1qpw4c95n972f4w2rk9l3llyp8r1vmci"; }; libName = "stackable_certs"; @@ -10877,7 +10877,7 @@ rec { workspace_member = null; src = pkgs.fetchgit { url = "https://github.com/stackabletech/operator-rs.git"; - rev = "754e444b8cc3daf6a59fb69d7eaf632249ccc3be"; + rev = "92e1a3655432e3899da735a9d86ded2e38030c0e"; sha256 = "0cgziqra8097hp05ynib1qpw4c95n972f4w2rk9l3llyp8r1vmci"; }; libName = "stackable_operator"; @@ -11076,7 +11076,7 @@ rec { workspace_member = null; src = pkgs.fetchgit { url = "https://github.com/stackabletech/operator-rs.git"; - rev = "754e444b8cc3daf6a59fb69d7eaf632249ccc3be"; + rev = "92e1a3655432e3899da735a9d86ded2e38030c0e"; sha256 = "0cgziqra8097hp05ynib1qpw4c95n972f4w2rk9l3llyp8r1vmci"; }; procMacro = true; @@ -11111,7 +11111,7 @@ rec { workspace_member = null; src = pkgs.fetchgit { url = "https://github.com/stackabletech/operator-rs.git"; - rev = "754e444b8cc3daf6a59fb69d7eaf632249ccc3be"; + rev = "92e1a3655432e3899da735a9d86ded2e38030c0e"; sha256 = "0cgziqra8097hp05ynib1qpw4c95n972f4w2rk9l3llyp8r1vmci"; }; libName = "stackable_shared"; @@ -11192,7 +11192,7 @@ rec { workspace_member = null; src = pkgs.fetchgit { url = "https://github.com/stackabletech/operator-rs.git"; - rev = "754e444b8cc3daf6a59fb69d7eaf632249ccc3be"; + rev = "92e1a3655432e3899da735a9d86ded2e38030c0e"; sha256 = "0cgziqra8097hp05ynib1qpw4c95n972f4w2rk9l3llyp8r1vmci"; }; libName = "stackable_telemetry"; @@ -11302,7 +11302,7 @@ rec { workspace_member = null; src = pkgs.fetchgit { url = "https://github.com/stackabletech/operator-rs.git"; - rev = "754e444b8cc3daf6a59fb69d7eaf632249ccc3be"; + rev = "92e1a3655432e3899da735a9d86ded2e38030c0e"; sha256 = "0cgziqra8097hp05ynib1qpw4c95n972f4w2rk9l3llyp8r1vmci"; }; libName = "stackable_versioned"; @@ -11352,7 +11352,7 @@ rec { workspace_member = null; src = pkgs.fetchgit { url = "https://github.com/stackabletech/operator-rs.git"; - rev = "754e444b8cc3daf6a59fb69d7eaf632249ccc3be"; + rev = "92e1a3655432e3899da735a9d86ded2e38030c0e"; sha256 = "0cgziqra8097hp05ynib1qpw4c95n972f4w2rk9l3llyp8r1vmci"; }; procMacro = true; @@ -11416,7 +11416,7 @@ rec { workspace_member = null; src = pkgs.fetchgit { url = "https://github.com/stackabletech/operator-rs.git"; - rev = "754e444b8cc3daf6a59fb69d7eaf632249ccc3be"; + rev = "92e1a3655432e3899da735a9d86ded2e38030c0e"; sha256 = "0cgziqra8097hp05ynib1qpw4c95n972f4w2rk9l3llyp8r1vmci"; }; libName = "stackable_webhook"; diff --git a/rust/operator-binary/src/controller.rs b/rust/operator-binary/src/controller.rs index 98eb8d541..66aedb632 100644 --- a/rust/operator-binary/src/controller.rs +++ b/rust/operator-binary/src/controller.rs @@ -15,12 +15,13 @@ use std::{ use const_format::concatcp; use snafu::{ResultExt, Snafu}; +use stackable_kafka_shared::agent::ClientSecurity; use stackable_operator::{ cli::OperatorEnvironmentOptions, cluster_resources::ClusterResourceApplyStrategy, commons::{networking::DomainName, product_image_selection::ResolvedProductImage}, constant, - crd::listener, + crd::{listener, platform_access::v1alpha1::TlsClientCredential}, k8s_openapi::api::{ apps::v1::{Deployment, StatefulSet}, core::v1::{ConfigMap, Service, ServiceAccount}, @@ -40,7 +41,7 @@ use stackable_operator::{ role_group_utils::ResourceNames, role_utils, types::{ - kubernetes::{ConfigMapName, ListenerName, NamespaceName, Uid}, + kubernetes::{ConfigMapName, ListenerName, NamespaceName, SecretClassName, Uid}, operator::{ClusterName, ControllerName, OperatorName, ProductName, ProductVersion}, }, }, @@ -346,7 +347,29 @@ impl ValidatedClusterConfig { /// The configuration of the agent deployed with the cluster. pub struct ValidatedAgentConfig { pub image: String, - pub authentication: v1alpha1::KafkaPlatformAccessAuthentication, + pub connection: AgentConnection, +} + +#[derive(Debug, PartialEq)] +pub enum AgentConnection { + Plaintext, + Tls { + server_ca_secret_class: SecretClassName, + }, + MutualTls { + server_ca_secret_class: SecretClassName, + client_credential: TlsClientCredential, + }, +} + +impl From<&AgentConnection> for ClientSecurity { + fn from(connection: &AgentConnection) -> Self { + match connection { + AgentConnection::Plaintext => Self::Plaintext, + AgentConnection::Tls { .. } => Self::Tls, + AgentConnection::MutualTls { .. } => Self::MutualTls, + } + } } /// Per-role configuration extracted during validation. @@ -529,16 +552,24 @@ pub fn error_policy( #[cfg(test)] pub(crate) mod test_support { - use std::collections::BTreeMap; + use std::{collections::BTreeMap, str::FromStr}; use stackable_operator::{ + builder::meta::ObjectMetaBuilder, cli::OperatorEnvironmentOptions, commons::networking::DomainName, - crd::listener, + crd::{ + authentication::{core, kerberos, tls}, + listener, + }, utils::{cluster_info::KubernetesClusterInfo, yaml_from_str_singleton_map}, + v2::types::kubernetes::SecretClassName, }; - use super::{ValidatedCluster, dereference::DereferencedObjects, validate::validate}; + use super::{ + ValidatedCluster, dereference::DereferencedObjects, security::ValidatedKafkaSecurity, + validate::validate, + }; use crate::crd::{authentication::ResolvedAuthenticationClasses, v1alpha1}; /// The expected `app.kubernetes.io/version` label value for the given product version. @@ -557,6 +588,96 @@ pub(crate) mod test_support { yaml_from_str_singleton_map(yaml).expect("invalid test KafkaCluster YAML") } + fn tls_auth_class() -> core::v1alpha1::AuthenticationClass { + core::v1alpha1::AuthenticationClass { + metadata: ObjectMetaBuilder::new().name("tls-auth").build(), + spec: core::v1alpha1::AuthenticationClassSpec { + provider: core::v1alpha1::AuthenticationClassProvider::Tls( + tls::v1alpha1::AuthenticationProvider { + client_cert_secret_class: Some("client-auth-secret-class".to_string()), + }, + ), + }, + } + } + + fn kerberos_auth_class() -> core::v1alpha1::AuthenticationClass { + core::v1alpha1::AuthenticationClass { + metadata: ObjectMetaBuilder::new().name("kerberos-auth").build(), + spec: core::v1alpha1::AuthenticationClassSpec { + provider: core::v1alpha1::AuthenticationClassProvider::Kerberos( + kerberos::v1alpha1::AuthenticationProvider { + kerberos_secret_class: "kerberos-secret-class".to_string(), + }, + ), + }, + } + } + + fn no_auth() -> ResolvedAuthenticationClasses { + ResolvedAuthenticationClasses::new(vec![]) + } + + /// Plaintext: no TLS, no authentication, no OPA. + pub fn plaintext() -> ValidatedKafkaSecurity { + ValidatedKafkaSecurity::new( + no_auth(), + SecretClassName::from_str("tls").expect("tls secret class name is valid"), + None, + None, + ) + } + + /// Server TLS only (encryption without client authentication). + pub fn server_tls() -> ValidatedKafkaSecurity { + ValidatedKafkaSecurity::new( + no_auth(), + SecretClassName::from_str("tls").expect("tls secret class name is valid"), + Some("tls".parse().unwrap()), + None, + ) + } + + /// Mutual TLS (client-certificate authentication). + pub fn client_auth_tls() -> ValidatedKafkaSecurity { + ValidatedKafkaSecurity::new( + ResolvedAuthenticationClasses::new(vec![tls_auth_class()]), + SecretClassName::from_str("tls").expect("tls secret class name is valid"), + Some("tls".parse().unwrap()), + None, + ) + } + + /// Kerberos, which also requires server and internal TLS. + pub fn kerberos() -> ValidatedKafkaSecurity { + ValidatedKafkaSecurity::new( + ResolvedAuthenticationClasses::new(vec![kerberos_auth_class()]), + SecretClassName::from_str("tls").expect("tls secret class name is valid"), + Some("tls".parse().unwrap()), + None, + ) + } + + /// Internal TLS only (broker/controller encryption without client TLS). + pub fn internal_tls() -> ValidatedKafkaSecurity { + ValidatedKafkaSecurity::new( + no_auth(), + SecretClassName::from_str("tls").expect("tls secret class name is valid"), + None, + None, + ) + } + + /// OPA authorization with a TLS truststore (internal + server TLS also enabled). + pub fn opa() -> ValidatedKafkaSecurity { + ValidatedKafkaSecurity::new( + no_auth(), + SecretClassName::from_str("tls").expect("tls secret class name is valid"), + Some("tls".parse().unwrap()), + Some("opa-tls".parse().unwrap()), + ) + } + fn cluster_info() -> KubernetesClusterInfo { KubernetesClusterInfo { cluster_domain: DomainName::try_from("cluster.local").expect("valid domain"), diff --git a/rust/operator-binary/src/controller/build/resource/agent.rs b/rust/operator-binary/src/controller/build/resource/agent.rs index bad1fb2a7..81c7e34a1 100644 --- a/rust/operator-binary/src/controller/build/resource/agent.rs +++ b/rust/operator-binary/src/controller/build/resource/agent.rs @@ -2,13 +2,22 @@ use std::str::FromStr; +use stackable_kafka_shared::agent::{ + ClientSecurity, DISCOVERY_DIR, ENV_CLIENT_SECURITY, ENV_CLUSTER_NAME, ENV_CLUSTER_NAMESPACE, + KAFKA_TLS_DIR, +}; use stackable_operator::{ builder::{ meta::ObjectMetaBuilder, pod::{ - PodBuilder, resources::ResourceRequirementsBuilder, security::PodSecurityContextBuilder, + PodBuilder, + container::ContainerBuilder, + resources::ResourceRequirementsBuilder, + security::PodSecurityContextBuilder, + volume::{SecretFormat, SecretOperatorVolumeSourceBuilder, VolumeBuilder}, }, }, + commons::secret_class::SecretClassVolumeProvisionParts, constant, k8s_openapi::{ api::{ @@ -20,20 +29,22 @@ use stackable_operator::{ }, kube::api::ObjectMeta, v2::{ - builder::{meta::ownerreference_from_resource, pod::container::new_container_builder}, - types::kubernetes::ContainerName, + builder::{ + meta::ownerreference_from_resource, + pod::container::{EnvVarSet, new_container_builder}, + }, + types::kubernetes::{ContainerName, SecretClassName, VolumeName}, }, }; -use crate::{ - controller::{ - ValidatedAgentConfig, ValidatedCluster, - build::{agent_selector, recommended_labels_for_agent_resources}, - }, - crd::v1alpha1::KafkaPlatformAccessAuthentication, +use crate::controller::{ + AgentConnection, ValidatedAgentConfig, ValidatedCluster, + build::{agent_selector, recommended_labels_for_agent_resources}, }; constant!(AGENT_CONTAINER_NAME: ContainerName = "agent"); +constant!(DISCOVERY_VOLUME_NAME: VolumeName = "discovery"); +constant!(KAFKA_TLS_VOLUME_NAME: VolumeName = "tls-kafka"); pub const AGENT_CLUSTER_ROLE_NAME: &str = "kafka-agent-clusterrole"; pub fn build_agent_deployment( @@ -43,16 +54,35 @@ pub fn build_agent_deployment( let mut cb_agent = new_container_builder(&AGENT_CONTAINER_NAME); let mut pod_builder = PodBuilder::new(); - match &agent_config.authentication { - KafkaPlatformAccessAuthentication::Tls(credential) => { - credential.add_volumes_and_mounts(&mut pod_builder, vec![&mut cb_agent]) + add_discovery_volume_and_mount(cluster, &mut pod_builder, &mut cb_agent); + match &agent_config.connection { + AgentConnection::Plaintext => {} + AgentConnection::Tls { + server_ca_secret_class, + } => { + add_kafka_tls_volume_and_mount(server_ca_secret_class, &mut pod_builder, &mut cb_agent) + } + AgentConnection::MutualTls { + server_ca_secret_class, + client_credential, + } => { + add_kafka_tls_volume_and_mount(server_ca_secret_class, &mut pod_builder, &mut cb_agent); + client_credential.add_volumes_and_mounts(&mut pod_builder, vec![&mut cb_agent]); } - KafkaPlatformAccessAuthentication::Anonymous(_) => {} } cb_agent .image(&agent_config.image) .args(vec!["run".to_owned()]) + .add_env_vars( + EnvVarSet::new() + .with_value(&ENV_CLUSTER_NAME, cluster.name.to_string()) + .with_value(&ENV_CLUSTER_NAMESPACE, cluster.namespace.to_string()) + .with_value( + &ENV_CLIENT_SECURITY, + ClientSecurity::from(&agent_config.connection).to_string(), + ), + ) .add_env_var_from_source( "KUBERNETES_NODE_NAME", EnvVarSource { @@ -128,6 +158,48 @@ pub fn build_agent_role_binding(cluster: &ValidatedCluster) -> RoleBinding { } } +fn add_discovery_volume_and_mount( + cluster: &ValidatedCluster, + pod_builder: &mut PodBuilder, + container_builder: &mut ContainerBuilder, +) { + pod_builder + .add_volume( + VolumeBuilder::new(&*DISCOVERY_VOLUME_NAME) + .with_config_map(cluster.name.to_string()) + .build(), + ) + .expect("the volume name is statically defined and must not collide"); + container_builder + .add_volume_mount(&*DISCOVERY_VOLUME_NAME, DISCOVERY_DIR) + .expect("the mount path is statically defined and must not collide"); +} + +fn add_kafka_tls_volume_and_mount( + secret_class: &SecretClassName, + pod_builder: &mut PodBuilder, + container_builder: &mut ContainerBuilder, +) { + pod_builder + .add_volume( + VolumeBuilder::new(&*KAFKA_TLS_VOLUME_NAME) + .ephemeral( + SecretOperatorVolumeSourceBuilder::new( + secret_class, + SecretClassVolumeProvisionParts::Public, + ) + .with_format(SecretFormat::TlsPem) + .build() + .expect("the annotations are built from a valid SecretClass name"), + ) + .build(), + ) + .expect("the volume name is statically defined and must not collide"); + container_builder + .add_volume_mount(&*KAFKA_TLS_VOLUME_NAME, KAFKA_TLS_DIR) + .expect("the mount path is statically defined and must not collide"); +} + /// The name of all agent resources. Kinds don't share a namespace, so unlike a suffix per kind, this /// cannot collide with the resources of a cluster named `-agent`. fn agent_name(cluster: &ValidatedCluster) -> String { @@ -240,12 +312,37 @@ mod tests { assert_eq!(pod_spec["containers"][0]["args"], json!(["run"])); assert_eq!( pod_spec["containers"][0]["volumeMounts"], - json!([{"mountPath": "/stackable/secrets/tls-client-cert", "name": "tls-client-cert"}]) + json!([ + {"mountPath": "/stackable/discovery", "name": "discovery"}, + {"mountPath": "/stackable/secrets/tls-kafka", "name": "tls-kafka"}, + {"mountPath": "/stackable/secrets/tls-platform-access", "name": "tls-platform-access"} + ]) ); - assert_eq!(pod_spec["volumes"][0]["name"], "tls-client-cert"); assert_eq!( - pod_spec["volumes"][0]["ephemeral"]["volumeClaimTemplate"]["metadata"]["annotations"]["secrets.stackable.tech/class"], - "tls" + pod_spec["volumes"][0]["configMap"], + json!({"name": "simple-kafka"}) + ); + assert_eq!( + pod_spec["volumes"][1]["ephemeral"]["volumeClaimTemplate"]["metadata"]["annotations"], + json!({ + "secrets.stackable.tech/class": "tls", + "secrets.stackable.tech/format": "tls-pem", + "secrets.stackable.tech/provision-parts": "public" + }) + ); + + assert_eq!( + pod_spec["containers"][0]["env"], + json!([ + {"name": "KAFKA_CLIENT_SECURITY", "value": "mutual-tls"}, + {"name": "KAFKA_CLUSTER_NAME", "value": "simple-kafka"}, + {"name": "KAFKA_CLUSTER_NAMESPACE", "value": "default"}, + { + "name": "KUBERNETES_NODE_NAME", + "valueFrom": {"fieldRef": {"fieldPath": "spec.nodeName"}} + }, + {"name": "KUBERNETES_CLUSTER_DOMAIN", "value": "cluster.local"} + ]) ); } @@ -255,18 +352,23 @@ mod tests { agent_deployment("{enabled: true, authentication: {tls: {secret: my-cert}}}"); assert_eq!( - deployment["spec"]["template"]["spec"]["volumes"], - json!([{"name": "tls-client-cert", "secret": {"secretName": "my-cert"}}]) + deployment["spec"]["template"]["spec"]["volumes"][2], + json!({"name": "tls-platform-access", "secret": {"secretName": "my-cert"}}) ); } #[test] - fn deployment_with_anonymous_authentication_mounts_no_credential() { + fn deployment_with_anonymous_authentication_verifies_brokers_without_credential() { let deployment = agent_deployment("{enabled: true, authentication: {anonymous: {}}}"); let pod_spec = &deployment["spec"]["template"]["spec"]; - assert_eq!(pod_spec["volumes"], Value::Null); - assert_eq!(pod_spec["containers"][0]["volumeMounts"], Value::Null); + assert_eq!(pod_spec["volumes"][0]["name"], "discovery"); + assert_eq!(pod_spec["volumes"][1]["name"], "tls-kafka"); + assert_eq!(pod_spec["volumes"][2], Value::Null); + assert_eq!( + pod_spec["containers"][0]["env"][0], + json!({"name": "KAFKA_CLIENT_SECURITY", "value": "tls"}) + ); } #[test] diff --git a/rust/operator-binary/src/controller/build/security.rs b/rust/operator-binary/src/controller/build/security.rs index b9eb650af..cdab2d9ec 100644 --- a/rust/operator-binary/src/controller/build/security.rs +++ b/rust/operator-binary/src/controller/build/security.rs @@ -17,7 +17,6 @@ use stackable_operator::{ }, commons::secret_class::SecretClassVolumeProvisionParts, constant, - crd::authentication::core, k8s_openapi::api::core::v1::Volume, shared::time::Duration, v2::types::kubernetes::VolumeName, @@ -250,7 +249,7 @@ pub fn add_broker_volume_and_volume_mounts( requested_secret_lifetime: &Duration, ) -> Result<(), Error> { // add tls (server or client authentication volumes) if required - if let Some(tls_server_secret_class) = tls_secret_class(security) { + if let Some(tls_server_secret_class) = security.tls_secret_class() { // We have to mount tls pem files for kcat's readiness-probe command (the mount can be // used directly) pod_builder @@ -572,19 +571,6 @@ pub fn controller_config_settings(security: &ValidatedKafkaSecurity) -> BTreeMap config } -/// Returns the `SecretClass` provided in a `AuthenticationClass` for TLS. -fn tls_secret_class(security: &ValidatedKafkaSecurity) -> Option<&str> { - security - .tls_client_authentication_class() - .and_then(|auth_class| match &auth_class.spec.provider { - core::v1alpha1::AuthenticationClassProvider::Tls(tls) => { - tls.client_cert_secret_class.as_deref() - } - _ => None, - }) - .or_else(|| security.tls_server_secret_class()) -} - /// Creates ephemeral volumes to mount the `SecretClass` into the Pods for kcat client fn create_kcat_tls_volume( volume_name: &VolumeName, @@ -676,107 +662,13 @@ fn kcat_client_sasl_ssl(cert_directory: &str, service_name: &str) -> Vec } #[cfg(test)] -pub(crate) mod tests { - use std::{collections::BTreeMap, str::FromStr}; - - use stackable_operator::{ - builder::meta::ObjectMetaBuilder, - crd::authentication::{core, kerberos, tls}, - v2::types::kubernetes::SecretClassName, - }; +mod tests { + use std::collections::BTreeMap; use super::*; - use crate::crd::authentication::ResolvedAuthenticationClasses; - - fn tls_auth_class() -> core::v1alpha1::AuthenticationClass { - core::v1alpha1::AuthenticationClass { - metadata: ObjectMetaBuilder::new().name("tls-auth").build(), - spec: core::v1alpha1::AuthenticationClassSpec { - provider: core::v1alpha1::AuthenticationClassProvider::Tls( - tls::v1alpha1::AuthenticationProvider { - client_cert_secret_class: Some("client-auth-secret-class".to_string()), - }, - ), - }, - } - } - - fn kerberos_auth_class() -> core::v1alpha1::AuthenticationClass { - core::v1alpha1::AuthenticationClass { - metadata: ObjectMetaBuilder::new().name("kerberos-auth").build(), - spec: core::v1alpha1::AuthenticationClassSpec { - provider: core::v1alpha1::AuthenticationClassProvider::Kerberos( - kerberos::v1alpha1::AuthenticationProvider { - kerberos_secret_class: "kerberos-secret-class".to_string(), - }, - ), - }, - } - } - - fn no_auth() -> ResolvedAuthenticationClasses { - ResolvedAuthenticationClasses::new(vec![]) - } - - /// Plaintext: no TLS, no authentication, no OPA. - fn plaintext() -> ValidatedKafkaSecurity { - ValidatedKafkaSecurity::new( - no_auth(), - SecretClassName::from_str("tls").expect("tls secret class name is valid"), - None, - None, - ) - } - - /// Server TLS only (encryption without client authentication). - fn server_tls() -> ValidatedKafkaSecurity { - ValidatedKafkaSecurity::new( - no_auth(), - SecretClassName::from_str("tls").expect("tls secret class name is valid"), - Some("tls".parse().unwrap()), - None, - ) - } - - /// Mutual TLS (client-certificate authentication). - fn client_auth_tls() -> ValidatedKafkaSecurity { - ValidatedKafkaSecurity::new( - ResolvedAuthenticationClasses::new(vec![tls_auth_class()]), - SecretClassName::from_str("tls").expect("tls secret class name is valid"), - Some("tls".parse().unwrap()), - None, - ) - } - - /// Kerberos, which also requires server and internal TLS. - pub(crate) fn kerberos() -> ValidatedKafkaSecurity { - ValidatedKafkaSecurity::new( - ResolvedAuthenticationClasses::new(vec![kerberos_auth_class()]), - SecretClassName::from_str("tls").expect("tls secret class name is valid"), - Some("tls".parse().unwrap()), - None, - ) - } - - /// Internal TLS only (broker/controller encryption without client TLS). - fn internal_tls() -> ValidatedKafkaSecurity { - ValidatedKafkaSecurity::new( - no_auth(), - SecretClassName::from_str("tls").expect("tls secret class name is valid"), - None, - None, - ) - } - - /// OPA authorization with a TLS truststore (internal + server TLS also enabled). - fn opa() -> ValidatedKafkaSecurity { - ValidatedKafkaSecurity::new( - no_auth(), - SecretClassName::from_str("tls").expect("tls secret class name is valid"), - Some("tls".parse().unwrap()), - Some("opa-tls".parse().unwrap()), - ) - } + use crate::controller::test_support::{ + client_auth_tls, internal_tls, kerberos, opa, plaintext, server_tls, + }; fn as_str_vec(commands: &[String]) -> Vec<&str> { commands.iter().map(String::as_str).collect() diff --git a/rust/operator-binary/src/controller/security.rs b/rust/operator-binary/src/controller/security.rs index 90bd62178..a97c732f2 100644 --- a/rust/operator-binary/src/controller/security.rs +++ b/rust/operator-binary/src/controller/security.rs @@ -101,6 +101,18 @@ impl ValidatedKafkaSecurity { .get_tls_authentication_class() } + /// The `SecretClass` of a TLS `AuthenticationClass`, otherwise the server `SecretClass`. + pub fn tls_secret_class(&self) -> Option<&str> { + self.tls_client_authentication_class() + .and_then(|auth_class| match &auth_class.spec.provider { + core::v1alpha1::AuthenticationClassProvider::Tls(tls) => { + tls.client_cert_secret_class.as_deref() + } + _ => None, + }) + .or_else(|| self.tls_server_secret_class()) + } + /// Retrieve the internal `SecretClass`. Internal (inter-broker) TLS is mandatory, so this is /// always set (defaults to `tls`). pub fn tls_internal_secret_class(&self) -> &str { diff --git a/rust/operator-binary/src/controller/validate.rs b/rust/operator-binary/src/controller/validate.rs index 62fb2bff8..f500b0595 100644 --- a/rust/operator-binary/src/controller/validate.rs +++ b/rust/operator-binary/src/controller/validate.rs @@ -6,7 +6,7 @@ use std::{collections::BTreeMap, str::FromStr}; use serde::Serialize; -use snafu::{OptionExt, ResultExt, Snafu}; +use snafu::{OptionExt, ResultExt, Snafu, ensure}; use stackable_operator::{ cli::OperatorEnvironmentOptions, commons::product_image_selection, @@ -24,14 +24,15 @@ use stackable_operator::{ validate_logging_configuration_for_container, }, role_utils::{JavaCommonConfig, Role, with_validated_config}, - types::kubernetes::ConfigMapName, + types::kubernetes::{ConfigMapName, SecretClassName}, }, }; use crate::{ controller::{ - RoleGroupName, ValidatedAgentConfig, ValidatedCluster, ValidatedClusterConfig, - ValidatedKafkaConfig, ValidatedRoleConfig, ValidatedRoleGroupConfig, + AgentConnection, RoleGroupName, ValidatedAgentConfig, ValidatedCluster, + ValidatedClusterConfig, ValidatedKafkaConfig, ValidatedRoleConfig, + ValidatedRoleGroupConfig, dereference::DereferencedObjects, security::{self, ValidatedKafkaSecurity}, }, @@ -115,6 +116,28 @@ pub enum Error { to join without it" ))] MissingKraftControllerRole, + + #[snafu(display( + "platform access does not support the cluster authentication method {authentication:?}" + ))] + UnsupportedClusterAuthentication { authentication: String }, + + #[snafu(display( + "the cluster requires client authentication, but platform access is anonymous" + ))] + AnonymousPlatformAccessWithClientAuthentication, + + #[snafu(display( + "the platform access authentication method {authentication:?} requires the cluster to use \ + TLS" + ))] + PlatformAccessRequiresTls { authentication: String }, + + #[snafu(display("the SecretClass {secret_class:?} of the client listener is invalid"))] + ParseClientListenerSecretClass { + source: stackable_operator::v2::macros::attributed_string_type::Error, + secret_class: String, + }, } /// Validated logging configuration for a Kafka role group's Kafka and (optional) Vector @@ -334,15 +357,16 @@ pub fn validate( .cluster_domain .clone(); - let agent_config = kafka - .spec - .platform_access - .as_ref() - .filter(|platform_access| platform_access.enabled) - .map(|platform_access| ValidatedAgentConfig { + let agent_config = match &kafka.spec.platform_access { + Some(platform_access) if platform_access.enabled => Some(ValidatedAgentConfig { image: agent_image.to_owned(), - authentication: platform_access.authentication.clone(), - }); + connection: validate_agent_connection( + &platform_access.authentication, + &kafka_security, + )?, + }), + _ => None, + }; Ok(ValidatedCluster::new( name, @@ -452,15 +476,164 @@ fn inject_cluster_id(env_overrides: EnvVarSet, cluster_id: Option<&str>) -> EnvV } } +fn validate_agent_connection( + authentication: &v1alpha1::KafkaPlatformAccessAuthentication, + security: &ValidatedKafkaSecurity, +) -> Result { + ensure!( + !security.has_kerberos_enabled(), + UnsupportedClusterAuthenticationSnafu { + authentication: "kerberos" + } + ); + + let server_ca_secret_class = security + .tls_secret_class() + .map(|secret_class| { + SecretClassName::from_str(secret_class) + .context(ParseClientListenerSecretClassSnafu { secret_class }) + }) + .transpose()?; + + match (authentication, server_ca_secret_class) { + (v1alpha1::KafkaPlatformAccessAuthentication::Anonymous(_), None) => { + Ok(AgentConnection::Plaintext) + } + ( + v1alpha1::KafkaPlatformAccessAuthentication::Anonymous(_), + Some(server_ca_secret_class), + ) => { + ensure!( + security.tls_client_authentication_class().is_none(), + AnonymousPlatformAccessWithClientAuthenticationSnafu + ); + Ok(AgentConnection::Tls { + server_ca_secret_class, + }) + } + (v1alpha1::KafkaPlatformAccessAuthentication::Tls(_), None) => { + PlatformAccessRequiresTlsSnafu { + authentication: "tls", + } + .fail() + } + ( + v1alpha1::KafkaPlatformAccessAuthentication::Tls(client_credential), + Some(server_ca_secret_class), + ) => Ok(AgentConnection::MutualTls { + server_ca_secret_class, + client_credential: client_credential.clone(), + }), + } +} + #[cfg(test)] mod tests { use std::str::FromStr; - use stackable_operator::v2::{ - builder::pod::container::EnvVarSet, types::operator::RoleGroupName, + use rstest::rstest; + use stackable_operator::{ + crd::platform_access::v1alpha1::{Anonymous, TlsClientCredential}, + v2::{ + builder::pod::container::EnvVarSet, + types::{ + kubernetes::{SecretClassName, SecretName}, + operator::RoleGroupName, + }, + }, }; - use super::{Error, KAFKA_CLUSTER_ID, inject_cluster_id}; + use super::{Error, KAFKA_CLUSTER_ID, inject_cluster_id, validate_agent_connection}; + use crate::{ + controller::{ + AgentConnection, + security::ValidatedKafkaSecurity, + test_support::{client_auth_tls, kerberos, plaintext, server_tls}, + }, + crd::v1alpha1::KafkaPlatformAccessAuthentication, + }; + + fn anonymous() -> KafkaPlatformAccessAuthentication { + KafkaPlatformAccessAuthentication::Anonymous(Anonymous::default()) + } + + fn agent_cert() -> TlsClientCredential { + TlsClientCredential::Secret( + SecretName::from_str("agent-cert").expect("the Secret name should be valid"), + ) + } + + fn tls() -> KafkaPlatformAccessAuthentication { + KafkaPlatformAccessAuthentication::Tls(agent_cert()) + } + + fn tls_secret_class(name: &str) -> SecretClassName { + SecretClassName::from_str(name).expect("the SecretClass name should be valid") + } + + #[rstest] + #[case::plaintext_anonymous(plaintext(), anonymous(), AgentConnection::Plaintext)] + #[case::server_tls_anonymous( + server_tls(), + anonymous(), + AgentConnection::Tls { server_ca_secret_class: tls_secret_class("tls") } + )] + #[case::server_tls_with_credential( + server_tls(), + tls(), + AgentConnection::MutualTls { + server_ca_secret_class: tls_secret_class("tls"), + client_credential: agent_cert(), + } + )] + #[case::client_auth_with_credential( + client_auth_tls(), + tls(), + AgentConnection::MutualTls { + server_ca_secret_class: tls_secret_class("client-auth-secret-class"), + client_credential: agent_cert(), + } + )] + fn agent_connection( + #[case] security: ValidatedKafkaSecurity, + #[case] authentication: KafkaPlatformAccessAuthentication, + #[case] expected: AgentConnection, + ) { + let connection = validate_agent_connection(&authentication, &security) + .expect("the agent connection should be valid"); + + assert_eq!(expected, connection); + } + + #[test] + fn reject_unsupported_cluster_authentication() { + let error = validate_agent_connection(&tls(), &kerberos()) + .expect_err("the agent connection should be invalid"); + + assert!(matches!( + error, + Error::UnsupportedClusterAuthentication { .. } + )); + } + + #[test] + fn reject_anonymous_agent_connection_with_client_authentication() { + let error = validate_agent_connection(&anonymous(), &client_auth_tls()) + .expect_err("the agent connection should be invalid"); + + assert!(matches!( + error, + Error::AnonymousPlatformAccessWithClientAuthentication + )); + } + + #[test] + fn reject_agent_connection_requiring_tls_without_tls() { + let error = validate_agent_connection(&tls(), &plaintext()) + .expect_err("the agent connection should be invalid"); + + assert!(matches!(error, Error::PlatformAccessRequiresTls { .. })); + } use crate::{ controller::test_support::{app_version_label, minimal_kafka, validated_cluster}, crd::role::KafkaRole, diff --git a/rust/shared/src/agent.rs b/rust/shared/src/agent.rs index 4af72434c..617360172 100644 --- a/rust/shared/src/agent.rs +++ b/rust/shared/src/agent.rs @@ -1,17 +1,18 @@ //! Paths and environment variables through which the operator configures the agent. +use std::str::FromStr; + +use stackable_operator::{constant, v2::builder::pod::container::EnvVarName}; use strum::{Display, EnumString}; pub const DISCOVERY_DIR: &str = "/stackable/discovery"; /// Directory of the CA certificate the Kafka brokers are verified with. -pub const SERVER_CA_DIR: &str = "/stackable/secrets/tls-server-ca"; - -pub const ENV_CLUSTER_NAME: &str = "KAFKA_CLUSTER_NAME"; - -pub const ENV_CLUSTER_NAMESPACE: &str = "KAFKA_CLUSTER_NAMESPACE"; +pub const KAFKA_TLS_DIR: &str = "/stackable/secrets/tls-kafka"; -pub const ENV_CLIENT_SECURITY: &str = "KAFKA_CLIENT_SECURITY"; +constant!(pub ENV_CLUSTER_NAME: EnvVarName = "KAFKA_CLUSTER_NAME"); +constant!(pub ENV_CLUSTER_NAMESPACE: EnvVarName = "KAFKA_CLUSTER_NAMESPACE"); +constant!(pub ENV_CLIENT_SECURITY: EnvVarName = "KAFKA_CLIENT_SECURITY"); /// How the agent connects to the brokers. #[derive(Clone, Copy, Debug, Display, EnumString, Eq, PartialEq)] From 9ba595dcc6eacc0b7e7f276d6d4e8e7661df4d84 Mon Sep 17 00:00:00 2001 From: Benedikt Labrenz Date: Fri, 16 Oct 2026 11:30:09 +0200 Subject: [PATCH 14/93] reject zookeeper-mode when platformAccess is configured --- .../src/controller/build/resource/agent.rs | 6 +- .../src/controller/validate.rs | 60 ++++++++++++++++--- 2 files changed, 58 insertions(+), 8 deletions(-) diff --git a/rust/operator-binary/src/controller/build/resource/agent.rs b/rust/operator-binary/src/controller/build/resource/agent.rs index 81c7e34a1..20c6323a3 100644 --- a/rust/operator-binary/src/controller/build/resource/agent.rs +++ b/rust/operator-binary/src/controller/build/resource/agent.rs @@ -241,8 +241,12 @@ mod tests { spec: image: {image} clusterConfig: - zookeeperConfigMapName: xyz + metadataManager: kraft platformAccess: {platform_access} + controllers: + roleGroups: + default: + replicas: 1 brokers: roleGroups: default: diff --git a/rust/operator-binary/src/controller/validate.rs b/rust/operator-binary/src/controller/validate.rs index f500b0595..ce9497f03 100644 --- a/rust/operator-binary/src/controller/validate.rs +++ b/rust/operator-binary/src/controller/validate.rs @@ -122,6 +122,9 @@ pub enum Error { ))] UnsupportedClusterAuthentication { authentication: String }, + #[snafu(display("platform access is only supported in KRaft mode"))] + PlatformAccessRequiresKraft, + #[snafu(display( "the cluster requires client authentication, but platform access is anonymous" ))] @@ -358,13 +361,19 @@ pub fn validate( .clone(); let agent_config = match &kafka.spec.platform_access { - Some(platform_access) if platform_access.enabled => Some(ValidatedAgentConfig { - image: agent_image.to_owned(), - connection: validate_agent_connection( - &platform_access.authentication, - &kafka_security, - )?, - }), + Some(platform_access) if platform_access.enabled => { + ensure!( + metadata_manager == crate::crd::MetadataManager::KRaft, + PlatformAccessRequiresKraftSnafu + ); + Some(ValidatedAgentConfig { + image: agent_image.to_owned(), + connection: validate_agent_connection( + &platform_access.authentication, + &kafka_security, + )?, + }) + } _ => None, }; @@ -929,4 +938,41 @@ mod tests { let _cluster = validated_cluster(&kafka); } + + #[test] + fn zookeeper_mode_rejects_platform_access() { + let kafka = minimal_kafka( + r#" + apiVersion: kafka.stackable.tech/v1alpha1 + kind: KafkaCluster + metadata: + name: simple-kafka + namespace: default + uid: 12345678-1234-1234-1234-123456789012 + spec: + image: + productVersion: 3.9.2 + clusterConfig: + zookeeperConfigMapName: zk-discovery + platformAccess: + enabled: true + authentication: + anonymous: {} + brokers: + roleGroups: + default: + replicas: 3 + "#, + ); + + let result = crate::controller::test_support::validate_err(&kafka); + let Err(error) = result else { + panic!("validate should reject platform access in ZooKeeper mode"); + }; + + assert!( + matches!(error, Error::PlatformAccessRequiresKraft), + "expected PlatformAccessRequiresKraft, got: {error:?}" + ); + } } From 2c730cad26e55395472a6c008f038e38deaa6ef1 Mon Sep 17 00:00:00 2001 From: Benedikt Labrenz Date: Thu, 15 Oct 2026 19:21:47 +0200 Subject: [PATCH 15/93] add config to agent: bootstrap servers from discovery cm, security mode, tls paths --- Cargo.lock | 5 ++ Cargo.nix | 22 +++++ rust/agent-binary/Cargo.toml | 7 ++ rust/agent-binary/src/config.rs | 141 ++++++++++++++++++++++++++++++++ rust/agent-binary/src/main.rs | 46 ++++++++++- 5 files changed, 219 insertions(+), 2 deletions(-) create mode 100644 rust/agent-binary/src/config.rs diff --git a/Cargo.lock b/Cargo.lock index f646c56ea..bfb174c5c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3258,6 +3258,11 @@ dependencies = [ "anyhow", "built", "clap", + "const_format", + "rstest", + "snafu 0.9.2", + "stackable-kafka-admin", + "stackable-kafka-shared", "stackable-operator", "tokio", "tracing", diff --git a/Cargo.nix b/Cargo.nix index 27002f0f9..15ce6ebba 100644 --- a/Cargo.nix +++ b/Cargo.nix @@ -10698,6 +10698,22 @@ rec { name = "clap"; packageId = "clap"; } + { + name = "const_format"; + packageId = "const_format"; + } + { + name = "snafu"; + packageId = "snafu 0.9.2"; + } + { + name = "stackable-kafka-admin"; + packageId = "stackable-kafka-admin"; + } + { + name = "stackable-kafka-shared"; + packageId = "stackable-kafka-shared"; + } { name = "stackable-operator"; packageId = "stackable-operator"; @@ -10720,6 +10736,12 @@ rec { features = [ "chrono" "git2" ]; } ]; + devDependencies = [ + { + name = "rstest"; + packageId = "rstest"; + } + ]; }; "stackable-kafka-operator" = rec { diff --git a/rust/agent-binary/Cargo.toml b/rust/agent-binary/Cargo.toml index c138e675b..925327a25 100644 --- a/rust/agent-binary/Cargo.toml +++ b/rust/agent-binary/Cargo.toml @@ -9,13 +9,20 @@ repository.workspace = true publish = false [dependencies] +stackable-kafka-admin.workspace = true +stackable-kafka-shared.workspace = true stackable-operator.workspace = true anyhow.workspace = true clap.workspace = true +const_format.workspace = true +snafu.workspace = true tokio.workspace = true tracing.workspace = true +[dev-dependencies] +rstest.workspace = true + [build-dependencies] built.workspace = true diff --git a/rust/agent-binary/src/config.rs b/rust/agent-binary/src/config.rs new file mode 100644 index 000000000..bdb2bc9f7 --- /dev/null +++ b/rust/agent-binary/src/config.rs @@ -0,0 +1,141 @@ +//! The agent's view of the files and settings the operator provides. + +use std::{ + fs, io, + path::{Path, PathBuf}, +}; + +use const_format::concatcp; +use snafu::{ResultExt, Snafu}; +use stackable_kafka_admin::{ + bootstrap::{self, BrokerAddress, parse_bootstrap_servers}, + connection::Security, +}; +use stackable_kafka_shared::agent::{ClientSecurity, DISCOVERY_DIR, KAFKA_TLS_DIR}; +use stackable_operator::crd::platform_access::tls::MOUNT_PATH; + +/// The key `KAFKA` of the discovery ConfigMap, holding `host:port,host:port` +pub const BOOTSTRAP_SERVERS_FILE: &str = concatcp!(DISCOVERY_DIR, "/KAFKA"); + +const CA_FILE: &str = concatcp!(KAFKA_TLS_DIR, "/ca.crt"); +const CERTIFICATE_FILE: &str = concatcp!(MOUNT_PATH, "/tls.crt"); +const KEY_FILE: &str = concatcp!(MOUNT_PATH, "/tls.key"); + +#[derive(Debug, Snafu)] +pub enum Error { + #[snafu(display("failed to read the bootstrap servers from {path:?}"))] + ReadBootstrapServers { source: io::Error, path: PathBuf }, + + #[snafu(display("the bootstrap servers in {path:?} are invalid"))] + ParseBootstrapServers { + source: bootstrap::Error, + path: PathBuf, + }, +} + +pub fn security(client_security: ClientSecurity) -> Security { + match client_security { + ClientSecurity::Plaintext => Security::Plaintext, + ClientSecurity::Tls => Security::Tls { ca: CA_FILE.into() }, + ClientSecurity::MutualTls => Security::MutualTls { + ca: CA_FILE.into(), + certificate: CERTIFICATE_FILE.into(), + key: KEY_FILE.into(), + }, + } +} + +/// Returns `None` while the operator has written no bootstrap servers to the discovery configmap yet. +pub fn read_bootstrap_servers(path: &Path) -> Result>, Error> { + let value = fs::read_to_string(path).context(ReadBootstrapServersSnafu { path })?; + if value.trim().is_empty() { + return Ok(None); + } + parse_bootstrap_servers(&value) + .map(Some) + .context(ParseBootstrapServersSnafu { path }) +} + +#[cfg(test)] +mod tests { + use rstest::rstest; + + use super::*; + + #[rstest] + #[case::plaintext(ClientSecurity::Plaintext, Security::Plaintext)] + #[case::tls( + ClientSecurity::Tls, + Security::Tls { + ca: "/stackable/secrets/tls-kafka/ca.crt".into(), + } + )] + #[case::mutual_tls( + ClientSecurity::MutualTls, + Security::MutualTls { + ca: "/stackable/secrets/tls-kafka/ca.crt".into(), + certificate: "/stackable/secrets/tls-platform-access/tls.crt".into(), + key: "/stackable/secrets/tls-platform-access/tls.key".into(), + } + )] + fn map_client_security(#[case] client_security: ClientSecurity, #[case] expected: Security) { + assert_eq!(expected, security(client_security)); + } + + #[rstest] + #[case::empty("", None)] + #[case::blank(" \n", None)] + #[case::servers( + "kafka-0:9093,kafka-1:9093", + Some(vec![ + BrokerAddress { host: "kafka-0".to_owned(), port: 9093 }, + BrokerAddress { host: "kafka-1".to_owned(), port: 9093 }, + ]) + )] + fn read_written_bootstrap_servers( + #[case] content: &str, + #[case] expected: Option>, + ) { + let path = temp_file(content); + + let bootstrap_servers = read_bootstrap_servers(&path); + fs::remove_file(&path).expect("the file should be removable"); + + assert_eq!( + expected, + bootstrap_servers.expect("the bootstrap servers should be readable") + ); + } + + #[test] + fn reject_missing_bootstrap_servers_file() { + let path = std::env::temp_dir().join("missing-bootstrap-servers"); + + assert!(matches!( + read_bootstrap_servers(&path).expect_err("a missing file should be an error"), + Error::ReadBootstrapServers { .. } + )); + } + + #[test] + fn reject_invalid_bootstrap_servers() { + let path = temp_file("kafka"); + + let error = read_bootstrap_servers(&path); + fs::remove_file(&path).expect("the file should be removable"); + + assert!(matches!( + error.expect_err("the bootstrap servers should be invalid"), + Error::ParseBootstrapServers { .. } + )); + } + + /// Writes the content to a file unique to this test thread. + fn temp_file(content: &str) -> PathBuf { + let thread = std::thread::current(); + let name = thread.name().unwrap_or("test").replace("::", "-"); + let path = std::env::temp_dir().join(format!("bootstrap-{}-{name}", std::process::id())); + fs::write(&path, content).expect("the file should be writable"); + path + } +} diff --git a/rust/agent-binary/src/main.rs b/rust/agent-binary/src/main.rs index 0934f803f..df9393ba6 100644 --- a/rust/agent-binary/src/main.rs +++ b/rust/agent-binary/src/main.rs @@ -1,12 +1,20 @@ //! The Stackable Kafka agent: a per-cluster controller that reconciles in-cluster resources like `KafkaTopic`s. +use std::path::Path; + use clap::Parser; +use stackable_kafka_shared::agent::{ + ClientSecurity, ENV_CLIENT_SECURITY, ENV_CLUSTER_NAME, ENV_CLUSTER_NAMESPACE, +}; use stackable_operator::{ cli::{Command, CommonOptions}, telemetry::Tracing, utils::signal::SignalWatcher, + v2::types::{kubernetes::NamespaceName, operator::ClusterName}, }; +mod config; + mod built_info { include!(concat!(env!("OUT_DIR"), "/built.rs")); } @@ -15,14 +23,37 @@ mod built_info { #[clap(about, author)] struct Opts { #[clap(subcommand)] - cmd: Command, + cmd: Command, +} + +#[derive(clap::Parser)] +struct KafkaAgentRun { + /// Name of the KafkaCluster this agent belongs to. + #[arg(long, env = AsRef::::as_ref(&*ENV_CLUSTER_NAME))] + cluster_name: ClusterName, + + /// Namespace of the KafkaCluster. + #[arg(long, env = AsRef::::as_ref(&*ENV_CLUSTER_NAMESPACE))] + cluster_namespace: NamespaceName, + + /// How to connect to the brokers: plaintext, tls or mutual-tls. + #[arg(long, env = AsRef::::as_ref(&*ENV_CLIENT_SECURITY))] + client_security: ClientSecurity, + + #[clap(flatten)] + common: CommonOptions, } #[tokio::main] async fn main() -> anyhow::Result<()> { match Opts::parse().cmd { Command::Crd => anyhow::bail!("this agent has no CRDs, they are owned by the operator"), - Command::Run(CommonOptions { telemetry, .. }) => { + Command::Run(KafkaAgentRun { + cluster_name, + cluster_namespace, + client_security, + common: CommonOptions { telemetry, .. }, + }) => { let _tracing_guard = Tracing::pre_configured(built_info::PKG_NAME, telemetry).init()?; tracing::info!( @@ -35,6 +66,17 @@ async fn main() -> anyhow::Result<()> { description = built_info::PKG_DESCRIPTION ); + let security = config::security(client_security); + let bootstrap_servers = + config::read_bootstrap_servers(Path::new(config::BOOTSTRAP_SERVERS_FILE)); + tracing::info!( + %cluster_name, + %cluster_namespace, + ?security, + ?bootstrap_servers, + "Configured the agent" + ); + let sigterm_watcher = SignalWatcher::sigterm()?; sigterm_watcher.handle().await; } From 5463640b82544b6c3831519da765d35e01d93716 Mon Sep 17 00:00:00 2001 From: Benedikt Labrenz Date: Fri, 16 Oct 2026 11:33:15 +0200 Subject: [PATCH 16/93] add stackable-kafka-adminto workspace dependencies --- Cargo.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/Cargo.toml b/Cargo.toml index dfe32d4b2..dabfa6f28 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -11,6 +11,7 @@ repository = "https://github.com/stackabletech/kafka-operator" [workspace.dependencies] stackable-operator = { git = "https://github.com/stackabletech/operator-rs.git", branch = "feat/platform-access", features = ["crds", "webhook"] } +stackable-kafka-admin = { path = "rust/kafka-admin" } stackable-kafka-shared = { path = "rust/shared" } anyhow = "1.0" From 87a547951f54a1a61c951ff064d5270da78bca2a Mon Sep 17 00:00:00 2001 From: Benedikt Labrenz Date: Fri, 16 Oct 2026 14:17:40 +0200 Subject: [PATCH 17/93] add support for list_partition_reassignments --- rust/kafka-admin/src/client.rs | 157 ++++++++++++++++++++++++++++++++- 1 file changed, 153 insertions(+), 4 deletions(-) diff --git a/rust/kafka-admin/src/client.rs b/rust/kafka-admin/src/client.rs index 39e5c390f..27f00b8ad 100644 --- a/rust/kafka-admin/src/client.rs +++ b/rust/kafka-admin/src/client.rs @@ -10,14 +10,16 @@ use kafka_protocol::{ ResponseError, error::ParseResponseErrorCode, messages::{ - CreatePartitionsRequest, CreateTopicsRequest, DeleteTopicsRequest, DeleteTopicsResponse, - DescribeConfigsRequest, DescribeConfigsResponse, IncrementalAlterConfigsRequest, - MetadataRequest, MetadataResponse, TopicName, + BrokerId, CreatePartitionsRequest, CreateTopicsRequest, DeleteTopicsRequest, + DeleteTopicsResponse, DescribeConfigsRequest, DescribeConfigsResponse, + IncrementalAlterConfigsRequest, ListPartitionReassignmentsRequest, + ListPartitionReassignmentsResponse, MetadataRequest, MetadataResponse, TopicName, create_partitions_request::CreatePartitionsTopic, create_topics_request::{CreatableTopic, CreatableTopicConfig}, delete_topics_request::DeleteTopicState, describe_configs_request::DescribeConfigsResource, incremental_alter_configs_request::{AlterConfigsResource, AlterableConfig}, + list_partition_reassignments_request::ListPartitionReassignmentsTopics, metadata_request::MetadataRequestTopic, }, protocol::{Request, StrBytes}, @@ -36,6 +38,7 @@ const DELETE_TOPICS_VERSION: i16 = 6; const CREATE_PARTITIONS_VERSION: i16 = 3; const DESCRIBE_CONFIGS_VERSION: i16 = 4; const INCREMENTAL_ALTER_CONFIGS_VERSION: i16 = 1; +const LIST_PARTITION_REASSIGNMENTS_VERSION: i16 = 0; /// Must stay below the connection's request timeout. const BROKER_TIMEOUT_MS: i32 = 25_000; @@ -115,6 +118,14 @@ pub struct Partition { pub replicas: Vec, } +/// An ongoing reassignment of one partition. +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct PartitionReassignment { + pub index: i32, + pub adding_replicas: Vec, + pub removing_replicas: Vec, +} + pub struct KafkaAdminClient { bootstrap_servers: Vec, security: Security, @@ -233,6 +244,22 @@ impl KafkaAdminClient { delete_topic_result(response, topic) } + /// Returns the ongoing reassignments of the given partitions, sorted by the partition index. + /// Needs `DESCRIBE` on the cluster. + pub async fn list_partition_reassignments( + &mut self, + topic: &str, + partitions: &[i32], + ) -> Result> { + let response = self + .send( + list_partition_reassignments_request(topic, partitions), + LIST_PARTITION_REASSIGNMENTS_VERSION, + ) + .await?; + list_partition_reassignments_result(response, topic) + } + /// Sends the request on the current connection or a new one. A failed or cancelled request /// drops the connection, and a connection idle for longer than [`CONNECTION_MAX_IDLE`] is /// replaced. @@ -289,7 +316,7 @@ fn describe_topic_result(response: MetadataResponse, topic: &str) -> Result>(); partitions.sort_by_key(|partition| partition.index); @@ -414,10 +441,55 @@ fn delete_topic_result(response: DeleteTopicsResponse, topic: &str) -> Result<() ) } +/// Only the given partitions are checked; an empty list checks none. +fn list_partition_reassignments_request( + topic: &str, + partitions: &[i32], +) -> ListPartitionReassignmentsRequest { + ListPartitionReassignmentsRequest::default() + .with_topics(Some(vec![ + ListPartitionReassignmentsTopics::default() + .with_name(topic_name(topic)) + .with_partition_indexes(partitions.to_vec()), + ])) + .with_timeout_ms(BROKER_TIMEOUT_MS) +} + +/// The response only contains partitions with an ongoing reassignment. +fn list_partition_reassignments_result( + response: ListPartitionReassignmentsResponse, + topic: &str, +) -> Result> { + check( + ListPartitionReassignmentsRequest::KEY, + topic, + response.error_code, + response.error_message, + )?; + + let mut reassignments = response + .topics + .into_iter() + .filter(|result| result.name == topic_name(topic)) + .flat_map(|result| result.partitions) + .map(|partition| PartitionReassignment { + index: partition.partition_index, + adding_replicas: broker_ids(partition.adding_replicas), + removing_replicas: broker_ids(partition.removing_replicas), + }) + .collect::>(); + reassignments.sort_by_key(|reassignment| reassignment.index); + Ok(reassignments) +} + fn topic_name(topic: &str) -> TopicName { TopicName::from(StrBytes::from_string(topic.to_owned())) } +fn broker_ids(ids: Vec) -> Vec { + ids.into_iter().map(|id| *id).collect() +} + // Used instead of [0] to correctly handle empty results & take ownership fn first(results: Vec, api_key: i16, topic: &str) -> Result { results @@ -444,6 +516,9 @@ mod tests { use kafka_protocol::messages::{ delete_topics_response::DeletableTopicResult, describe_configs_response::{DescribeConfigsResourceResult, DescribeConfigsResult}, + list_partition_reassignments_response::{ + OngoingPartitionReassignment, OngoingTopicReassignment, + }, metadata_response::{MetadataResponsePartition, MetadataResponseTopic}, }; use rstest::rstest; @@ -645,6 +720,80 @@ mod tests { ); } + #[test] + fn list_reassignments_of_the_given_partitions() { + let request = list_partition_reassignments_request(TOPIC, &[0, 1, 2]); + + let topics = request.topics.expect("the topics should be requested"); + assert_eq!(topic_name(TOPIC), topics[0].name); + assert_eq!(vec![0, 1, 2], topics[0].partition_indexes); + assert_eq!(BROKER_TIMEOUT_MS, request.timeout_ms); + } + + #[test] + fn read_ongoing_reassignments() { + let reassignment = |index: i32, adding: Vec, removing: Vec| { + OngoingPartitionReassignment::default() + .with_partition_index(index) + .with_adding_replicas(adding.into_iter().map(Into::into).collect()) + .with_removing_replicas(removing.into_iter().map(Into::into).collect()) + }; + let response = ListPartitionReassignmentsResponse::default().with_topics(vec![ + OngoingTopicReassignment::default() + .with_name(topic_name("other")) + .with_partitions(vec![reassignment(0, vec![9], vec![8])]), + OngoingTopicReassignment::default() + .with_name(topic_name(TOPIC)) + .with_partitions(vec![ + reassignment(2, vec![4], vec![]), + reassignment(0, vec![4, 5], vec![1, 2]), + ]), + ]); + + assert_eq!( + vec![ + PartitionReassignment { + index: 0, + adding_replicas: vec![4, 5], + removing_replicas: vec![1, 2], + }, + PartitionReassignment { + index: 2, + adding_replicas: vec![4], + removing_replicas: vec![], + }, + ], + list_partition_reassignments_result(response, TOPIC) + .expect("the reassignments should be readable") + ); + } + + #[test] + fn read_no_ongoing_reassignments() { + assert_eq!( + Vec::::new(), + list_partition_reassignments_result( + ListPartitionReassignmentsResponse::default(), + TOPIC + ) + .expect("no reassignments should not be an error") + ); + } + + #[test] + fn reject_reassignments_without_cluster_permission() { + let response = ListPartitionReassignmentsResponse::default() + .with_error_code(ResponseError::ClusterAuthorizationFailed.code()); + + let error = list_partition_reassignments_result(response, TOPIC) + .expect_err("the request should be rejected"); + + assert_eq!( + Some(ResponseError::ClusterAuthorizationFailed), + error.kafka_error() + ); + } + #[test] fn delete_topic_by_name() { let request = delete_topic_request(TOPIC); From 798619e1006956181091cf144c9583a007613567 Mon Sep 17 00:00:00 2001 From: Benedikt Labrenz Date: Fri, 16 Oct 2026 16:00:41 +0200 Subject: [PATCH 18/93] replace kafka_error with semantic error checks in the kafka admin client --- rust/kafka-admin/src/client.rs | 43 ++++++++++++++++++++++++++++++++-- 1 file changed, 41 insertions(+), 2 deletions(-) diff --git a/rust/kafka-admin/src/client.rs b/rust/kafka-admin/src/client.rs index 27f00b8ad..8320048e3 100644 --- a/rust/kafka-admin/src/client.rs +++ b/rust/kafka-admin/src/client.rs @@ -93,7 +93,21 @@ impl Error { } } - pub fn kafka_error(&self) -> Option { + /// Whether the broker answered with an error code, as opposed to a connection problem. + pub fn is_rejected_by_kafka(&self) -> bool { + self.kafka_error().is_some() + } + + pub fn is_topic_already_exists(&self) -> bool { + self.kafka_error() == Some(ResponseError::TopicAlreadyExists) + } + + /// Kafka rejects a partition count that is not larger than the current one. + pub fn is_partition_count_reached(&self) -> bool { + self.kafka_error() == Some(ResponseError::InvalidPartitions) + } + + fn kafka_error(&self) -> Option { match self { Error::Kafka { error, .. } => Some(*error), _ => None, @@ -824,11 +838,36 @@ mod tests { ) .expect_err("the request should be rejected"); - assert_eq!(Some(ResponseError::TopicAlreadyExists), error.kafka_error()); + assert!(error.is_rejected_by_kafka()); + assert!(error.is_topic_already_exists()); + assert!(!error.is_partition_count_reached()); assert!(!error.is_retriable()); assert!(error.to_string().contains("Topic 'orders' already exists.")); } + #[test] + fn recognize_a_reached_partition_count() { + let error = check( + CreatePartitionsRequest::KEY, + TOPIC, + ResponseError::InvalidPartitions.code(), + Some(StrBytes::from_static_str( + "Topic already has 6 partition(s).", + )), + ) + .expect_err("the request should be rejected"); + + assert!(error.is_partition_count_reached()); + assert!(!error.is_topic_already_exists()); + } + + #[test] + fn do_not_treat_connection_errors_as_rejected_by_kafka() { + let error = Error::NoBootstrapServers; + + assert!(!error.is_rejected_by_kafka()); + } + #[test] fn reject_response_without_result() { let error = first( From d0a7abbf7ab4fe06dbcbec6aaa51c62f65599901 Mon Sep 17 00:00:00 2001 From: Benedikt Labrenz Date: Fri, 16 Oct 2026 17:01:41 +0200 Subject: [PATCH 19/93] remove is retryable checks for kafka errors --- rust/kafka-admin/src/client.rs | 9 --------- rust/kafka-admin/src/connection.rs | 17 ----------------- 2 files changed, 26 deletions(-) diff --git a/rust/kafka-admin/src/client.rs b/rust/kafka-admin/src/client.rs index 8320048e3..00f7f093a 100644 --- a/rust/kafka-admin/src/client.rs +++ b/rust/kafka-admin/src/client.rs @@ -85,14 +85,6 @@ pub enum Error { type Result = std::result::Result; impl Error { - pub fn is_retriable(&self) -> bool { - match self { - Error::Connect { source, .. } | Error::Send { source } => source.is_retriable(), - Error::Kafka { error, .. } => error.is_retriable(), - _ => false, - } - } - /// Whether the broker answered with an error code, as opposed to a connection problem. pub fn is_rejected_by_kafka(&self) -> bool { self.kafka_error().is_some() @@ -841,7 +833,6 @@ mod tests { assert!(error.is_rejected_by_kafka()); assert!(error.is_topic_already_exists()); assert!(!error.is_partition_count_reached()); - assert!(!error.is_retriable()); assert!(error.to_string().contains("Topic 'orders' already exists.")); } diff --git a/rust/kafka-admin/src/connection.rs b/rust/kafka-admin/src/connection.rs index d4a29ef2e..b6f97c0be 100644 --- a/rust/kafka-admin/src/connection.rs +++ b/rust/kafka-admin/src/connection.rs @@ -84,23 +84,6 @@ pub enum Error { type Result = std::result::Result; -impl Error { - pub fn is_retriable(&self) -> bool { - match self { - Error::Connect { .. } - | Error::ConnectTimeout { .. } - | Error::TlsHandshake { .. } - | Error::Write { .. } - | Error::Read { .. } - | Error::ConnectionClosed - | Error::ConnectionBroken - | Error::RequestTimeout { .. } => true, - Error::Kafka { error, .. } => error.is_retriable(), - _ => false, - } - } -} - /// How to connect to the broker. Paths point to PEM files. #[derive(Clone, Debug, Eq, PartialEq)] pub enum Security { From 95581f035dfcb394f1807faa0d40ac13c6f10336 Mon Sep 17 00:00:00 2001 From: Benedikt Labrenz Date: Fri, 16 Oct 2026 19:38:16 +0200 Subject: [PATCH 20/93] simplify kafka error handling --- rust/kafka-admin/src/client.rs | 47 ++++++---------------------------- 1 file changed, 8 insertions(+), 39 deletions(-) diff --git a/rust/kafka-admin/src/client.rs b/rust/kafka-admin/src/client.rs index 00f7f093a..ce041188d 100644 --- a/rust/kafka-admin/src/client.rs +++ b/rust/kafka-admin/src/client.rs @@ -87,23 +87,7 @@ type Result = std::result::Result; impl Error { /// Whether the broker answered with an error code, as opposed to a connection problem. pub fn is_rejected_by_kafka(&self) -> bool { - self.kafka_error().is_some() - } - - pub fn is_topic_already_exists(&self) -> bool { - self.kafka_error() == Some(ResponseError::TopicAlreadyExists) - } - - /// Kafka rejects a partition count that is not larger than the current one. - pub fn is_partition_count_reached(&self) -> bool { - self.kafka_error() == Some(ResponseError::InvalidPartitions) - } - - fn kafka_error(&self) -> Option { - match self { - Error::Kafka { error, .. } => Some(*error), - _ => None, - } + matches!(self, Error::Kafka { .. }) } } @@ -794,10 +778,13 @@ mod tests { let error = list_partition_reassignments_result(response, TOPIC) .expect_err("the request should be rejected"); - assert_eq!( - Some(ResponseError::ClusterAuthorizationFailed), - error.kafka_error() - ); + assert!(matches!( + error, + Error::Kafka { + error: ResponseError::ClusterAuthorizationFailed, + .. + } + )); } #[test] @@ -831,27 +818,9 @@ mod tests { .expect_err("the request should be rejected"); assert!(error.is_rejected_by_kafka()); - assert!(error.is_topic_already_exists()); - assert!(!error.is_partition_count_reached()); assert!(error.to_string().contains("Topic 'orders' already exists.")); } - #[test] - fn recognize_a_reached_partition_count() { - let error = check( - CreatePartitionsRequest::KEY, - TOPIC, - ResponseError::InvalidPartitions.code(), - Some(StrBytes::from_static_str( - "Topic already has 6 partition(s).", - )), - ) - .expect_err("the request should be rejected"); - - assert!(error.is_partition_count_reached()); - assert!(!error.is_topic_already_exists()); - } - #[test] fn do_not_treat_connection_errors_as_rejected_by_kafka() { let error = Error::NoBootstrapServers; From 69ed88e3286122133e086a34f3f40675acbf76b2 Mon Sep 17 00:00:00 2001 From: Benedikt Labrenz Date: Tue, 20 Oct 2026 14:59:37 +0200 Subject: [PATCH 21/93] add a flag for topic already exists errors --- rust/kafka-admin/src/client.rs | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/rust/kafka-admin/src/client.rs b/rust/kafka-admin/src/client.rs index ce041188d..f26dbcefd 100644 --- a/rust/kafka-admin/src/client.rs +++ b/rust/kafka-admin/src/client.rs @@ -89,6 +89,18 @@ impl Error { pub fn is_rejected_by_kafka(&self) -> bool { matches!(self, Error::Kafka { .. }) } + + /// Whether a CreateTopics request was rejected because the topic already exists. + pub fn is_topic_already_exists(&self) -> bool { + matches!( + self, + Error::Kafka { + api_key: CreateTopicsRequest::KEY, + error: ResponseError::TopicAlreadyExists, + .. + } + ) + } } #[derive(Clone, Debug, Eq, PartialEq)] @@ -821,6 +833,21 @@ mod tests { assert!(error.to_string().contains("Topic 'orders' already exists.")); } + #[rstest] + #[case::create_topics(CreateTopicsRequest::KEY, ResponseError::TopicAlreadyExists, true)] + #[case::other_error(CreateTopicsRequest::KEY, ResponseError::InvalidPartitions, false)] + #[case::other_request(CreatePartitionsRequest::KEY, ResponseError::TopicAlreadyExists, false)] + fn recognize_an_existing_topic_on_create( + #[case] api_key: i16, + #[case] error: ResponseError, + #[case] expected: bool, + ) { + let error = + check(api_key, TOPIC, error.code(), None).expect_err("the request should be rejected"); + + assert_eq!(expected, error.is_topic_already_exists()); + } + #[test] fn do_not_treat_connection_errors_as_rejected_by_kafka() { let error = Error::NoBootstrapServers; From e61fcdbf5be15d7b0b6945710360c5f8771796bd Mon Sep 17 00:00:00 2001 From: Benedikt Labrenz Date: Wed, 21 Oct 2026 13:56:40 +0200 Subject: [PATCH 22/93] add kafka controller --- Cargo.lock | 3 + Cargo.nix | 13 + .../templates/clusterrole-agent.yaml | 24 + extra/crds.yaml | 25 + rust/agent-binary/Cargo.toml | 3 + rust/agent-binary/src/config.rs | 2 +- rust/agent-binary/src/main.rs | 82 +- rust/agent-binary/src/topic/condition.rs | 302 ++++++++ rust/agent-binary/src/topic/controller.rs | 702 ++++++++++++++++++ .../src/topic/managed_config_keys.rs | 69 ++ rust/agent-binary/src/topic/mod.rs | 8 + rust/agent-binary/src/topic/plan.rs | 324 ++++++++ rust/agent-binary/src/topic/test_support.rs | 23 + rust/shared/src/crd/topic.rs | 18 + 14 files changed, 1589 insertions(+), 9 deletions(-) create mode 100644 rust/agent-binary/src/topic/condition.rs create mode 100644 rust/agent-binary/src/topic/controller.rs create mode 100644 rust/agent-binary/src/topic/managed_config_keys.rs create mode 100644 rust/agent-binary/src/topic/mod.rs create mode 100644 rust/agent-binary/src/topic/plan.rs create mode 100644 rust/agent-binary/src/topic/test_support.rs diff --git a/Cargo.lock b/Cargo.lock index bfb174c5c..10295ba60 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3259,11 +3259,14 @@ dependencies = [ "built", "clap", "const_format", + "futures", "rstest", + "serde_json", "snafu 0.9.2", "stackable-kafka-admin", "stackable-kafka-shared", "stackable-operator", + "strum", "tokio", "tracing", ] diff --git a/Cargo.nix b/Cargo.nix index 15ce6ebba..32a7143c6 100644 --- a/Cargo.nix +++ b/Cargo.nix @@ -10702,6 +10702,14 @@ rec { name = "const_format"; packageId = "const_format"; } + { + name = "futures"; + packageId = "futures"; + } + { + name = "serde_json"; + packageId = "serde_json"; + } { name = "snafu"; packageId = "snafu 0.9.2"; @@ -10719,6 +10727,11 @@ rec { packageId = "stackable-operator"; features = [ "crds" "webhook" ]; } + { + name = "strum"; + packageId = "strum"; + features = [ "derive" ]; + } { name = "tokio"; packageId = "tokio"; diff --git a/deploy/helm/kafka-operator/templates/clusterrole-agent.yaml b/deploy/helm/kafka-operator/templates/clusterrole-agent.yaml index a17015062..e138ef017 100644 --- a/deploy/helm/kafka-operator/templates/clusterrole-agent.yaml +++ b/deploy/helm/kafka-operator/templates/clusterrole-agent.yaml @@ -8,6 +8,30 @@ metadata: labels: {{- include "kafka-operator.labels" . | nindent 4 }} rules: + # KafkaTopics of the agent's cluster: watched, annotated (ownership, managed config keys) and finalizers. + - apiGroups: + - {{ include "kafka-operator.name" . }}.stackable.tech + resources: + - kafkatopics + verbs: + - get + - list + - watch + - patch + - apiGroups: + - {{ include "kafka-operator.name" . }}.stackable.tech + resources: + - kafkatopics/status + verbs: + - patch + # Required to report reconciliation errors as events on the KafkaTopic objects. + - apiGroups: + - events.k8s.io + resources: + - events + verbs: + - create + - patch {{ if .Capabilities.APIVersions.Has "security.openshift.io/v1" }} # On OpenShift, the agent pod must be allowed to use the nonroot-v2 SCC - apiGroups: diff --git a/extra/crds.yaml b/extra/crds.yaml index 3c8248781..a778f97bf 100644 --- a/extra/crds.yaml +++ b/extra/crds.yaml @@ -2365,6 +2365,31 @@ spec: - type type: object type: array + partitions: + default: [] + description: The partitions of the topic in Kafka. + items: + properties: + index: + format: int32 + type: integer + replicas: + description: |- + Broker ids holding a replica, the preferred leader first. During a reassignment, this + includes the replicas being added and removed. + items: + format: int32 + type: integer + type: array + required: + - index + - replicas + type: object + type: array + topicId: + description: The topic id in Kafka, as shown by `kafka-topics.sh`. + nullable: true + type: string type: object required: - spec diff --git a/rust/agent-binary/Cargo.toml b/rust/agent-binary/Cargo.toml index 925327a25..eb1fe94cd 100644 --- a/rust/agent-binary/Cargo.toml +++ b/rust/agent-binary/Cargo.toml @@ -16,7 +16,10 @@ stackable-operator.workspace = true anyhow.workspace = true clap.workspace = true const_format.workspace = true +futures.workspace = true +serde_json.workspace = true snafu.workspace = true +strum.workspace = true tokio.workspace = true tracing.workspace = true diff --git a/rust/agent-binary/src/config.rs b/rust/agent-binary/src/config.rs index bdb2bc9f7..41cdf48a7 100644 --- a/rust/agent-binary/src/config.rs +++ b/rust/agent-binary/src/config.rs @@ -45,7 +45,7 @@ pub fn security(client_security: ClientSecurity) -> Security { } } -/// Returns `None` while the operator has written no bootstrap servers to the discovery configmap yet. +/// Returns `None` while the operator has written no bootstrap servers to the discovery ConfigMap yet. pub fn read_bootstrap_servers(path: &Path) -> Result>, Error> { let value = fs::read_to_string(path).context(ReadBootstrapServersSnafu { path })?; if value.trim().is_empty() { diff --git a/rust/agent-binary/src/main.rs b/rust/agent-binary/src/main.rs index df9393ba6..b0bbe1aec 100644 --- a/rust/agent-binary/src/main.rs +++ b/rust/agent-binary/src/main.rs @@ -1,19 +1,38 @@ //! The Stackable Kafka agent: a per-cluster controller that reconciles in-cluster resources like `KafkaTopic`s. -use std::path::Path; +#![allow(clippy::large_enum_variant, clippy::result_large_err)] + +use std::sync::Arc; use clap::Parser; -use stackable_kafka_shared::agent::{ - ClientSecurity, ENV_CLIENT_SECURITY, ENV_CLUSTER_NAME, ENV_CLUSTER_NAMESPACE, +use futures::StreamExt; +use stackable_kafka_shared::{ + agent::{ClientSecurity, ENV_CLIENT_SECURITY, ENV_CLUSTER_NAME, ENV_CLUSTER_NAMESPACE}, + crd::topic::v1alpha1::KafkaTopic, }; use stackable_operator::{ cli::{Command, CommonOptions}, + client, + kube::{ + core::DeserializeGuard, + runtime::{ + Controller, + events::{Recorder, Reporter}, + watcher, + }, + }, + logging::controller::report_controller_reconciled, telemetry::Tracing, utils::signal::SignalWatcher, v2::types::{kubernetes::NamespaceName, operator::ClusterName}, }; +use crate::topic::controller::{self, TOPIC_FULL_CONTROLLER_NAME}; + mod config; +mod topic; + +pub const AGENT_NAME: &str = "kafka-agent.stackable.tech"; mod built_info { include!(concat!(env!("OUT_DIR"), "/built.rs")); @@ -52,7 +71,11 @@ async fn main() -> anyhow::Result<()> { cluster_name, cluster_namespace, client_security, - common: CommonOptions { telemetry, .. }, + common: + CommonOptions { + telemetry, + cluster_info, + }, }) => { let _tracing_guard = Tracing::pre_configured(built_info::PKG_NAME, telemetry).init()?; @@ -67,18 +90,61 @@ async fn main() -> anyhow::Result<()> { ); let security = config::security(client_security); - let bootstrap_servers = - config::read_bootstrap_servers(Path::new(config::BOOTSTRAP_SERVERS_FILE)); tracing::info!( %cluster_name, %cluster_namespace, ?security, - ?bootstrap_servers, "Configured the agent" ); + // Watches for the SIGTERM signal and gracefully shuts down the controller. let sigterm_watcher = SignalWatcher::sigterm()?; - sigterm_watcher.handle().await; + + let client = + client::initialize_operator(Some(AGENT_NAME.to_owned()), &cluster_info).await?; + + let event_recorder = Arc::new(Recorder::new( + client.as_kube_client(), + Reporter { + controller: TOPIC_FULL_CONTROLLER_NAME.to_string(), + instance: None, + }, + )); + + let topic_controller = Controller::new( + client.get_api::>(cluster_namespace.as_ref()), + watcher::Config::default(), + ); + let topic_store = topic_controller.store(); + topic_controller + .graceful_shutdown_on(sigterm_watcher.handle()) + .run( + controller::reconcile_topic, + controller::error_policy, + Arc::new(controller::Ctx::new( + client, + cluster_name, + security, + topic_store, + )), + ) + .for_each_concurrent( + 16, // concurrency limit + move |result| { + // The event_recorder needs to be shared across all invocations, so that + // events are correctly aggregated + let event_recorder = event_recorder.clone(); + async move { + report_controller_reconciled( + &event_recorder, + TOPIC_FULL_CONTROLLER_NAME, + &result, + ) + .await; + } + }, + ) + .await; } } diff --git a/rust/agent-binary/src/topic/condition.rs b/rust/agent-binary/src/topic/condition.rs new file mode 100644 index 000000000..d29c651d3 --- /dev/null +++ b/rust/agent-binary/src/topic/condition.rs @@ -0,0 +1,302 @@ +//! The `Ready` and `Synced` conditions of a `KafkaTopic` for the outcome of a sync. + +use stackable_kafka_admin::client; +use stackable_operator::commons::platform_access::condition::{ + ManagedCondition, ManagedConditionType, +}; + +const MAX_LISTED_PARTITIONS: usize = 10; + +#[derive(Debug, Eq, PartialEq)] +pub enum TopicSyncOutcome { + /// The topic matches the spec + Synced { + partitions: usize, + reassigning_partitions: Vec, + }, + UnsupportedChange { + messages: Vec, + }, + ClusterNotReady, + InvalidSpec { + message: String, + }, + /// Another `KafkaTopic` manages the same topic + Conflict { + message: String, + }, + /// The topic exists in Kafka, but this `KafkaTopic` does not own it and is not allowed to adopt + /// it + AlreadyExists { + message: String, + }, + KafkaRejected { + message: String, + }, + ConnectionFailed { + message: String, + }, +} + +impl TopicSyncOutcome { + pub fn from_client_error(error: &client::Error) -> Self { + let message = error_chain(error); + if error.is_rejected_by_kafka() { + TopicSyncOutcome::KafkaRejected { message } + } else { + TopicSyncOutcome::ConnectionFailed { message } + } + } +} + +/// The conditions to update. `Ready` says whether the topic is usable through this `KafkaTopic` +/// and is only updated if that is known. +pub fn conditions(outcome: &TopicSyncOutcome) -> Vec { + let available = ready(true, "Available", "the topic exists in Kafka"); + match outcome { + TopicSyncOutcome::Synced { + partitions, + reassigning_partitions, + } => { + let synced = if reassigning_partitions.is_empty() { + synced(true, "Synced", "the topic matches the spec") + } else { + synced( + true, + "ReassignmentInProgress", + &reassignment_message(reassigning_partitions, *partitions), + ) + }; + vec![available, synced] + } + TopicSyncOutcome::UnsupportedChange { messages } => vec![ + available, + synced(false, "UnsupportedChange", &messages.join("; ")), + ], + TopicSyncOutcome::ClusterNotReady => { + let message = "the KafkaCluster has no bootstrap servers yet"; + vec![ + ready(false, "ClusterNotReady", message), + synced(false, "ClusterNotReady", message), + ] + } + TopicSyncOutcome::Conflict { message } => vec![ + ready(false, "Conflict", "another KafkaTopic manages the topic"), + synced(false, "Conflict", message), + ], + TopicSyncOutcome::AlreadyExists { message } => vec![ + ready( + false, + "AlreadyExists", + "the topic exists in Kafka, but is not owned by this KafkaTopic", + ), + synced(false, "AlreadyExists", message), + ], + TopicSyncOutcome::InvalidSpec { message } => vec![synced(false, "InvalidSpec", message)], + TopicSyncOutcome::KafkaRejected { message } => { + vec![synced(false, "KafkaRejected", message)] + } + TopicSyncOutcome::ConnectionFailed { message } => { + vec![synced(false, "ConnectionFailed", message)] + } + } +} + +/// Renders the error and its sources, skipping a source whose message the previous error already +/// contains. +fn error_chain(error: &dyn std::error::Error) -> String { + snafu::CleanedErrorText::new(error) + .map(|(_, text, _)| text) + .filter(|text| !text.is_empty()) + .collect::>() + .join(": ") +} + +fn reassignment_message(reassigning_partitions: &[i32], partitions: usize) -> String { + let listed = reassigning_partitions + .iter() + .take(MAX_LISTED_PARTITIONS) + .map(i32::to_string) + .collect::>() + .join(", "); + let more = reassigning_partitions + .len() + .saturating_sub(MAX_LISTED_PARTITIONS); + if more > 0 { + format!("partitions {listed} and {more} more of {partitions} are being reassigned") + } else { + format!("partitions {listed} of {partitions} are being reassigned") + } +} + +fn ready(status: bool, reason: &str, message: &str) -> ManagedCondition { + condition(ManagedConditionType::Ready, status, reason, message) +} + +fn synced(status: bool, reason: &str, message: &str) -> ManagedCondition { + condition(ManagedConditionType::Synced, status, reason, message) +} + +fn condition( + type_: ManagedConditionType, + status: bool, + reason: &str, + message: &str, +) -> ManagedCondition { + ManagedCondition { + type_, + status, + reason: reason.to_owned(), + message: message.to_owned(), + } +} + +#[cfg(test)] +mod tests { + use snafu::Snafu; + + use super::*; + + fn summary(conditions: &[ManagedCondition]) -> Vec<(String, bool, &str)> { + conditions + .iter() + .map(|condition| { + ( + condition.type_.to_string(), + condition.status, + condition.reason.as_str(), + ) + }) + .collect() + } + + #[test] + fn mark_a_synced_topic() { + let conditions = conditions(&TopicSyncOutcome::Synced { + partitions: 6, + reassigning_partitions: vec![], + }); + + assert_eq!( + vec![ + ("Ready".to_owned(), true, "Available"), + ("Synced".to_owned(), true, "Synced") + ], + summary(&conditions) + ); + } + + #[test] + fn keep_a_reassigned_topic_synced() { + let conditions = conditions(&TopicSyncOutcome::Synced { + partitions: 6, + reassigning_partitions: vec![0, 2], + }); + + assert_eq!( + vec![ + ("Ready".to_owned(), true, "Available"), + ("Synced".to_owned(), true, "ReassignmentInProgress") + ], + summary(&conditions) + ); + assert_eq!( + "partitions 0, 2 of 6 are being reassigned", + conditions[1].message + ); + } + + #[test] + fn limit_the_listed_reassigning_partitions() { + let message = reassignment_message(&(0..12).collect::>(), 20); + + assert_eq!( + "partitions 0, 1, 2, 3, 4, 5, 6, 7, 8, 9 and 2 more of 20 are being reassigned", + message + ); + } + + #[test] + fn only_update_synced_after_an_error() { + let conditions = conditions(&TopicSyncOutcome::ConnectionFailed { + message: "timed out".to_owned(), + }); + + assert_eq!( + vec![("Synced".to_owned(), false, "ConnectionFailed")], + summary(&conditions) + ); + } + + #[test] + fn mark_an_unsupported_change() { + let conditions = conditions(&TopicSyncOutcome::UnsupportedChange { + messages: vec!["first".to_owned(), "second".to_owned()], + }); + + assert_eq!( + vec![ + ("Ready".to_owned(), true, "Available"), + ("Synced".to_owned(), false, "UnsupportedChange") + ], + summary(&conditions) + ); + assert_eq!("first; second", conditions[1].message); + } + + #[test] + fn leave_ready_untouched_for_an_invalid_spec() { + let conditions = conditions(&TopicSyncOutcome::InvalidSpec { + message: "invalid".to_owned(), + }); + + assert_eq!( + vec![("Synced".to_owned(), false, "InvalidSpec")], + summary(&conditions) + ); + } + + #[test] + fn keep_the_ready_message_fixed_for_a_foreign_topic() { + let conditions = conditions(&TopicSyncOutcome::AlreadyExists { + message: "set managementPolicy.adoption to Adopt".to_owned(), + }); + + assert_eq!( + vec![ + ("Ready".to_owned(), false, "AlreadyExists"), + ("Synced".to_owned(), false, "AlreadyExists") + ], + summary(&conditions) + ); + assert_eq!( + "the topic exists in Kafka, but is not owned by this KafkaTopic", + conditions[0].message + ); + assert_eq!( + "set managementPolicy.adoption to Adopt", + conditions[1].message + ); + } + + #[derive(Debug, Snafu)] + enum Outer { + #[snafu(display("failed to connect to the bootstrap server kafka-0:9093"))] + Connect { source: Inner }, + } + + #[derive(Debug, Snafu)] + #[snafu(display("invalid peer certificate: UnknownIssuer"))] + struct Inner; + + #[test] + fn render_the_error_chain() { + let error = Outer::Connect { source: Inner }; + + assert_eq!( + "failed to connect to the bootstrap server kafka-0:9093: invalid peer certificate: \ + UnknownIssuer", + error_chain(&error) + ); + } +} diff --git a/rust/agent-binary/src/topic/controller.rs b/rust/agent-binary/src/topic/controller.rs new file mode 100644 index 000000000..484bd9e13 --- /dev/null +++ b/rust/agent-binary/src/topic/controller.rs @@ -0,0 +1,702 @@ +//! The `KafkaTopic` controller: reconcile, cleanup on deletion and error policy. + +use std::{collections::BTreeSet, path::Path, sync::Arc, time::Duration}; + +use const_format::concatcp; +use serde_json::json; +use snafu::{ResultExt, Snafu}; +use stackable_kafka_admin::{ + bootstrap::BrokerAddress, + client::{self, KafkaAdminClient, Topic}, + connection::Security, +}; +use stackable_kafka_shared::crd::topic::v1alpha1::{ + KafkaTopic, KafkaTopicPartitionStatus, KafkaTopicStatus, +}; +use stackable_operator::{ + commons::platform_access::{ + condition::update_conditions, + ownership::{OWNED_SINCE_ANNOTATION, is_owned}, + }, + k8s_openapi::{apimachinery::pkg::apis::meta::v1::Time, jiff::Timestamp}, + kube::{ + ResourceExt, + core::{DeserializeGuard, error_boundary}, + runtime::{controller::Action, reflector::Store}, + }, + logging::controller::ReconcilerError, + v2::types::operator::ClusterName, +}; +use strum::{EnumDiscriminants, IntoStaticStr}; +use tokio::sync::Mutex; +use tracing::info; + +use super::{ + condition::{TopicSyncOutcome, conditions}, + managed_config_keys::{MANAGED_CONFIG_KEYS_ANNOTATION, managed_config_keys, render}, + plan::{ObservedTopic, TopicPlan, plan}, +}; +use crate::{ + AGENT_NAME, + config::{BOOTSTRAP_SERVERS_FILE, read_bootstrap_servers}, +}; + +pub const TOPIC_CONTROLLER_NAME: &str = "kafkatopic"; +pub const TOPIC_FULL_CONTROLLER_NAME: &str = concatcp!(TOPIC_CONTROLLER_NAME, ".", AGENT_NAME); +const FINALIZER: &str = "kafka.stackable.tech/kafkatopic"; +const KAFKA_CLIENT_ID: &str = "stackable-kafka-agent"; + +/// Set by `kafka-reassign-partitions` and Cruise Control for the duration of a reassignment. +const THROTTLE_CONFIG_KEYS: [&str; 2] = [ + "leader.replication.throttled.replicas", + "follower.replication.throttled.replicas", +]; + +const RESYNC_INTERVAL: Duration = Duration::from_secs(5 * 60); +const RETRY_INTERVAL: Duration = Duration::from_secs(10); +const ERROR_RETRY_INTERVAL: Duration = Duration::from_secs(30); + +#[derive(Debug, Snafu, EnumDiscriminants)] +#[strum_discriminants(derive(IntoStaticStr))] +pub enum Error { + #[snafu(display("the KafkaTopic is invalid"))] + InvalidKafkaTopic { + source: error_boundary::InvalidObject, + }, + + #[snafu(display("failed to read the bootstrap servers"))] + ReadBootstrapServers { source: crate::config::Error }, + + #[snafu(display("failed to talk to Kafka"))] + Kafka { source: client::Error }, + + #[snafu(display("failed to annotate the KafkaTopic"))] + Annotate { + source: stackable_operator::client::Error, + }, + + #[snafu(display("failed to update the status of the KafkaTopic"))] + UpdateStatus { + source: stackable_operator::client::Error, + }, + + #[snafu(display("failed to update the finalizers of the KafkaTopic"))] + UpdateFinalizers { + source: stackable_operator::client::Error, + }, +} + +type Result = std::result::Result; + +impl ReconcilerError for Error { + fn category(&self) -> &'static str { + ErrorDiscriminants::from(self).into() + } +} + +pub struct Ctx { + client: stackable_operator::client::Client, + cluster_name: ClusterName, + security: Security, + topics: Store>, + /// One client per agent, rebuilt when the bootstrap servers change + kafka: Mutex>, +} + +impl Ctx { + /// `topics` is the store of the controller, used to detect conflicts between `KafkaTopic`s. + pub fn new( + client: stackable_operator::client::Client, + cluster_name: ClusterName, + security: Security, + topics: Store>, + ) -> Self { + Self { + client, + cluster_name, + security, + topics, + kafka: Mutex::new(None), + } + } +} + +struct CachedClient { + bootstrap_servers: Vec, + client: KafkaAdminClient, +} + +pub async fn reconcile_topic( + topic: Arc>, + ctx: Arc, +) -> Result { + info!("Starting reconcile"); + + // dereference + let topic = topic + .0 + .as_ref() + .map_err(error_boundary::InvalidObject::clone) + .context(InvalidKafkaTopicSnafu)?; + if topic.spec.cluster_name != ctx.cluster_name { + return Ok(Action::await_change()); + } + + // finalizer (Kubernetes) + if let Some(action) = reconcile_finalizer(topic, &ctx).await? { + return Ok(action); + } + + // sync (Kafka + Kubernetes) + let (outcome, observed) = match sync(topic, &ctx).await { + Ok(synced) => synced, + Err(Error::Kafka { source }) => { + let outcome = TopicSyncOutcome::from_client_error(&source); + update_status(topic, &ctx, &outcome, None).await?; + return Err(Error::Kafka { source }); + } + Err(error) => return Err(error), + }; + + // update status (Kubernetes) + update_status(topic, &ctx, &outcome, observed.as_ref()).await?; + Ok(requeue(&outcome, observed.is_some())) +} + +/// Returns `Some` if the reconcile ends here. +async fn reconcile_finalizer(topic: &KafkaTopic, ctx: &Ctx) -> Result> { + let has_finalizer = topic + .finalizers() + .iter() + .any(|finalizer| finalizer == FINALIZER); + if topic.metadata.deletion_timestamp.is_some() { + if has_finalizer { + cleanup(topic, ctx).await?; + set_finalizer(topic, ctx, false).await?; + } + return Ok(Some(Action::await_change())); + } + + let needs_finalizer = topic.spec.management_policy.deletes_on_cleanup(); + if needs_finalizer != has_finalizer { + // The patch triggers another reconcile with the updated object. + set_finalizer(topic, ctx, needs_finalizer).await?; + return Ok(Some(Action::await_change())); + } + Ok(None) +} + +fn requeue(outcome: &TopicSyncOutcome, observed: bool) -> Action { + match outcome { + TopicSyncOutcome::InvalidSpec { .. } => Action::await_change(), + // After a create or a grow, the next reconcile fills in the topic id and partitions. + TopicSyncOutcome::Synced { .. } | TopicSyncOutcome::UnsupportedChange { .. } + if !observed => + { + Action::requeue(RETRY_INTERVAL) + } + TopicSyncOutcome::ClusterNotReady => Action::requeue(RETRY_INTERVAL), + _ => Action::requeue(RESYNC_INTERVAL), + } +} + +pub fn error_policy( + _topic: Arc>, + error: &Error, + _ctx: Arc, +) -> Action { + match error { + Error::InvalidKafkaTopic { .. } => Action::await_change(), + _ => Action::requeue(ERROR_RETRY_INTERVAL), + } +} + +/// Returns the outcome and, if the `KafkaTopic` owns the topic, the described topic. After a +/// partition grow, the topic is left out, because the description is outdated. +async fn sync(topic: &KafkaTopic, ctx: &Ctx) -> Result<(TopicSyncOutcome, Option)> { + let topic_name = topic.spec.topic_name.as_ref(); + + // validate (no client required) + if let Some(outcome) = validate(topic, &ctx.topics) { + return Ok((outcome, None)); + } + + // observe (Kafka) + let Some(bootstrap_servers) = read_bootstrap_servers(Path::new(BOOTSTRAP_SERVERS_FILE)) + .context(ReadBootstrapServersSnafu)? + else { + return Ok((TopicSyncOutcome::ClusterNotReady, None)); + }; + let mut cache = ctx.kafka.lock().await; + let kafka = client_for(&mut cache, bootstrap_servers, &ctx.security); + let observed = observe(kafka, topic_name).await?; + + // claim ownership (Kubernetes) + let claimed = if is_owned(&topic.metadata) { + false + } else { + if observed.is_some() && !topic.spec.management_policy.adopts_existing() { + let outcome = TopicSyncOutcome::AlreadyExists { + message: format!( + "the topic {topic_name:?} already exists in Kafka; set \ + managementPolicy.adoption to Adopt to take it over" + ), + }; + return Ok((outcome, None)); + } + claim_ownership(topic, ctx).await?; + true + }; + + // plan (no client required) + let recorded_keys = managed_config_keys(topic); + let topic_plan = plan(topic, observed.as_ref(), &recorded_keys); + let grows_partitions = matches!( + &topic_plan, + TopicPlan::Update(update) if update.grow_partitions_to.is_some() + ); + + // execute (Kafka + Kubernetes) + let outcome = match execute(topic, ctx, kafka, topic_plan, recorded_keys).await { + // The topic was created by someone else after it was observed. + Err(Error::Kafka { source }) if claimed && source.is_topic_already_exists() => { + release_ownership(topic, ctx).await?; + return Err(Error::Kafka { source }); + } + result => result?, + }; + + let observed = observed + .filter(|_| !grows_partitions) + .map(|observed| observed.topic); + Ok((outcome, observed)) +} + +/// Returns why the `KafkaTopic` cannot be synced, if it cannot. +fn validate( + topic: &KafkaTopic, + topics: &Store>, +) -> Option { + let throttle_keys = THROTTLE_CONFIG_KEYS + .iter() + .filter(|key| topic.spec.config.contains_key(**key)) + .copied() + .collect::>(); + if !throttle_keys.is_empty() { + return Some(TopicSyncOutcome::InvalidSpec { + message: format!( + "the config keys {throttle_keys:?} are managed by partition reassignments and \ + cannot be set" + ), + }); + } + conflicting_owner(topic, topics).map(|owner| TopicSyncOutcome::Conflict { + message: format!( + "the topic {:?} is managed by the KafkaTopic {owner:?}", + topic.spec.topic_name.as_ref() + ), + }) +} + +async fn observe(kafka: &mut KafkaAdminClient, topic_name: &str) -> Result> { + let Some(topic) = kafka.describe_topic(topic_name).await.context(KafkaSnafu)? else { + return Ok(None); + }; + let partitions = topic + .partitions + .iter() + .map(|partition| partition.index) + .collect::>(); + Ok(Some(ObservedTopic { + configs: kafka + .describe_topic_configs(topic_name) + .await + .context(KafkaSnafu)?, + reassignments: kafka + .list_partition_reassignments(topic_name, &partitions) + .await + .context(KafkaSnafu)?, + topic, + })) +} + +async fn execute( + topic: &KafkaTopic, + ctx: &Ctx, + kafka: &mut KafkaAdminClient, + topic_plan: TopicPlan, + mut recorded_keys: BTreeSet, +) -> Result { + let topic_name = topic.spec.topic_name.as_ref(); + let spec_keys = topic.spec.config.keys().cloned().collect::>(); + + let changes_configs = match &topic_plan { + TopicPlan::Create { .. } => true, + TopicPlan::Update(update) => { + !update.set_configs.is_empty() || !update.delete_configs.is_empty() + } + }; + let intended_keys = &recorded_keys | &spec_keys; + if changes_configs && intended_keys != recorded_keys { + record_managed_config_keys(topic, ctx, &intended_keys).await?; + recorded_keys = intended_keys; + } + + let outcome = match topic_plan { + TopicPlan::Create { + partitions, + replication_factor, + configs, + } => { + kafka + .create_topic(topic_name, partitions, replication_factor, &configs) + .await + .context(KafkaSnafu)?; + info!( + topic = topic_name, + partitions, + replication_factor, + ?configs, + kafka_topic = %topic.name_any(), + "Created the topic" + ); + TopicSyncOutcome::Synced { + partitions: partitions as usize, + reassigning_partitions: Vec::new(), + } + } + TopicPlan::Update(update) => { + if let Some(partitions) = update.grow_partitions_to { + kafka + .create_partitions(topic_name, partitions) + .await + .context(KafkaSnafu)?; + info!( + topic = topic_name, + partitions, + kafka_topic = %topic.name_any(), + "Increased the number of partitions" + ); + } + if changes_configs { + kafka + .incremental_alter_configs( + topic_name, + &update.set_configs, + &update.delete_configs, + ) + .await + .context(KafkaSnafu)?; + info!( + topic = topic_name, + set = ?update.set_configs, + reset = ?update.delete_configs, + kafka_topic = %topic.name_any(), + "Changed the topic configs" + ); + } + if update.unsupported.is_empty() { + TopicSyncOutcome::Synced { + partitions: topic.spec.partitions as usize, + reassigning_partitions: update.reassigning_partitions, + } + } else { + TopicSyncOutcome::UnsupportedChange { + messages: update.unsupported, + } + } + } + }; + + if recorded_keys != spec_keys { + record_managed_config_keys(topic, ctx, &spec_keys).await?; + } + + Ok(outcome) +} + +async fn cleanup(topic: &KafkaTopic, ctx: &Ctx) -> Result<()> { + if topic.spec.management_policy.deletes_on_cleanup() + && is_owned(&topic.metadata) + && conflicting_owner(topic, &ctx.topics).is_none() + { + let bootstrap_servers = read_bootstrap_servers(Path::new(BOOTSTRAP_SERVERS_FILE)) + .context(ReadBootstrapServersSnafu)? + .unwrap_or_default(); + let mut cache = ctx.kafka.lock().await; + client_for(&mut cache, bootstrap_servers, &ctx.security) + .delete_topic(topic.spec.topic_name.as_ref()) + .await + .context(KafkaSnafu)?; + info!( + topic = topic.spec.topic_name.as_ref(), + kafka_topic = %topic.name_any(), + "Deleted the topic" + ); + } + Ok(()) +} + +/// Fails if the `KafkaTopic` was changed since it was read, so concurrent changes to the +/// finalizers are not lost. +async fn set_finalizer(topic: &KafkaTopic, ctx: &Ctx, present: bool) -> Result<()> { + let mut finalizers = topic + .finalizers() + .iter() + .filter(|finalizer| *finalizer != FINALIZER) + .cloned() + .collect::>(); + if present { + finalizers.push(FINALIZER.to_owned()); + } + let patch = json!({ + "metadata": { + "resourceVersion": topic.metadata.resource_version, + "finalizers": finalizers, + } + }); + ctx.client + .merge_patch(topic, patch) + .await + .context(UpdateFinalizersSnafu)?; + Ok(()) +} + +/// Returns the cached client, rebuilt if the bootstrap servers changed. +fn client_for<'a>( + cache: &'a mut Option, + bootstrap_servers: Vec, + security: &Security, +) -> &'a mut KafkaAdminClient { + if cache + .as_ref() + .is_some_and(|cached| cached.bootstrap_servers != bootstrap_servers) + { + *cache = None; + } + let cached = cache.get_or_insert_with(|| { + info!(?bootstrap_servers, "Connecting to new bootstrap servers"); + CachedClient { + bootstrap_servers: bootstrap_servers.clone(), + client: KafkaAdminClient::new(bootstrap_servers, security.clone(), KAFKA_CLIENT_ID), + } + }); + &mut cached.client +} + +/// The oldest other `KafkaTopic` for the same topic of the same cluster, by creation time and +/// name. Checked against all `KafkaTopic`s in the store, so the result does not depend on the +/// reconcile order. +fn conflicting_owner( + topic: &KafkaTopic, + topics: &Store>, +) -> Option { + let key = |topic: &KafkaTopic| (topic.metadata.creation_timestamp.clone(), topic.name_any()); + topics + .state() + .iter() + .filter_map(|other| other.0.as_ref().ok()) + .filter(|other| { + other.spec.cluster_name == topic.spec.cluster_name + && other.spec.topic_name == topic.spec.topic_name + }) + .min_by_key(|other| key(other)) + .filter(|oldest| key(oldest) != key(topic)) + .map(ResourceExt::name_any) +} + +/// Fails if the `KafkaTopic` was changed since it was read, so ownership is never claimed twice. +async fn claim_ownership(topic: &KafkaTopic, ctx: &Ctx) -> Result<()> { + let patch = json!({ + "metadata": { + "resourceVersion": topic.metadata.resource_version, + "annotations": { OWNED_SINCE_ANNOTATION: Timestamp::now().to_string() }, + } + }); + ctx.client + .merge_patch(topic, patch) + .await + .context(AnnotateSnafu)?; + Ok(()) +} + +async fn release_ownership(topic: &KafkaTopic, ctx: &Ctx) -> Result<()> { + let patch = json!({ + "metadata": { + "annotations": { + OWNED_SINCE_ANNOTATION: null, + MANAGED_CONFIG_KEYS_ANNOTATION: null, + }, + } + }); + ctx.client + .merge_patch(topic, patch) + .await + .context(AnnotateSnafu)?; + Ok(()) +} + +async fn record_managed_config_keys( + topic: &KafkaTopic, + ctx: &Ctx, + keys: &BTreeSet, +) -> Result<()> { + let patch = json!({ + "metadata": { "annotations": { MANAGED_CONFIG_KEYS_ANNOTATION: render(keys) } } + }); + ctx.client + .merge_patch(topic, patch) + .await + .context(AnnotateSnafu)?; + Ok(()) +} + +/// Patches the status only if it changed, so the agent does not trigger its own watch. +async fn update_status( + topic: &KafkaTopic, + ctx: &Ctx, + outcome: &TopicSyncOutcome, + observed: Option<&Topic>, +) -> Result<()> { + let current = topic.status.clone().unwrap_or_default(); + let mut status = KafkaTopicStatus { + conditions: update_conditions( + ¤t.conditions, + conditions(outcome), + &Time(Timestamp::now()), + ), + ..current.clone() + }; + match (outcome, observed) { + (_, Some(observed)) => { + status.topic_id = Some(observed.id.clone()); + status.partitions = observed + .partitions + .iter() + .map(|partition| KafkaTopicPartitionStatus { + index: partition.index, + replicas: partition.replicas.clone(), + }) + .collect(); + } + (TopicSyncOutcome::AlreadyExists { .. } | TopicSyncOutcome::Conflict { .. }, None) => { + status.topic_id = None; + status.partitions = Vec::new(); + } + _ => {} + } + + if status != current { + ctx.client + .apply_patch_status(AGENT_NAME, topic, &status) + .await + .context(UpdateStatusSnafu)?; + } + Ok(()) +} + +#[cfg(test)] +mod tests { + use stackable_operator::{ + k8s_openapi::apimachinery::pkg::apis::meta::v1::Time, kube::runtime::watcher, + }; + + use super::*; + use crate::topic::test_support::kafka_topic; + + fn created_topic( + name: &str, + created_at: i64, + topic_name: &str, + ) -> DeserializeGuard { + let mut topic = kafka_topic(name, topic_name); + topic.metadata.creation_timestamp = Some(Time( + Timestamp::from_second(created_at).expect("the timestamp should be valid"), + )); + DeserializeGuard(Ok(topic)) + } + + fn store(topics: Vec>) -> Store> { + let (store, mut writer) = stackable_operator::kube::runtime::reflector::store(); + for topic in topics { + writer.apply_watcher_event(&watcher::Event::Apply(topic)); + } + store + } + + fn inner(topic: &DeserializeGuard) -> &KafkaTopic { + topic.0.as_ref().expect("the topic should be valid") + } + + #[test] + fn the_oldest_kafka_topic_owns_the_topic() { + let older = created_topic("older", 10, "orders"); + let newer = created_topic("newer", 20, "orders"); + let other_topic = created_topic("other", 5, "payments"); + let topics = store(vec![older.clone(), newer.clone(), other_topic]); + + assert_eq!(None, conflicting_owner(inner(&older), &topics)); + assert_eq!( + Some("older".to_owned()), + conflicting_owner(inner(&newer), &topics) + ); + } + + #[test] + fn the_name_decides_between_kafka_topics_of_the_same_age() { + let first = created_topic("a", 10, "orders"); + let second = created_topic("b", 10, "orders"); + let topics = store(vec![second.clone(), first.clone()]); + + assert_eq!(None, conflicting_owner(inner(&first), &topics)); + assert_eq!( + Some("a".to_owned()), + conflicting_owner(inner(&second), &topics) + ); + } + + #[rstest::rstest] + #[case::no_throttle(&[("retention.ms", "1000")], false)] + #[case::throttle(&[("follower.replication.throttled.replicas", "*")], true)] + fn reject_throttle_config_keys(#[case] config: &[(&str, &str)], #[case] invalid: bool) { + let mut topic = kafka_topic("orders", "orders"); + topic.spec.config = config + .iter() + .map(|(key, value)| (key.to_string(), value.to_string())) + .collect(); + + assert_eq!( + invalid, + matches!( + validate(&topic, &store(vec![])), + Some(TopicSyncOutcome::InvalidSpec { .. }) + ) + ); + } + + fn synced() -> TopicSyncOutcome { + TopicSyncOutcome::Synced { + partitions: 1, + reassigning_partitions: vec![], + } + } + + #[rstest::rstest] + #[case::invalid_spec(TopicSyncOutcome::InvalidSpec { message: String::new() }, false, Action::await_change())] + #[case::created(synced(), false, Action::requeue(RETRY_INTERVAL))] + #[case::cluster_not_ready( + TopicSyncOutcome::ClusterNotReady, + false, + Action::requeue(RETRY_INTERVAL) + )] + #[case::synced(synced(), true, Action::requeue(RESYNC_INTERVAL))] + #[case::grown_with_unsupported_change(TopicSyncOutcome::UnsupportedChange { messages: vec![] }, false, Action::requeue(RETRY_INTERVAL))] + #[case::unsupported_change(TopicSyncOutcome::UnsupportedChange { messages: vec![] }, true, Action::requeue(RESYNC_INTERVAL))] + fn requeue_by_outcome( + #[case] outcome: TopicSyncOutcome, + #[case] observed: bool, + #[case] expected: Action, + ) { + assert_eq!(expected, requeue(&outcome, observed)); + } +} diff --git a/rust/agent-binary/src/topic/managed_config_keys.rs b/rust/agent-binary/src/topic/managed_config_keys.rs new file mode 100644 index 000000000..069d3230d --- /dev/null +++ b/rust/agent-binary/src/topic/managed_config_keys.rs @@ -0,0 +1,69 @@ +//! The topic config keys the agent manages, recorded on the `KafkaTopic`. +//! +//! Keys removed from the spec are reset in Kafka, but only if the agent set them before. Kafka +//! only accepts its known topic config names, none of which contain a comma, so the keys are +//! stored comma-separated. + +use std::collections::BTreeSet; + +use stackable_kafka_shared::crd::topic::v1alpha1::KafkaTopic; +use stackable_operator::kube::ResourceExt; + +pub const MANAGED_CONFIG_KEYS_ANNOTATION: &str = "kafka.stackable.tech/managed-config-keys"; + +pub fn managed_config_keys(topic: &KafkaTopic) -> BTreeSet { + topic + .annotations() + .get(MANAGED_CONFIG_KEYS_ANNOTATION) + .map(|value| { + value + .split(',') + .map(str::trim) + .filter(|key| !key.is_empty()) + .map(str::to_owned) + .collect() + }) + .unwrap_or_default() +} + +pub fn render(keys: &BTreeSet) -> String { + keys.iter() + .map(String::as_str) + .collect::>() + .join(",") +} + +#[cfg(test)] +mod tests { + use std::collections::BTreeMap; + + use rstest::rstest; + + use super::*; + use crate::topic::test_support::kafka_topic; + + #[rstest] + #[case::no_annotation(None, &[])] + #[case::empty(Some(""), &[])] + #[case::keys(Some("retention.ms,cleanup.policy"), &["cleanup.policy", "retention.ms"])] + #[case::spaces_and_empty_entries(Some(" retention.ms, ,"), &["retention.ms"])] + fn read_managed_config_keys(#[case] annotation: Option<&str>, #[case] expected: &[&str]) { + let mut topic = kafka_topic("orders", "orders"); + topic.metadata.annotations = annotation.map(|value| { + BTreeMap::from([(MANAGED_CONFIG_KEYS_ANNOTATION.to_owned(), value.to_owned())]) + }); + + let expected = expected + .iter() + .map(|key| key.to_string()) + .collect::>(); + assert_eq!(expected, managed_config_keys(&topic)); + } + + #[test] + fn render_sorted_keys() { + let keys = BTreeSet::from(["retention.ms".to_owned(), "cleanup.policy".to_owned()]); + + assert_eq!("cleanup.policy,retention.ms", render(&keys)); + } +} diff --git a/rust/agent-binary/src/topic/mod.rs b/rust/agent-binary/src/topic/mod.rs new file mode 100644 index 000000000..a1ef9ffd9 --- /dev/null +++ b/rust/agent-binary/src/topic/mod.rs @@ -0,0 +1,8 @@ +//! Reconciles `KafkaTopic`s into topics in Kafka. + +mod condition; +pub mod controller; +mod managed_config_keys; +mod plan; +#[cfg(test)] +mod test_support; diff --git a/rust/agent-binary/src/topic/plan.rs b/rust/agent-binary/src/topic/plan.rs new file mode 100644 index 000000000..44e5f71ff --- /dev/null +++ b/rust/agent-binary/src/topic/plan.rs @@ -0,0 +1,324 @@ +//! What the agent has to do in Kafka to bring a topic to the state of its `KafkaTopic` spec. + +use std::collections::{BTreeMap, BTreeSet}; + +use stackable_kafka_admin::client::{PartitionReassignment, Topic}; +use stackable_kafka_shared::crd::topic::v1alpha1::KafkaTopic; + +/// The state of an existing topic in Kafka. +pub struct ObservedTopic { + pub topic: Topic, + pub configs: BTreeMap, + pub reassignments: Vec, +} + +#[derive(Debug, Eq, PartialEq)] +pub enum TopicPlan { + Create { + partitions: i32, + replication_factor: i16, + configs: BTreeMap, + }, + Update(TopicUpdate), +} + +/// Changes to an existing topic. No grow, configs or unsupported differences means the topic +/// matches the spec. +#[derive(Debug, Default, Eq, PartialEq)] +pub struct TopicUpdate { + pub grow_partitions_to: Option, + pub set_configs: BTreeMap, + pub delete_configs: BTreeSet, + /// Differences the agent does not apply, as messages for the status + pub unsupported: Vec, + pub reassigning_partitions: Vec, +} + +pub fn plan( + topic: &KafkaTopic, + observed: Option<&ObservedTopic>, + managed_config_keys: &BTreeSet, +) -> TopicPlan { + let spec = &topic.spec; + let Some(observed) = observed else { + return TopicPlan::Create { + partitions: spec.partitions, + replication_factor: spec.replication_factor, + configs: spec.config.clone(), + }; + }; + + let mut update = TopicUpdate::default(); + + let observed_partitions = observed.topic.partitions.len() as i32; + if spec.partitions > observed_partitions { + update.grow_partitions_to = Some(spec.partitions); + } else if spec.partitions < observed_partitions { + update.unsupported.push(format!( + "the topic has {observed_partitions} partitions, but the spec requests {}; partitions cannot be \ + removed", + spec.partitions + )); + } + + if let Some(message) = replication_factor_difference( + spec.replication_factor, + &target_replication_factors(observed), + ) { + update.unsupported.push(message); + } + + update.set_configs = spec + .config + .iter() + .filter(|(key, value)| observed.configs.get(*key) != Some(*value)) + .map(|(key, value)| (key.clone(), value.clone())) + .collect(); + update.delete_configs = managed_config_keys + .iter() + .filter(|key| !spec.config.contains_key(*key) && observed.configs.contains_key(*key)) + .cloned() + .collect(); + + update.reassigning_partitions = observed + .reassignments + .iter() + .map(|reassignment| reassignment.index) + .collect(); + + TopicPlan::Update(update) +} + +/// The replication factor of each partition, excluding replicas a reassignment removes. +fn target_replication_factors(observed: &ObservedTopic) -> BTreeSet { + observed + .topic + .partitions + .iter() + .map(|partition| { + let removing = observed + .reassignments + .iter() + .find(|reassignment| reassignment.index == partition.index) + .map_or(0, |reassignment| reassignment.removing_replicas.len()); + partition.replicas.len().saturating_sub(removing) + }) + .collect() +} + +fn replication_factor_difference( + desired: i16, + replication_factors: &BTreeSet, +) -> Option { + let (Some(min), Some(max)) = (replication_factors.first(), replication_factors.last()) else { + return None; + }; + + let hint = "change it with kafka-reassign-partitions"; + if min != max { + Some(format!( + "the partitions have between {min} and {max} replicas, but the spec requests a \ + replication factor of {desired}; {hint}" + )) + } else if *min != desired as usize { + Some(format!( + "the topic has a replication factor of {min}, but the spec requests {desired}; {hint}" + )) + } else { + None + } +} + +#[cfg(test)] +mod tests { + use stackable_kafka_admin::client::Partition; + + use super::*; + use crate::topic::test_support::kafka_topic; + + fn topic(partitions: i32, replication_factor: i16, config: &[(&str, &str)]) -> KafkaTopic { + let mut topic = kafka_topic("orders", "orders"); + topic.spec.partitions = partitions; + topic.spec.replication_factor = replication_factor; + topic.spec.config = configs(config); + topic + } + + fn configs(entries: &[(&str, &str)]) -> BTreeMap { + entries + .iter() + .map(|(key, value)| (key.to_string(), value.to_string())) + .collect() + } + + fn keys(keys: &[&str]) -> BTreeSet { + keys.iter().map(|key| key.to_string()).collect() + } + + /// A topic whose partitions have the given replicas + fn observed(replicas: &[&[i32]], config: &[(&str, &str)]) -> ObservedTopic { + ObservedTopic { + topic: Topic { + id: "UXyU9i5ARn6W00ON2taeWA".to_owned(), + partitions: replicas + .iter() + .zip(0..) + .map(|(replicas, index)| Partition { + index, + replicas: replicas.to_vec(), + }) + .collect(), + }, + configs: configs(config), + reassignments: vec![], + } + } + + #[test] + fn create_missing_topic() { + let plan = plan(&topic(3, 2, &[("retention.ms", "1000")]), None, &keys(&[])); + + assert_eq!( + TopicPlan::Create { + partitions: 3, + replication_factor: 2, + configs: configs(&[("retention.ms", "1000")]), + }, + plan + ); + } + + #[test] + fn do_nothing_for_a_matching_topic() { + let observed = observed(&[&[1, 2], &[2, 3]], &[("retention.ms", "1000")]); + + let plan = plan( + &topic(2, 2, &[("retention.ms", "1000")]), + Some(&observed), + &keys(&["retention.ms"]), + ); + + assert_eq!(TopicPlan::Update(TopicUpdate::default()), plan); + } + + #[test] + fn grow_partitions() { + let observed = observed(&[&[1], &[2]], &[]); + + let plan = plan(&topic(4, 1, &[]), Some(&observed), &keys(&[])); + + assert_eq!( + TopicPlan::Update(TopicUpdate { + grow_partitions_to: Some(4), + ..TopicUpdate::default() + }), + plan + ); + } + + #[test] + fn report_fewer_partitions_as_unsupported() { + let observed = observed(&[&[1], &[2], &[3]], &[]); + + let TopicPlan::Update(update) = plan(&topic(2, 1, &[]), Some(&observed), &keys(&[])) else { + panic!("the topic exists"); + }; + + assert_eq!(None, update.grow_partitions_to); + assert_eq!(1, update.unsupported.len()); + assert!(update.unsupported[0].contains("cannot be removed")); + } + + #[test] + fn set_changed_and_reset_removed_configs() { + let observed = observed( + &[&[1]], + &[ + ("retention.ms", "1000"), + ("cleanup.policy", "compact"), + ("segment.bytes", "1024"), + ("leader.replication.throttled.replicas", "0:1"), + ], + ); + + let plan = plan( + &topic( + 1, + 1, + &[("retention.ms", "2000"), ("max.message.bytes", "10")], + ), + Some(&observed), + &keys(&["retention.ms", "cleanup.policy", "min.insync.replicas"]), + ); + + assert_eq!( + TopicPlan::Update(TopicUpdate { + set_configs: configs(&[("max.message.bytes", "10"), ("retention.ms", "2000")]), + // `segment.bytes` and the throttle were set by others, `min.insync.replicas` is + // already gone + delete_configs: keys(&["cleanup.policy"]), + ..TopicUpdate::default() + }), + plan + ); + } + + #[test] + fn report_a_different_replication_factor_as_unsupported() { + let observed = observed(&[&[1, 2, 3], &[2, 3, 4]], &[]); + + let TopicPlan::Update(update) = plan(&topic(2, 2, &[]), Some(&observed), &keys(&[])) else { + panic!("the topic exists"); + }; + + assert_eq!(1, update.unsupported.len()); + assert!(update.unsupported[0].contains("replication factor of 3")); + } + + #[test] + fn report_uneven_replica_counts_as_unsupported() { + let observed = observed(&[&[1, 2], &[2, 3, 4]], &[]); + + let TopicPlan::Update(update) = plan(&topic(2, 2, &[]), Some(&observed), &keys(&[])) else { + panic!("the topic exists"); + }; + + assert_eq!(1, update.unsupported.len()); + assert!(update.unsupported[0].contains("between 2 and 3 replicas")); + } + + #[test] + fn accept_a_reassignment_towards_the_desired_replication_factor() { + let mut observed = observed(&[&[4, 5, 1, 2], &[2, 3]], &[]); + observed.reassignments = vec![PartitionReassignment { + index: 0, + adding_replicas: vec![4, 5], + removing_replicas: vec![1, 2], + }]; + + let plan = plan(&topic(2, 2, &[]), Some(&observed), &keys(&[])); + + assert_eq!( + TopicPlan::Update(TopicUpdate { + reassigning_partitions: vec![0], + ..TopicUpdate::default() + }), + plan + ); + } + + #[test] + fn exclude_removed_replicas_from_the_replication_factor() { + let mut observed = observed(&[&[4, 5, 1, 2], &[2, 3, 4]], &[]); + observed.reassignments = vec![PartitionReassignment { + index: 0, + adding_replicas: vec![4, 5], + removing_replicas: vec![1, 2], + }]; + + assert_eq!( + BTreeSet::from([2, 3]), + target_replication_factors(&observed) + ); + } +} diff --git a/rust/agent-binary/src/topic/test_support.rs b/rust/agent-binary/src/topic/test_support.rs new file mode 100644 index 000000000..44830af98 --- /dev/null +++ b/rust/agent-binary/src/topic/test_support.rs @@ -0,0 +1,23 @@ +use std::str::FromStr; + +use stackable_kafka_shared::crd::topic::{ + TopicName, + v1alpha1::{KafkaTopic, KafkaTopicSpec}, +}; +use stackable_operator::v2::types::operator::ClusterName; + +/// A `KafkaTopic` of the cluster `simple-kafka` with one partition, one replica and no configs. +pub fn kafka_topic(name: &str, topic_name: &str) -> KafkaTopic { + KafkaTopic::new( + name, + KafkaTopicSpec { + cluster_name: ClusterName::from_str("simple-kafka") + .expect("the cluster name should be valid"), + management_policy: Default::default(), + topic_name: TopicName::from_str(topic_name).expect("the topic name should be valid"), + partitions: 1, + replication_factor: 1, + config: Default::default(), + }, + ) +} diff --git a/rust/shared/src/crd/topic.rs b/rust/shared/src/crd/topic.rs index 7b855a8e4..c919c3cd3 100644 --- a/rust/shared/src/crd/topic.rs +++ b/rust/shared/src/crd/topic.rs @@ -82,6 +82,24 @@ pub mod versioned { /// `Ready` and `Synced`. #[serde(default)] pub conditions: Vec, + + /// The topic id in Kafka, as shown by `kafka-topics.sh`. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub topic_id: Option, + + /// The partitions of the topic in Kafka. + #[serde(default)] + pub partitions: Vec, + } + + #[derive(Clone, Debug, Deserialize, JsonSchema, PartialEq, Serialize)] + #[serde(rename_all = "camelCase")] + pub struct KafkaTopicPartitionStatus { + pub index: i32, + + /// Broker ids holding a replica, the preferred leader first. During a reassignment, this + /// includes the replicas being added and removed. + pub replicas: Vec, } } From 773d3bdb83011bd1fa8f4fc63955d9848d257b99 Mon Sep 17 00:00:00 2001 From: Benedikt Labrenz Date: Wed, 21 Oct 2026 13:59:59 +0200 Subject: [PATCH 23/93] use the recreate strategy for the agent deployment --- .../src/controller/build/resource/agent.rs | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/rust/operator-binary/src/controller/build/resource/agent.rs b/rust/operator-binary/src/controller/build/resource/agent.rs index bad1fb2a7..7af6b6d39 100644 --- a/rust/operator-binary/src/controller/build/resource/agent.rs +++ b/rust/operator-binary/src/controller/build/resource/agent.rs @@ -12,7 +12,7 @@ use stackable_operator::{ constant, k8s_openapi::{ api::{ - apps::v1::{Deployment, DeploymentSpec}, + apps::v1::{Deployment, DeploymentSpec, DeploymentStrategy}, core::v1::{EnvVarSource, ObjectFieldSelector, ServiceAccount}, rbac::v1::{RoleBinding, RoleRef, Subject}, }, @@ -92,6 +92,11 @@ pub fn build_agent_deployment( metadata: agent_metadata(cluster), spec: Some(DeploymentSpec { replicas: Some(1), + // Never run two agents at the same time, as for the operator itself. + strategy: Some(DeploymentStrategy { + type_: Some("Recreate".to_owned()), + ..DeploymentStrategy::default() + }), selector: LabelSelector { match_labels: Some(agent_selector(cluster).into()), ..LabelSelector::default() @@ -210,6 +215,7 @@ mod tests { assert_eq!(deployment["metadata"]["name"], "simple-kafka-agent"); assert_eq!(deployment["spec"]["replicas"], 1); + assert_eq!(deployment["spec"]["strategy"]["type"], "Recreate"); // The selector must not match the broker or controller Pods. assert_eq!( deployment["spec"]["selector"]["matchLabels"], From 679a24a7b37ba08e55eb19138fb204ec0a855936 Mon Sep 17 00:00:00 2001 From: Benedikt Labrenz Date: Wed, 21 Oct 2026 14:16:02 +0200 Subject: [PATCH 24/93] ship agent binary in operator image --- .github/workflows/build.yaml | 42 ++--- .gitignore | 1 - Tiltfile | 17 +- default.nix | 36 +--- .../kafka-operator/templates/_helpers.tpl | 9 +- .../kafka-operator/templates/deployment.yaml | 8 - docker/Dockerfile | 2 +- docker/Dockerfile.agent | 158 ------------------ nix/meta.json | 2 +- result-agent | 1 + rust/operator-binary/src/controller.rs | 17 +- .../src/controller/build/resource/agent.rs | 13 +- .../src/controller/validate.rs | 67 +++++++- rust/operator-binary/src/crd/mod.rs | 9 - rust/operator-binary/src/main.rs | 13 +- 15 files changed, 106 insertions(+), 289 deletions(-) delete mode 100644 docker/Dockerfile.agent create mode 120000 result-agent diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 2c6cc0009..7251f20b1 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -62,22 +62,8 @@ jobs: - '**/Cargo.toml' - 'Cargo.lock' - '**/*.rs' - - 'nix/meta.json' - - # Operators that ship an agent (see `agent` in nix/meta.json) also build and publish its image. - - name: Determine Container Images - id: images - shell: bash - run: | - set -euo pipefail - IMAGES=$(jq -c --arg operator "$OPERATOR_NAME" ' - [{component: "operator", name: $operator, "container-file": "docker/Dockerfile"}] - + if .agent then [{component: "agent", name: .agent.name, "container-file": "docker/Dockerfile.agent"}] else [] end - ' nix/meta.json) - echo "IMAGES=$IMAGES" | tee -a "$GITHUB_OUTPUT" outputs: detected: ${{ steps.check.outputs.detected }} - images: ${{ steps.images.outputs.IMAGES }} helm-lint: name: Lint Helm Chart @@ -145,7 +131,7 @@ jobs: run: cargo udeps --workspace --all-targets build-container-image: - name: Build/Publish ${{ matrix.image.component }} ${{ matrix.runner.arch }} Image + name: Build/Publish ${{ matrix.runner.arch }} Image if: | github.repository_owner == 'stackabletech' && (github.event_name != 'merge_group') @@ -160,7 +146,6 @@ jobs: runner: - { name: "ubuntu-latest", arch: "amd64" } - { name: "ubicloud-standard-8-arm", arch: "arm64" } - image: ${{ fromJSON(needs.detect-changes.outputs.images) }} runs-on: ${{ matrix.runner.name }} outputs: operator-version: ${{ steps.version.outputs.OPERATOR_VERSION }} @@ -223,10 +208,10 @@ jobs: id: build uses: stackabletech/actions/build-container-image@34a64229959b15db6c5f307db8809805ba7edc55 # v0.18.3 with: - image-name: ${{ matrix.image.name }} + image-name: ${{ env.OPERATOR_NAME }} image-index-manifest-tag: ${{ steps.version.outputs.OPERATOR_VERSION }} build-arguments: VERSION=${{ steps.version.outputs.OPERATOR_VERSION }} - container-file: ${{ matrix.image.container-file }} + container-file: docker/Dockerfile - name: Publish Container Image to oci.stackable.tech if: ${{ !github.event.pull_request.head.repo.fork }} @@ -235,7 +220,7 @@ jobs: image-registry-uri: oci.stackable.tech image-registry-username: robot$sdp+github-action-build image-registry-password: ${{ secrets.HARBOR_ROBOT_SDP_GITHUB_ACTION_BUILD_SECRET }} - image-repository: sdp/${{ matrix.image.name }} + image-repository: sdp/${{ env.OPERATOR_NAME }} canonical-image-manifest-tag: ${{ steps.build.outputs.image-manifest-tag }} canonical-source-image-uri: ${{ steps.build.outputs.image-manifest-uri }} @@ -246,12 +231,12 @@ jobs: image-registry-uri: quay.io image-registry-username: stackable+robot_sdp_github_action_build image-registry-password: ${{ secrets.QUAY_ROBOT_SDP_GITHUB_ACTION_BUILD_SECRET }} - image-repository: stackable/sdp/${{ matrix.image.name }} + image-repository: stackable/sdp/${{ env.OPERATOR_NAME }} canonical-image-manifest-tag: ${{ steps.build.outputs.image-manifest-tag }} canonical-source-image-uri: ${{ steps.build.outputs.image-manifest-uri }} publish-index-manifest: - name: Publish/Sign ${{ matrix.image.component }} ${{ needs.build-container-image.outputs.operator-version }} Index + name: Publish/Sign ${{ needs.build-container-image.outputs.operator-version }} Index if: | github.repository_owner == 'stackabletech' && (github.event_name != 'merge_group') @@ -263,10 +248,6 @@ jobs: permissions: contents: read id-token: write - strategy: - fail-fast: false - matrix: - image: ${{ fromJSON(needs.detect-changes.outputs.images) }} runs-on: ubuntu-latest steps: - name: Checkout Repository @@ -280,7 +261,7 @@ jobs: image-registry-uri: oci.stackable.tech image-registry-username: robot$sdp+github-action-build image-registry-password: ${{ secrets.HARBOR_ROBOT_SDP_GITHUB_ACTION_BUILD_SECRET }} - image-repository: sdp/${{ matrix.image.name }} + image-repository: sdp/${{ env.OPERATOR_NAME }} canonical-image-index-manifest-tag: ${{ needs.build-container-image.outputs.operator-version }} - name: Publish and Sign Image Index to quay.io @@ -289,7 +270,7 @@ jobs: image-registry-uri: quay.io image-registry-username: stackable+robot_sdp_github_action_build image-registry-password: ${{ secrets.QUAY_ROBOT_SDP_GITHUB_ACTION_BUILD_SECRET }} - image-repository: stackable/sdp/${{ matrix.image.name }} + image-repository: stackable/sdp/${{ env.OPERATOR_NAME }} canonical-image-index-manifest-tag: ${{ needs.build-container-image.outputs.operator-version }} publish-helm-chart: @@ -339,7 +320,7 @@ jobs: helm-version: v3.17.4 # This is currently the latest version which supports pushing to quay.io openshift-preflight-check: - name: Run OpenShift Preflight Check for ${{ matrix.image.component }} ${{ needs.build-container-image.outputs.operator-version }}-${{ matrix.arch }} + name: Run OpenShift Preflight Check for ${{ needs.build-container-image.outputs.operator-version }}-${{ matrix.arch }} if: | github.repository_owner == 'stackabletech' && (github.event_name != 'merge_group') @@ -355,19 +336,18 @@ jobs: arch: - amd64 - arm64 - image: ${{ fromJSON(needs.detect-changes.outputs.images) }} runs-on: ubuntu-latest steps: - name: Run OpenShift Preflight Check for oci.stackable.tech uses: stackabletech/actions/run-openshift-preflight@34a64229959b15db6c5f307db8809805ba7edc55 # v0.18.3 with: - image-index-uri: oci.stackable.tech/sdp/${{ matrix.image.name }}:${{ needs.build-container-image.outputs.operator-version }} + image-index-uri: oci.stackable.tech/sdp/${{ env.OPERATOR_NAME }}:${{ needs.build-container-image.outputs.operator-version }} image-architecture: ${{ matrix.arch }} - name: Run OpenShift Preflight Check for quay.io uses: stackabletech/actions/run-openshift-preflight@34a64229959b15db6c5f307db8809805ba7edc55 # v0.18.3 with: - image-index-uri: quay.io/stackable/sdp/${{ matrix.image.name }}:${{ needs.build-container-image.outputs.operator-version }} + image-index-uri: quay.io/stackable/sdp/${{ env.OPERATOR_NAME }}:${{ needs.build-container-image.outputs.operator-version }} image-architecture: ${{ matrix.arch }} # This job is a required check in GitHub Settings for this repository. diff --git a/.gitignore b/.gitignore index cc2e5c12e..411cc85da 100644 --- a/.gitignore +++ b/.gitignore @@ -13,7 +13,6 @@ target/ *.tgz result -result-agent image.tar tilt_options.json diff --git a/Tiltfile b/Tiltfile index fba0c44f2..1769d19a5 100644 --- a/Tiltfile +++ b/Tiltfile @@ -34,22 +34,7 @@ custom_build( # We need to set the correct image annotation on the operator Deployment to use e.g. # oci.stackable.tech/sandbox/opa-operator:7y19m3d8clwxlv34v5q2x4p7v536s00g instead of # oci.stackable.tech/sandbox/opa-operator:0.0.0-dev (which does not exist) -deployment_image_json_paths = ['{.spec.template.metadata.annotations.internal\\.stackable\\.tech/image}'] - -# Operators that ship an agent (see `agent` in nix/meta.json) also build the agent image and pass -# it to the operator via the `internal.stackable.tech/agent-image` annotation. -if 'agent' in meta: - agent_image_name = operator_repository + '/' + meta['agent']['name'] - custom_build( - agent_image_name, - 'nix-build . -A dockerAgent --argstr dockerNameAgent "' + agent_image_name + '" -o result-agent && ./result-agent/load-image | docker load', - deps=['rust', 'Cargo.toml', 'Cargo.lock', 'default.nix', "nix", 'build.rs', 'vendor'], - ignore=['*.~undo-tree~'], - outputs_image_ref_to='result-agent/ref', - ) - deployment_image_json_paths.append('{.spec.template.metadata.annotations.internal\\.stackable\\.tech/agent-image}') - -k8s_kind('Deployment', image_json_path=deployment_image_json_paths) +k8s_kind('Deployment', image_json_path='{.spec.template.metadata.annotations.internal\\.stackable\\.tech/image}') k8s_kind('DaemonSet', image_json_path='{.spec.template.metadata.annotations.internal\\.stackable\\.tech/image}') # Optionally specify a custom Helm values file to be passed to the Helm deployment below. diff --git a/default.nix b/default.nix index abaa026b6..6feb190a3 100644 --- a/default.nix +++ b/default.nix @@ -101,8 +101,6 @@ } , meta ? pkgsLocal.lib.importJSON ./nix/meta.json , dockerName ? "oci.stackable.tech/sandbox/${meta.operator.name}" -# Only used by operators that ship an agent (see `agent` in nix/meta.json) -, dockerNameAgent ? "oci.stackable.tech/sandbox/${meta.agent.name}" , dockerTag ? null # Controls the amount of debug information included in the built operator binaries, # see https://doc.rust-lang.org/rustc/codegen-options/index.html#debuginfo @@ -127,11 +125,7 @@ rec { inherit cargo sources pkgsLocal pkgsTarget meta; inherit (pkgsLocal) lib; pkgs = lib.warn "pkgs is not cross-compilation-aware, explicitly use either pkgsLocal or pkgsTarget" pkgsLocal; - # Operators that ship an agent (see `agent` in nix/meta.json) keep the agent binary out of the - # operator image, as it has its own image (`dockerAgent`). - build = if meta ? agent - then cargo.workspaceMembers."stackable-${meta.operator.name}".build - else cargo.allWorkspaceMembers; + build = cargo.allWorkspaceMembers; entrypoint = build+"/bin/stackable-${meta.operator.name}"; # Run crds in the target environment, to avoid compiling everything twice crds = pkgsTarget.runCommand "${meta.operator.name}-crds.yaml" {} @@ -149,13 +143,13 @@ rec { # build it in the local environment so that the generated load-image # can run locally. # That's still fine, as long as we only refer to pkgsTarget *inside* of the image. - dockerImageArgs = { + dockerImage = pkgsLocal.dockerTools.streamLayeredImage { name = dockerName; tag = dockerTag; contents = [ # Kerberos 5 must be installed globally to load plugins correctly pkgsTarget.krb5 - # Make the operator binaries available on $PATH + # Make the whole cargo workspace available on $PATH build ] ++ lib.optional includeShell [ pkgsTarget.bashInteractive @@ -195,7 +189,6 @@ rec { User = toString stackableUserUid; }; }; - dockerImage = pkgsLocal.dockerTools.streamLayeredImage dockerImageArgs; docker = pkgsLocal.linkFarm "${dockerImage.name}-docker" [ { name = "load-image"; @@ -219,29 +212,6 @@ rec { } ]; - # The image of the agent, only built for operators that ship one. - dockerAgent = if meta ? agent then - let - agentBuild = cargo.workspaceMembers."stackable-${meta.agent.name}".build; - agentImage = pkgsLocal.dockerTools.streamLayeredImage (dockerImageArgs // { - name = dockerNameAgent; - contents = [ agentBuild ] ++ lib.optional includeShell [ - pkgsTarget.bashInteractive - pkgsTarget.coreutils - pkgsTarget.util-linuxMinimal - ]; - config = dockerImageArgs.config // { - Entrypoint = [ "${agentBuild}/bin/stackable-${meta.agent.name}" ]; - }; - }); - in pkgsLocal.linkFarm "${agentImage.name}-docker" [ - { name = "load-image"; path = agentImage; } - { name = "ref"; path = pkgsLocal.writeText "${agentImage.name}-image-tag" "${agentImage.imageName}:${agentImage.imageTag}"; } - { name = "image-repo"; path = pkgsLocal.writeText "${agentImage.name}-repo" agentImage.imageName; } - { name = "image-tag"; path = pkgsLocal.writeText "${agentImage.name}-tag" agentImage.imageTag; } - ] - else null; - # need to use vendored crate2nix because of https://github.com/kolloch/crate2nix/issues/264 crate2nix = import sources.crate2nix { pkgs = pkgsLocal; }; tilt = pkgsLocal.tilt; diff --git a/deploy/helm/kafka-operator/templates/_helpers.tpl b/deploy/helm/kafka-operator/templates/_helpers.tpl index 281aa5495..6f6d19411 100644 --- a/deploy/helm/kafka-operator/templates/_helpers.tpl +++ b/deploy/helm/kafka-operator/templates/_helpers.tpl @@ -7,7 +7,7 @@ Expand the name of the chart. {{/* Fixed product name. Unlike kafka-operator.name it ignores nameOverride, because it must match -names hardcoded in the operator and the published image names. +names hardcoded in the operator. */}} {{- define "kafka-operator.productName" -}} kafka @@ -92,10 +92,3 @@ Build the full operator container image reference. {{- define "kafka-operator.image" -}} {{- printf "%s/%s:%s" .Values.image.repository .Chart.Name (.Values.image.tag | default .Chart.AppVersion) -}} {{- end }} - -{{/* -Build the full agent container image reference. The agent is released together with the operator. -*/}} -{{- define "kafka-operator.agentImage" -}} -{{- printf "%s/%s-agent:%s" .Values.image.repository (include "kafka-operator.productName" .) (.Values.image.tag | default .Chart.AppVersion) -}} -{{- end }} diff --git a/deploy/helm/kafka-operator/templates/deployment.yaml b/deploy/helm/kafka-operator/templates/deployment.yaml index 9fac4f33b..6f7163dcd 100644 --- a/deploy/helm/kafka-operator/templates/deployment.yaml +++ b/deploy/helm/kafka-operator/templates/deployment.yaml @@ -16,7 +16,6 @@ spec: metadata: annotations: internal.stackable.tech/image: {{ include "kafka-operator.image" . }} - internal.stackable.tech/agent-image: {{ include "kafka-operator.agentImage" . }} {{- with .Values.podAnnotations }} {{- toYaml . | nindent 8 }} {{- end }} @@ -71,13 +70,6 @@ spec: - name: IMAGE_REPOSITORY value: {{ .Values.image.productRepository | default .Values.image.repository }} - # The image of the agent the operator deploys alongside a cluster. - - name: AGENT_IMAGE - # Tilt can use annotations as image paths, but not env variables - valueFrom: - fieldRef: - fieldPath: metadata.annotations['internal.stackable.tech/agent-image'] - # Operators need to know the node name they are running on, to e.g. discover the # Kubernetes domain name from the kubelet API. - name: KUBERNETES_NODE_NAME diff --git a/docker/Dockerfile b/docker/Dockerfile index a96fc6111..84458c3a2 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -190,7 +190,7 @@ EOF COPY LICENSE /licenses/LICENSE -COPY --from=builder --chown=${STACKABLE_USER_UID}:0 /app/stackable-kafka-operator* /app/stackable-src.tar.gz /usr/local/bin/ +COPY --from=builder --chown=${STACKABLE_USER_UID}:0 /app/* /usr/local/bin/ USER ${STACKABLE_USER_UID} diff --git a/docker/Dockerfile.agent b/docker/Dockerfile.agent deleted file mode 100644 index 04f55b38d..000000000 --- a/docker/Dockerfile.agent +++ /dev/null @@ -1,158 +0,0 @@ -# syntax=docker/dockerfile:1.16.0@sha256:e2dd261f92e4b763d789984f6eab84be66ab4f5f08052316d8eb8f173593acf7 -# NOTE: The syntax directive needs to be the first line in a Dockerfile -# Find the latest versions here: https://hub.docker.com/r/docker/dockerfile/tags -# And the changelogs: https://docs.docker.com/build/buildkit/dockerfile-release-notes/ or https://github.com/moby/buildkit/releases - -# https://docs.docker.com/build/checks/#fail-build-on-check-violations -# check=error=true - -# We want to automatically use the latest. We also don't tag our images with a version. -# hadolint ignore=DL3007 -FROM oci.stackable.tech/sdp/ubi10-rust-builder:latest AS builder - - -# We want to automatically use the latest. -# hadolint ignore=DL3007 -FROM registry.access.redhat.com/ubi10/ubi-minimal:latest AS agent - -ARG VERSION -# NOTE (@Techassi): This is required for OpenShift/Red Hat certification -# Keeping this as "1" seems to be fine since a couple of years /shrug -ARG RELEASE="1" - -# These are chosen at random and are this high on purpose to have very little chance to clash with an existing user or group on the host system -# NOTE: Please also update default.nix accordingly! -ARG STACKABLE_USER_GID="574654813" -ARG STACKABLE_USER_UID="782252253" -ARG STACKABLE_USER_NAME="stackable" - -# Sets the default shell to Bash with strict error handling and robust pipeline processing. -# "-e": Exits immediately if a command exits with a non-zero status -# "-u": Treats unset variables as an error, preventing unexpected behavior from undefined variables. -# "-o pipefail": Causes a pipeline to return the exit status of the last command in the pipe that failed, ensuring errors in any part of a pipeline are not ignored. -# "-c": Allows the execution of commands passed as a string -SHELL ["/bin/bash", "-euo", "pipefail", "-c"] - -# These labels have mostly been superseded by the OpenContainer spec annotations below but it doesn't hurt to include them -# http://label-schema.org/rc1/ -LABEL name="Stackable Agent for Apache Kafka" -LABEL maintainer="info@stackable.tech" -LABEL vendor="Stackable GmbH" -LABEL version="${VERSION}" -LABEL release="${RELEASE}" -LABEL summary="Manage resources inside Apache Kafka clusters." -LABEL description="Manage resources inside Apache Kafka clusters." - -# Overwriting/Pinning UBI labels -# https://github.com/projectatomic/ContainerApplicationGenericLabels -LABEL vcs-ref="" -LABEL distribution-scope="public" -LABEL url="https://stackable.tech" -ARG TARGETARCH -LABEL architecture="${TARGETARCH}" -LABEL com.redhat.component="" -# It complains about it being an invalid label but RedHat uses it and we want to override it and it works.... -# hadolint ignore=DL3048 -LABEL com.redhat.license_terms="" -LABEL io.buildah.version="" -LABEL io.openshift.expose-services="" - -# https://github.com/opencontainers/image-spec/blob/64294bd7a2bf2537e1a6a34d687caae70300b0c4/annotations.md#annotations -LABEL org.opencontainers.image.authors="info@stackable.tech" -LABEL org.opencontainers.image.url="https://stackable.tech" -LABEL org.opencontainers.image.vendor="Stackable GmbH" -LABEL org.opencontainers.image.licenses="OSL-3.0" -LABEL org.opencontainers.image.documentation="https://docs.stackable.tech/home/stable/kafka/" -LABEL org.opencontainers.image.version="${VERSION}" -LABEL org.opencontainers.image.revision="${RELEASE}" -LABEL org.opencontainers.image.title="Stackable Agent for Apache Kafka" -LABEL org.opencontainers.image.description="Manage resources inside Apache Kafka clusters." - -# https://docs.openshift.com/container-platform/4.16/openshift_images/create-images.html#defining-image-metadata -# https://github.com/projectatomic/ContainerApplicationGenericLabels/blob/master/vendor/redhat/labels.md -LABEL io.openshift.tags="ubi10,stackable,sdp,kafka" -LABEL io.k8s.description="Manage resources inside Apache Kafka clusters." -LABEL io.k8s.display-name="Stackable Agent for Apache Kafka" - -COPY <> /stackable/.bashrc - -echo -e "if [ -f ~/.bashrc ]; then\n\tsource ~/.bashrc\nfi" >> /stackable/.profile - -chown ${STACKABLE_USER_UID}:0 /stackable/.bashrc -chown ${STACKABLE_USER_UID}:0 /stackable/.profile - -# All files and folders owned by root to support running as arbitrary users -# This is best practice as all container users will belong to the root group (0) -chown -R ${STACKABLE_USER_UID}:0 /stackable -chmod -R g=u /stackable -EOF - -COPY <, } #[derive(Snafu, Debug, EnumDiscriminants)] @@ -481,7 +481,7 @@ pub async fn reconcile_kafka( kafka, dereferenced_objects, &ctx.operator_environment, - &ctx.agent_image, + ctx.operator_image.as_deref(), ) .context(ValidateClusterSnafu)?; @@ -541,6 +541,8 @@ pub(crate) mod test_support { use super::{ValidatedCluster, dereference::DereferencedObjects, validate::validate}; use crate::crd::{authentication::ResolvedAuthenticationClasses, v1alpha1}; + pub const OPERATOR_IMAGE: &str = "oci.stackable.tech/sdp/kafka-operator:0.0.0-dev"; + /// The expected `app.kubernetes.io/version` label value for the given product version. /// /// The `-stackable` suffix carries the operator's own version, which is `0.0.0-dev` on main @@ -633,6 +635,13 @@ pub(crate) mod test_support { /// the result. Used for tests asserting on a specific validation failure. pub fn validate_err( kafka: &v1alpha1::KafkaCluster, + ) -> Result { + validate_with_operator_image(kafka, Some(OPERATOR_IMAGE)) + } + + pub fn validate_with_operator_image( + kafka: &v1alpha1::KafkaCluster, + operator_image: Option<&str>, ) -> Result { validate( kafka, @@ -643,7 +652,7 @@ pub(crate) mod test_support { bootstrap_listeners: Vec::new(), }, &operator_environment(), - &crate::crd::default_agent_image(&operator_environment().image_repository), + operator_image, ) } } @@ -796,7 +805,7 @@ spec: {} operator_service_name: "kafka-operator".to_owned(), image_repository: "oci.stackable.tech/sdp".to_owned(), }, - agent_image: crate::crd::default_agent_image("oci.stackable.tech/sdp"), + operator_image: None, }); reconcile_kafka(Arc::new(kafka), ctx).await diff --git a/rust/operator-binary/src/controller/build/resource/agent.rs b/rust/operator-binary/src/controller/build/resource/agent.rs index 7af6b6d39..fe677e317 100644 --- a/rust/operator-binary/src/controller/build/resource/agent.rs +++ b/rust/operator-binary/src/controller/build/resource/agent.rs @@ -52,6 +52,7 @@ pub fn build_agent_deployment( cb_agent .image(&agent_config.image) + .command(vec!["stackable-kafka-agent".to_owned()]) .args(vec!["run".to_owned()]) .add_env_var_from_source( "KUBERNETES_NODE_NAME", @@ -153,9 +154,8 @@ mod tests { use serde_json::{Value, json}; use super::*; - use crate::{ - controller::test_support::{app_version_label, minimal_kafka, validated_cluster}, - crd::default_agent_image, + use crate::controller::test_support::{ + OPERATOR_IMAGE, app_version_label, minimal_kafka, validated_cluster, }; fn cluster(platform_access: &str) -> ValidatedCluster { @@ -205,7 +205,7 @@ mod tests { let cluster = cluster("{enabled: true, authentication: {tls: {secretClass: tls}}}"); let agent_config = cluster.agent_config.expect("platform access is enabled"); - assert_eq!(agent_config.image, default_agent_image("oci.example.org")); + assert_eq!(agent_config.image, OPERATOR_IMAGE); } #[test] @@ -239,9 +239,10 @@ mod tests { let pod_spec = &deployment["spec"]["template"]["spec"]; assert_eq!(pod_spec["serviceAccountName"], "simple-kafka-agent"); + assert_eq!(pod_spec["containers"][0]["image"], OPERATOR_IMAGE); assert_eq!( - pod_spec["containers"][0]["image"], - default_agent_image("oci.example.org") + pod_spec["containers"][0]["command"], + json!(["stackable-kafka-agent"]) ); assert_eq!(pod_spec["containers"][0]["args"], json!(["run"])); assert_eq!( diff --git a/rust/operator-binary/src/controller/validate.rs b/rust/operator-binary/src/controller/validate.rs index 62fb2bff8..9865afeca 100644 --- a/rust/operator-binary/src/controller/validate.rs +++ b/rust/operator-binary/src/controller/validate.rs @@ -115,6 +115,12 @@ pub enum Error { to join without it" ))] MissingKraftControllerRole, + + #[snafu(display( + "platform access requires the operator image, which also contains the agent; set the \ + OPERATOR_IMAGE environment variable of the operator" + ))] + PlatformAccessRequiresOperatorImage, } /// Validated logging configuration for a Kafka role group's Kafka and (optional) Vector @@ -199,7 +205,7 @@ pub fn validate( kafka: &v1alpha1::KafkaCluster, dereferenced_objects: DereferencedObjects, operator_environment: &OperatorEnvironmentOptions, - agent_image: &str, + operator_image: Option<&str>, ) -> Result { let image = kafka .spec @@ -339,10 +345,15 @@ pub fn validate( .platform_access .as_ref() .filter(|platform_access| platform_access.enabled) - .map(|platform_access| ValidatedAgentConfig { - image: agent_image.to_owned(), - authentication: platform_access.authentication.clone(), - }); + .map(|platform_access| { + Ok(ValidatedAgentConfig { + image: operator_image + .context(PlatformAccessRequiresOperatorImageSnafu)? + .to_owned(), + authentication: platform_access.authentication.clone(), + }) + }) + .transpose()?; Ok(ValidatedCluster::new( name, @@ -462,7 +473,9 @@ mod tests { use super::{Error, KAFKA_CLUSTER_ID, inject_cluster_id}; use crate::{ - controller::test_support::{app_version_label, minimal_kafka, validated_cluster}, + controller::test_support::{ + app_version_label, minimal_kafka, validate_with_operator_image, validated_cluster, + }, crd::role::KafkaRole, }; @@ -564,6 +577,48 @@ mod tests { } } + #[rstest::rstest] + #[case::enabled(true, false)] + #[case::disabled(false, true)] + fn platform_access_requires_the_operator_image(#[case] enabled: bool, #[case] valid: bool) { + let kafka = minimal_kafka(&format!( + r#" + apiVersion: kafka.stackable.tech/v1alpha1 + kind: KafkaCluster + metadata: + name: simple-kafka + namespace: default + uid: 12345678-1234-1234-1234-123456789012 + spec: + image: + productVersion: 3.9.2 + clusterConfig: + zookeeperConfigMapName: xyz + platformAccess: + enabled: {enabled} + authentication: + tls: + secretClass: tls + brokers: + roleGroups: + default: + replicas: 1 + "# + )); + + let result = validate_with_operator_image(&kafka, None); + + if valid { + assert!(result.is_ok(), "expected Ok, got: {:?}", result.err()); + } else { + assert!( + matches!(result, Err(Error::PlatformAccessRequiresOperatorImage)), + "expected PlatformAccessRequiresOperatorImage, got: {:?}", + result.err() + ); + } + } + #[test] fn injects_cluster_id_when_absent() { let env = inject_cluster_id(EnvVarSet::new(), Some("my-id")); diff --git a/rust/operator-binary/src/crd/mod.rs b/rust/operator-binary/src/crd/mod.rs index 8274f1b20..840fdc175 100644 --- a/rust/operator-binary/src/crd/mod.rs +++ b/rust/operator-binary/src/crd/mod.rs @@ -46,15 +46,6 @@ use crate::crd::{ }; pub const CONTAINER_IMAGE_BASE_NAME: &str = "kafka"; -pub const AGENT_IMAGE_BASE_NAME: &str = "kafka-agent"; - -/// The agent image of the same release as the operator. -pub fn default_agent_image(image_repository: &str) -> String { - format!( - "{image_repository}/{AGENT_IMAGE_BASE_NAME}:{}", - crate::built_info::PKG_VERSION - ) -} pub const APP_NAME: &str = "kafka"; pub const KAFKA_OPERATOR_NAME: &str = "kafka.stackable.tech"; pub const FIELD_MANAGER: &str = "kafka-operator"; diff --git a/rust/operator-binary/src/main.rs b/rust/operator-binary/src/main.rs index 809bd5664..eb28ac0f4 100644 --- a/rust/operator-binary/src/main.rs +++ b/rust/operator-binary/src/main.rs @@ -37,7 +37,7 @@ use stackable_operator::{ use crate::{ controller::KAFKA_FULL_CONTROLLER_NAME, - crd::{KAFKA_OPERATOR_NAME, KafkaCluster, KafkaClusterVersion, default_agent_image, v1alpha1}, + crd::{KAFKA_OPERATOR_NAME, KafkaCluster, KafkaClusterVersion, v1alpha1}, webhooks::conversion::create_webhook_server, }; @@ -68,9 +68,10 @@ struct KafkaRun { #[clap(flatten)] common: RunArguments, - /// The agent image. Defaults to the agent image belonging to this operator release. + /// The image of this operator, which also contains the agent. Required to enable platform + /// access on a KafkaCluster. #[arg(long, env)] - agent_image: Option, + operator_image: Option, } #[tokio::main] @@ -87,7 +88,7 @@ async fn main() -> anyhow::Result<()> { maintenance, common, }, - agent_image, + operator_image, }) => { // NOTE (@NickLarsenNZ): Before stackable-telemetry was used: // - The console log level was set by `KAFKA_OPERATOR_LOG`, and is now `CONSOLE_LOG` (when using Tracing::pre_configured). @@ -196,10 +197,8 @@ async fn main() -> anyhow::Result<()> { controller::error_policy, Arc::new(controller::Ctx { client: client.clone(), - agent_image: agent_image.unwrap_or_else(|| { - default_agent_image(&operator_environment.image_repository) - }), operator_environment, + operator_image, }), ) // We can let the reporting happen in the background From 85716c905bd6c38f8d91bb20b1a3d01d97d6ed9b Mon Sep 17 00:00:00 2001 From: Benedikt Labrenz Date: Wed, 21 Oct 2026 15:12:41 +0200 Subject: [PATCH 25/93] cleanup & only watch deployments with agent labels --- .../kafka-operator/templates/_helpers.tpl | 8 ---- .../templates/clusterrole-agent.yaml | 2 +- .../templates/clusterrole-operator.yaml | 2 +- result-agent | 1 - .../src/controller/build/mod.rs | 2 +- .../src/controller/build/resource/agent.rs | 37 +++++++++---------- .../src/controller/validate.rs | 12 +++--- rust/operator-binary/src/main.rs | 4 +- 8 files changed, 29 insertions(+), 39 deletions(-) delete mode 120000 result-agent diff --git a/deploy/helm/kafka-operator/templates/_helpers.tpl b/deploy/helm/kafka-operator/templates/_helpers.tpl index 6f6d19411..9c61cd511 100644 --- a/deploy/helm/kafka-operator/templates/_helpers.tpl +++ b/deploy/helm/kafka-operator/templates/_helpers.tpl @@ -5,14 +5,6 @@ Expand the name of the chart. {{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-operator" }} {{- end }} -{{/* -Fixed product name. Unlike kafka-operator.name it ignores nameOverride, because it must match -names hardcoded in the operator. -*/}} -{{- define "kafka-operator.productName" -}} -kafka -{{- end }} - {{/* Expand the name of the chart. */}} diff --git a/deploy/helm/kafka-operator/templates/clusterrole-agent.yaml b/deploy/helm/kafka-operator/templates/clusterrole-agent.yaml index a17015062..8288ab33a 100644 --- a/deploy/helm/kafka-operator/templates/clusterrole-agent.yaml +++ b/deploy/helm/kafka-operator/templates/clusterrole-agent.yaml @@ -4,7 +4,7 @@ apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: - name: {{ include "kafka-operator.productName" . }}-agent-clusterrole + name: {{ include "kafka-operator.name" . }}-agent-clusterrole labels: {{- include "kafka-operator.labels" . | nindent 4 }} rules: diff --git a/deploy/helm/kafka-operator/templates/clusterrole-operator.yaml b/deploy/helm/kafka-operator/templates/clusterrole-operator.yaml index cabc631e4..96045367c 100644 --- a/deploy/helm/kafka-operator/templates/clusterrole-operator.yaml +++ b/deploy/helm/kafka-operator/templates/clusterrole-operator.yaml @@ -50,7 +50,7 @@ rules: - bind resourceNames: - {{ include "kafka-operator.name" . }}-clusterrole - - {{ include "kafka-operator.productName" . }}-agent-clusterrole + - {{ include "kafka-operator.name" . }}-agent-clusterrole # StatefulSet created per role group (broker, KRaft controller). Applied via # SSA, tracked for orphan cleanup, and owned by the controller. - apiGroups: diff --git a/result-agent b/result-agent deleted file mode 120000 index 9ac2d309c..000000000 --- a/result-agent +++ /dev/null @@ -1 +0,0 @@ -/nix/store/6m2nq8gfixyb147i629qrj894b3bxazv-stream-kafka-agent-docker \ No newline at end of file diff --git a/rust/operator-binary/src/controller/build/mod.rs b/rust/operator-binary/src/controller/build/mod.rs index 45764219d..8ddcf577e 100644 --- a/rust/operator-binary/src/controller/build/mod.rs +++ b/rust/operator-binary/src/controller/build/mod.rs @@ -239,11 +239,11 @@ pub(crate) fn recommended_labels_for_unversioned_role_group_resources( ) } -/// Selector labels matching the pods of a role group. pub(crate) fn agent_selector(cluster: &ValidatedCluster) -> Labels { label::agent_selector(&cluster.name, &PRODUCT_NAME) } +/// Selector labels matching the pods of a role group. pub(crate) fn role_group_selector( cluster: &ValidatedCluster, role_name: &RoleName, diff --git a/rust/operator-binary/src/controller/build/resource/agent.rs b/rust/operator-binary/src/controller/build/resource/agent.rs index ac9bec6e8..8e09b28ab 100644 --- a/rust/operator-binary/src/controller/build/resource/agent.rs +++ b/rust/operator-binary/src/controller/build/resource/agent.rs @@ -11,10 +11,11 @@ use stackable_operator::{ }, constant, k8s_openapi::{ + Resource, api::{ apps::v1::{Deployment, DeploymentSpec, DeploymentStrategy}, core::v1::{EnvVarSource, ObjectFieldSelector, ServiceAccount}, - rbac::v1::{RoleBinding, RoleRef, Subject}, + rbac::v1::{ClusterRole, RoleBinding, RoleRef, Subject}, }, apimachinery::pkg::apis::meta::v1::LabelSelector, }, @@ -121,12 +122,12 @@ pub fn build_agent_role_binding(cluster: &ValidatedCluster) -> RoleBinding { RoleBinding { metadata: agent_metadata(cluster), role_ref: RoleRef { - api_group: Some("rbac.authorization.k8s.io".to_owned()), - kind: "ClusterRole".to_owned(), + api_group: Some(ClusterRole::GROUP.to_owned()), + kind: ClusterRole::KIND.to_owned(), name: AGENT_CLUSTER_ROLE_NAME.to_owned(), }, subjects: Some(vec![Subject { - kind: "ServiceAccount".to_owned(), + kind: ServiceAccount::KIND.to_owned(), name: agent_name(cluster), namespace: Some(cluster.namespace.to_string()), ..Subject::default() @@ -185,13 +186,12 @@ mod tests { validated_cluster(&kafka) } - fn agent_deployment(platform_access: &str) -> Value { - let cluster = cluster(platform_access); + fn agent_deployment(cluster: &ValidatedCluster) -> Value { let agent_config = cluster .agent_config .as_ref() .expect("platform access is enabled"); - serde_json::to_value(build_agent_deployment(&cluster, agent_config)) + serde_json::to_value(build_agent_deployment(cluster, agent_config)) .expect("must be serializable") } @@ -210,8 +210,9 @@ mod tests { #[test] fn test_deployment() { - let deployment = - agent_deployment("{enabled: true, authentication: {tls: {secretClass: tls}}}"); + let deployment = agent_deployment(&cluster( + "{enabled: true, authentication: {tls: {secretClass: tls}}}", + )); assert_eq!(deployment["metadata"]["name"], "simple-kafka-agent"); assert_eq!(deployment["spec"]["replicas"], 1); @@ -258,8 +259,9 @@ mod tests { #[test] fn test_deployment_with_static_secret() { - let deployment = - agent_deployment("{enabled: true, authentication: {tls: {secret: my-cert}}}"); + let deployment = agent_deployment(&cluster( + "{enabled: true, authentication: {tls: {secret: my-cert}}}", + )); assert_eq!( deployment["spec"]["template"]["spec"]["volumes"], @@ -269,7 +271,8 @@ mod tests { #[test] fn deployment_with_anonymous_authentication_mounts_no_credential() { - let deployment = agent_deployment("{enabled: true, authentication: {anonymous: {}}}"); + let deployment = + agent_deployment(&cluster("{enabled: true, authentication: {anonymous: {}}}")); let pod_spec = &deployment["spec"]["template"]["spec"]; assert_eq!(pod_spec["volumes"], Value::Null); @@ -278,16 +281,10 @@ mod tests { #[test] fn test_deployment_uses_product_image_pull_secrets() { - let cluster = cluster_with_image( + let deployment = agent_deployment(&cluster_with_image( "{productVersion: 3.9.2, pullSecrets: [{name: regcred}]}", "{enabled: true, authentication: {tls: {secretClass: tls}}}", - ); - let agent_config = cluster - .agent_config - .as_ref() - .expect("platform access is enabled"); - let deployment = serde_json::to_value(build_agent_deployment(&cluster, agent_config)) - .expect("must be serializable"); + )); assert_eq!( deployment["spec"]["template"]["spec"]["imagePullSecrets"], diff --git a/rust/operator-binary/src/controller/validate.rs b/rust/operator-binary/src/controller/validate.rs index 8ca80138d..2a2dabd26 100644 --- a/rust/operator-binary/src/controller/validate.rs +++ b/rust/operator-binary/src/controller/validate.rs @@ -582,9 +582,9 @@ mod tests { } #[rstest::rstest] - #[case::enabled(true, false)] - #[case::disabled(false, true)] - fn platform_access_requires_the_operator_image(#[case] enabled: bool, #[case] valid: bool) { + #[case::enabled(true)] + #[case::disabled(false)] + fn platform_access_requires_the_operator_image(#[case] enabled: bool) { let kafka = minimal_kafka(&format!( r#" apiVersion: kafka.stackable.tech/v1alpha1 @@ -612,14 +612,14 @@ mod tests { let result = validate_with_operator_image(&kafka, None); - if valid { - assert!(result.is_ok(), "expected Ok, got: {:?}", result.err()); - } else { + if enabled { assert!( matches!(result, Err(Error::PlatformAccessRequiresOperatorImage)), "expected PlatformAccessRequiresOperatorImage, got: {:?}", result.err() ); + } else { + assert!(result.is_ok(), "expected Ok, got: {:?}", result.err()); } } diff --git a/rust/operator-binary/src/main.rs b/rust/operator-binary/src/main.rs index a24f5df5e..817b07ac1 100644 --- a/rust/operator-binary/src/main.rs +++ b/rust/operator-binary/src/main.rs @@ -186,7 +186,9 @@ async fn main() -> anyhow::Result<()> { ) .owns( watch_namespace.get_api::>(&client), - watcher::Config::default(), + // The agent is a the only Deployment + watcher::Config::default() + .labels("app.kubernetes.io/name=kafka,app.kubernetes.io/component=agent"), ) .watches( watch_namespace.get_api::>(&client), From 58e33939617912d3bf9fc79e25982377984dc9fb Mon Sep 17 00:00:00 2001 From: Benedikt Labrenz Date: Wed, 21 Oct 2026 15:53:29 +0200 Subject: [PATCH 26/93] use BrokerAddress in more places and remove awaiting_response --- rust/kafka-admin/src/client.rs | 7 +- rust/kafka-admin/src/connection.rs | 162 ++++++++++++----------------- 2 files changed, 69 insertions(+), 100 deletions(-) diff --git a/rust/kafka-admin/src/client.rs b/rust/kafka-admin/src/client.rs index f26dbcefd..c24c39f95 100644 --- a/rust/kafka-admin/src/client.rs +++ b/rust/kafka-admin/src/client.rs @@ -42,6 +42,7 @@ const LIST_PARTITION_REASSIGNMENTS_VERSION: i16 = 0; /// Must stay below the connection's request timeout. const BROKER_TIMEOUT_MS: i32 = 25_000; +const _: () = assert!((BROKER_TIMEOUT_MS as u128) < connection::REQUEST_TIMEOUT.as_millis()); /// Must stay below the broker's `connections.max.idle.ms` (10 minutes by default). const CONNECTION_MAX_IDLE: Duration = Duration::from_secs(5 * 60); @@ -282,9 +283,7 @@ impl KafkaAdminClient { async fn connect(&self) -> Result { let mut last_error = Error::NoBootstrapServers; for server in &self.bootstrap_servers { - match Connection::connect(&server.host, server.port, &self.security, &self.client_id) - .await - { + match Connection::connect(server, &self.security, &self.client_id).await { Ok(connection) => return Ok(connection), Err(source) => { last_error = Error::Connect { @@ -472,7 +471,7 @@ fn list_partition_reassignments_result( let mut reassignments = response .topics .into_iter() - .filter(|result| result.name == topic_name(topic)) + .filter(|result| result.name.as_str() == topic) .flat_map(|result| result.partitions) .map(|partition| PartitionReassignment { index: partition.partition_index, diff --git a/rust/kafka-admin/src/connection.rs b/rust/kafka-admin/src/connection.rs index b6f97c0be..13fe0d3a3 100644 --- a/rust/kafka-admin/src/connection.rs +++ b/rust/kafka-admin/src/connection.rs @@ -22,22 +22,29 @@ use tokio_util::{ }; use crate::{ + bootstrap::BrokerAddress, codec::{self, decode_response, encode_request}, tls, }; const CONNECT_TIMEOUT: Duration = Duration::from_secs(10); -const REQUEST_TIMEOUT: Duration = Duration::from_secs(30); +pub(crate) const REQUEST_TIMEOUT: Duration = Duration::from_secs(30); const MAX_FRAME_SIZE: usize = 64 * 1024 * 1024; const API_VERSIONS_VERSION: i16 = 3; #[derive(Debug, Snafu)] pub enum Error { - #[snafu(display("failed to connect to {address:?}"))] - Connect { source: io::Error, address: String }, + #[snafu(display("failed to connect to {address}"))] + Connect { + source: io::Error, + address: BrokerAddress, + }, - #[snafu(display("timed out connecting to {address:?}"))] - ConnectTimeout { source: Elapsed, address: String }, + #[snafu(display("timed out connecting to {address}"))] + ConnectTimeout { + source: Elapsed, + address: BrokerAddress, + }, #[snafu(display("failed to configure TLS"))] Tls { source: tls::Error }, @@ -45,8 +52,11 @@ pub enum Error { #[snafu(display("{host:?} is not a valid TLS server name"))] InvalidServerName { host: String }, - #[snafu(display("failed the TLS handshake with {address:?}"))] - TlsHandshake { source: io::Error, address: String }, + #[snafu(display("failed the TLS handshake with {address}"))] + TlsHandshake { + source: io::Error, + address: BrokerAddress, + }, #[snafu(display("failed to write to the broker"))] Write { source: io::Error }, @@ -57,9 +67,6 @@ pub enum Error { #[snafu(display("the broker closed the connection"))] ConnectionClosed, - #[snafu(display("the connection is broken by an earlier failed or cancelled request"))] - ConnectionBroken, - #[snafu(display("timed out waiting for the response to the request with API key {api_key}"))] RequestTimeout { source: Elapsed, api_key: i16 }, @@ -98,27 +105,25 @@ pub enum Security { }, } -pub struct Connection { +/// One request at a time. After a failed or cancelled request, the client drops the connection. +pub(crate) struct Connection { stream: Framed>, LengthDelimitedCodec>, client_id: String, correlation_id: i32, - awaiting_response: bool, broker_api_versions: BTreeMap, } impl Connection { /// Connects to the broker and fetches the API versions it supports. - pub async fn connect( - host: &str, - port: u16, + pub(crate) async fn connect( + address: &BrokerAddress, security: &Security, client_id: &str, ) -> Result { - let address = format!("{host}:{port}"); - let tcp_stream = connect_tcp(host, port, &address).await?; + let tcp_stream = connect_tcp(address).await?; let stream = match tls::client_config(security).context(TlsSnafu)? { None => Either::Left(tcp_stream), - Some(config) => Either::Right(tls_handshake(tcp_stream, host, &address, config).await?), + Some(config) => Either::Right(tls_handshake(tcp_stream, address, config).await?), }; let mut connection = Self::new(stream, client_id); @@ -138,13 +143,16 @@ impl Connection { stream: Framed::new(stream, codec), client_id: client_id.to_owned(), correlation_id: 0, - awaiting_response: false, broker_api_versions: BTreeMap::new(), } } /// Sends the request with the given version, if the broker supports it. - pub async fn send(&mut self, request: R, version: i16) -> Result { + pub(crate) async fn send( + &mut self, + request: R, + version: i16, + ) -> Result { // Each API that is relevant for us has a version that is supported by all of our Kafka versions. // We use these versions and pin them in client.rs. // We will have to add API version negotiation at some point in the future. @@ -157,9 +165,6 @@ impl Connection { request: R, version: i16, ) -> Result { - ensure!(!self.awaiting_response, ConnectionBrokenSnafu); - self.awaiting_response = true; - self.correlation_id = self.correlation_id.wrapping_add(1); let frame = encode_request(&request, version, self.correlation_id, &self.client_id) .context(CodecSnafu)?; @@ -175,19 +180,22 @@ impl Connection { .await .context(RequestTimeoutSnafu { api_key: R::KEY })??; - let response = decode_response::(response.freeze(), version, self.correlation_id) - .context(CodecSnafu)?; - - self.awaiting_response = false; - Ok(response) + decode_response::(response.freeze(), version, self.correlation_id).context(CodecSnafu) } } -async fn connect_tcp(host: &str, port: u16, address: &str) -> Result { - timeout(CONNECT_TIMEOUT, TcpStream::connect((host, port))) - .await - .context(ConnectTimeoutSnafu { address })? - .context(ConnectSnafu { address }) +async fn connect_tcp(address: &BrokerAddress) -> Result { + timeout( + CONNECT_TIMEOUT, + TcpStream::connect((address.host.as_str(), address.port)), + ) + .await + .context(ConnectTimeoutSnafu { + address: address.clone(), + })? + .context(ConnectSnafu { + address: address.clone(), + }) } fn api_versions_request() -> ApiVersionsRequest { @@ -241,11 +249,11 @@ fn check_version( async fn tls_handshake( tcp_stream: TcpStream, - host: &str, - address: &str, + address: &BrokerAddress, config: ClientConfig, ) -> Result> { - let server_name = ServerName::try_from(host.to_owned()) + let host = &address.host; + let server_name = ServerName::try_from(host.clone()) .ok() .context(InvalidServerNameSnafu { host })?; timeout( @@ -253,8 +261,12 @@ async fn tls_handshake( TlsConnector::from(Arc::new(config)).connect(server_name, tcp_stream), ) .await - .context(ConnectTimeoutSnafu { address })? - .context(TlsHandshakeSnafu { address }) + .context(ConnectTimeoutSnafu { + address: address.clone(), + })? + .context(TlsHandshakeSnafu { + address: address.clone(), + }) } /// A mock Kafka broker for tests, speaking the wire protocol over a local TCP socket. @@ -275,7 +287,7 @@ pub(crate) mod test_support { use super::API_VERSIONS_VERSION; - pub async fn listen() -> (TcpListener, u16) { + pub(crate) async fn listen() -> (TcpListener, u16) { let listener = TcpListener::bind("127.0.0.1:0") .await .expect("the mock broker should be able to listen"); @@ -288,7 +300,7 @@ pub(crate) mod test_support { /// Accepts a connection and answers its ApiVersions request with the given /// `(api_key, min_version, max_version)` ranges. - pub async fn accept(listener: &TcpListener, apis: &[(i16, i16, i16)]) -> TcpStream { + pub(crate) async fn accept(listener: &TcpListener, apis: &[(i16, i16, i16)]) -> TcpStream { let (mut broker, _) = listener .accept() .await @@ -306,7 +318,7 @@ pub(crate) mod test_support { broker } - pub async fn read_request(broker: &mut TcpStream) -> (RequestHeader, Bytes) { + pub(crate) async fn read_request(broker: &mut TcpStream) -> (RequestHeader, Bytes) { let size = broker .read_i32() .await @@ -322,7 +334,9 @@ pub(crate) mod test_support { (header, frame) } - pub async fn read_request_body(broker: &mut TcpStream) -> (RequestHeader, R) { + pub(crate) async fn read_request_body( + broker: &mut TcpStream, + ) -> (RequestHeader, R) { let (header, mut body) = read_request(broker).await; assert_eq!(R::KEY, header.request_api_key); let request = R::decode(&mut body, header.request_api_version) @@ -330,7 +344,7 @@ pub(crate) mod test_support { (header, request) } - pub async fn write_response( + pub(crate) async fn write_response( broker: &mut TcpStream, request: &RequestHeader, correlation_id: i32, @@ -353,7 +367,7 @@ pub(crate) mod test_support { .expect("the response should be writable"); } - fn api_versions(apis: &[(i16, i16, i16)]) -> ApiVersionsResponse { + pub(crate) fn api_versions(apis: &[(i16, i16, i16)]) -> ApiVersionsResponse { ApiVersionsResponse::default().with_api_keys( apis.iter() .map(|(api_key, min_version, max_version)| { @@ -369,21 +383,23 @@ pub(crate) mod test_support { #[cfg(test)] mod tests { - use kafka_protocol::messages::{ - MetadataRequest, MetadataResponse, api_versions_response::ApiVersion, - }; + use kafka_protocol::messages::{MetadataRequest, MetadataResponse}; use tokio::io::AsyncWriteExt; use super::{ - test_support::{accept, listen, read_request, write_response}, + test_support::{accept, api_versions, listen, read_request, write_response}, *, }; /// Connects to a mock broker supporting Metadata versions 0 to 13. async fn connect() -> (Connection, TcpStream) { let (listener, port) = listen().await; + let address = BrokerAddress { + host: "127.0.0.1".to_owned(), + port, + }; let (connection, broker) = tokio::join!( - Connection::connect("127.0.0.1", port, &Security::Plaintext, "test"), + Connection::connect(&address, &Security::Plaintext, "test"), accept(&listener, &[(MetadataRequest::KEY, 0, 13)]) ); (connection.expect("the connection should succeed"), broker) @@ -391,12 +407,7 @@ mod tests { #[test] fn read_api_versions() { - let response = ApiVersionsResponse::default().with_api_keys(vec![ - ApiVersion::default() - .with_api_key(MetadataRequest::KEY) - .with_min_version(0) - .with_max_version(13), - ]); + let response = api_versions(&[(MetadataRequest::KEY, 0, 13)]); assert_eq!( BTreeMap::from([(MetadataRequest::KEY, VersionRange { min: 0, max: 13 })]), @@ -537,45 +548,4 @@ mod tests { Error::RequestTimeout { .. } )); } - - #[tokio::test] - async fn reject_requests_after_a_failed_request() { - let (mut connection, mut broker) = connect().await; - - let broker_side = async { - read_request(&mut broker).await; - drop(broker); - }; - let (failed, ()) = - tokio::join!(connection.send(MetadataRequest::default(), 12), broker_side); - assert!(matches!( - failed.expect_err("the broker should have closed the connection"), - Error::ConnectionClosed - )); - - let error = connection - .send(MetadataRequest::default(), 12) - .await - .expect_err("the connection should be broken"); - - assert!(matches!(error, Error::ConnectionBroken)); - } - - #[tokio::test] - async fn reject_requests_after_a_cancelled_request() { - let (mut connection, _broker) = connect().await; - - { - let request = connection.send(MetadataRequest::default(), 12); - tokio::pin!(request); - let _ = futures::poll!(request.as_mut()); - } - - let error = connection - .send(MetadataRequest::default(), 12) - .await - .expect_err("the connection should be broken"); - - assert!(matches!(error, Error::ConnectionBroken)); - } } From 198e800761b6b5eefbd98c887f7407acb84cb738 Mon Sep 17 00:00:00 2001 From: Benedikt Labrenz Date: Wed, 21 Oct 2026 16:16:11 +0200 Subject: [PATCH 27/93] fix flaky unit test --- rust/kafka-admin/src/client.rs | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/rust/kafka-admin/src/client.rs b/rust/kafka-admin/src/client.rs index c24c39f95..626574ef7 100644 --- a/rust/kafka-admin/src/client.rs +++ b/rust/kafka-admin/src/client.rs @@ -523,7 +523,7 @@ mod tests { metadata_response::{MetadataResponsePartition, MetadataResponseTopic}, }; use rstest::rstest; - use tokio::net::TcpStream; + use tokio::net::{TcpSocket, TcpStream}; use super::*; use crate::connection::test_support::{accept, listen, read_request_body, write_response}; @@ -533,8 +533,15 @@ mod tests { #[tokio::test] async fn try_the_next_bootstrap_server() { - let (unreachable, unreachable_port) = listen().await; - drop(unreachable); + // Use a port that is always rejected + let unreachable = TcpSocket::new_v4().expect("the socket should be creatable"); + unreachable + .bind("127.0.0.1:0".parse().expect("the address should be valid")) + .expect("the socket should be bindable"); + let unreachable_port = unreachable + .local_addr() + .expect("the socket should have an address") + .port(); let (listener, port) = listen().await; let mut client = kafka_admin_client(&[unreachable_port, port]); From 81a49a1dd3ebbb93fdd737d19c6704d876afea63 Mon Sep 17 00:00:00 2001 From: Benedikt Labrenz Date: Fri, 23 Oct 2026 14:36:23 +0200 Subject: [PATCH 28/93] wip: add kuttl tests --- Cargo.nix | 18 ++--- crate-hashes.json | 18 ++--- .../topic-authentication/00-patch-ns.yaml.j2 | 9 +++ .../05-create-agent-credential.yaml.j2 | 29 +++++++ .../kuttl/topic-authentication/10-assert.yaml | 20 +++++ .../10-install-kafka.yaml | 7 ++ .../topic-authentication/10_kafka.yaml.j2 | 77 +++++++++++++++++++ .../topic-authentication/11-assert.yaml.j2 | 14 ++++ .../kuttl/topic-authentication/20-assert.yaml | 8 ++ .../topic-authentication/20-create-topic.yaml | 10 +++ .../kuttl/topic-authentication/kafka-cli.sh | 13 ++++ .../topic-authentication/topic-condition.sh | 11 +++ .../templates/kuttl/topic/00-patch-ns.yaml.j2 | 9 +++ tests/templates/kuttl/topic/10-assert.yaml | 42 ++++++++++ .../kuttl/topic/10-install-kafka.yaml.j2 | 34 ++++++++ tests/templates/kuttl/topic/20-assert.yaml | 12 +++ .../kuttl/topic/20-create-topic.yaml | 13 ++++ tests/templates/kuttl/topic/30-assert.yaml | 9 +++ .../kuttl/topic/30-change-topic.yaml | 13 ++++ .../topic/40-set-config-out-of-band.yaml | 6 ++ tests/templates/kuttl/topic/41-assert.yaml | 12 +++ .../kuttl/topic/41-remove-config-key.yaml | 8 ++ tests/templates/kuttl/topic/50-assert.yaml | 9 +++ .../kuttl/topic/50-unsupported-change.yaml | 13 ++++ tests/templates/kuttl/topic/51-assert.yaml | 6 ++ .../topic/51-revert-unsupported-change.yaml | 12 +++ tests/templates/kuttl/topic/60-assert.yaml | 11 +++ .../kuttl/topic/60-invalid-spec.yaml | 13 ++++ .../kuttl/topic/70-create-foreign-topic.yaml | 6 ++ tests/templates/kuttl/topic/71-assert.yaml | 8 ++ .../kuttl/topic/71-refuse-foreign-topic.yaml | 10 +++ .../kuttl/topic/72-adopt-foreign-topic.yaml | 12 +++ tests/templates/kuttl/topic/72-assert.yaml | 8 ++ tests/templates/kuttl/topic/80-assert.yaml | 9 +++ .../topic/80-create-conflicting-topic.yaml | 11 +++ tests/templates/kuttl/topic/81-assert.yaml | 6 ++ .../topic/81-delete-conflicting-topic.yaml | 7 ++ tests/templates/kuttl/topic/90-assert.yaml | 8 ++ .../90-delete-legacy-topic-in-kafka.yaml | 13 ++++ tests/templates/kuttl/topic/91-assert.yaml | 11 +++ .../kuttl/topic/91-delete-topics.yaml | 13 ++++ .../topic/95-disable-platform-access.yaml.j2 | 34 ++++++++ tests/templates/kuttl/topic/95-errors.yaml | 16 ++++ tests/templates/kuttl/topic/kafka-cli.sh | 13 ++++ .../templates/kuttl/topic/topic-condition.sh | 11 +++ tests/test-definition.yaml | 17 +++- 46 files changed, 650 insertions(+), 19 deletions(-) create mode 100644 tests/templates/kuttl/topic-authentication/00-patch-ns.yaml.j2 create mode 100644 tests/templates/kuttl/topic-authentication/05-create-agent-credential.yaml.j2 create mode 100644 tests/templates/kuttl/topic-authentication/10-assert.yaml create mode 100644 tests/templates/kuttl/topic-authentication/10-install-kafka.yaml create mode 100644 tests/templates/kuttl/topic-authentication/10_kafka.yaml.j2 create mode 100644 tests/templates/kuttl/topic-authentication/11-assert.yaml.j2 create mode 100644 tests/templates/kuttl/topic-authentication/20-assert.yaml create mode 100644 tests/templates/kuttl/topic-authentication/20-create-topic.yaml create mode 100644 tests/templates/kuttl/topic-authentication/kafka-cli.sh create mode 100644 tests/templates/kuttl/topic-authentication/topic-condition.sh create mode 100644 tests/templates/kuttl/topic/00-patch-ns.yaml.j2 create mode 100644 tests/templates/kuttl/topic/10-assert.yaml create mode 100644 tests/templates/kuttl/topic/10-install-kafka.yaml.j2 create mode 100644 tests/templates/kuttl/topic/20-assert.yaml create mode 100644 tests/templates/kuttl/topic/20-create-topic.yaml create mode 100644 tests/templates/kuttl/topic/30-assert.yaml create mode 100644 tests/templates/kuttl/topic/30-change-topic.yaml create mode 100644 tests/templates/kuttl/topic/40-set-config-out-of-band.yaml create mode 100644 tests/templates/kuttl/topic/41-assert.yaml create mode 100644 tests/templates/kuttl/topic/41-remove-config-key.yaml create mode 100644 tests/templates/kuttl/topic/50-assert.yaml create mode 100644 tests/templates/kuttl/topic/50-unsupported-change.yaml create mode 100644 tests/templates/kuttl/topic/51-assert.yaml create mode 100644 tests/templates/kuttl/topic/51-revert-unsupported-change.yaml create mode 100644 tests/templates/kuttl/topic/60-assert.yaml create mode 100644 tests/templates/kuttl/topic/60-invalid-spec.yaml create mode 100644 tests/templates/kuttl/topic/70-create-foreign-topic.yaml create mode 100644 tests/templates/kuttl/topic/71-assert.yaml create mode 100644 tests/templates/kuttl/topic/71-refuse-foreign-topic.yaml create mode 100644 tests/templates/kuttl/topic/72-adopt-foreign-topic.yaml create mode 100644 tests/templates/kuttl/topic/72-assert.yaml create mode 100644 tests/templates/kuttl/topic/80-assert.yaml create mode 100644 tests/templates/kuttl/topic/80-create-conflicting-topic.yaml create mode 100644 tests/templates/kuttl/topic/81-assert.yaml create mode 100644 tests/templates/kuttl/topic/81-delete-conflicting-topic.yaml create mode 100644 tests/templates/kuttl/topic/90-assert.yaml create mode 100644 tests/templates/kuttl/topic/90-delete-legacy-topic-in-kafka.yaml create mode 100644 tests/templates/kuttl/topic/91-assert.yaml create mode 100644 tests/templates/kuttl/topic/91-delete-topics.yaml create mode 100644 tests/templates/kuttl/topic/95-disable-platform-access.yaml.j2 create mode 100644 tests/templates/kuttl/topic/95-errors.yaml create mode 100644 tests/templates/kuttl/topic/kafka-cli.sh create mode 100644 tests/templates/kuttl/topic/topic-condition.sh diff --git a/Cargo.nix b/Cargo.nix index 3abdc326f..dcd03807f 100644 --- a/Cargo.nix +++ b/Cargo.nix @@ -5510,7 +5510,7 @@ rec { src = pkgs.fetchgit { url = "https://github.com/stackabletech/operator-rs.git"; rev = "92e1a3655432e3899da735a9d86ded2e38030c0e"; - sha256 = "0cgziqra8097hp05ynib1qpw4c95n972f4w2rk9l3llyp8r1vmci"; + sha256 = "132z49z2xg839grsn5kryh55p1k069rhil4dl5brmvivkh5i8ffd"; }; libName = "k8s_version"; authors = [ @@ -10508,7 +10508,7 @@ rec { src = pkgs.fetchgit { url = "https://github.com/stackabletech/operator-rs.git"; rev = "92e1a3655432e3899da735a9d86ded2e38030c0e"; - sha256 = "0cgziqra8097hp05ynib1qpw4c95n972f4w2rk9l3llyp8r1vmci"; + sha256 = "132z49z2xg839grsn5kryh55p1k069rhil4dl5brmvivkh5i8ffd"; }; libName = "stackable_certs"; authors = [ @@ -10910,7 +10910,7 @@ rec { src = pkgs.fetchgit { url = "https://github.com/stackabletech/operator-rs.git"; rev = "92e1a3655432e3899da735a9d86ded2e38030c0e"; - sha256 = "0cgziqra8097hp05ynib1qpw4c95n972f4w2rk9l3llyp8r1vmci"; + sha256 = "132z49z2xg839grsn5kryh55p1k069rhil4dl5brmvivkh5i8ffd"; }; libName = "stackable_operator"; authors = [ @@ -11109,7 +11109,7 @@ rec { src = pkgs.fetchgit { url = "https://github.com/stackabletech/operator-rs.git"; rev = "92e1a3655432e3899da735a9d86ded2e38030c0e"; - sha256 = "0cgziqra8097hp05ynib1qpw4c95n972f4w2rk9l3llyp8r1vmci"; + sha256 = "132z49z2xg839grsn5kryh55p1k069rhil4dl5brmvivkh5i8ffd"; }; procMacro = true; libName = "stackable_operator_derive"; @@ -11144,7 +11144,7 @@ rec { src = pkgs.fetchgit { url = "https://github.com/stackabletech/operator-rs.git"; rev = "92e1a3655432e3899da735a9d86ded2e38030c0e"; - sha256 = "0cgziqra8097hp05ynib1qpw4c95n972f4w2rk9l3llyp8r1vmci"; + sha256 = "132z49z2xg839grsn5kryh55p1k069rhil4dl5brmvivkh5i8ffd"; }; libName = "stackable_shared"; authors = [ @@ -11225,7 +11225,7 @@ rec { src = pkgs.fetchgit { url = "https://github.com/stackabletech/operator-rs.git"; rev = "92e1a3655432e3899da735a9d86ded2e38030c0e"; - sha256 = "0cgziqra8097hp05ynib1qpw4c95n972f4w2rk9l3llyp8r1vmci"; + sha256 = "132z49z2xg839grsn5kryh55p1k069rhil4dl5brmvivkh5i8ffd"; }; libName = "stackable_telemetry"; authors = [ @@ -11335,7 +11335,7 @@ rec { src = pkgs.fetchgit { url = "https://github.com/stackabletech/operator-rs.git"; rev = "92e1a3655432e3899da735a9d86ded2e38030c0e"; - sha256 = "0cgziqra8097hp05ynib1qpw4c95n972f4w2rk9l3llyp8r1vmci"; + sha256 = "132z49z2xg839grsn5kryh55p1k069rhil4dl5brmvivkh5i8ffd"; }; libName = "stackable_versioned"; authors = [ @@ -11385,7 +11385,7 @@ rec { src = pkgs.fetchgit { url = "https://github.com/stackabletech/operator-rs.git"; rev = "92e1a3655432e3899da735a9d86ded2e38030c0e"; - sha256 = "0cgziqra8097hp05ynib1qpw4c95n972f4w2rk9l3llyp8r1vmci"; + sha256 = "132z49z2xg839grsn5kryh55p1k069rhil4dl5brmvivkh5i8ffd"; }; procMacro = true; libName = "stackable_versioned_macros"; @@ -11449,7 +11449,7 @@ rec { src = pkgs.fetchgit { url = "https://github.com/stackabletech/operator-rs.git"; rev = "92e1a3655432e3899da735a9d86ded2e38030c0e"; - sha256 = "0cgziqra8097hp05ynib1qpw4c95n972f4w2rk9l3llyp8r1vmci"; + sha256 = "132z49z2xg839grsn5kryh55p1k069rhil4dl5brmvivkh5i8ffd"; }; libName = "stackable_webhook"; authors = [ diff --git a/crate-hashes.json b/crate-hashes.json index e4f5f7b1d..b5d1f4835 100644 --- a/crate-hashes.json +++ b/crate-hashes.json @@ -1,11 +1,11 @@ { - "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#k8s-version@0.1.3": "0cgziqra8097hp05ynib1qpw4c95n972f4w2rk9l3llyp8r1vmci", - "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#stackable-certs@0.4.1": "0cgziqra8097hp05ynib1qpw4c95n972f4w2rk9l3llyp8r1vmci", - "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#stackable-operator-derive@0.3.1": "0cgziqra8097hp05ynib1qpw4c95n972f4w2rk9l3llyp8r1vmci", - "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#stackable-operator@0.119.0": "0cgziqra8097hp05ynib1qpw4c95n972f4w2rk9l3llyp8r1vmci", - "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#stackable-shared@0.1.2": "0cgziqra8097hp05ynib1qpw4c95n972f4w2rk9l3llyp8r1vmci", - "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#stackable-telemetry@0.6.5": "0cgziqra8097hp05ynib1qpw4c95n972f4w2rk9l3llyp8r1vmci", - "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#stackable-versioned-macros@0.11.1": "0cgziqra8097hp05ynib1qpw4c95n972f4w2rk9l3llyp8r1vmci", - "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#stackable-versioned@0.11.1": "0cgziqra8097hp05ynib1qpw4c95n972f4w2rk9l3llyp8r1vmci", - "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#stackable-webhook@0.10.0": "0cgziqra8097hp05ynib1qpw4c95n972f4w2rk9l3llyp8r1vmci" + "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#k8s-version@0.1.3": "132z49z2xg839grsn5kryh55p1k069rhil4dl5brmvivkh5i8ffd", + "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#stackable-certs@0.4.1": "132z49z2xg839grsn5kryh55p1k069rhil4dl5brmvivkh5i8ffd", + "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#stackable-operator-derive@0.3.1": "132z49z2xg839grsn5kryh55p1k069rhil4dl5brmvivkh5i8ffd", + "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#stackable-operator@0.119.0": "132z49z2xg839grsn5kryh55p1k069rhil4dl5brmvivkh5i8ffd", + "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#stackable-shared@0.1.2": "132z49z2xg839grsn5kryh55p1k069rhil4dl5brmvivkh5i8ffd", + "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#stackable-telemetry@0.6.5": "132z49z2xg839grsn5kryh55p1k069rhil4dl5brmvivkh5i8ffd", + "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#stackable-versioned-macros@0.11.1": "132z49z2xg839grsn5kryh55p1k069rhil4dl5brmvivkh5i8ffd", + "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#stackable-versioned@0.11.1": "132z49z2xg839grsn5kryh55p1k069rhil4dl5brmvivkh5i8ffd", + "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#stackable-webhook@0.10.0": "132z49z2xg839grsn5kryh55p1k069rhil4dl5brmvivkh5i8ffd" } \ No newline at end of file diff --git a/tests/templates/kuttl/topic-authentication/00-patch-ns.yaml.j2 b/tests/templates/kuttl/topic-authentication/00-patch-ns.yaml.j2 new file mode 100644 index 000000000..67185acfd --- /dev/null +++ b/tests/templates/kuttl/topic-authentication/00-patch-ns.yaml.j2 @@ -0,0 +1,9 @@ +{% if test_scenario['values']['openshift'] == 'true' %} +# see https://github.com/stackabletech/issues/issues/566 +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +commands: + - script: kubectl patch namespace $NAMESPACE -p '{"metadata":{"labels":{"pod-security.kubernetes.io/enforce":"privileged"}}}' + timeout: 120 +{% endif %} diff --git a/tests/templates/kuttl/topic-authentication/05-create-agent-credential.yaml.j2 b/tests/templates/kuttl/topic-authentication/05-create-agent-credential.yaml.j2 new file mode 100644 index 000000000..40c0d2fd1 --- /dev/null +++ b/tests/templates/kuttl/topic-authentication/05-create-agent-credential.yaml.j2 @@ -0,0 +1,29 @@ +{% if test_scenario['values']['platform-access-authentication'] == 'secret' %} +--- +# A CA trusted by the brokers for client authentication, and a certificate for the agent signed by +# it, as a user would provide them +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +commands: + - script: | + set -eu + dir=$(mktemp -d) + printf "basicConstraints=CA:FALSE\nextendedKeyUsage=clientAuth\n" > "$dir/extensions.cnf" + openssl req -x509 -newkey rsa:2048 -nodes -days 30 \ + -subj "/CN=Test client CA" \ + -addext "basicConstraints=critical,CA:TRUE" \ + -addext "keyUsage=critical,keyCertSign,cRLSign" \ + -keyout "$dir/ca.key" -out "$dir/ca.crt" + openssl req -newkey rsa:2048 -nodes \ + -subj "/CN=kafka-agent" \ + -keyout "$dir/tls.key" -out "$dir/tls.csr" + openssl x509 -req -days 30 -in "$dir/tls.csr" \ + -CA "$dir/ca.crt" -CAkey "$dir/ca.key" -CAcreateserial \ + -extfile "$dir/extensions.cnf" \ + -out "$dir/tls.crt" + kubectl create secret generic client-auth-ca --namespace "$NAMESPACE" \ + --from-file=ca.crt="$dir/ca.crt" --from-file=ca.key="$dir/ca.key" + kubectl create secret tls kafka-agent-credential --namespace "$NAMESPACE" \ + --cert="$dir/tls.crt" --key="$dir/tls.key" + rm -r "$dir" +{% endif %} diff --git a/tests/templates/kuttl/topic-authentication/10-assert.yaml b/tests/templates/kuttl/topic-authentication/10-assert.yaml new file mode 100644 index 000000000..5b1be639f --- /dev/null +++ b/tests/templates/kuttl/topic-authentication/10-assert.yaml @@ -0,0 +1,20 @@ +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +timeout: 600 +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: test-kafka-broker-default +status: + readyReplicas: 1 + replicas: 1 +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: test-kafka-agent +status: + readyReplicas: 1 + replicas: 1 diff --git a/tests/templates/kuttl/topic-authentication/10-install-kafka.yaml b/tests/templates/kuttl/topic-authentication/10-install-kafka.yaml new file mode 100644 index 000000000..4652b959e --- /dev/null +++ b/tests/templates/kuttl/topic-authentication/10-install-kafka.yaml @@ -0,0 +1,7 @@ +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +commands: + - script: > + envsubst '$NAMESPACE' < 10_kafka.yaml | + kubectl apply --namespace $NAMESPACE -f - diff --git a/tests/templates/kuttl/topic-authentication/10_kafka.yaml.j2 b/tests/templates/kuttl/topic-authentication/10_kafka.yaml.j2 new file mode 100644 index 000000000..e49e34ee5 --- /dev/null +++ b/tests/templates/kuttl/topic-authentication/10_kafka.yaml.j2 @@ -0,0 +1,77 @@ +{% set authentication = test_scenario['values']['platform-access-authentication'] %} +{% if authentication != 'anonymous' %} +--- +# SecretClasses and AuthenticationClasses are cluster-scoped, so the namespace keeps their names +# unique between parallel test runs. +apiVersion: secrets.stackable.tech/v1alpha1 +kind: SecretClass +metadata: + name: client-auth-$NAMESPACE +spec: + backend: + autoTls: + ca: + secret: + name: client-auth-ca + namespace: $NAMESPACE +{% if authentication == 'secret' %} + autoGenerate: false +{% else %} + autoGenerate: true +{% endif %} +--- +apiVersion: authentication.stackable.tech/v1alpha1 +kind: AuthenticationClass +metadata: + name: client-auth-$NAMESPACE +spec: + provider: + tls: + clientCertSecretClass: client-auth-$NAMESPACE +{% endif %} +--- +apiVersion: kafka.stackable.tech/v1alpha1 +kind: KafkaCluster +metadata: + name: test-kafka +spec: + image: +{% if test_scenario['values']['kafka-kraft'].find(",") > 0 %} + custom: "{{ test_scenario['values']['kafka-kraft'].split(',')[1] }}" + productVersion: "{{ test_scenario['values']['kafka-kraft'].split(',')[0] }}" +{% else %} + productVersion: "{{ test_scenario['values']['kafka-kraft'] }}" +{% endif %} + pullPolicy: IfNotPresent + clusterConfig: + metadataManager: KRaft + tls: + serverSecretClass: tls +{% if authentication != 'anonymous' %} + authentication: + - authenticationClass: client-auth-$NAMESPACE +{% endif %} + platformAccess: + enabled: true + authentication: +{% if authentication == 'anonymous' %} + anonymous: {} +{% elif authentication == 'secret-class' %} + tls: + secretClass: client-auth-$NAMESPACE +{% else %} + tls: + secret: kafka-agent-credential +{% endif %} + controllers: + config: + gracefulShutdownTimeout: 30s # speed up tests + roleGroups: + default: + replicas: 1 + brokers: + config: + gracefulShutdownTimeout: 30s # speed up tests + roleGroups: + default: + replicas: 1 diff --git a/tests/templates/kuttl/topic-authentication/11-assert.yaml.j2 b/tests/templates/kuttl/topic-authentication/11-assert.yaml.j2 new file mode 100644 index 000000000..41267d436 --- /dev/null +++ b/tests/templates/kuttl/topic-authentication/11-assert.yaml.j2 @@ -0,0 +1,14 @@ +{% set authentication = test_scenario['values']['platform-access-authentication'] %} +--- +# Runs after the agent is ready, so the commands are not polled during the startup. +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +timeout: 60 +commands: +{% if authentication == 'anonymous' %} + - script: test "$(kubectl get deployment --namespace $NAMESPACE test-kafka-agent --output jsonpath='{.spec.template.spec.volumes[*].name}')" = "discovery tls-kafka" + - script: test "$(kubectl get deployment --namespace $NAMESPACE test-kafka-agent --output jsonpath='{.spec.template.spec.containers[0].env[?(@.name=="KAFKA_CLIENT_SECURITY")].value}')" = tls +{% else %} + - script: test "$(kubectl get deployment --namespace $NAMESPACE test-kafka-agent --output jsonpath='{.spec.template.spec.volumes[*].name}')" = "discovery tls-kafka tls-platform-access" + - script: test "$(kubectl get deployment --namespace $NAMESPACE test-kafka-agent --output jsonpath='{.spec.template.spec.containers[0].env[?(@.name=="KAFKA_CLIENT_SECURITY")].value}')" = mutual-tls +{% endif %} diff --git a/tests/templates/kuttl/topic-authentication/20-assert.yaml b/tests/templates/kuttl/topic-authentication/20-assert.yaml new file mode 100644 index 000000000..fcdd3b419 --- /dev/null +++ b/tests/templates/kuttl/topic-authentication/20-assert.yaml @@ -0,0 +1,8 @@ +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +timeout: 300 +commands: + - script: bash topic-condition.sh orders Ready True Available + - script: bash topic-condition.sh orders Synced True Synced + - script: 'bash kafka-cli.sh kafka-topics.sh --describe --topic orders | grep "PartitionCount: 3"' diff --git a/tests/templates/kuttl/topic-authentication/20-create-topic.yaml b/tests/templates/kuttl/topic-authentication/20-create-topic.yaml new file mode 100644 index 000000000..a0020413f --- /dev/null +++ b/tests/templates/kuttl/topic-authentication/20-create-topic.yaml @@ -0,0 +1,10 @@ +--- +apiVersion: kafka.stackable.tech/v1alpha1 +kind: KafkaTopic +metadata: + name: orders +spec: + clusterName: test-kafka + topicName: orders + partitions: 3 + replicationFactor: 1 diff --git a/tests/templates/kuttl/topic-authentication/kafka-cli.sh b/tests/templates/kuttl/topic-authentication/kafka-cli.sh new file mode 100644 index 000000000..316274375 --- /dev/null +++ b/tests/templates/kuttl/topic-authentication/kafka-cli.sh @@ -0,0 +1,13 @@ +#!/usr/bin/env bash +# Usage: kafka-cli.sh , e.g. kafka-cli.sh kafka-topics.sh --describe --topic orders +# Runs a Kafka CLI tool in the broker Pod against the brokers. +set -euo pipefail + +# The script is expanded in the broker Pod, not here. +# shellcheck disable=SC2016 +kubectl exec --namespace "$NAMESPACE" test-kafka-broker-default-0 --container kafka -- bash -c ' + bootstrap="$(cat /stackable/listener-broker/default-address/address):$(cat /stackable/listener-broker/default-address/ports/kafka-tls)" + tool=$1 + shift + "/stackable/kafka/bin/$tool" --bootstrap-server "$bootstrap" --command-config /stackable/config/client.properties "$@" +' kafka-cli "$@" diff --git a/tests/templates/kuttl/topic-authentication/topic-condition.sh b/tests/templates/kuttl/topic-authentication/topic-condition.sh new file mode 100644 index 000000000..59a05a0d7 --- /dev/null +++ b/tests/templates/kuttl/topic-authentication/topic-condition.sh @@ -0,0 +1,11 @@ +#!/usr/bin/env bash +# Usage: topic-condition.sh +# Fails unless the condition of the KafkaTopic has the given status and reason. +set -euo pipefail + +actual=$(kubectl get kafkatopic --namespace "$NAMESPACE" "$1" \ + --output "jsonpath={.status.conditions[?(@.type==\"$2\")].status}/{.status.conditions[?(@.type==\"$2\")].reason}") +if [ "$actual" != "$3/$4" ]; then + echo "$1: expected the condition $2 to be $3/$4, but it is \"$actual\"" + exit 1 +fi diff --git a/tests/templates/kuttl/topic/00-patch-ns.yaml.j2 b/tests/templates/kuttl/topic/00-patch-ns.yaml.j2 new file mode 100644 index 000000000..67185acfd --- /dev/null +++ b/tests/templates/kuttl/topic/00-patch-ns.yaml.j2 @@ -0,0 +1,9 @@ +{% if test_scenario['values']['openshift'] == 'true' %} +# see https://github.com/stackabletech/issues/issues/566 +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +commands: + - script: kubectl patch namespace $NAMESPACE -p '{"metadata":{"labels":{"pod-security.kubernetes.io/enforce":"privileged"}}}' + timeout: 120 +{% endif %} diff --git a/tests/templates/kuttl/topic/10-assert.yaml b/tests/templates/kuttl/topic/10-assert.yaml new file mode 100644 index 000000000..dcb6e2619 --- /dev/null +++ b/tests/templates/kuttl/topic/10-assert.yaml @@ -0,0 +1,42 @@ +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +timeout: 600 +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: test-kafka-broker-default +status: + readyReplicas: 1 + replicas: 1 +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: test-kafka-controller-default +status: + readyReplicas: 1 + replicas: 1 +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: test-kafka-agent +status: + readyReplicas: 1 + replicas: 1 +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: test-kafka-agent +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: test-kafka-agent +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: kafka-agent-clusterrole diff --git a/tests/templates/kuttl/topic/10-install-kafka.yaml.j2 b/tests/templates/kuttl/topic/10-install-kafka.yaml.j2 new file mode 100644 index 000000000..feec661ab --- /dev/null +++ b/tests/templates/kuttl/topic/10-install-kafka.yaml.j2 @@ -0,0 +1,34 @@ +--- +apiVersion: kafka.stackable.tech/v1alpha1 +kind: KafkaCluster +metadata: + name: test-kafka +spec: + image: +{% if test_scenario['values']['kafka-kraft'].find(",") > 0 %} + custom: "{{ test_scenario['values']['kafka-kraft'].split(',')[1] }}" + productVersion: "{{ test_scenario['values']['kafka-kraft'].split(',')[0] }}" +{% else %} + productVersion: "{{ test_scenario['values']['kafka-kraft'] }}" +{% endif %} + pullPolicy: IfNotPresent + clusterConfig: + metadataManager: KRaft + tls: + serverSecretClass: tls + platformAccess: + enabled: true + authentication: + anonymous: {} + controllers: + config: + gracefulShutdownTimeout: 30s # speed up tests + roleGroups: + default: + replicas: 1 + brokers: + config: + gracefulShutdownTimeout: 30s # speed up tests + roleGroups: + default: + replicas: 1 diff --git a/tests/templates/kuttl/topic/20-assert.yaml b/tests/templates/kuttl/topic/20-assert.yaml new file mode 100644 index 000000000..fcc58d2c5 --- /dev/null +++ b/tests/templates/kuttl/topic/20-assert.yaml @@ -0,0 +1,12 @@ +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +timeout: 300 +commands: + - script: bash topic-condition.sh orders Ready True Available + - script: bash topic-condition.sh orders Synced True Synced + - script: test -n "$(kubectl get kafkatopic --namespace $NAMESPACE orders --output jsonpath='{.status.topicId}')" + - script: test "$(kubectl get kafkatopic --namespace $NAMESPACE orders --output jsonpath='{.status.partitions[*].index}')" = "0 1 2" + - script: 'bash kafka-cli.sh kafka-topics.sh --describe --topic orders | grep "PartitionCount: 3"' + - script: bash kafka-cli.sh kafka-configs.sh --describe --entity-type topics --entity-name orders | grep "retention.ms=3600000" + - script: bash kafka-cli.sh kafka-configs.sh --describe --entity-type topics --entity-name orders | grep "max.message.bytes=2097152" diff --git a/tests/templates/kuttl/topic/20-create-topic.yaml b/tests/templates/kuttl/topic/20-create-topic.yaml new file mode 100644 index 000000000..e91eecfb7 --- /dev/null +++ b/tests/templates/kuttl/topic/20-create-topic.yaml @@ -0,0 +1,13 @@ +--- +apiVersion: kafka.stackable.tech/v1alpha1 +kind: KafkaTopic +metadata: + name: orders +spec: + clusterName: test-kafka + topicName: orders + partitions: 3 + replicationFactor: 1 + config: + retention.ms: "3600000" + max.message.bytes: "2097152" diff --git a/tests/templates/kuttl/topic/30-assert.yaml b/tests/templates/kuttl/topic/30-assert.yaml new file mode 100644 index 000000000..f8eaa1371 --- /dev/null +++ b/tests/templates/kuttl/topic/30-assert.yaml @@ -0,0 +1,9 @@ +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +timeout: 300 +commands: + - script: bash topic-condition.sh orders Synced True Synced + - script: test "$(kubectl get kafkatopic --namespace $NAMESPACE orders --output jsonpath='{.status.partitions[*].index}')" = "0 1 2 3 4" + - script: 'bash kafka-cli.sh kafka-topics.sh --describe --topic orders | grep "PartitionCount: 5"' + - script: bash kafka-cli.sh kafka-configs.sh --describe --entity-type topics --entity-name orders | grep "retention.ms=7200000" diff --git a/tests/templates/kuttl/topic/30-change-topic.yaml b/tests/templates/kuttl/topic/30-change-topic.yaml new file mode 100644 index 000000000..9b4b26679 --- /dev/null +++ b/tests/templates/kuttl/topic/30-change-topic.yaml @@ -0,0 +1,13 @@ +--- +apiVersion: kafka.stackable.tech/v1alpha1 +kind: KafkaTopic +metadata: + name: orders +spec: + clusterName: test-kafka + topicName: orders + partitions: 5 + replicationFactor: 1 + config: + retention.ms: "7200000" + max.message.bytes: "2097152" diff --git a/tests/templates/kuttl/topic/40-set-config-out-of-band.yaml b/tests/templates/kuttl/topic/40-set-config-out-of-band.yaml new file mode 100644 index 000000000..2a125a288 --- /dev/null +++ b/tests/templates/kuttl/topic/40-set-config-out-of-band.yaml @@ -0,0 +1,6 @@ +--- +# A config set by someone else than the agent, e.g. an administrator with kafka-configs.sh +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +commands: + - script: bash kafka-cli.sh kafka-configs.sh --alter --entity-type topics --entity-name orders --add-config segment.ms=86400000 diff --git a/tests/templates/kuttl/topic/41-assert.yaml b/tests/templates/kuttl/topic/41-assert.yaml new file mode 100644 index 000000000..796e09f85 --- /dev/null +++ b/tests/templates/kuttl/topic/41-assert.yaml @@ -0,0 +1,12 @@ +--- +# The key removed from the spec is reset, the key set out of band stays. +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +timeout: 300 +commands: + - script: bash topic-condition.sh orders Synced True Synced + - script: | + configs=$(bash kafka-cli.sh kafka-configs.sh --describe --entity-type topics --entity-name orders) + ! echo "$configs" | grep max.message.bytes + - script: bash kafka-cli.sh kafka-configs.sh --describe --entity-type topics --entity-name orders | grep "segment.ms=86400000" + - script: bash kafka-cli.sh kafka-configs.sh --describe --entity-type topics --entity-name orders | grep "retention.ms=7200000" diff --git a/tests/templates/kuttl/topic/41-remove-config-key.yaml b/tests/templates/kuttl/topic/41-remove-config-key.yaml new file mode 100644 index 000000000..880129971 --- /dev/null +++ b/tests/templates/kuttl/topic/41-remove-config-key.yaml @@ -0,0 +1,8 @@ +--- +# kuttl merges applied objects into the existing ones, so a removed key has to be removed explicitly. +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +commands: + - script: >- + kubectl patch kafkatopic orders --namespace $NAMESPACE --type json + --patch '[{"op": "remove", "path": "/spec/config/max.message.bytes"}]' diff --git a/tests/templates/kuttl/topic/50-assert.yaml b/tests/templates/kuttl/topic/50-assert.yaml new file mode 100644 index 000000000..443a7e5a8 --- /dev/null +++ b/tests/templates/kuttl/topic/50-assert.yaml @@ -0,0 +1,9 @@ +--- +# The topic stays usable and unchanged. +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +timeout: 300 +commands: + - script: bash topic-condition.sh orders Synced False UnsupportedChange + - script: bash topic-condition.sh orders Ready True Available + - script: 'bash kafka-cli.sh kafka-topics.sh --describe --topic orders | grep "PartitionCount: 5"' diff --git a/tests/templates/kuttl/topic/50-unsupported-change.yaml b/tests/templates/kuttl/topic/50-unsupported-change.yaml new file mode 100644 index 000000000..72f427bc8 --- /dev/null +++ b/tests/templates/kuttl/topic/50-unsupported-change.yaml @@ -0,0 +1,13 @@ +--- +# Partitions cannot be removed, and the agent does not reassign replicas. +apiVersion: kafka.stackable.tech/v1alpha1 +kind: KafkaTopic +metadata: + name: orders +spec: + clusterName: test-kafka + topicName: orders + partitions: 2 + replicationFactor: 3 + config: + retention.ms: "7200000" diff --git a/tests/templates/kuttl/topic/51-assert.yaml b/tests/templates/kuttl/topic/51-assert.yaml new file mode 100644 index 000000000..3f2cfc72e --- /dev/null +++ b/tests/templates/kuttl/topic/51-assert.yaml @@ -0,0 +1,6 @@ +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +timeout: 300 +commands: + - script: bash topic-condition.sh orders Synced True Synced diff --git a/tests/templates/kuttl/topic/51-revert-unsupported-change.yaml b/tests/templates/kuttl/topic/51-revert-unsupported-change.yaml new file mode 100644 index 000000000..4a1fa1998 --- /dev/null +++ b/tests/templates/kuttl/topic/51-revert-unsupported-change.yaml @@ -0,0 +1,12 @@ +--- +apiVersion: kafka.stackable.tech/v1alpha1 +kind: KafkaTopic +metadata: + name: orders +spec: + clusterName: test-kafka + topicName: orders + partitions: 5 + replicationFactor: 1 + config: + retention.ms: "7200000" diff --git a/tests/templates/kuttl/topic/60-assert.yaml b/tests/templates/kuttl/topic/60-assert.yaml new file mode 100644 index 000000000..d212ade8a --- /dev/null +++ b/tests/templates/kuttl/topic/60-assert.yaml @@ -0,0 +1,11 @@ +--- +# Ready stays unknown, because the agent did not look at the topic in Kafka. +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +timeout: 300 +commands: + - script: bash topic-condition.sh throttled Synced False InvalidSpec + - script: bash topic-condition.sh throttled Ready "" "" + - script: | + topics=$(bash kafka-cli.sh kafka-topics.sh --list) + ! echo "$topics" | grep -x throttled diff --git a/tests/templates/kuttl/topic/60-invalid-spec.yaml b/tests/templates/kuttl/topic/60-invalid-spec.yaml new file mode 100644 index 000000000..effcf8bc3 --- /dev/null +++ b/tests/templates/kuttl/topic/60-invalid-spec.yaml @@ -0,0 +1,13 @@ +--- +# Throttles are set by partition reassignments and cannot be managed by a KafkaTopic. +apiVersion: kafka.stackable.tech/v1alpha1 +kind: KafkaTopic +metadata: + name: throttled +spec: + clusterName: test-kafka + topicName: throttled + partitions: 1 + replicationFactor: 1 + config: + leader.replication.throttled.replicas: "*" diff --git a/tests/templates/kuttl/topic/70-create-foreign-topic.yaml b/tests/templates/kuttl/topic/70-create-foreign-topic.yaml new file mode 100644 index 000000000..da72f46ac --- /dev/null +++ b/tests/templates/kuttl/topic/70-create-foreign-topic.yaml @@ -0,0 +1,6 @@ +--- +# A topic created outside of KafkaTopics, e.g. by an application or an administrator +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +commands: + - script: bash kafka-cli.sh kafka-topics.sh --create --topic legacy --partitions 1 --replication-factor 1 diff --git a/tests/templates/kuttl/topic/71-assert.yaml b/tests/templates/kuttl/topic/71-assert.yaml new file mode 100644 index 000000000..3ed15c319 --- /dev/null +++ b/tests/templates/kuttl/topic/71-assert.yaml @@ -0,0 +1,8 @@ +--- +# The adoption policy defaults to Refuse. +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +timeout: 300 +commands: + - script: bash topic-condition.sh legacy Ready False AlreadyExists + - script: test -z "$(kubectl get kafkatopic --namespace $NAMESPACE legacy --output jsonpath='{.status.topicId}')" diff --git a/tests/templates/kuttl/topic/71-refuse-foreign-topic.yaml b/tests/templates/kuttl/topic/71-refuse-foreign-topic.yaml new file mode 100644 index 000000000..fe7eef33e --- /dev/null +++ b/tests/templates/kuttl/topic/71-refuse-foreign-topic.yaml @@ -0,0 +1,10 @@ +--- +apiVersion: kafka.stackable.tech/v1alpha1 +kind: KafkaTopic +metadata: + name: legacy +spec: + clusterName: test-kafka + topicName: legacy + partitions: 1 + replicationFactor: 1 diff --git a/tests/templates/kuttl/topic/72-adopt-foreign-topic.yaml b/tests/templates/kuttl/topic/72-adopt-foreign-topic.yaml new file mode 100644 index 000000000..ef3a851da --- /dev/null +++ b/tests/templates/kuttl/topic/72-adopt-foreign-topic.yaml @@ -0,0 +1,12 @@ +--- +apiVersion: kafka.stackable.tech/v1alpha1 +kind: KafkaTopic +metadata: + name: legacy +spec: + clusterName: test-kafka + topicName: legacy + partitions: 1 + replicationFactor: 1 + managementPolicy: + adoption: Adopt diff --git a/tests/templates/kuttl/topic/72-assert.yaml b/tests/templates/kuttl/topic/72-assert.yaml new file mode 100644 index 000000000..02e00c9f0 --- /dev/null +++ b/tests/templates/kuttl/topic/72-assert.yaml @@ -0,0 +1,8 @@ +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +timeout: 300 +commands: + - script: bash topic-condition.sh legacy Ready True Available + - script: bash topic-condition.sh legacy Synced True Synced + - script: test -n "$(kubectl get kafkatopic --namespace $NAMESPACE legacy --output jsonpath='{.metadata.annotations.stackable\.tech/owned-since}')" diff --git a/tests/templates/kuttl/topic/80-assert.yaml b/tests/templates/kuttl/topic/80-assert.yaml new file mode 100644 index 000000000..ab8423f0a --- /dev/null +++ b/tests/templates/kuttl/topic/80-assert.yaml @@ -0,0 +1,9 @@ +--- +# The older KafkaTopic keeps managing the topic. +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +timeout: 300 +commands: + - script: bash topic-condition.sh orders-copy Ready False Conflict + - script: bash topic-condition.sh orders Ready True Available + - script: bash kafka-cli.sh kafka-configs.sh --describe --entity-type topics --entity-name orders | grep "retention.ms=7200000" diff --git a/tests/templates/kuttl/topic/80-create-conflicting-topic.yaml b/tests/templates/kuttl/topic/80-create-conflicting-topic.yaml new file mode 100644 index 000000000..3a2b28c54 --- /dev/null +++ b/tests/templates/kuttl/topic/80-create-conflicting-topic.yaml @@ -0,0 +1,11 @@ +--- +# A second KafkaTopic for the topic already managed by the KafkaTopic orders +apiVersion: kafka.stackable.tech/v1alpha1 +kind: KafkaTopic +metadata: + name: orders-copy +spec: + clusterName: test-kafka + topicName: orders + partitions: 5 + replicationFactor: 1 diff --git a/tests/templates/kuttl/topic/81-assert.yaml b/tests/templates/kuttl/topic/81-assert.yaml new file mode 100644 index 000000000..091073c7d --- /dev/null +++ b/tests/templates/kuttl/topic/81-assert.yaml @@ -0,0 +1,6 @@ +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +timeout: 300 +commands: + - script: bash kafka-cli.sh kafka-topics.sh --list | grep -x orders diff --git a/tests/templates/kuttl/topic/81-delete-conflicting-topic.yaml b/tests/templates/kuttl/topic/81-delete-conflicting-topic.yaml new file mode 100644 index 000000000..a9805a169 --- /dev/null +++ b/tests/templates/kuttl/topic/81-delete-conflicting-topic.yaml @@ -0,0 +1,7 @@ +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +delete: + - apiVersion: kafka.stackable.tech/v1alpha1 + kind: KafkaTopic + name: orders-copy diff --git a/tests/templates/kuttl/topic/90-assert.yaml b/tests/templates/kuttl/topic/90-assert.yaml new file mode 100644 index 000000000..b2bebc87c --- /dev/null +++ b/tests/templates/kuttl/topic/90-assert.yaml @@ -0,0 +1,8 @@ +--- +# Only KafkaTopics that delete their topic in Kafka have a finalizer. +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +timeout: 300 +commands: + - script: test "$(kubectl get kafkatopic --namespace $NAMESPACE legacy --output jsonpath='{.metadata.finalizers}')" = '["kafka.stackable.tech/kafkatopic"]' + - script: test -z "$(kubectl get kafkatopic --namespace $NAMESPACE orders --output jsonpath='{.metadata.finalizers}')" diff --git a/tests/templates/kuttl/topic/90-delete-legacy-topic-in-kafka.yaml b/tests/templates/kuttl/topic/90-delete-legacy-topic-in-kafka.yaml new file mode 100644 index 000000000..2ea201fe0 --- /dev/null +++ b/tests/templates/kuttl/topic/90-delete-legacy-topic-in-kafka.yaml @@ -0,0 +1,13 @@ +--- +apiVersion: kafka.stackable.tech/v1alpha1 +kind: KafkaTopic +metadata: + name: legacy +spec: + clusterName: test-kafka + topicName: legacy + partitions: 1 + replicationFactor: 1 + managementPolicy: + adoption: Adopt + deletion: Delete diff --git a/tests/templates/kuttl/topic/91-assert.yaml b/tests/templates/kuttl/topic/91-assert.yaml new file mode 100644 index 000000000..537525c93 --- /dev/null +++ b/tests/templates/kuttl/topic/91-assert.yaml @@ -0,0 +1,11 @@ +--- +# orders has the default deletion policy Retain, legacy has Delete. +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +timeout: 300 +commands: + - script: test -z "$(kubectl get kafkatopics --namespace $NAMESPACE --output name)" + - script: bash kafka-cli.sh kafka-topics.sh --list | grep -x orders + - script: | + topics=$(bash kafka-cli.sh kafka-topics.sh --list) + ! echo "$topics" | grep -x legacy diff --git a/tests/templates/kuttl/topic/91-delete-topics.yaml b/tests/templates/kuttl/topic/91-delete-topics.yaml new file mode 100644 index 000000000..430f58415 --- /dev/null +++ b/tests/templates/kuttl/topic/91-delete-topics.yaml @@ -0,0 +1,13 @@ +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +delete: + - apiVersion: kafka.stackable.tech/v1alpha1 + kind: KafkaTopic + name: orders + - apiVersion: kafka.stackable.tech/v1alpha1 + kind: KafkaTopic + name: legacy + - apiVersion: kafka.stackable.tech/v1alpha1 + kind: KafkaTopic + name: throttled diff --git a/tests/templates/kuttl/topic/95-disable-platform-access.yaml.j2 b/tests/templates/kuttl/topic/95-disable-platform-access.yaml.j2 new file mode 100644 index 000000000..5e9b635e8 --- /dev/null +++ b/tests/templates/kuttl/topic/95-disable-platform-access.yaml.j2 @@ -0,0 +1,34 @@ +--- +apiVersion: kafka.stackable.tech/v1alpha1 +kind: KafkaCluster +metadata: + name: test-kafka +spec: + image: +{% if test_scenario['values']['kafka-kraft'].find(",") > 0 %} + custom: "{{ test_scenario['values']['kafka-kraft'].split(',')[1] }}" + productVersion: "{{ test_scenario['values']['kafka-kraft'].split(',')[0] }}" +{% else %} + productVersion: "{{ test_scenario['values']['kafka-kraft'] }}" +{% endif %} + pullPolicy: IfNotPresent + clusterConfig: + metadataManager: KRaft + tls: + serverSecretClass: tls + platformAccess: + enabled: false + authentication: + anonymous: {} + controllers: + config: + gracefulShutdownTimeout: 30s # speed up tests + roleGroups: + default: + replicas: 1 + brokers: + config: + gracefulShutdownTimeout: 30s # speed up tests + roleGroups: + default: + replicas: 1 diff --git a/tests/templates/kuttl/topic/95-errors.yaml b/tests/templates/kuttl/topic/95-errors.yaml new file mode 100644 index 000000000..1769b9a57 --- /dev/null +++ b/tests/templates/kuttl/topic/95-errors.yaml @@ -0,0 +1,16 @@ +--- +# The agent resources are removed when platform access is disabled. +apiVersion: apps/v1 +kind: Deployment +metadata: + name: test-kafka-agent +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: test-kafka-agent +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: test-kafka-agent diff --git a/tests/templates/kuttl/topic/kafka-cli.sh b/tests/templates/kuttl/topic/kafka-cli.sh new file mode 100644 index 000000000..316274375 --- /dev/null +++ b/tests/templates/kuttl/topic/kafka-cli.sh @@ -0,0 +1,13 @@ +#!/usr/bin/env bash +# Usage: kafka-cli.sh , e.g. kafka-cli.sh kafka-topics.sh --describe --topic orders +# Runs a Kafka CLI tool in the broker Pod against the brokers. +set -euo pipefail + +# The script is expanded in the broker Pod, not here. +# shellcheck disable=SC2016 +kubectl exec --namespace "$NAMESPACE" test-kafka-broker-default-0 --container kafka -- bash -c ' + bootstrap="$(cat /stackable/listener-broker/default-address/address):$(cat /stackable/listener-broker/default-address/ports/kafka-tls)" + tool=$1 + shift + "/stackable/kafka/bin/$tool" --bootstrap-server "$bootstrap" --command-config /stackable/config/client.properties "$@" +' kafka-cli "$@" diff --git a/tests/templates/kuttl/topic/topic-condition.sh b/tests/templates/kuttl/topic/topic-condition.sh new file mode 100644 index 000000000..59a05a0d7 --- /dev/null +++ b/tests/templates/kuttl/topic/topic-condition.sh @@ -0,0 +1,11 @@ +#!/usr/bin/env bash +# Usage: topic-condition.sh +# Fails unless the condition of the KafkaTopic has the given status and reason. +set -euo pipefail + +actual=$(kubectl get kafkatopic --namespace "$NAMESPACE" "$1" \ + --output "jsonpath={.status.conditions[?(@.type==\"$2\")].status}/{.status.conditions[?(@.type==\"$2\")].reason}") +if [ "$actual" != "$3/$4" ]; then + echo "$1: expected the condition $2 to be $3/$4, but it is \"$actual\"" + exit 1 +fi diff --git a/tests/test-definition.yaml b/tests/test-definition.yaml index c4c294ffd..6d8352584 100644 --- a/tests/test-definition.yaml +++ b/tests/test-definition.yaml @@ -7,7 +7,7 @@ dimensions: - name: kafka-kraft values: - 3.9.2 - - 4.3.1 + # - 4.3.1 - name: kafka values: - 3.9.2 @@ -72,6 +72,12 @@ dimensions: - "cluster-internal" - "external-stable" - "external-unstable" + # How the agent authenticates to Kafka + - name: platform-access-authentication + values: + - anonymous + - secret-class + - secret tests: - name: operations-kraft dimensions: @@ -81,6 +87,15 @@ tests: dimensions: - kafka-kraft - openshift + - name: topic + dimensions: + - kafka-kraft + - openshift + - name: topic-authentication + dimensions: + - kafka-kraft + - platform-access-authentication + - openshift - name: smoke dimensions: - kafka From 668119f8d8ff8baa7b13f9c9aeeccd2adda9a68c Mon Sep 17 00:00:00 2001 From: Benedikt Labrenz Date: Fri, 23 Oct 2026 15:42:14 +0200 Subject: [PATCH 29/93] return the created topic and all configs with their source from the kafka admin client --- rust/kafka-admin/src/client.rs | 345 ++++++++++++++++++++++++--------- 1 file changed, 253 insertions(+), 92 deletions(-) diff --git a/rust/kafka-admin/src/client.rs b/rust/kafka-admin/src/client.rs index 626574ef7..f5145ff54 100644 --- a/rust/kafka-admin/src/client.rs +++ b/rust/kafka-admin/src/client.rs @@ -2,6 +2,7 @@ use std::{ collections::{BTreeMap, BTreeSet}, + fmt, time::Duration, }; @@ -10,7 +11,7 @@ use kafka_protocol::{ ResponseError, error::ParseResponseErrorCode, messages::{ - BrokerId, CreatePartitionsRequest, CreateTopicsRequest, DeleteTopicsRequest, + CreatePartitionsRequest, CreateTopicsRequest, CreateTopicsResponse, DeleteTopicsRequest, DeleteTopicsResponse, DescribeConfigsRequest, DescribeConfigsResponse, IncrementalAlterConfigsRequest, ListPartitionReassignmentsRequest, ListPartitionReassignmentsResponse, MetadataRequest, MetadataResponse, TopicName, @@ -47,8 +48,10 @@ const _: () = assert!((BROKER_TIMEOUT_MS as u128) < connection::REQUEST_TIMEOUT. /// Must stay below the broker's `connections.max.idle.ms` (10 minutes by default). const CONNECTION_MAX_IDLE: Duration = Duration::from_secs(5 * 60); +/// Lets the broker apply `num.partitions` or `default.replication.factor` when creating a topic. +const BROKER_DEFAULT: i32 = -1; + const RESOURCE_TYPE_TOPIC: i8 = 2; -const CONFIG_SOURCE_DYNAMIC_TOPIC: i8 = 1; const CONFIG_OPERATION_SET: i8 = 0; const CONFIG_OPERATION_DELETE: i8 = 1; @@ -90,17 +93,61 @@ impl Error { pub fn is_rejected_by_kafka(&self) -> bool { matches!(self, Error::Kafka { .. }) } +} - /// Whether a CreateTopics request was rejected because the topic already exists. - pub fn is_topic_already_exists(&self) -> bool { - matches!( - self, - Error::Kafka { - api_key: CreateTopicsRequest::KEY, - error: ResponseError::TopicAlreadyExists, - .. - } - ) +/// A topic as created by Kafka, with the broker defaults applied to unset values. +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct CreatedTopic { + /// The topic id in Kafka's string format + pub id: String, + pub partitions: i32, + pub replication_factor: i16, + /// All configs of the topic, as [`KafkaAdminClient::describe_topic_configs`] returns them + pub configs: BTreeMap, +} + +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct TopicConfig { + /// `None` for sensitive configs + pub value: Option, + pub source: ConfigSource, +} + +/// Where the value of a topic config comes from. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum ConfigSource { + DynamicTopic, + DynamicBroker, + DynamicDefaultBroker, + StaticBroker, + Default, + Unknown, +} + +impl ConfigSource { + fn from_code(code: i8) -> Self { + match code { + 1 => Self::DynamicTopic, + 2 => Self::DynamicBroker, + 3 => Self::DynamicDefaultBroker, + 4 => Self::StaticBroker, + 5 => Self::Default, + _ => Self::Unknown, + } + } +} + +/// The names used by Kafka's admin client and `kafka-configs.sh`. +impl fmt::Display for ConfigSource { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str(match self { + Self::DynamicTopic => "DYNAMIC_TOPIC_CONFIG", + Self::DynamicBroker => "DYNAMIC_BROKER_CONFIG", + Self::DynamicDefaultBroker => "DYNAMIC_DEFAULT_BROKER_CONFIG", + Self::StaticBroker => "STATIC_BROKER_CONFIG", + Self::Default => "DEFAULT_CONFIG", + Self::Unknown => "UNKNOWN", + }) } } @@ -110,23 +157,22 @@ pub struct Topic { /// `kafka-topics.sh` pub id: String, - /// Sorted by the partition index pub partitions: Vec, } #[derive(Clone, Debug, Eq, PartialEq)] pub struct Partition { pub index: i32, - /// Broker ids, including replicas that are added or removed by a reassignment - pub replicas: Vec, + /// The number of replicas, including those added or removed by a reassignment + pub replicas: usize, } /// An ongoing reassignment of one partition. #[derive(Clone, Debug, Eq, PartialEq)] pub struct PartitionReassignment { pub index: i32, - pub adding_replicas: Vec, - pub removing_replicas: Vec, + /// The number of replicas the reassignment removes + pub removing_replicas: usize, } pub struct KafkaAdminClient { @@ -156,11 +202,11 @@ impl KafkaAdminClient { describe_topic_result(response, topic) } - /// Returns the configs set on the topic itself, without broker or default values. + /// Returns all configs of the topic, including broker and default values. pub async fn describe_topic_configs( &mut self, topic: &str, - ) -> Result> { + ) -> Result> { let response = self .send( describe_topic_configs_request(topic), @@ -170,27 +216,21 @@ impl KafkaAdminClient { describe_topic_configs_result(response, topic) } + /// `None` uses the broker default for the number of partitions or the replication factor. pub async fn create_topic( &mut self, topic: &str, - partitions: i32, - replication_factor: i16, + partitions: Option, + replication_factor: Option, configs: &BTreeMap, - ) -> Result<()> { + ) -> Result { let response = self .send( create_topic_request(topic, partitions, replication_factor, configs), CREATE_TOPICS_VERSION, ) .await?; - - let result = first(response.topics, CreateTopicsRequest::KEY, topic)?; - check( - CreateTopicsRequest::KEY, - topic, - result.error_code, - result.error_message, - ) + create_topic_result(response, topic) } /// Increases the number of partitions to `partitions`. @@ -247,8 +287,7 @@ impl KafkaAdminClient { delete_topic_result(response, topic) } - /// Returns the ongoing reassignments of the given partitions, sorted by the partition index. - /// Needs `DESCRIBE` on the cluster. + /// Returns the ongoing reassignments of the given partitions. Needs `DESCRIBE` on the cluster. pub async fn list_partition_reassignments( &mut self, topic: &str, @@ -312,21 +351,25 @@ fn describe_topic_result(response: MetadataResponse, topic: &str) -> Result>(); - partitions.sort_by_key(|partition| partition.index); + .collect(); Ok(Some(Topic { - id: URL_SAFE_NO_PAD.encode(result.topic_id.as_bytes()), + id: topic_id(result.topic_id.as_bytes()), partitions, })) } +/// Kafka's string format of a topic id: URL-safe base64 without padding, as `Uuid.toString()`. +fn topic_id(id: &[u8; 16]) -> String { + URL_SAFE_NO_PAD.encode(id) +} + fn describe_topic_configs_request(topic: &str) -> DescribeConfigsRequest { DescribeConfigsRequest::default().with_resources(vec![ DescribeConfigsResource::default() @@ -338,7 +381,7 @@ fn describe_topic_configs_request(topic: &str) -> DescribeConfigsRequest { fn describe_topic_configs_result( response: DescribeConfigsResponse, topic: &str, -) -> Result> { +) -> Result> { let result = first(response.results, DescribeConfigsRequest::KEY, topic)?; check( DescribeConfigsRequest::KEY, @@ -350,15 +393,26 @@ fn describe_topic_configs_result( Ok(result .configs .into_iter() - .filter(|config| config.config_source == CONFIG_SOURCE_DYNAMIC_TOPIC) - .filter_map(|config| Some((config.name.to_string(), config.value?.to_string()))) + .map(|config| { + ( + config.name.to_string(), + topic_config(config.value, config.config_source), + ) + }) .collect()) } +fn topic_config(value: Option, source: i8) -> TopicConfig { + TopicConfig { + value: value.map(|value| value.to_string()), + source: ConfigSource::from_code(source), + } +} + fn create_topic_request( topic: &str, - partitions: i32, - replication_factor: i16, + partitions: Option, + replication_factor: Option, configs: &BTreeMap, ) -> CreateTopicsRequest { let configs = configs @@ -373,13 +427,46 @@ fn create_topic_request( .with_topics(vec![ CreatableTopic::default() .with_name(topic_name(topic)) - .with_num_partitions(partitions) - .with_replication_factor(replication_factor) + .with_num_partitions(partitions.unwrap_or(BROKER_DEFAULT)) + .with_replication_factor(replication_factor.unwrap_or(BROKER_DEFAULT as i16)) .with_configs(configs), ]) .with_timeout_ms(BROKER_TIMEOUT_MS) } +fn create_topic_result(response: CreateTopicsResponse, topic: &str) -> Result { + let result = first(response.topics, CreateTopicsRequest::KEY, topic)?; + check( + CreateTopicsRequest::KEY, + topic, + result.error_code, + result.error_message, + )?; + // Without the details, e.g. if `DESCRIBE_CONFIGS` is not allowed, the topic was still created. + check( + CreateTopicsRequest::KEY, + topic, + result.topic_config_error_code, + None, + )?; + Ok(CreatedTopic { + id: topic_id(result.topic_id.as_bytes()), + partitions: result.num_partitions, + replication_factor: result.replication_factor, + configs: result + .configs + .unwrap_or_default() + .into_iter() + .map(|config| { + ( + config.name.to_string(), + topic_config(config.value, config.config_source), + ) + }) + .collect(), + }) +} + fn create_partitions_request(topic: &str, partitions: i32) -> CreatePartitionsRequest { CreatePartitionsRequest::default() .with_topics(vec![ @@ -468,29 +555,22 @@ fn list_partition_reassignments_result( response.error_message, )?; - let mut reassignments = response + Ok(response .topics .into_iter() .filter(|result| result.name.as_str() == topic) .flat_map(|result| result.partitions) .map(|partition| PartitionReassignment { index: partition.partition_index, - adding_replicas: broker_ids(partition.adding_replicas), - removing_replicas: broker_ids(partition.removing_replicas), + removing_replicas: partition.removing_replicas.len(), }) - .collect::>(); - reassignments.sort_by_key(|reassignment| reassignment.index); - Ok(reassignments) + .collect()) } fn topic_name(topic: &str) -> TopicName { TopicName::from(StrBytes::from_string(topic.to_owned())) } -fn broker_ids(ids: Vec) -> Vec { - ids.into_iter().map(|id| *id).collect() -} - // Used instead of [0] to correctly handle empty results & take ownership fn first(results: Vec, api_key: i16, topic: &str) -> Result { results @@ -515,6 +595,7 @@ fn check(api_key: i16, topic: &str, error_code: i16, message: Option) #[cfg(test)] mod tests { use kafka_protocol::messages::{ + create_topics_response::{CreatableTopicConfigs, CreatableTopicResult}, delete_topics_response::DeletableTopicResult, describe_configs_response::{DescribeConfigsResourceResult, DescribeConfigsResult}, list_partition_reassignments_response::{ @@ -599,12 +680,12 @@ mod tests { .expect("the topic id should be valid"), ) .with_partitions(vec![ - MetadataResponsePartition::default() - .with_partition_index(1) - .with_replica_nodes(vec![2.into(), 3.into()]), MetadataResponsePartition::default() .with_partition_index(0) .with_replica_nodes(vec![1.into(), 2.into()]), + MetadataResponsePartition::default() + .with_partition_index(1) + .with_replica_nodes(vec![2.into(), 3.into(), 4.into()]), ]), ]); @@ -614,11 +695,11 @@ mod tests { partitions: vec![ Partition { index: 0, - replicas: vec![1, 2] + replicas: 2 }, Partition { index: 1, - replicas: vec![2, 3] + replicas: 3 }, ] }), @@ -648,32 +729,68 @@ mod tests { } #[test] - fn describe_only_dynamic_topic_configs() { - let config = |name: &'static str, value: &'static str, source: i8| { + fn describe_all_topic_configs_with_their_source() { + let config = |name: &'static str, value: Option<&'static str>, source: i8| { DescribeConfigsResourceResult::default() .with_name(StrBytes::from_static_str(name)) - .with_value(Some(StrBytes::from_static_str(value))) + .with_value(value.map(StrBytes::from_static_str)) .with_config_source(source) }; let response = DescribeConfigsResponse::default().with_results(vec![ DescribeConfigsResult::default().with_configs(vec![ - config("retention.ms", "1000", CONFIG_SOURCE_DYNAMIC_TOPIC), - config("cleanup.policy", "delete", 5), + config("retention.ms", Some("1000"), 1), + config("cleanup.policy", Some("delete"), 5), + config("sasl.secret", None, 4), ]), ]); assert_eq!( - BTreeMap::from([("retention.ms".to_owned(), "1000".to_owned())]), + BTreeMap::from([ + ( + "retention.ms".to_owned(), + TopicConfig { + value: Some("1000".to_owned()), + source: ConfigSource::DynamicTopic, + } + ), + ( + "cleanup.policy".to_owned(), + TopicConfig { + value: Some("delete".to_owned()), + source: ConfigSource::Default, + } + ), + ( + "sasl.secret".to_owned(), + TopicConfig { + value: None, + source: ConfigSource::StaticBroker, + } + ), + ]), describe_topic_configs_result(response, TOPIC) .expect("the configs should be described") ); } + #[rstest] + #[case::dynamic_topic(1, ConfigSource::DynamicTopic, "DYNAMIC_TOPIC_CONFIG")] + #[case::default(5, ConfigSource::Default, "DEFAULT_CONFIG")] + #[case::not_for_topics(8, ConfigSource::Unknown, "UNKNOWN")] + fn name_config_sources_like_kafka( + #[case] code: i8, + #[case] source: ConfigSource, + #[case] name: &str, + ) { + assert_eq!(source, ConfigSource::from_code(code)); + assert_eq!(name, source.to_string()); + } + #[test] fn create_topic_with_configs() { let configs = BTreeMap::from([("retention.ms".to_owned(), "1000".to_owned())]); - let request = create_topic_request(TOPIC, 3, 2, &configs); + let request = create_topic_request(TOPIC, Some(3), Some(2), &configs); let topic = &request.topics[0]; assert_eq!(topic_name(TOPIC), topic.name); @@ -688,6 +805,71 @@ mod tests { assert_eq!(BROKER_TIMEOUT_MS, request.timeout_ms); } + #[test] + fn create_topic_with_broker_defaults() { + let request = create_topic_request(TOPIC, None, None, &BTreeMap::new()); + + assert_eq!(-1, request.topics[0].num_partitions); + assert_eq!(-1, request.topics[0].replication_factor); + } + + #[test] + fn read_the_created_topic() { + let response = CreateTopicsResponse::default().with_topics(vec![ + CreatableTopicResult::default() + .with_name(topic_name(TOPIC)) + .with_topic_id( + "517c94f6-2e40-467e-96d3-438ddad69e58" + .parse() + .expect("the topic id should be valid"), + ) + .with_num_partitions(6) + .with_replication_factor(3) + .with_configs(Some(vec![ + CreatableTopicConfigs::default() + .with_name(StrBytes::from_static_str("retention.ms")) + .with_value(Some(StrBytes::from_static_str("1000"))) + .with_config_source(1), + ])), + ]); + + assert_eq!( + CreatedTopic { + id: "UXyU9i5ARn6W00ON2taeWA".to_owned(), + partitions: 6, + replication_factor: 3, + configs: BTreeMap::from([( + "retention.ms".to_owned(), + TopicConfig { + value: Some("1000".to_owned()), + source: ConfigSource::DynamicTopic, + } + )]), + }, + create_topic_result(response, TOPIC).expect("the topic should be created") + ); + } + + #[test] + fn reject_a_created_topic_without_details() { + let response = CreateTopicsResponse::default().with_topics(vec![ + CreatableTopicResult::default() + .with_name(topic_name(TOPIC)) + .with_topic_config_error_code(ResponseError::TopicAuthorizationFailed.code()), + ]); + + let error = + create_topic_result(response, TOPIC).expect_err("the details should be missing"); + + assert!(matches!( + error, + Error::Kafka { + error: ResponseError::TopicAuthorizationFailed, + .. + } + )); + } + #[test] fn create_partitions_without_assignments() { let request = create_partitions_request(TOPIC, 6); @@ -740,35 +922,29 @@ mod tests { #[test] fn read_ongoing_reassignments() { - let reassignment = |index: i32, adding: Vec, removing: Vec| { + let reassignment = |index: i32, removing: Vec| { OngoingPartitionReassignment::default() .with_partition_index(index) - .with_adding_replicas(adding.into_iter().map(Into::into).collect()) .with_removing_replicas(removing.into_iter().map(Into::into).collect()) }; let response = ListPartitionReassignmentsResponse::default().with_topics(vec![ OngoingTopicReassignment::default() .with_name(topic_name("other")) - .with_partitions(vec![reassignment(0, vec![9], vec![8])]), + .with_partitions(vec![reassignment(0, vec![8])]), OngoingTopicReassignment::default() .with_name(topic_name(TOPIC)) - .with_partitions(vec![ - reassignment(2, vec![4], vec![]), - reassignment(0, vec![4, 5], vec![1, 2]), - ]), + .with_partitions(vec![reassignment(0, vec![1, 2]), reassignment(2, vec![])]), ]); assert_eq!( vec![ PartitionReassignment { index: 0, - adding_replicas: vec![4, 5], - removing_replicas: vec![1, 2], + removing_replicas: 2, }, PartitionReassignment { index: 2, - adding_replicas: vec![4], - removing_replicas: vec![], + removing_replicas: 0, }, ], list_partition_reassignments_result(response, TOPIC) @@ -839,21 +1015,6 @@ mod tests { assert!(error.to_string().contains("Topic 'orders' already exists.")); } - #[rstest] - #[case::create_topics(CreateTopicsRequest::KEY, ResponseError::TopicAlreadyExists, true)] - #[case::other_error(CreateTopicsRequest::KEY, ResponseError::InvalidPartitions, false)] - #[case::other_request(CreatePartitionsRequest::KEY, ResponseError::TopicAlreadyExists, false)] - fn recognize_an_existing_topic_on_create( - #[case] api_key: i16, - #[case] error: ResponseError, - #[case] expected: bool, - ) { - let error = - check(api_key, TOPIC, error.code(), None).expect_err("the request should be rejected"); - - assert_eq!(expected, error.is_topic_already_exists()); - } - #[test] fn do_not_treat_connection_errors_as_rejected_by_kafka() { let error = Error::NoBootstrapServers; From 146602d939e09339b1f3d5f3dae9fc0f38ec12a7 Mon Sep 17 00:00:00 2001 From: Benedikt Labrenz Date: Fri, 23 Oct 2026 18:20:12 +0200 Subject: [PATCH 30/93] set fsGroup on the agent pod so it can read secret-operator volumes --- .../src/controller/build/resource/agent.rs | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/rust/operator-binary/src/controller/build/resource/agent.rs b/rust/operator-binary/src/controller/build/resource/agent.rs index 8e09b28ab..c22ebc625 100644 --- a/rust/operator-binary/src/controller/build/resource/agent.rs +++ b/rust/operator-binary/src/controller/build/resource/agent.rs @@ -88,7 +88,11 @@ pub fn build_agent_deployment( .image_pull_secrets_from_product_image(&cluster.image) .add_container(cb_agent.build()) .service_account_name(agent_name(cluster)) - .security_context(PodSecurityContextBuilder::with_stackable_defaults().build()); + .security_context( + PodSecurityContextBuilder::with_stackable_defaults() + .fs_group(1000) + .build(), + ); Deployment { metadata: agent_metadata(cluster), @@ -217,6 +221,11 @@ mod tests { assert_eq!(deployment["metadata"]["name"], "simple-kafka-agent"); assert_eq!(deployment["spec"]["replicas"], 1); assert_eq!(deployment["spec"]["strategy"]["type"], "Recreate"); + // secret-operator volumes are only readable by the fsGroup. + assert_eq!( + deployment["spec"]["template"]["spec"]["securityContext"]["fsGroup"], + 1000 + ); // The selector must not match the broker or controller Pods. assert_eq!( deployment["spec"]["selector"]["matchLabels"], From 60f012398538c12828d6701c05800d6faafc5d10 Mon Sep 17 00:00:00 2001 From: Benedikt Labrenz Date: Fri, 23 Oct 2026 20:17:19 +0200 Subject: [PATCH 31/93] align CRD and validation with operator-rs and decision outcome --- Cargo.lock | 71 ++++--- Cargo.nix | 144 ++++++------- crate-hashes.json | 10 +- .../templates/clusterrole-agent.yaml | 2 +- .../templates/clusterrole-operator.yaml | 2 +- extra/crds.yaml | 100 ++++----- rust/operator-binary/src/controller.rs | 86 +++++++- .../src/controller/build/mod.rs | 2 +- .../src/controller/build/resource/agent.rs | 72 ++++--- .../src/controller/validate.rs | 194 +++++++++++++++--- rust/operator-binary/src/crd/mod.rs | 59 +++--- rust/operator-binary/src/main.rs | 7 +- 12 files changed, 501 insertions(+), 248 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index ab828f1eb..42c5394f6 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1711,8 +1711,8 @@ dependencies = [ [[package]] name = "k8s-version" -version = "0.1.3" -source = "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#92e1a3655432e3899da735a9d86ded2e38030c0e" +version = "0.1.4" +source = "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#a2a7d0caf7b00c332cd299768eea19b1a3a00a51" dependencies = [ "darling 0.24.1", "regex", @@ -1876,6 +1876,12 @@ version = "0.2.16" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b6d2cec3eae94f9f509c767b45932f1ada8350c4bdb85af2fcab4a3c14807981" +[[package]] +name = "libyaml-rs" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2e126dda6f34391ab7b444f9922055facc83c07a910da3eb16f1e4d9c45dc777" + [[package]] name = "libz-sys" version = "1.1.30" @@ -2994,19 +3000,6 @@ dependencies = [ "serde", ] -[[package]] -name = "serde_yaml" -version = "0.9.34+deprecated" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6a8b1a1a2ebf674015cc02edccce75287f1a0130d394307b36743c2f5d504b47" -dependencies = [ - "indexmap", - "itoa", - "ryu", - "serde", - "unsafe-libyaml", -] - [[package]] name = "sha1" version = "0.10.7" @@ -3175,7 +3168,7 @@ checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596" [[package]] name = "stackable-certs" version = "0.4.1" -source = "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#92e1a3655432e3899da735a9d86ded2e38030c0e" +source = "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#a2a7d0caf7b00c332cd299768eea19b1a3a00a51" dependencies = [ "const-oid", "ecdsa", @@ -3231,8 +3224,8 @@ dependencies = [ [[package]] name = "stackable-operator" -version = "0.119.0" -source = "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#92e1a3655432e3899da735a9d86ded2e38030c0e" +version = "0.121.0" +source = "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#a2a7d0caf7b00c332cd299768eea19b1a3a00a51" dependencies = [ "base64 0.23.1", "clap", @@ -3255,7 +3248,6 @@ dependencies = [ "semver", "serde", "serde_json", - "serde_yaml", "sha2", "snafu 0.9.2", "stackable-operator-derive", @@ -3271,12 +3263,13 @@ dependencies = [ "url", "uuid", "xml", + "yaml_serde", ] [[package]] name = "stackable-operator-derive" version = "0.3.1" -source = "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#92e1a3655432e3899da735a9d86ded2e38030c0e" +source = "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#a2a7d0caf7b00c332cd299768eea19b1a3a00a51" dependencies = [ "darling 0.24.1", "proc-macro2", @@ -3286,8 +3279,8 @@ dependencies = [ [[package]] name = "stackable-shared" -version = "0.1.2" -source = "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#92e1a3655432e3899da735a9d86ded2e38030c0e" +version = "0.2.0" +source = "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#a2a7d0caf7b00c332cd299768eea19b1a3a00a51" dependencies = [ "jiff", "k8s-openapi", @@ -3295,16 +3288,16 @@ dependencies = [ "schemars", "semver", "serde", - "serde_yaml", "snafu 0.9.2", "strum", "time", + "yaml_serde", ] [[package]] name = "stackable-telemetry" version = "0.6.5" -source = "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#92e1a3655432e3899da735a9d86ded2e38030c0e" +source = "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#a2a7d0caf7b00c332cd299768eea19b1a3a00a51" dependencies = [ "axum", "clap", @@ -3327,22 +3320,21 @@ dependencies = [ [[package]] name = "stackable-versioned" -version = "0.11.1" -source = "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#92e1a3655432e3899da735a9d86ded2e38030c0e" +version = "0.13.0" +source = "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#a2a7d0caf7b00c332cd299768eea19b1a3a00a51" dependencies = [ "kube", "schemars", "serde", "serde_json", - "serde_yaml", "snafu 0.9.2", "stackable-versioned-macros", ] [[package]] name = "stackable-versioned-macros" -version = "0.11.1" -source = "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#92e1a3655432e3899da735a9d86ded2e38030c0e" +version = "0.13.0" +source = "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#a2a7d0caf7b00c332cd299768eea19b1a3a00a51" dependencies = [ "convert_case", "darling 0.24.1", @@ -3359,7 +3351,7 @@ dependencies = [ [[package]] name = "stackable-webhook" version = "0.10.0" -source = "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#92e1a3655432e3899da735a9d86ded2e38030c0e" +source = "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#a2a7d0caf7b00c332cd299768eea19b1a3a00a51" dependencies = [ "arc-swap", "async-trait", @@ -3961,12 +3953,6 @@ version = "0.2.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ebc1c04c71510c7f702b52b7c350734c9ff1295c464a03335b00bb84fc54f853" -[[package]] -name = "unsafe-libyaml" -version = "0.2.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "673aac59facbab8a9007c7f6108d11f63b603f7cabff99fabf650fea5c32b861" - [[package]] name = "untrusted" version = "0.9.0" @@ -4336,6 +4322,19 @@ version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2f45bb2c13fec6a6cb4c0f76a7e94839e110a14ec803ec2940777a94c347bc52" +[[package]] +name = "yaml_serde" +version = "0.10.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "33b729a08a9a6be689bbad3e2bf8015926db54b6622cc89c3a5f7dc174b9e918" +dependencies = [ + "indexmap", + "itoa", + "libyaml-rs", + "ryu", + "serde", +] + [[package]] name = "yoke" version = "0.8.3" diff --git a/Cargo.nix b/Cargo.nix index ff21a3b66..7b49d7c1a 100644 --- a/Cargo.nix +++ b/Cargo.nix @@ -5440,12 +5440,12 @@ rec { }; "k8s-version" = rec { crateName = "k8s-version"; - version = "0.1.3"; + version = "0.1.4"; edition = "2024"; workspace_member = null; src = pkgs.fetchgit { url = "https://github.com/stackabletech/operator-rs.git"; - rev = "92e1a3655432e3899da735a9d86ded2e38030c0e"; + rev = "a2a7d0caf7b00c332cd299768eea19b1a3a00a51"; sha256 = "0cgziqra8097hp05ynib1qpw4c95n972f4w2rk9l3llyp8r1vmci"; }; libName = "k8s_version"; @@ -6177,6 +6177,19 @@ rec { }; resolvedDefaultFeatures = [ "arch" "default" ]; }; + "libyaml-rs" = rec { + crateName = "libyaml-rs"; + version = "0.3.0"; + edition = "2021"; + crateBin = []; + sha256 = "0xy7bp2dkr7i2vms63cigb087k7salh95ya4njvilf9ldzd6s4if"; + libName = "libyaml_rs"; + authors = [ + "David Tolnay " + "YAML Organization " + ]; + + }; "libz-sys" = rec { crateName = "libz-sys"; version = "1.1.30"; @@ -9885,38 +9898,6 @@ rec { } ]; - }; - "serde_yaml" = rec { - crateName = "serde_yaml"; - version = "0.9.34+deprecated"; - edition = "2021"; - sha256 = "0isba1fjyg3l6rxk156k600ilzr8fp7crv82rhal0rxz5qd1m2va"; - authors = [ - "David Tolnay " - ]; - dependencies = [ - { - name = "indexmap"; - packageId = "indexmap"; - } - { - name = "itoa"; - packageId = "itoa"; - } - { - name = "ryu"; - packageId = "ryu"; - } - { - name = "serde"; - packageId = "serde"; - } - { - name = "unsafe-libyaml"; - packageId = "unsafe-libyaml"; - } - ]; - }; "sha1" = rec { crateName = "sha1"; @@ -10397,7 +10378,7 @@ rec { workspace_member = null; src = pkgs.fetchgit { url = "https://github.com/stackabletech/operator-rs.git"; - rev = "92e1a3655432e3899da735a9d86ded2e38030c0e"; + rev = "a2a7d0caf7b00c332cd299768eea19b1a3a00a51"; sha256 = "0cgziqra8097hp05ynib1qpw4c95n972f4w2rk9l3llyp8r1vmci"; }; libName = "stackable_certs"; @@ -10631,12 +10612,12 @@ rec { }; "stackable-operator" = rec { crateName = "stackable-operator"; - version = "0.119.0"; + version = "0.121.0"; edition = "2024"; workspace_member = null; src = pkgs.fetchgit { url = "https://github.com/stackabletech/operator-rs.git"; - rev = "92e1a3655432e3899da735a9d86ded2e38030c0e"; + rev = "a2a7d0caf7b00c332cd299768eea19b1a3a00a51"; sha256 = "0cgziqra8097hp05ynib1qpw4c95n972f4w2rk9l3llyp8r1vmci"; }; libName = "stackable_operator"; @@ -10739,10 +10720,6 @@ rec { name = "serde_json"; packageId = "serde_json"; } - { - name = "serde_yaml"; - packageId = "serde_yaml"; - } { name = "sha2"; packageId = "sha2"; @@ -10812,6 +10789,10 @@ rec { name = "xml"; packageId = "xml"; } + { + name = "yaml_serde"; + packageId = "yaml_serde"; + } ]; features = { "certs" = [ "dep:stackable-certs" ]; @@ -10835,7 +10816,7 @@ rec { workspace_member = null; src = pkgs.fetchgit { url = "https://github.com/stackabletech/operator-rs.git"; - rev = "92e1a3655432e3899da735a9d86ded2e38030c0e"; + rev = "a2a7d0caf7b00c332cd299768eea19b1a3a00a51"; sha256 = "0cgziqra8097hp05ynib1qpw4c95n972f4w2rk9l3llyp8r1vmci"; }; procMacro = true; @@ -10865,12 +10846,12 @@ rec { }; "stackable-shared" = rec { crateName = "stackable-shared"; - version = "0.1.2"; + version = "0.2.0"; edition = "2024"; workspace_member = null; src = pkgs.fetchgit { url = "https://github.com/stackabletech/operator-rs.git"; - rev = "92e1a3655432e3899da735a9d86ded2e38030c0e"; + rev = "a2a7d0caf7b00c332cd299768eea19b1a3a00a51"; sha256 = "0cgziqra8097hp05ynib1qpw4c95n972f4w2rk9l3llyp8r1vmci"; }; libName = "stackable_shared"; @@ -10909,10 +10890,6 @@ rec { packageId = "serde"; features = [ "derive" ]; } - { - name = "serde_yaml"; - packageId = "serde_yaml"; - } { name = "snafu"; packageId = "snafu 0.9.2"; @@ -10927,6 +10904,10 @@ rec { packageId = "time"; optional = true; } + { + name = "yaml_serde"; + packageId = "yaml_serde"; + } ]; devDependencies = [ { @@ -10951,7 +10932,7 @@ rec { workspace_member = null; src = pkgs.fetchgit { url = "https://github.com/stackabletech/operator-rs.git"; - rev = "92e1a3655432e3899da735a9d86ded2e38030c0e"; + rev = "a2a7d0caf7b00c332cd299768eea19b1a3a00a51"; sha256 = "0cgziqra8097hp05ynib1qpw4c95n972f4w2rk9l3llyp8r1vmci"; }; libName = "stackable_telemetry"; @@ -11056,12 +11037,12 @@ rec { }; "stackable-versioned" = rec { crateName = "stackable-versioned"; - version = "0.11.1"; + version = "0.13.0"; edition = "2024"; workspace_member = null; src = pkgs.fetchgit { url = "https://github.com/stackabletech/operator-rs.git"; - rev = "92e1a3655432e3899da735a9d86ded2e38030c0e"; + rev = "a2a7d0caf7b00c332cd299768eea19b1a3a00a51"; sha256 = "0cgziqra8097hp05ynib1qpw4c95n972f4w2rk9l3llyp8r1vmci"; }; libName = "stackable_versioned"; @@ -11089,10 +11070,6 @@ rec { name = "serde_json"; packageId = "serde_json"; } - { - name = "serde_yaml"; - packageId = "serde_yaml"; - } { name = "snafu"; packageId = "snafu 0.9.2"; @@ -11106,12 +11083,12 @@ rec { }; "stackable-versioned-macros" = rec { crateName = "stackable-versioned-macros"; - version = "0.11.1"; + version = "0.13.0"; edition = "2024"; workspace_member = null; src = pkgs.fetchgit { url = "https://github.com/stackabletech/operator-rs.git"; - rev = "92e1a3655432e3899da735a9d86ded2e38030c0e"; + rev = "a2a7d0caf7b00c332cd299768eea19b1a3a00a51"; sha256 = "0cgziqra8097hp05ynib1qpw4c95n972f4w2rk9l3llyp8r1vmci"; }; procMacro = true; @@ -11175,7 +11152,7 @@ rec { workspace_member = null; src = pkgs.fetchgit { url = "https://github.com/stackabletech/operator-rs.git"; - rev = "92e1a3655432e3899da735a9d86ded2e38030c0e"; + rev = "a2a7d0caf7b00c332cd299768eea19b1a3a00a51"; sha256 = "0cgziqra8097hp05ynib1qpw4c95n972f4w2rk9l3llyp8r1vmci"; }; libName = "stackable_webhook"; @@ -13313,18 +13290,6 @@ rec { }; resolvedDefaultFeatures = [ "default" ]; }; - "unsafe-libyaml" = rec { - crateName = "unsafe-libyaml"; - version = "0.2.11"; - edition = "2021"; - crateBin = []; - sha256 = "0qdq69ffl3v5pzx9kzxbghzn0fzn266i1xn70y88maybz9csqfk7"; - libName = "unsafe_libyaml"; - authors = [ - "David Tolnay " - ]; - - }; "untrusted" = rec { crateName = "untrusted"; version = "0.9.0"; @@ -15203,6 +15168,45 @@ rec { ]; }; + "yaml_serde" = rec { + crateName = "yaml_serde"; + version = "0.10.7"; + edition = "2021"; + sha256 = "0679p5sc2zaz7afchb32nradn9jr07w2ngmdpf4ycswsiah2kdrk"; + authors = [ + "YAML Organization " + ]; + dependencies = [ + { + name = "indexmap"; + packageId = "indexmap"; + usesDefaultFeatures = false; + } + { + name = "itoa"; + packageId = "itoa"; + } + { + name = "libyaml-rs"; + packageId = "libyaml-rs"; + } + { + name = "ryu"; + packageId = "ryu"; + } + { + name = "serde"; + packageId = "serde"; + usesDefaultFeatures = false; + features = [ "alloc" ]; + } + ]; + features = { + "default" = [ "std" ]; + "std" = [ "serde/std" "indexmap/std" ]; + }; + resolvedDefaultFeatures = [ "default" "std" ]; + }; "yoke" = rec { crateName = "yoke"; version = "0.8.3"; diff --git a/crate-hashes.json b/crate-hashes.json index e4f5f7b1d..a5ee97020 100644 --- a/crate-hashes.json +++ b/crate-hashes.json @@ -1,11 +1,11 @@ { - "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#k8s-version@0.1.3": "0cgziqra8097hp05ynib1qpw4c95n972f4w2rk9l3llyp8r1vmci", + "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#k8s-version@0.1.4": "0cgziqra8097hp05ynib1qpw4c95n972f4w2rk9l3llyp8r1vmci", "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#stackable-certs@0.4.1": "0cgziqra8097hp05ynib1qpw4c95n972f4w2rk9l3llyp8r1vmci", "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#stackable-operator-derive@0.3.1": "0cgziqra8097hp05ynib1qpw4c95n972f4w2rk9l3llyp8r1vmci", - "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#stackable-operator@0.119.0": "0cgziqra8097hp05ynib1qpw4c95n972f4w2rk9l3llyp8r1vmci", - "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#stackable-shared@0.1.2": "0cgziqra8097hp05ynib1qpw4c95n972f4w2rk9l3llyp8r1vmci", + "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#stackable-operator@0.121.0": "0cgziqra8097hp05ynib1qpw4c95n972f4w2rk9l3llyp8r1vmci", + "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#stackable-shared@0.2.0": "0cgziqra8097hp05ynib1qpw4c95n972f4w2rk9l3llyp8r1vmci", "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#stackable-telemetry@0.6.5": "0cgziqra8097hp05ynib1qpw4c95n972f4w2rk9l3llyp8r1vmci", - "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#stackable-versioned-macros@0.11.1": "0cgziqra8097hp05ynib1qpw4c95n972f4w2rk9l3llyp8r1vmci", - "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#stackable-versioned@0.11.1": "0cgziqra8097hp05ynib1qpw4c95n972f4w2rk9l3llyp8r1vmci", + "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#stackable-versioned-macros@0.13.0": "0cgziqra8097hp05ynib1qpw4c95n972f4w2rk9l3llyp8r1vmci", + "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#stackable-versioned@0.13.0": "0cgziqra8097hp05ynib1qpw4c95n972f4w2rk9l3llyp8r1vmci", "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#stackable-webhook@0.10.0": "0cgziqra8097hp05ynib1qpw4c95n972f4w2rk9l3llyp8r1vmci" } \ No newline at end of file diff --git a/deploy/helm/kafka-operator/templates/clusterrole-agent.yaml b/deploy/helm/kafka-operator/templates/clusterrole-agent.yaml index 8288ab33a..fb3f2995e 100644 --- a/deploy/helm/kafka-operator/templates/clusterrole-agent.yaml +++ b/deploy/helm/kafka-operator/templates/clusterrole-agent.yaml @@ -1,6 +1,6 @@ --- # Agent ClusterRole: bound (via per KafkaCluster RoleBinding) to the ServiceAccount that the -# platform-access agent pod runs as. +# agent pod runs as. apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: diff --git a/deploy/helm/kafka-operator/templates/clusterrole-operator.yaml b/deploy/helm/kafka-operator/templates/clusterrole-operator.yaml index 96045367c..aafed503c 100644 --- a/deploy/helm/kafka-operator/templates/clusterrole-operator.yaml +++ b/deploy/helm/kafka-operator/templates/clusterrole-operator.yaml @@ -64,7 +64,7 @@ rules: - list - patch - watch - # Deployment of the platform-access agent. + # Deployment of the agent. - apiGroups: - apps resources: diff --git a/extra/crds.yaml b/extra/crds.yaml index 00d43f9b1..d814d144f 100644 --- a/extra/crds.yaml +++ b/extra/crds.yaml @@ -1024,6 +1024,60 @@ spec: nullable: true pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ type: string + managementAccess: + description: |- + Management access of the Stackable Data Platform to this Kafka cluster, used to manage + resources like `KafkaTopic`s. The credential must be authorized in Kafka by the user. + nullable: true + properties: + authentication: + description: How the Stackable Data Platform authenticates to Kafka. + properties: + userProvided: + description: A credential provided by the user. + oneOf: + - required: + - anonymous + - required: + - tls + properties: + anonymous: + description: 'No credential: the product does not require authentication.' + type: object + tls: + description: 'Source of a TLS client certificate: a secret-operator SecretClass or a static Secret.' + oneOf: + - required: + - secretClassName + - required: + - secretName + properties: + secretClassName: + description: An AutoTLS SecretClass used to provision the certificate. + maxLength: 253 + minLength: 1 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ + type: string + secretName: + description: |- + A static Secret holding the certificate in the keys `tls.crt` and `tls.key` (PEM), e.g. as + a Secret of type `kubernetes.io/tls`. + maxLength: 253 + minLength: 1 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ + type: string + type: object + type: object + required: + - userProvided + type: object + enabled: + description: Whether management access is enabled. + type: boolean + required: + - authentication + - enabled + type: object metadataManager: description: |- Metadata manager to use for the Kafka cluster. @@ -2139,52 +2193,6 @@ spec: type: object x-kubernetes-preserve-unknown-fields: true type: array - platformAccess: - description: Access of the Stackable Data Platform to this cluster. - nullable: true - properties: - authentication: - description: How the agent authenticates to Kafka. - oneOf: - - required: - - tls - - required: - - anonymous - properties: - anonymous: - description: 'No credential: the product does not authenticate the agent.' - type: object - tls: - description: 'Source of a TLS client certificate: a secret-operator SecretClass or a static Secret.' - oneOf: - - required: - - secretClass - - required: - - secret - properties: - secret: - description: |- - A static Secret holding the certificate in the keys `tls.crt` and `tls.key` (PEM), e.g. as - a Secret of type `kubernetes.io/tls`. - maxLength: 253 - minLength: 1 - pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ - type: string - secretClass: - description: An AutoTLS SecretClass used to provision the certificate. - maxLength: 253 - minLength: 1 - pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ - type: string - type: object - type: object - enabled: - default: false - description: Whether the operator deploys the agent. False by default. - type: boolean - required: - - authentication - type: object required: - brokers - image diff --git a/rust/operator-binary/src/controller.rs b/rust/operator-binary/src/controller.rs index 4540de068..5986e4ee8 100644 --- a/rust/operator-binary/src/controller.rs +++ b/rust/operator-binary/src/controller.rs @@ -346,7 +346,7 @@ impl ValidatedClusterConfig { /// The configuration of the agent deployed with the cluster. pub struct ValidatedAgentConfig { pub image: String, - pub authentication: v1alpha1::KafkaPlatformAccessAuthentication, + pub authentication: v1alpha1::UserProvidedAuthentication, } /// Per-role configuration extracted during validation. @@ -529,16 +529,24 @@ pub fn error_policy( #[cfg(test)] pub(crate) mod test_support { - use std::collections::BTreeMap; + use std::{collections::BTreeMap, str::FromStr}; use stackable_operator::{ + builder::meta::ObjectMetaBuilder, cli::OperatorEnvironmentOptions, commons::networking::DomainName, - crd::listener, + crd::{ + authentication::{core, kerberos, tls}, + listener, + }, utils::{cluster_info::KubernetesClusterInfo, yaml_from_str_singleton_map}, + v2::types::kubernetes::SecretClassName, }; - use super::{ValidatedCluster, dereference::DereferencedObjects, validate::validate}; + use super::{ + ValidatedCluster, dereference::DereferencedObjects, security::ValidatedKafkaSecurity, + validate::validate, + }; use crate::crd::{authentication::ResolvedAuthenticationClasses, v1alpha1}; pub const OPERATOR_IMAGE: &str = "oci.stackable.tech/sdp/kafka-operator:0.0.0-dev"; @@ -559,6 +567,76 @@ pub(crate) mod test_support { yaml_from_str_singleton_map(yaml).expect("invalid test KafkaCluster YAML") } + fn tls_auth_class() -> core::v1alpha1::AuthenticationClass { + core::v1alpha1::AuthenticationClass { + metadata: ObjectMetaBuilder::new().name("tls-auth").build(), + spec: core::v1alpha1::AuthenticationClassSpec { + provider: core::v1alpha1::AuthenticationClassProvider::Tls( + tls::v1alpha1::AuthenticationProvider { + client_cert_secret_class: Some("client-auth-secret-class".to_string()), + }, + ), + }, + } + } + + fn kerberos_auth_class() -> core::v1alpha1::AuthenticationClass { + core::v1alpha1::AuthenticationClass { + metadata: ObjectMetaBuilder::new().name("kerberos-auth").build(), + spec: core::v1alpha1::AuthenticationClassSpec { + provider: core::v1alpha1::AuthenticationClassProvider::Kerberos( + kerberos::v1alpha1::AuthenticationProvider { + kerberos_secret_class: "kerberos-secret-class".to_string(), + }, + ), + }, + } + } + + fn no_auth() -> ResolvedAuthenticationClasses { + ResolvedAuthenticationClasses::new(vec![]) + } + + /// Plaintext: no TLS, no authentication, no OPA. + pub fn plaintext() -> ValidatedKafkaSecurity { + ValidatedKafkaSecurity::new( + no_auth(), + SecretClassName::from_str("tls").expect("tls secret class name is valid"), + None, + None, + ) + } + + /// Server TLS only (encryption without client authentication). + pub fn server_tls() -> ValidatedKafkaSecurity { + ValidatedKafkaSecurity::new( + no_auth(), + SecretClassName::from_str("tls").expect("tls secret class name is valid"), + Some("tls".parse().unwrap()), + None, + ) + } + + /// Mutual TLS (client-certificate authentication). + pub fn client_auth_tls() -> ValidatedKafkaSecurity { + ValidatedKafkaSecurity::new( + ResolvedAuthenticationClasses::new(vec![tls_auth_class()]), + SecretClassName::from_str("tls").expect("tls secret class name is valid"), + Some("tls".parse().unwrap()), + None, + ) + } + + /// Kerberos, which also requires server and internal TLS. + pub fn kerberos() -> ValidatedKafkaSecurity { + ValidatedKafkaSecurity::new( + ResolvedAuthenticationClasses::new(vec![kerberos_auth_class()]), + SecretClassName::from_str("tls").expect("tls secret class name is valid"), + Some("tls".parse().unwrap()), + None, + ) + } + fn cluster_info() -> KubernetesClusterInfo { KubernetesClusterInfo { cluster_domain: DomainName::try_from("cluster.local").expect("valid domain"), diff --git a/rust/operator-binary/src/controller/build/mod.rs b/rust/operator-binary/src/controller/build/mod.rs index 8ddcf577e..e30188571 100644 --- a/rust/operator-binary/src/controller/build/mod.rs +++ b/rust/operator-binary/src/controller/build/mod.rs @@ -390,7 +390,7 @@ mod tests { sorted_names(&resources.role_bindings), ["simple-kafka-rolebinding"] ); - // No agent without platform access. + // No agent without management access. assert!(resources.deployments.is_empty()); } diff --git a/rust/operator-binary/src/controller/build/resource/agent.rs b/rust/operator-binary/src/controller/build/resource/agent.rs index c22ebc625..a903da6fc 100644 --- a/rust/operator-binary/src/controller/build/resource/agent.rs +++ b/rust/operator-binary/src/controller/build/resource/agent.rs @@ -1,4 +1,4 @@ -//! Builds the resources of the platform-access agent deployed alongside the cluster. +//! Builds the resources of the agent deployed alongside the cluster. use std::str::FromStr; @@ -31,7 +31,7 @@ use crate::{ ValidatedAgentConfig, ValidatedCluster, build::{agent_selector, recommended_labels_for_agent_resources}, }, - crd::v1alpha1::KafkaPlatformAccessAuthentication, + crd::v1alpha1::UserProvidedAuthentication, }; constant!(AGENT_CONTAINER_NAME: ContainerName = "agent"); @@ -45,10 +45,10 @@ pub fn build_agent_deployment( let mut pod_builder = PodBuilder::new(); match &agent_config.authentication { - KafkaPlatformAccessAuthentication::Tls(credential) => { - credential.add_volumes_and_mounts(&mut pod_builder, vec![&mut cb_agent]) - } - KafkaPlatformAccessAuthentication::Anonymous(_) => {} + UserProvidedAuthentication::Anonymous(_) => {} + UserProvidedAuthentication::Tls(credential) => credential + .add_volumes_and_mounts(&mut pod_builder, vec![&mut cb_agent]) + .expect("the agent Pod has no other volume or mount for the TLS client certificate"), } cb_agent @@ -163,11 +163,14 @@ mod tests { OPERATOR_IMAGE, app_version_label, minimal_kafka, validated_cluster, }; - fn cluster(platform_access: &str) -> ValidatedCluster { - cluster_with_image("{productVersion: 3.9.2}", platform_access) + const TLS_SECRET_CLASS: &str = + "{enabled: true, authentication: {userProvided: {tls: {secretClassName: tls}}}}"; + + fn cluster(management_access: &str) -> ValidatedCluster { + cluster_with_image("{productVersion: 3.9.2}", management_access) } - fn cluster_with_image(image: &str, platform_access: &str) -> ValidatedCluster { + fn cluster_with_image(image: &str, management_access: &str) -> ValidatedCluster { let kafka = minimal_kafka(&format!( r#" apiVersion: kafka.stackable.tech/v1alpha1 @@ -179,8 +182,12 @@ mod tests { spec: image: {image} clusterConfig: - zookeeperConfigMapName: xyz - platformAccess: {platform_access} + metadataManager: kraft + managementAccess: {management_access} + controllers: + roleGroups: + default: + replicas: 1 brokers: roleGroups: default: @@ -194,29 +201,29 @@ mod tests { let agent_config = cluster .agent_config .as_ref() - .expect("platform access is enabled"); + .expect("management access is enabled"); serde_json::to_value(build_agent_deployment(cluster, agent_config)) .expect("must be serializable") } #[test] - fn agent_config_is_only_resolved_if_platform_access_is_enabled() { + fn agent_config_is_only_resolved_if_management_access_is_enabled() { assert!( - cluster("{enabled: false, authentication: {tls: {secretClass: tls}}}") - .agent_config - .is_none() + cluster( + "{enabled: false, authentication: {userProvided: {tls: {secretClassName: tls}}}}" + ) + .agent_config + .is_none() ); - let cluster = cluster("{enabled: true, authentication: {tls: {secretClass: tls}}}"); - let agent_config = cluster.agent_config.expect("platform access is enabled"); + let cluster = cluster(TLS_SECRET_CLASS); + let agent_config = cluster.agent_config.expect("management access is enabled"); assert_eq!(agent_config.image, OPERATOR_IMAGE); } #[test] fn test_deployment() { - let deployment = agent_deployment(&cluster( - "{enabled: true, authentication: {tls: {secretClass: tls}}}", - )); + let deployment = agent_deployment(&cluster(TLS_SECRET_CLASS)); assert_eq!(deployment["metadata"]["name"], "simple-kafka-agent"); assert_eq!(deployment["spec"]["replicas"], 1); @@ -230,7 +237,7 @@ mod tests { assert_eq!( deployment["spec"]["selector"]["matchLabels"], json!({ - "app.kubernetes.io/component": "agent", + "app.kubernetes.io/component": "kafka-agent", "app.kubernetes.io/instance": "simple-kafka", "app.kubernetes.io/name": "kafka" }) @@ -238,7 +245,7 @@ mod tests { assert_eq!( deployment["spec"]["template"]["metadata"]["labels"], json!({ - "app.kubernetes.io/component": "agent", + "app.kubernetes.io/component": "kafka-agent", "app.kubernetes.io/instance": "simple-kafka", "app.kubernetes.io/managed-by": "kafka.stackable.tech_kafkacluster", "app.kubernetes.io/name": "kafka", @@ -257,9 +264,9 @@ mod tests { assert_eq!(pod_spec["containers"][0]["args"], json!(["run"])); assert_eq!( pod_spec["containers"][0]["volumeMounts"], - json!([{"mountPath": "/stackable/secrets/tls-platform-access", "name": "tls-platform-access"}]) + json!([{"mountPath": "/stackable/secrets/tls-management-access", "name": "tls-management-access"}]) ); - assert_eq!(pod_spec["volumes"][0]["name"], "tls-platform-access"); + assert_eq!(pod_spec["volumes"][0]["name"], "tls-management-access"); assert_eq!( pod_spec["volumes"][0]["ephemeral"]["volumeClaimTemplate"]["metadata"]["annotations"]["secrets.stackable.tech/class"], "tls" @@ -269,19 +276,20 @@ mod tests { #[test] fn test_deployment_with_static_secret() { let deployment = agent_deployment(&cluster( - "{enabled: true, authentication: {tls: {secret: my-cert}}}", + "{enabled: true, authentication: {userProvided: {tls: {secretName: my-cert}}}}", )); assert_eq!( deployment["spec"]["template"]["spec"]["volumes"], - json!([{"name": "tls-platform-access", "secret": {"secretName": "my-cert"}}]) + json!([{"name": "tls-management-access", "secret": {"optional": false, "secretName": "my-cert"}}]) ); } #[test] fn deployment_with_anonymous_authentication_mounts_no_credential() { - let deployment = - agent_deployment(&cluster("{enabled: true, authentication: {anonymous: {}}}")); + let deployment = agent_deployment(&cluster( + "{enabled: true, authentication: {userProvided: {anonymous: {}}}}", + )); let pod_spec = &deployment["spec"]["template"]["spec"]; assert_eq!(pod_spec["volumes"], Value::Null); @@ -292,7 +300,7 @@ mod tests { fn test_deployment_uses_product_image_pull_secrets() { let deployment = agent_deployment(&cluster_with_image( "{productVersion: 3.9.2, pullSecrets: [{name: regcred}]}", - "{enabled: true, authentication: {tls: {secretClass: tls}}}", + TLS_SECRET_CLASS, )); assert_eq!( @@ -303,7 +311,7 @@ mod tests { #[test] fn test_role_binding() { - let cluster = cluster("{enabled: true, authentication: {tls: {secretClass: tls}}}"); + let cluster = cluster(TLS_SECRET_CLASS); assert_eq!( json!({ @@ -311,7 +319,7 @@ mod tests { "kind": "RoleBinding", "metadata": { "labels": { - "app.kubernetes.io/component": "agent", + "app.kubernetes.io/component": "kafka-agent", "app.kubernetes.io/instance": "simple-kafka", "app.kubernetes.io/managed-by": "kafka.stackable.tech_kafkacluster", "app.kubernetes.io/name": "kafka", diff --git a/rust/operator-binary/src/controller/validate.rs b/rust/operator-binary/src/controller/validate.rs index 2a2dabd26..1280baf0b 100644 --- a/rust/operator-binary/src/controller/validate.rs +++ b/rust/operator-binary/src/controller/validate.rs @@ -6,7 +6,7 @@ use std::{collections::BTreeMap, str::FromStr}; use serde::Serialize; -use snafu::{OptionExt, ResultExt, Snafu}; +use snafu::{OptionExt, ResultExt, Snafu, ensure}; use stackable_operator::{ cli::OperatorEnvironmentOptions, commons::product_image_selection, @@ -117,10 +117,24 @@ pub enum Error { MissingKraftControllerRole, #[snafu(display( - "platform access requires the operator image, which also contains the agent; set the \ + "management access requires the operator image, which also contains the agent; set the \ OPERATOR_IMAGE environment variable of the operator" ))] - PlatformAccessRequiresOperatorImage, + ManagementAccessRequiresOperatorImage, + + #[snafu(display("management access is only supported in KRaft mode"))] + ManagementAccessRequiresKraft, + + #[snafu(display("management access does not support clusters with Kerberos authentication"))] + ManagementAccessWithKerberos, + + #[snafu(display( + "the cluster requires client authentication, but management access is anonymous" + ))] + AnonymousManagementAccessWithClientAuthentication, + + #[snafu(display("TLS management access requires the cluster to use TLS"))] + TlsManagementAccessRequiresTls, } /// Validated logging configuration for a Kafka role group's Kafka and (optional) Vector @@ -346,15 +360,24 @@ pub fn validate( let agent_config = kafka .spec - .platform_access + .cluster_config + .management_access .as_ref() - .filter(|platform_access| platform_access.enabled) - .map(|platform_access| { + .filter(|management_access| management_access.enabled) + .map(|management_access| { + ensure!( + metadata_manager == crate::crd::MetadataManager::KRaft, + ManagementAccessRequiresKraftSnafu + ); + validate_management_access( + &management_access.authentication.user_provided, + &kafka_security, + )?; Ok(ValidatedAgentConfig { image: operator_image - .context(PlatformAccessRequiresOperatorImageSnafu)? + .context(ManagementAccessRequiresOperatorImageSnafu)? .to_owned(), - authentication: platform_access.authentication.clone(), + authentication: management_access.authentication.user_provided.clone(), }) }) .transpose()?; @@ -467,22 +490,106 @@ fn inject_cluster_id(env_overrides: EnvVarSet, cluster_id: Option<&str>) -> EnvV } } +/// Rejects management access authentication config that cannot connect to the cluster. +fn validate_management_access( + authentication: &v1alpha1::UserProvidedAuthentication, + security: &ValidatedKafkaSecurity, +) -> Result<()> { + ensure!( + !security.has_kerberos_enabled(), + ManagementAccessWithKerberosSnafu + ); + + match authentication { + v1alpha1::UserProvidedAuthentication::Anonymous(_) => ensure!( + security.tls_client_authentication_class().is_none(), + AnonymousManagementAccessWithClientAuthenticationSnafu + ), + v1alpha1::UserProvidedAuthentication::Tls(_) => { + ensure!(security.tls_enabled(), TlsManagementAccessRequiresTlsSnafu) + } + } + + Ok(()) +} + #[cfg(test)] mod tests { use std::str::FromStr; - use stackable_operator::v2::{ - builder::pod::container::EnvVarSet, types::operator::RoleGroupName, + use rstest::rstest; + use stackable_operator::{ + crd::management_access::v1alpha1::{Anonymous, TlsClientCredential}, + v2::{ + builder::pod::container::EnvVarSet, + types::{kubernetes::SecretName, operator::RoleGroupName}, + }, }; - use super::{Error, KAFKA_CLUSTER_ID, inject_cluster_id}; + use super::{Error, KAFKA_CLUSTER_ID, inject_cluster_id, validate_management_access}; use crate::{ - controller::test_support::{ - app_version_label, minimal_kafka, validate_with_operator_image, validated_cluster, + controller::{ + security::ValidatedKafkaSecurity, + test_support::{ + app_version_label, client_auth_tls, kerberos, minimal_kafka, plaintext, server_tls, + validate_with_operator_image, validated_cluster, + }, }, - crd::role::KafkaRole, + crd::{role::KafkaRole, v1alpha1::UserProvidedAuthentication}, }; + fn anonymous() -> UserProvidedAuthentication { + UserProvidedAuthentication::Anonymous(Anonymous::default()) + } + + fn tls() -> UserProvidedAuthentication { + UserProvidedAuthentication::Tls(TlsClientCredential::SecretName( + SecretName::from_str("agent-cert").expect("the Secret name should be valid"), + )) + } + + #[rstest] + #[case::plaintext_anonymous(plaintext(), anonymous())] + #[case::server_tls_anonymous(server_tls(), anonymous())] + #[case::server_tls_with_credential(server_tls(), tls())] + #[case::client_auth_with_credential(client_auth_tls(), tls())] + fn accept_management_access( + #[case] security: ValidatedKafkaSecurity, + #[case] authentication: UserProvidedAuthentication, + ) { + validate_management_access(&authentication, &security) + .expect("the management access should be valid"); + } + + #[rstest] + #[case::anonymous(anonymous())] + #[case::tls(tls())] + fn reject_management_access_with_kerberos(#[case] authentication: UserProvidedAuthentication) { + let error = validate_management_access(&authentication, &kerberos()) + .expect_err("the management access should be invalid"); + + assert!(matches!(error, Error::ManagementAccessWithKerberos)); + } + + #[test] + fn reject_anonymous_management_access_with_client_authentication() { + let error = validate_management_access(&anonymous(), &client_auth_tls()) + .expect_err("the management access should be invalid"); + + assert!(matches!( + error, + Error::AnonymousManagementAccessWithClientAuthentication + )); + } + + #[test] + fn reject_tls_management_access_without_tls() { + let error = validate_management_access(&tls(), &plaintext()) + .expect_err("the management access should be invalid"); + + assert!(matches!(error, Error::TlsManagementAccessRequiresTls)); + } + #[test] fn test_constants() { // Test that dereferencing the constants does not panic. @@ -584,7 +691,7 @@ mod tests { #[rstest::rstest] #[case::enabled(true)] #[case::disabled(false)] - fn platform_access_requires_the_operator_image(#[case] enabled: bool) { + fn management_access_requires_the_operator_image(#[case] enabled: bool) { let kafka = minimal_kafka(&format!( r#" apiVersion: kafka.stackable.tech/v1alpha1 @@ -597,12 +704,17 @@ mod tests { image: productVersion: 3.9.2 clusterConfig: - zookeeperConfigMapName: xyz - platformAccess: - enabled: {enabled} - authentication: - tls: - secretClass: tls + metadataManager: kraft + managementAccess: + enabled: {enabled} + authentication: + userProvided: + tls: + secretClassName: tls + controllers: + roleGroups: + default: + replicas: 1 brokers: roleGroups: default: @@ -614,8 +726,8 @@ mod tests { if enabled { assert!( - matches!(result, Err(Error::PlatformAccessRequiresOperatorImage)), - "expected PlatformAccessRequiresOperatorImage, got: {:?}", + matches!(result, Err(Error::ManagementAccessRequiresOperatorImage)), + "expected ManagementAccessRequiresOperatorImage, got: {:?}", result.err() ); } else { @@ -623,6 +735,42 @@ mod tests { } } + #[test] + fn zookeeper_mode_rejects_management_access() { + let kafka = minimal_kafka( + r#" + apiVersion: kafka.stackable.tech/v1alpha1 + kind: KafkaCluster + metadata: + name: simple-kafka + namespace: default + uid: 12345678-1234-1234-1234-123456789012 + spec: + image: + productVersion: 3.9.2 + clusterConfig: + zookeeperConfigMapName: zk-discovery + managementAccess: + enabled: true + authentication: + userProvided: + anonymous: {} + brokers: + roleGroups: + default: + replicas: 3 + "#, + ); + + let result = crate::controller::test_support::validate_err(&kafka); + + assert!( + matches!(result, Err(Error::ManagementAccessRequiresKraft)), + "expected ManagementAccessRequiresKraft, got: {:?}", + result.err() + ); + } + #[test] fn injects_cluster_id_when_absent() { let env = inject_cluster_id(EnvVarSet::new(), Some("my-id")); diff --git a/rust/operator-binary/src/crd/mod.rs b/rust/operator-binary/src/crd/mod.rs index 7b43745f5..3faba948e 100644 --- a/rust/operator-binary/src/crd/mod.rs +++ b/rust/operator-binary/src/crd/mod.rs @@ -18,7 +18,7 @@ use stackable_operator::{ }, config::merge::Merge, constant, - crd::platform_access::v1alpha1::{Anonymous, TlsClientCredential}, + crd::management_access::v1alpha1::{Anonymous, TlsClientCredential}, deep_merger::ObjectOverrides, kube::{CustomResource, runtime::reflector::ObjectRef}, role_utils::GenericRoleConfig, @@ -150,10 +150,6 @@ pub mod versioned { #[serde(default)] pub cluster_config: v1alpha1::KafkaClusterConfig, - /// Access of the Stackable Data Platform to this cluster. - #[serde(default, skip_serializing_if = "Option::is_none")] - pub platform_access: Option, - // no doc - docs in ClusterOperation struct. #[serde(default)] pub cluster_operation: ClusterOperation, @@ -163,27 +159,6 @@ pub mod versioned { pub object_overrides: ObjectOverrides, } - /// Access of the Stackable Data Platform to this Kafka cluster, used by an agent to manage - /// resources like `KafkaTopic`s. The credential must be authorized in Kafka by the user. - #[derive(Clone, Debug, Deserialize, Eq, JsonSchema, PartialEq, Serialize)] - #[serde(rename_all = "camelCase")] - pub struct KafkaPlatformAccess { - /// Whether the operator deploys the agent. False by default. - #[serde(default)] - pub enabled: bool, - - /// How the agent authenticates to Kafka. - pub authentication: v1alpha1::KafkaPlatformAccessAuthentication, - } - - /// How the agent authenticates to Kafka. - #[derive(Clone, Debug, Deserialize, Eq, JsonSchema, PartialEq, Serialize)] - #[serde(rename_all = "camelCase")] - pub enum KafkaPlatformAccessAuthentication { - Tls(TlsClientCredential), - Anonymous(Anonymous), - } - #[derive(Clone, Deserialize, Debug, Eq, JsonSchema, PartialEq, Serialize)] #[serde(rename_all = "camelCase")] pub struct KafkaClusterConfig { @@ -267,6 +242,37 @@ pub mod versioned { /// #[serde(skip_serializing_if = "Option::is_none")] pub broker_id_pod_config_map_name: Option, + + // no doc - docs in KafkaManagementAccess struct. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub management_access: Option, + } + + /// Management access of the Stackable Data Platform to this Kafka cluster, used to manage + /// resources like `KafkaTopic`s. The credential must be authorized in Kafka by the user. + #[derive(Clone, Debug, Deserialize, Eq, JsonSchema, PartialEq, Serialize)] + #[serde(rename_all = "camelCase")] + pub struct KafkaManagementAccess { + /// Whether management access is enabled. + pub enabled: bool, + + /// How the Stackable Data Platform authenticates to Kafka. + pub authentication: v1alpha1::KafkaManagementAccessAuthentication, + } + + #[derive(Clone, Debug, Deserialize, Eq, JsonSchema, PartialEq, Serialize)] + #[serde(rename_all = "camelCase")] + pub struct KafkaManagementAccessAuthentication { + // no doc - docs in UserProvidedAuthentication enum. + pub user_provided: v1alpha1::UserProvidedAuthentication, + } + + /// A credential provided by the user. + #[derive(Clone, Debug, Deserialize, Eq, JsonSchema, PartialEq, Serialize)] + #[serde(rename_all = "camelCase")] + pub enum UserProvidedAuthentication { + Anonymous(Anonymous), + Tls(TlsClientCredential), } #[derive(Clone, Debug, Default, Deserialize, JsonSchema, Merge, PartialEq, Serialize)] @@ -300,6 +306,7 @@ impl Default for v1alpha1::KafkaClusterConfig { zookeeper_config_map_name: None, metadata_manager: None, broker_id_pod_config_map_name: None, + management_access: None, } } } diff --git a/rust/operator-binary/src/main.rs b/rust/operator-binary/src/main.rs index 817b07ac1..8ef5b8398 100644 --- a/rust/operator-binary/src/main.rs +++ b/rust/operator-binary/src/main.rs @@ -186,9 +186,10 @@ async fn main() -> anyhow::Result<()> { ) .owns( watch_namespace.get_api::>(&client), - // The agent is a the only Deployment - watcher::Config::default() - .labels("app.kubernetes.io/name=kafka,app.kubernetes.io/component=agent"), + // The agent is the only Deployment + watcher::Config::default().labels( + "app.kubernetes.io/name=kafka,app.kubernetes.io/component=kafka-agent", + ), ) .watches( watch_namespace.get_api::>(&client), From 1d684766e91abf28f9695e07e5c44f9c3803cc2f Mon Sep 17 00:00:00 2001 From: Benedikt Labrenz Date: Fri, 23 Oct 2026 20:52:16 +0200 Subject: [PATCH 32/93] configure the agent's connection to Kafka via a shared contract crate --- Cargo.lock | 10 + Cargo.nix | 43 ++++ Cargo.toml | 3 +- rust/operator-binary/Cargo.toml | 1 + rust/operator-binary/src/controller.rs | 49 ++++- .../src/controller/build/command.rs | 2 +- .../src/controller/build/resource/agent.rs | 196 ++++++++++++++---- .../controller/build/resource/statefulset.rs | 2 +- .../src/controller/build/security.rs | 120 +---------- .../src/controller/security.rs | 12 ++ .../src/controller/validate.rs | 129 +++++++++--- rust/shared/Cargo.toml | 17 ++ rust/shared/src/agent.rs | 42 ++++ rust/shared/src/lib.rs | 1 + 14 files changed, 434 insertions(+), 193 deletions(-) create mode 100644 rust/shared/Cargo.toml create mode 100644 rust/shared/src/agent.rs create mode 100644 rust/shared/src/lib.rs diff --git a/Cargo.lock b/Cargo.lock index 42c5394f6..f092e03d4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3216,12 +3216,22 @@ dependencies = [ "serde", "serde_json", "snafu 0.9.2", + "stackable-kafka-shared", "stackable-operator", "strum", "tokio", "tracing", ] +[[package]] +name = "stackable-kafka-shared" +version = "0.0.0-dev" +dependencies = [ + "rstest", + "stackable-operator", + "strum", +] + [[package]] name = "stackable-operator" version = "0.121.0" diff --git a/Cargo.nix b/Cargo.nix index 7b49d7c1a..e3bd3c098 100644 --- a/Cargo.nix +++ b/Cargo.nix @@ -59,6 +59,16 @@ rec { packageId = "stackable-kafka-operator"; }; + # Debug support which might change between releases. + # File a bug if you depend on any for non-debug work! + debug = internal.debugCrate { inherit packageId; }; + }; + "stackable-kafka-shared" = rec { + packageId = "stackable-kafka-shared"; + build = internal.buildRustCrateWithFeatures { + packageId = "stackable-kafka-shared"; + }; + # Debug support which might change between releases. # File a bug if you depend on any for non-debug work! debug = internal.debugCrate { inherit packageId; }; @@ -10575,6 +10585,10 @@ rec { name = "snafu"; packageId = "snafu 0.9.2"; } + { + name = "stackable-kafka-shared"; + packageId = "stackable-kafka-shared"; + } { name = "stackable-operator"; packageId = "stackable-operator"; @@ -10609,6 +10623,35 @@ rec { } ]; + }; + "stackable-kafka-shared" = rec { + crateName = "stackable-kafka-shared"; + version = "0.0.0-dev"; + edition = "2024"; + src = lib.cleanSourceWith { filter = sourceFilter; src = ./rust/shared; }; + libName = "stackable_kafka_shared"; + authors = [ + "Stackable GmbH " + ]; + dependencies = [ + { + name = "stackable-operator"; + packageId = "stackable-operator"; + features = [ "crds" "webhook" ]; + } + { + name = "strum"; + packageId = "strum"; + features = [ "derive" ]; + } + ]; + devDependencies = [ + { + name = "rstest"; + packageId = "rstest"; + } + ]; + }; "stackable-operator" = rec { crateName = "stackable-operator"; diff --git a/Cargo.toml b/Cargo.toml index 975fdca36..81beda9ec 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,5 +1,5 @@ [workspace] -members = ["rust/operator-binary", "rust/agent-binary"] +members = ["rust/operator-binary", "rust/agent-binary", "rust/shared"] resolver = "2" [workspace.package] @@ -11,6 +11,7 @@ repository = "https://github.com/stackabletech/kafka-operator" [workspace.dependencies] stackable-operator = { git = "https://github.com/stackabletech/operator-rs.git", branch = "feat/platform-access", features = ["crds", "webhook"] } +stackable-kafka-shared = { path = "rust/shared" } anyhow = "1.0" built = { version = "0.8", features = ["chrono", "git2"] } diff --git a/rust/operator-binary/Cargo.toml b/rust/operator-binary/Cargo.toml index 38e9e1e18..ca41d66db 100644 --- a/rust/operator-binary/Cargo.toml +++ b/rust/operator-binary/Cargo.toml @@ -9,6 +9,7 @@ repository.workspace = true publish = false [dependencies] +stackable-kafka-shared.workspace = true stackable-operator.workspace = true indoc.workspace = true diff --git a/rust/operator-binary/src/controller.rs b/rust/operator-binary/src/controller.rs index 5986e4ee8..368bbdf2b 100644 --- a/rust/operator-binary/src/controller.rs +++ b/rust/operator-binary/src/controller.rs @@ -15,12 +15,13 @@ use std::{ use const_format::concatcp; use snafu::{ResultExt, Snafu}; +use stackable_kafka_shared::agent::ClientSecurity; use stackable_operator::{ cli::OperatorEnvironmentOptions, cluster_resources::ClusterResourceApplyStrategy, commons::{networking::DomainName, product_image_selection::ResolvedProductImage}, constant, - crd::listener, + crd::{listener, management_access::v1alpha1::TlsClientCredential}, k8s_openapi::api::{ apps::v1::{Deployment, StatefulSet}, core::v1::{ConfigMap, Service, ServiceAccount}, @@ -40,7 +41,7 @@ use stackable_operator::{ role_group_utils::ResourceNames, role_utils, types::{ - kubernetes::{ConfigMapName, ListenerName, NamespaceName, Uid}, + kubernetes::{ConfigMapName, ListenerName, NamespaceName, SecretClassName, Uid}, operator::{ClusterName, ControllerName, OperatorName, ProductName, ProductVersion}, }, }, @@ -346,7 +347,29 @@ impl ValidatedClusterConfig { /// The configuration of the agent deployed with the cluster. pub struct ValidatedAgentConfig { pub image: String, - pub authentication: v1alpha1::UserProvidedAuthentication, + pub connection: AgentConnection, +} + +#[derive(Debug, PartialEq)] +pub enum AgentConnection { + Plaintext, + Tls { + server_ca_secret_class: SecretClassName, + }, + MutualTls { + server_ca_secret_class: SecretClassName, + client_credential: TlsClientCredential, + }, +} + +impl From<&AgentConnection> for ClientSecurity { + fn from(connection: &AgentConnection) -> Self { + match connection { + AgentConnection::Plaintext => Self::Plaintext, + AgentConnection::Tls { .. } => Self::Tls, + AgentConnection::MutualTls { .. } => Self::MutualTls, + } + } } /// Per-role configuration extracted during validation. @@ -637,6 +660,26 @@ pub(crate) mod test_support { ) } + /// Internal TLS only (broker/controller encryption without client TLS). + pub fn internal_tls() -> ValidatedKafkaSecurity { + ValidatedKafkaSecurity::new( + no_auth(), + SecretClassName::from_str("tls").expect("tls secret class name is valid"), + None, + None, + ) + } + + /// OPA authorization with a TLS truststore (internal + server TLS also enabled). + pub fn opa() -> ValidatedKafkaSecurity { + ValidatedKafkaSecurity::new( + no_auth(), + SecretClassName::from_str("tls").expect("tls secret class name is valid"), + Some("tls".parse().unwrap()), + Some("opa-tls".parse().unwrap()), + ) + } + fn cluster_info() -> KubernetesClusterInfo { KubernetesClusterInfo { cluster_domain: DomainName::try_from("cluster.local").expect("valid domain"), diff --git a/rust/operator-binary/src/controller/build/command.rs b/rust/operator-binary/src/controller/build/command.rs index ebe59764d..34ff1e6ad 100644 --- a/rust/operator-binary/src/controller/build/command.rs +++ b/rust/operator-binary/src/controller/build/command.rs @@ -400,7 +400,7 @@ mod tests { use indoc::indoc; use super::*; - use crate::controller::build::security::tests::{kerberos, plaintext}; + use crate::controller::test_support::{kerberos, plaintext}; #[test] fn controller_command_does_not_export_a_realm_without_kerberos() { diff --git a/rust/operator-binary/src/controller/build/resource/agent.rs b/rust/operator-binary/src/controller/build/resource/agent.rs index a903da6fc..19fcd4332 100644 --- a/rust/operator-binary/src/controller/build/resource/agent.rs +++ b/rust/operator-binary/src/controller/build/resource/agent.rs @@ -2,39 +2,52 @@ use std::str::FromStr; +use stackable_kafka_shared::agent::{ + ClientSecurity, DISCOVERY_DIR, ENV_CLIENT_SECURITY, ENV_CLUSTER_NAME, ENV_CLUSTER_NAMESPACE, + KAFKA_TLS_DIR, +}; use stackable_operator::{ builder::{ meta::ObjectMetaBuilder, pod::{ - PodBuilder, resources::ResourceRequirementsBuilder, security::PodSecurityContextBuilder, + PodBuilder, + container::ContainerBuilder, + resources::ResourceRequirementsBuilder, + security::PodSecurityContextBuilder, + volume::{SecretFormat, SecretOperatorVolumeSourceBuilder, VolumeBuilder}, }, }, + commons::secret_class::SecretClassVolumeProvisionParts, constant, k8s_openapi::{ Resource, api::{ apps::v1::{Deployment, DeploymentSpec, DeploymentStrategy}, - core::v1::{EnvVarSource, ObjectFieldSelector, ServiceAccount}, + core::v1::{EnvVar, EnvVarSource, ObjectFieldSelector, ServiceAccount}, rbac::v1::{ClusterRole, RoleBinding, RoleRef, Subject}, }, apimachinery::pkg::apis::meta::v1::LabelSelector, }, kube::api::ObjectMeta, v2::{ - builder::{meta::ownerreference_from_resource, pod::container::new_container_builder}, - types::kubernetes::ContainerName, + builder::{ + meta::ownerreference_from_resource, + pod::container::{EnvVarName, EnvVarSet, new_container_builder}, + }, + types::kubernetes::{ContainerName, SecretClassName, VolumeName}, }, }; -use crate::{ - controller::{ - ValidatedAgentConfig, ValidatedCluster, - build::{agent_selector, recommended_labels_for_agent_resources}, - }, - crd::v1alpha1::UserProvidedAuthentication, +use crate::controller::{ + AgentConnection, ValidatedAgentConfig, ValidatedCluster, + build::{agent_selector, recommended_labels_for_agent_resources}, }; constant!(AGENT_CONTAINER_NAME: ContainerName = "agent"); +constant!(DISCOVERY_VOLUME_NAME: VolumeName = "discovery"); +constant!(KAFKA_TLS_VOLUME_NAME: VolumeName = "tls-kafka"); +constant!(KUBERNETES_NODE_NAME: EnvVarName = "KUBERNETES_NODE_NAME"); +constant!(KUBERNETES_CLUSTER_DOMAIN: EnvVarName = "KUBERNETES_CLUSTER_DOMAIN"); pub const AGENT_CLUSTER_ROLE_NAME: &str = "kafka-agent-clusterrole"; pub fn build_agent_deployment( @@ -44,32 +57,32 @@ pub fn build_agent_deployment( let mut cb_agent = new_container_builder(&AGENT_CONTAINER_NAME); let mut pod_builder = PodBuilder::new(); - match &agent_config.authentication { - UserProvidedAuthentication::Anonymous(_) => {} - UserProvidedAuthentication::Tls(credential) => credential - .add_volumes_and_mounts(&mut pod_builder, vec![&mut cb_agent]) - .expect("the agent Pod has no other volume or mount for the TLS client certificate"), + add_discovery_volume_and_mount(cluster, &mut pod_builder, &mut cb_agent); + match &agent_config.connection { + AgentConnection::Plaintext => {} + AgentConnection::Tls { + server_ca_secret_class, + } => { + add_kafka_tls_volume_and_mount(server_ca_secret_class, &mut pod_builder, &mut cb_agent) + } + AgentConnection::MutualTls { + server_ca_secret_class, + client_credential, + } => { + add_kafka_tls_volume_and_mount(server_ca_secret_class, &mut pod_builder, &mut cb_agent); + client_credential + .add_volumes_and_mounts(&mut pod_builder, vec![&mut cb_agent]) + .expect( + "the agent Pod has no other volume or mount for the TLS client certificate", + ); + } } cb_agent .image(&agent_config.image) .command(vec!["stackable-kafka-agent".to_owned()]) .args(vec!["run".to_owned()]) - .add_env_var_from_source( - "KUBERNETES_NODE_NAME", - EnvVarSource { - field_ref: Some(ObjectFieldSelector { - field_path: "spec.nodeName".to_owned(), - ..ObjectFieldSelector::default() - }), - ..EnvVarSource::default() - }, - ) - // Saves the agent from looking the domain up via the kubelet, which needs extra RBAC. - .add_env_var( - "KUBERNETES_CLUSTER_DOMAIN", - cluster.cluster_domain.to_string(), - ) + .add_env_vars(agent_env_vars(cluster, agent_config)) .resources( ResourceRequirementsBuilder::new() .with_cpu_request("100m") @@ -139,6 +152,80 @@ pub fn build_agent_role_binding(cluster: &ValidatedCluster) -> RoleBinding { } } +fn agent_env_vars(cluster: &ValidatedCluster, agent_config: &ValidatedAgentConfig) -> EnvVarSet { + let node_name_env_var = EnvVar { + name: KUBERNETES_NODE_NAME.to_string(), + value: None, + value_from: Some(EnvVarSource { + field_ref: Some(ObjectFieldSelector { + field_path: "spec.nodeName".to_owned(), + ..ObjectFieldSelector::default() + }), + ..EnvVarSource::default() + }), + }; + + EnvVarSet::new() + .with_value(&ENV_CLUSTER_NAME, cluster.name.to_string()) + .with_value(&ENV_CLUSTER_NAMESPACE, cluster.namespace.to_string()) + .with_value( + &ENV_CLIENT_SECURITY, + ClientSecurity::from(&agent_config.connection).to_string(), + ) + // Saves the agent from looking the domain up via the kubelet, which needs extra RBAC. + .with_value( + &KUBERNETES_CLUSTER_DOMAIN, + cluster.cluster_domain.to_string(), + ) + .with_env_var(node_name_env_var) + .expect("the env var name is a valid EnvVarName") +} + +fn add_discovery_volume_and_mount( + cluster: &ValidatedCluster, + pod_builder: &mut PodBuilder, + container_builder: &mut ContainerBuilder, +) { + pod_builder + .add_volume( + VolumeBuilder::new(&*DISCOVERY_VOLUME_NAME) + .with_config_map(cluster.name.to_string()) + .build(), + ) + .expect("the volume name is statically defined and must not collide"); + container_builder + .add_volume_mount(&*DISCOVERY_VOLUME_NAME, DISCOVERY_DIR) + .expect("the mount path is statically defined and must not collide"); +} + +fn add_kafka_tls_volume_and_mount( + secret_class: &SecretClassName, + pod_builder: &mut PodBuilder, + container_builder: &mut ContainerBuilder, +) { + pod_builder + .add_volume( + VolumeBuilder::new(&*KAFKA_TLS_VOLUME_NAME) + .ephemeral( + SecretOperatorVolumeSourceBuilder::new( + secret_class, + SecretClassVolumeProvisionParts::Public, + ) + .with_format(SecretFormat::TlsPem) + .build() + .expect( + "the annotations are built from a valid SecretClass name and constant \ + values", + ), + ) + .build(), + ) + .expect("the volume name is statically defined and must not collide"); + container_builder + .add_volume_mount(&*KAFKA_TLS_VOLUME_NAME, KAFKA_TLS_DIR) + .expect("the mount path is statically defined and must not collide"); +} + /// The name of all agent resources. Kinds don't share a namespace, so unlike a suffix per kind, this /// cannot collide with the resources of a cluster named `-agent`. fn agent_name(cluster: &ValidatedCluster) -> String { @@ -264,12 +351,38 @@ mod tests { assert_eq!(pod_spec["containers"][0]["args"], json!(["run"])); assert_eq!( pod_spec["containers"][0]["volumeMounts"], - json!([{"mountPath": "/stackable/secrets/tls-management-access", "name": "tls-management-access"}]) + json!([ + {"mountPath": "/stackable/discovery", "name": "discovery"}, + {"mountPath": "/stackable/secrets/tls-kafka", "name": "tls-kafka"}, + {"mountPath": "/stackable/secrets/tls-management-access", "name": "tls-management-access"} + ]) ); - assert_eq!(pod_spec["volumes"][0]["name"], "tls-management-access"); assert_eq!( - pod_spec["volumes"][0]["ephemeral"]["volumeClaimTemplate"]["metadata"]["annotations"]["secrets.stackable.tech/class"], - "tls" + pod_spec["volumes"][0]["configMap"], + json!({"name": "simple-kafka"}) + ); + assert_eq!( + pod_spec["volumes"][1]["ephemeral"]["volumeClaimTemplate"]["metadata"]["annotations"], + json!({ + "secrets.stackable.tech/class": "tls", + "secrets.stackable.tech/format": "tls-pem", + "secrets.stackable.tech/provision-parts": "public" + }) + ); + assert_eq!(pod_spec["volumes"][2]["name"], "tls-management-access"); + + assert_eq!( + pod_spec["containers"][0]["env"], + json!([ + {"name": "KAFKA_CLIENT_SECURITY", "value": "mutual-tls"}, + {"name": "KAFKA_CLUSTER_NAME", "value": "simple-kafka"}, + {"name": "KAFKA_CLUSTER_NAMESPACE", "value": "default"}, + {"name": "KUBERNETES_CLUSTER_DOMAIN", "value": "cluster.local"}, + { + "name": "KUBERNETES_NODE_NAME", + "valueFrom": {"fieldRef": {"fieldPath": "spec.nodeName"}} + } + ]) ); } @@ -280,20 +393,25 @@ mod tests { )); assert_eq!( - deployment["spec"]["template"]["spec"]["volumes"], - json!([{"name": "tls-management-access", "secret": {"optional": false, "secretName": "my-cert"}}]) + deployment["spec"]["template"]["spec"]["volumes"][2], + json!({"name": "tls-management-access", "secret": {"optional": false, "secretName": "my-cert"}}) ); } #[test] - fn deployment_with_anonymous_authentication_mounts_no_credential() { + fn deployment_with_anonymous_authentication_verifies_brokers_without_credential() { let deployment = agent_deployment(&cluster( "{enabled: true, authentication: {userProvided: {anonymous: {}}}}", )); let pod_spec = &deployment["spec"]["template"]["spec"]; - assert_eq!(pod_spec["volumes"], Value::Null); - assert_eq!(pod_spec["containers"][0]["volumeMounts"], Value::Null); + assert_eq!(pod_spec["volumes"][0]["name"], "discovery"); + assert_eq!(pod_spec["volumes"][1]["name"], "tls-kafka"); + assert_eq!(pod_spec["volumes"][2], Value::Null); + assert_eq!( + pod_spec["containers"][0]["env"][0], + json!({"name": "KAFKA_CLIENT_SECURITY", "value": "tls"}) + ); } #[test] diff --git a/rust/operator-binary/src/controller/build/resource/statefulset.rs b/rust/operator-binary/src/controller/build/resource/statefulset.rs index 4ea3b886f..ed56b0928 100644 --- a/rust/operator-binary/src/controller/build/resource/statefulset.rs +++ b/rust/operator-binary/src/controller/build/resource/statefulset.rs @@ -982,7 +982,7 @@ mod tests { #[test] fn quorum_manager_container_env_can_be_overridden() { let resolved_product_image = test_resolved_product_image(); - let kafka_security = crate::controller::build::security::tests::kerberos(); + let kafka_security = crate::controller::test_support::kerberos(); assert!( kafka_security.has_kerberos_enabled(), "sanity check failed: the fixture must have Kerberos enabled so that KRB5_CONFIG \ diff --git a/rust/operator-binary/src/controller/build/security.rs b/rust/operator-binary/src/controller/build/security.rs index 0906bf7ef..74dbc0149 100644 --- a/rust/operator-binary/src/controller/build/security.rs +++ b/rust/operator-binary/src/controller/build/security.rs @@ -17,7 +17,6 @@ use stackable_operator::{ }, commons::secret_class::SecretClassVolumeProvisionParts, constant, - crd::authentication::core, k8s_openapi::api::core::v1::Volume, shared::time::Duration, v2::types::kubernetes::VolumeName, @@ -277,7 +276,7 @@ pub fn add_broker_volume_and_volume_mounts( requested_secret_lifetime: &Duration, ) -> Result<(), Error> { // add tls (server or client authentication volumes) if required - if let Some(tls_server_secret_class) = tls_secret_class(security) { + if let Some(tls_server_secret_class) = security.tls_secret_class() { // We have to mount tls pem files for kcat's readiness-probe command (the mount can be // used directly) pod_builder @@ -607,19 +606,6 @@ pub fn controller_config_settings(security: &ValidatedKafkaSecurity) -> BTreeMap config } -/// Returns the `SecretClass` provided in a `AuthenticationClass` for TLS. -fn tls_secret_class(security: &ValidatedKafkaSecurity) -> Option<&str> { - security - .tls_client_authentication_class() - .and_then(|auth_class| match &auth_class.spec.provider { - core::v1alpha1::AuthenticationClassProvider::Tls(tls) => { - tls.client_cert_secret_class.as_deref() - } - _ => None, - }) - .or_else(|| security.tls_server_secret_class()) -} - /// Creates ephemeral volumes to mount the `SecretClass` into the Pods for kcat client fn create_kcat_tls_volume( volume_name: &VolumeName, @@ -711,107 +697,13 @@ fn kcat_client_sasl_ssl(cert_directory: &str, service_name: &str) -> Vec } #[cfg(test)] -pub(crate) mod tests { - use std::{collections::BTreeMap, str::FromStr}; - - use stackable_operator::{ - builder::meta::ObjectMetaBuilder, - crd::authentication::{core, kerberos, tls}, - v2::types::kubernetes::SecretClassName, - }; +mod tests { + use std::collections::BTreeMap; use super::*; - use crate::crd::authentication::ResolvedAuthenticationClasses; - - fn tls_auth_class() -> core::v1alpha1::AuthenticationClass { - core::v1alpha1::AuthenticationClass { - metadata: ObjectMetaBuilder::new().name("tls-auth").build(), - spec: core::v1alpha1::AuthenticationClassSpec { - provider: core::v1alpha1::AuthenticationClassProvider::Tls( - tls::v1alpha1::AuthenticationProvider { - client_cert_secret_class: Some("client-auth-secret-class".to_string()), - }, - ), - }, - } - } - - fn kerberos_auth_class() -> core::v1alpha1::AuthenticationClass { - core::v1alpha1::AuthenticationClass { - metadata: ObjectMetaBuilder::new().name("kerberos-auth").build(), - spec: core::v1alpha1::AuthenticationClassSpec { - provider: core::v1alpha1::AuthenticationClassProvider::Kerberos( - kerberos::v1alpha1::AuthenticationProvider { - kerberos_secret_class: "kerberos-secret-class".to_string(), - }, - ), - }, - } - } - - fn no_auth() -> ResolvedAuthenticationClasses { - ResolvedAuthenticationClasses::new(vec![]) - } - - /// Plaintext: no TLS, no authentication, no OPA. - pub(crate) fn plaintext() -> ValidatedKafkaSecurity { - ValidatedKafkaSecurity::new( - no_auth(), - SecretClassName::from_str("tls").expect("tls secret class name is valid"), - None, - None, - ) - } - - /// Server TLS only (encryption without client authentication). - fn server_tls() -> ValidatedKafkaSecurity { - ValidatedKafkaSecurity::new( - no_auth(), - SecretClassName::from_str("tls").expect("tls secret class name is valid"), - Some("tls".parse().unwrap()), - None, - ) - } - - /// Mutual TLS (client-certificate authentication). - fn client_auth_tls() -> ValidatedKafkaSecurity { - ValidatedKafkaSecurity::new( - ResolvedAuthenticationClasses::new(vec![tls_auth_class()]), - SecretClassName::from_str("tls").expect("tls secret class name is valid"), - Some("tls".parse().unwrap()), - None, - ) - } - - /// Kerberos, which also requires server and internal TLS. - pub(crate) fn kerberos() -> ValidatedKafkaSecurity { - ValidatedKafkaSecurity::new( - ResolvedAuthenticationClasses::new(vec![kerberos_auth_class()]), - SecretClassName::from_str("tls").expect("tls secret class name is valid"), - Some("tls".parse().unwrap()), - None, - ) - } - - /// Internal TLS only (broker/controller encryption without client TLS). - fn internal_tls() -> ValidatedKafkaSecurity { - ValidatedKafkaSecurity::new( - no_auth(), - SecretClassName::from_str("tls").expect("tls secret class name is valid"), - None, - None, - ) - } - - /// OPA authorization with a TLS truststore (internal + server TLS also enabled). - fn opa() -> ValidatedKafkaSecurity { - ValidatedKafkaSecurity::new( - no_auth(), - SecretClassName::from_str("tls").expect("tls secret class name is valid"), - Some("tls".parse().unwrap()), - Some("opa-tls".parse().unwrap()), - ) - } + use crate::controller::test_support::{ + client_auth_tls, internal_tls, kerberos, opa, plaintext, server_tls, + }; fn as_str_vec(commands: &[String]) -> Vec<&str> { commands.iter().map(String::as_str).collect() diff --git a/rust/operator-binary/src/controller/security.rs b/rust/operator-binary/src/controller/security.rs index 90bd62178..a97c732f2 100644 --- a/rust/operator-binary/src/controller/security.rs +++ b/rust/operator-binary/src/controller/security.rs @@ -101,6 +101,18 @@ impl ValidatedKafkaSecurity { .get_tls_authentication_class() } + /// The `SecretClass` of a TLS `AuthenticationClass`, otherwise the server `SecretClass`. + pub fn tls_secret_class(&self) -> Option<&str> { + self.tls_client_authentication_class() + .and_then(|auth_class| match &auth_class.spec.provider { + core::v1alpha1::AuthenticationClassProvider::Tls(tls) => { + tls.client_cert_secret_class.as_deref() + } + _ => None, + }) + .or_else(|| self.tls_server_secret_class()) + } + /// Retrieve the internal `SecretClass`. Internal (inter-broker) TLS is mandatory, so this is /// always set (defaults to `tls`). pub fn tls_internal_secret_class(&self) -> &str { diff --git a/rust/operator-binary/src/controller/validate.rs b/rust/operator-binary/src/controller/validate.rs index 1280baf0b..6d91613ed 100644 --- a/rust/operator-binary/src/controller/validate.rs +++ b/rust/operator-binary/src/controller/validate.rs @@ -24,14 +24,15 @@ use stackable_operator::{ validate_logging_configuration_for_container, }, role_utils::{JavaCommonConfig, Role, with_validated_config}, - types::kubernetes::ConfigMapName, + types::kubernetes::{ConfigMapName, SecretClassName}, }, }; use crate::{ controller::{ - RoleGroupName, ValidatedAgentConfig, ValidatedCluster, ValidatedClusterConfig, - ValidatedKafkaConfig, ValidatedRoleConfig, ValidatedRoleGroupConfig, + AgentConnection, RoleGroupName, ValidatedAgentConfig, ValidatedCluster, + ValidatedClusterConfig, ValidatedKafkaConfig, ValidatedRoleConfig, + ValidatedRoleGroupConfig, dereference::DereferencedObjects, security::{self, ValidatedKafkaSecurity}, }, @@ -135,6 +136,12 @@ pub enum Error { #[snafu(display("TLS management access requires the cluster to use TLS"))] TlsManagementAccessRequiresTls, + + #[snafu(display("the SecretClass {secret_class:?} of the client listener is invalid"))] + ParseClientListenerSecretClass { + source: stackable_operator::v2::macros::attributed_string_type::Error, + secret_class: String, + }, } /// Validated logging configuration for a Kafka role group's Kafka and (optional) Vector @@ -369,15 +376,14 @@ pub fn validate( metadata_manager == crate::crd::MetadataManager::KRaft, ManagementAccessRequiresKraftSnafu ); - validate_management_access( - &management_access.authentication.user_provided, - &kafka_security, - )?; Ok(ValidatedAgentConfig { image: operator_image .context(ManagementAccessRequiresOperatorImageSnafu)? .to_owned(), - authentication: management_access.authentication.user_provided.clone(), + connection: validate_agent_connection( + &management_access.authentication.user_provided, + &kafka_security, + )?, }) }) .transpose()?; @@ -490,27 +496,49 @@ fn inject_cluster_id(env_overrides: EnvVarSet, cluster_id: Option<&str>) -> EnvV } } -/// Rejects management access authentication config that cannot connect to the cluster. -fn validate_management_access( +/// Determines how the agent connects to the cluster, and rejects management access authentication +/// config that cannot connect to it. +fn validate_agent_connection( authentication: &v1alpha1::UserProvidedAuthentication, security: &ValidatedKafkaSecurity, -) -> Result<()> { +) -> Result { ensure!( !security.has_kerberos_enabled(), ManagementAccessWithKerberosSnafu ); - match authentication { - v1alpha1::UserProvidedAuthentication::Anonymous(_) => ensure!( - security.tls_client_authentication_class().is_none(), - AnonymousManagementAccessWithClientAuthenticationSnafu - ), - v1alpha1::UserProvidedAuthentication::Tls(_) => { - ensure!(security.tls_enabled(), TlsManagementAccessRequiresTlsSnafu) + let server_ca_secret_class = security + .tls_secret_class() + .map(|secret_class| { + SecretClassName::from_str(secret_class) + .context(ParseClientListenerSecretClassSnafu { secret_class }) + }) + .transpose()?; + + match (authentication, server_ca_secret_class) { + (v1alpha1::UserProvidedAuthentication::Anonymous(_), server_ca_secret_class) => { + ensure!( + security.tls_client_authentication_class().is_none(), + AnonymousManagementAccessWithClientAuthenticationSnafu + ); + Ok(match server_ca_secret_class { + None => AgentConnection::Plaintext, + Some(server_ca_secret_class) => AgentConnection::Tls { + server_ca_secret_class, + }, + }) + } + (v1alpha1::UserProvidedAuthentication::Tls(_), None) => { + TlsManagementAccessRequiresTlsSnafu.fail() } + ( + v1alpha1::UserProvidedAuthentication::Tls(client_credential), + Some(server_ca_secret_class), + ) => Ok(AgentConnection::MutualTls { + server_ca_secret_class, + client_credential: client_credential.clone(), + }), } - - Ok(()) } #[cfg(test)] @@ -522,13 +550,17 @@ mod tests { crd::management_access::v1alpha1::{Anonymous, TlsClientCredential}, v2::{ builder::pod::container::EnvVarSet, - types::{kubernetes::SecretName, operator::RoleGroupName}, + types::{ + kubernetes::{SecretClassName, SecretName}, + operator::RoleGroupName, + }, }, }; - use super::{Error, KAFKA_CLUSTER_ID, inject_cluster_id, validate_management_access}; + use super::{Error, KAFKA_CLUSTER_ID, inject_cluster_id, validate_agent_connection}; use crate::{ controller::{ + AgentConnection, security::ValidatedKafkaSecurity, test_support::{ app_version_label, client_auth_tls, kerberos, minimal_kafka, plaintext, server_tls, @@ -542,30 +574,59 @@ mod tests { UserProvidedAuthentication::Anonymous(Anonymous::default()) } - fn tls() -> UserProvidedAuthentication { - UserProvidedAuthentication::Tls(TlsClientCredential::SecretName( + fn agent_cert() -> TlsClientCredential { + TlsClientCredential::SecretName( SecretName::from_str("agent-cert").expect("the Secret name should be valid"), - )) + ) + } + + fn tls() -> UserProvidedAuthentication { + UserProvidedAuthentication::Tls(agent_cert()) + } + + fn tls_secret_class(name: &str) -> SecretClassName { + SecretClassName::from_str(name).expect("the SecretClass name should be valid") } #[rstest] - #[case::plaintext_anonymous(plaintext(), anonymous())] - #[case::server_tls_anonymous(server_tls(), anonymous())] - #[case::server_tls_with_credential(server_tls(), tls())] - #[case::client_auth_with_credential(client_auth_tls(), tls())] - fn accept_management_access( + #[case::plaintext_anonymous(plaintext(), anonymous(), AgentConnection::Plaintext)] + #[case::server_tls_anonymous( + server_tls(), + anonymous(), + AgentConnection::Tls { server_ca_secret_class: tls_secret_class("tls") } + )] + #[case::server_tls_with_credential( + server_tls(), + tls(), + AgentConnection::MutualTls { + server_ca_secret_class: tls_secret_class("tls"), + client_credential: agent_cert(), + } + )] + #[case::client_auth_with_credential( + client_auth_tls(), + tls(), + AgentConnection::MutualTls { + server_ca_secret_class: tls_secret_class("client-auth-secret-class"), + client_credential: agent_cert(), + } + )] + fn agent_connection( #[case] security: ValidatedKafkaSecurity, #[case] authentication: UserProvidedAuthentication, + #[case] expected: AgentConnection, ) { - validate_management_access(&authentication, &security) + let connection = validate_agent_connection(&authentication, &security) .expect("the management access should be valid"); + + assert_eq!(expected, connection); } #[rstest] #[case::anonymous(anonymous())] #[case::tls(tls())] fn reject_management_access_with_kerberos(#[case] authentication: UserProvidedAuthentication) { - let error = validate_management_access(&authentication, &kerberos()) + let error = validate_agent_connection(&authentication, &kerberos()) .expect_err("the management access should be invalid"); assert!(matches!(error, Error::ManagementAccessWithKerberos)); @@ -573,7 +634,7 @@ mod tests { #[test] fn reject_anonymous_management_access_with_client_authentication() { - let error = validate_management_access(&anonymous(), &client_auth_tls()) + let error = validate_agent_connection(&anonymous(), &client_auth_tls()) .expect_err("the management access should be invalid"); assert!(matches!( @@ -584,7 +645,7 @@ mod tests { #[test] fn reject_tls_management_access_without_tls() { - let error = validate_management_access(&tls(), &plaintext()) + let error = validate_agent_connection(&tls(), &plaintext()) .expect_err("the management access should be invalid"); assert!(matches!(error, Error::TlsManagementAccessRequiresTls)); diff --git a/rust/shared/Cargo.toml b/rust/shared/Cargo.toml new file mode 100644 index 000000000..a26223f1c --- /dev/null +++ b/rust/shared/Cargo.toml @@ -0,0 +1,17 @@ +[package] +name = "stackable-kafka-shared" +description = "Types and contracts shared by the Stackable operator and agent for Apache Kafka" +version.workspace = true +authors.workspace = true +license.workspace = true +edition.workspace = true +repository.workspace = true +publish = false + +[dependencies] +stackable-operator.workspace = true + +strum.workspace = true + +[dev-dependencies] +rstest.workspace = true diff --git a/rust/shared/src/agent.rs b/rust/shared/src/agent.rs new file mode 100644 index 000000000..617360172 --- /dev/null +++ b/rust/shared/src/agent.rs @@ -0,0 +1,42 @@ +//! Paths and environment variables through which the operator configures the agent. + +use std::str::FromStr; + +use stackable_operator::{constant, v2::builder::pod::container::EnvVarName}; +use strum::{Display, EnumString}; + +pub const DISCOVERY_DIR: &str = "/stackable/discovery"; + +/// Directory of the CA certificate the Kafka brokers are verified with. +pub const KAFKA_TLS_DIR: &str = "/stackable/secrets/tls-kafka"; + +constant!(pub ENV_CLUSTER_NAME: EnvVarName = "KAFKA_CLUSTER_NAME"); +constant!(pub ENV_CLUSTER_NAMESPACE: EnvVarName = "KAFKA_CLUSTER_NAMESPACE"); +constant!(pub ENV_CLIENT_SECURITY: EnvVarName = "KAFKA_CLIENT_SECURITY"); + +/// How the agent connects to the brokers. +#[derive(Clone, Copy, Debug, Display, EnumString, Eq, PartialEq)] +#[strum(serialize_all = "kebab-case")] +pub enum ClientSecurity { + Plaintext, + Tls, + MutualTls, +} + +#[cfg(test)] +mod tests { + use std::str::FromStr; + + use rstest::rstest; + + use super::*; + + #[rstest] + #[case::plaintext(ClientSecurity::Plaintext, "plaintext")] + #[case::tls(ClientSecurity::Tls, "tls")] + #[case::mutual_tls(ClientSecurity::MutualTls, "mutual-tls")] + fn roundtrip(#[case] security: ClientSecurity, #[case] value: &str) { + assert_eq!(value, security.to_string()); + assert_eq!(Ok(security), ClientSecurity::from_str(value)); + } +} diff --git a/rust/shared/src/lib.rs b/rust/shared/src/lib.rs new file mode 100644 index 000000000..f17bc55db --- /dev/null +++ b/rust/shared/src/lib.rs @@ -0,0 +1 @@ +pub mod agent; From 87a290eb566deda46e36e3f0d6c9d5eb560a5a00 Mon Sep 17 00:00:00 2001 From: Benedikt Labrenz Date: Mon, 26 Oct 2026 11:30:18 +0100 Subject: [PATCH 33/93] use AgentResourceNames from operator-rs --- Cargo.lock | 18 +++--- Cargo.nix | 18 +++--- rust/operator-binary/src/controller.rs | 9 +++ .../src/controller/build/resource/agent.rs | 59 +++++++------------ .../controller/build/resource/statefulset.rs | 1 + rust/operator-binary/src/main.rs | 2 +- 6 files changed, 49 insertions(+), 58 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index f092e03d4..12e294d61 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1712,7 +1712,7 @@ dependencies = [ [[package]] name = "k8s-version" version = "0.1.4" -source = "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#a2a7d0caf7b00c332cd299768eea19b1a3a00a51" +source = "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#079f7413186c44b0bf9723c1361ea955613c39b1" dependencies = [ "darling 0.24.1", "regex", @@ -3168,7 +3168,7 @@ checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596" [[package]] name = "stackable-certs" version = "0.4.1" -source = "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#a2a7d0caf7b00c332cd299768eea19b1a3a00a51" +source = "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#079f7413186c44b0bf9723c1361ea955613c39b1" dependencies = [ "const-oid", "ecdsa", @@ -3235,7 +3235,7 @@ dependencies = [ [[package]] name = "stackable-operator" version = "0.121.0" -source = "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#a2a7d0caf7b00c332cd299768eea19b1a3a00a51" +source = "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#079f7413186c44b0bf9723c1361ea955613c39b1" dependencies = [ "base64 0.23.1", "clap", @@ -3279,7 +3279,7 @@ dependencies = [ [[package]] name = "stackable-operator-derive" version = "0.3.1" -source = "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#a2a7d0caf7b00c332cd299768eea19b1a3a00a51" +source = "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#079f7413186c44b0bf9723c1361ea955613c39b1" dependencies = [ "darling 0.24.1", "proc-macro2", @@ -3290,7 +3290,7 @@ dependencies = [ [[package]] name = "stackable-shared" version = "0.2.0" -source = "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#a2a7d0caf7b00c332cd299768eea19b1a3a00a51" +source = "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#079f7413186c44b0bf9723c1361ea955613c39b1" dependencies = [ "jiff", "k8s-openapi", @@ -3307,7 +3307,7 @@ dependencies = [ [[package]] name = "stackable-telemetry" version = "0.6.5" -source = "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#a2a7d0caf7b00c332cd299768eea19b1a3a00a51" +source = "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#079f7413186c44b0bf9723c1361ea955613c39b1" dependencies = [ "axum", "clap", @@ -3331,7 +3331,7 @@ dependencies = [ [[package]] name = "stackable-versioned" version = "0.13.0" -source = "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#a2a7d0caf7b00c332cd299768eea19b1a3a00a51" +source = "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#079f7413186c44b0bf9723c1361ea955613c39b1" dependencies = [ "kube", "schemars", @@ -3344,7 +3344,7 @@ dependencies = [ [[package]] name = "stackable-versioned-macros" version = "0.13.0" -source = "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#a2a7d0caf7b00c332cd299768eea19b1a3a00a51" +source = "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#079f7413186c44b0bf9723c1361ea955613c39b1" dependencies = [ "convert_case", "darling 0.24.1", @@ -3361,7 +3361,7 @@ dependencies = [ [[package]] name = "stackable-webhook" version = "0.10.0" -source = "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#a2a7d0caf7b00c332cd299768eea19b1a3a00a51" +source = "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#079f7413186c44b0bf9723c1361ea955613c39b1" dependencies = [ "arc-swap", "async-trait", diff --git a/Cargo.nix b/Cargo.nix index e3bd3c098..3494b58fe 100644 --- a/Cargo.nix +++ b/Cargo.nix @@ -5455,7 +5455,7 @@ rec { workspace_member = null; src = pkgs.fetchgit { url = "https://github.com/stackabletech/operator-rs.git"; - rev = "a2a7d0caf7b00c332cd299768eea19b1a3a00a51"; + rev = "079f7413186c44b0bf9723c1361ea955613c39b1"; sha256 = "0cgziqra8097hp05ynib1qpw4c95n972f4w2rk9l3llyp8r1vmci"; }; libName = "k8s_version"; @@ -10388,7 +10388,7 @@ rec { workspace_member = null; src = pkgs.fetchgit { url = "https://github.com/stackabletech/operator-rs.git"; - rev = "a2a7d0caf7b00c332cd299768eea19b1a3a00a51"; + rev = "079f7413186c44b0bf9723c1361ea955613c39b1"; sha256 = "0cgziqra8097hp05ynib1qpw4c95n972f4w2rk9l3llyp8r1vmci"; }; libName = "stackable_certs"; @@ -10660,7 +10660,7 @@ rec { workspace_member = null; src = pkgs.fetchgit { url = "https://github.com/stackabletech/operator-rs.git"; - rev = "a2a7d0caf7b00c332cd299768eea19b1a3a00a51"; + rev = "079f7413186c44b0bf9723c1361ea955613c39b1"; sha256 = "0cgziqra8097hp05ynib1qpw4c95n972f4w2rk9l3llyp8r1vmci"; }; libName = "stackable_operator"; @@ -10859,7 +10859,7 @@ rec { workspace_member = null; src = pkgs.fetchgit { url = "https://github.com/stackabletech/operator-rs.git"; - rev = "a2a7d0caf7b00c332cd299768eea19b1a3a00a51"; + rev = "079f7413186c44b0bf9723c1361ea955613c39b1"; sha256 = "0cgziqra8097hp05ynib1qpw4c95n972f4w2rk9l3llyp8r1vmci"; }; procMacro = true; @@ -10894,7 +10894,7 @@ rec { workspace_member = null; src = pkgs.fetchgit { url = "https://github.com/stackabletech/operator-rs.git"; - rev = "a2a7d0caf7b00c332cd299768eea19b1a3a00a51"; + rev = "079f7413186c44b0bf9723c1361ea955613c39b1"; sha256 = "0cgziqra8097hp05ynib1qpw4c95n972f4w2rk9l3llyp8r1vmci"; }; libName = "stackable_shared"; @@ -10975,7 +10975,7 @@ rec { workspace_member = null; src = pkgs.fetchgit { url = "https://github.com/stackabletech/operator-rs.git"; - rev = "a2a7d0caf7b00c332cd299768eea19b1a3a00a51"; + rev = "079f7413186c44b0bf9723c1361ea955613c39b1"; sha256 = "0cgziqra8097hp05ynib1qpw4c95n972f4w2rk9l3llyp8r1vmci"; }; libName = "stackable_telemetry"; @@ -11085,7 +11085,7 @@ rec { workspace_member = null; src = pkgs.fetchgit { url = "https://github.com/stackabletech/operator-rs.git"; - rev = "a2a7d0caf7b00c332cd299768eea19b1a3a00a51"; + rev = "079f7413186c44b0bf9723c1361ea955613c39b1"; sha256 = "0cgziqra8097hp05ynib1qpw4c95n972f4w2rk9l3llyp8r1vmci"; }; libName = "stackable_versioned"; @@ -11131,7 +11131,7 @@ rec { workspace_member = null; src = pkgs.fetchgit { url = "https://github.com/stackabletech/operator-rs.git"; - rev = "a2a7d0caf7b00c332cd299768eea19b1a3a00a51"; + rev = "079f7413186c44b0bf9723c1361ea955613c39b1"; sha256 = "0cgziqra8097hp05ynib1qpw4c95n972f4w2rk9l3llyp8r1vmci"; }; procMacro = true; @@ -11195,7 +11195,7 @@ rec { workspace_member = null; src = pkgs.fetchgit { url = "https://github.com/stackabletech/operator-rs.git"; - rev = "a2a7d0caf7b00c332cd299768eea19b1a3a00a51"; + rev = "079f7413186c44b0bf9723c1361ea955613c39b1"; sha256 = "0cgziqra8097hp05ynib1qpw4c95n972f4w2rk9l3llyp8r1vmci"; }; libName = "stackable_webhook"; diff --git a/rust/operator-binary/src/controller.rs b/rust/operator-binary/src/controller.rs index 368bbdf2b..82ff790a2 100644 --- a/rust/operator-binary/src/controller.rs +++ b/rust/operator-binary/src/controller.rs @@ -38,6 +38,7 @@ use stackable_operator::{ shared::time::Duration, v2::{ HasName, HasUid, NameIsValidLabelValue, + agent_utils::AgentResourceNames, role_group_utils::ResourceNames, role_utils, types::{ @@ -263,6 +264,14 @@ impl ValidatedCluster { } } + /// Type-safe names for the resources of the agent. + pub fn agent_resource_names(&self) -> AgentResourceNames { + AgentResourceNames { + cluster_name: self.name.clone(), + product_name: PRODUCT_NAME.clone(), + } + } + /// Type-safe names for the resources of a given role group. pub(crate) fn role_group_resource_names( &self, diff --git a/rust/operator-binary/src/controller/build/resource/agent.rs b/rust/operator-binary/src/controller/build/resource/agent.rs index 19fcd4332..0f288f964 100644 --- a/rust/operator-binary/src/controller/build/resource/agent.rs +++ b/rust/operator-binary/src/controller/build/resource/agent.rs @@ -20,20 +20,19 @@ use stackable_operator::{ commons::secret_class::SecretClassVolumeProvisionParts, constant, k8s_openapi::{ - Resource, api::{ apps::v1::{Deployment, DeploymentSpec, DeploymentStrategy}, core::v1::{EnvVar, EnvVarSource, ObjectFieldSelector, ServiceAccount}, - rbac::v1::{ClusterRole, RoleBinding, RoleRef, Subject}, + rbac::v1::RoleBinding, }, apimachinery::pkg::apis::meta::v1::LabelSelector, }, - kube::api::ObjectMeta, v2::{ builder::{ meta::ownerreference_from_resource, pod::container::{EnvVarName, EnvVarSet, new_container_builder}, }, + rbac::{self, RbacResourceNames}, types::kubernetes::{ContainerName, SecretClassName, VolumeName}, }, }; @@ -48,12 +47,12 @@ constant!(DISCOVERY_VOLUME_NAME: VolumeName = "discovery"); constant!(KAFKA_TLS_VOLUME_NAME: VolumeName = "tls-kafka"); constant!(KUBERNETES_NODE_NAME: EnvVarName = "KUBERNETES_NODE_NAME"); constant!(KUBERNETES_CLUSTER_DOMAIN: EnvVarName = "KUBERNETES_CLUSTER_DOMAIN"); -pub const AGENT_CLUSTER_ROLE_NAME: &str = "kafka-agent-clusterrole"; pub fn build_agent_deployment( cluster: &ValidatedCluster, agent_config: &ValidatedAgentConfig, ) -> Deployment { + let resource_names = cluster.agent_resource_names(); let mut cb_agent = new_container_builder(&AGENT_CONTAINER_NAME); let mut pod_builder = PodBuilder::new(); @@ -100,7 +99,7 @@ pub fn build_agent_deployment( ) .image_pull_secrets_from_product_image(&cluster.image) .add_container(cb_agent.build()) - .service_account_name(agent_name(cluster)) + .service_account_name(resource_names.service_account_name().to_string()) .security_context( PodSecurityContextBuilder::with_stackable_defaults() .fs_group(1000) @@ -108,7 +107,12 @@ pub fn build_agent_deployment( ); Deployment { - metadata: agent_metadata(cluster), + metadata: ObjectMetaBuilder::new() + .name_and_namespace(cluster) + .name(resource_names.deployment_name().to_string()) + .ownerreference(ownerreference_from_resource(cluster, None, Some(true))) + .with_labels(recommended_labels_for_agent_resources(cluster)) + .build(), spec: Some(DeploymentSpec { replicas: Some(1), // Never run two agents at the same time, as for the operator itself. @@ -128,28 +132,20 @@ pub fn build_agent_deployment( } pub fn build_agent_service_account(cluster: &ValidatedCluster) -> ServiceAccount { - ServiceAccount { - metadata: agent_metadata(cluster), - ..ServiceAccount::default() - } + rbac::build_service_account( + cluster, + &cluster.agent_resource_names(), + recommended_labels_for_agent_resources(cluster), + ) } /// Binds the agent ServiceAccount to the agent ClusterRole deployed by the Helm chart. pub fn build_agent_role_binding(cluster: &ValidatedCluster) -> RoleBinding { - RoleBinding { - metadata: agent_metadata(cluster), - role_ref: RoleRef { - api_group: Some(ClusterRole::GROUP.to_owned()), - kind: ClusterRole::KIND.to_owned(), - name: AGENT_CLUSTER_ROLE_NAME.to_owned(), - }, - subjects: Some(vec![Subject { - kind: ServiceAccount::KIND.to_owned(), - name: agent_name(cluster), - namespace: Some(cluster.namespace.to_string()), - ..Subject::default() - }]), - } + rbac::build_role_binding( + cluster, + &cluster.agent_resource_names(), + recommended_labels_for_agent_resources(cluster), + ) } fn agent_env_vars(cluster: &ValidatedCluster, agent_config: &ValidatedAgentConfig) -> EnvVarSet { @@ -226,21 +222,6 @@ fn add_kafka_tls_volume_and_mount( .expect("the mount path is statically defined and must not collide"); } -/// The name of all agent resources. Kinds don't share a namespace, so unlike a suffix per kind, this -/// cannot collide with the resources of a cluster named `-agent`. -fn agent_name(cluster: &ValidatedCluster) -> String { - format!("{}-agent", cluster.name) -} - -fn agent_metadata(cluster: &ValidatedCluster) -> ObjectMeta { - ObjectMetaBuilder::new() - .name_and_namespace(cluster) - .name(agent_name(cluster)) - .ownerreference(ownerreference_from_resource(cluster, None, Some(true))) - .with_labels(recommended_labels_for_agent_resources(cluster)) - .build() -} - #[cfg(test)] mod tests { use serde_json::{Value, json}; diff --git a/rust/operator-binary/src/controller/build/resource/statefulset.rs b/rust/operator-binary/src/controller/build/resource/statefulset.rs index ed56b0928..b0da6a2de 100644 --- a/rust/operator-binary/src/controller/build/resource/statefulset.rs +++ b/rust/operator-binary/src/controller/build/resource/statefulset.rs @@ -35,6 +35,7 @@ use stackable_operator::{ product_logging::framework::{ STACKABLE_LOG_DIR, ValidatedContainerLogConfigChoice, vector_container, }, + rbac::RbacResourceNames, role_group_utils::ResourceNames, types::kubernetes::{ConfigMapKey, ContainerName, VolumeName}, }, diff --git a/rust/operator-binary/src/main.rs b/rust/operator-binary/src/main.rs index 8ef5b8398..73367def6 100644 --- a/rust/operator-binary/src/main.rs +++ b/rust/operator-binary/src/main.rs @@ -69,7 +69,7 @@ struct KafkaRun { #[clap(flatten)] common: RunArguments, - /// The image of this operator, which also contains the agent. Required to enable platform + /// The image of this operator, which also contains the agent. Required to enable management /// access on a KafkaCluster. #[arg(long, env)] operator_image: Option, From f094a2317a3c4bd2c23fef15f40181dbd6113a5e Mon Sep 17 00:00:00 2001 From: Benedikt Labrenz Date: Mon, 26 Oct 2026 14:56:33 +0100 Subject: [PATCH 34/93] update the CRD to reflect operator-rs changes and the CRD decision --- extra/crds.yaml | 150 ++++++++++++++----------- rust/shared/src/crd/topic.rs | 210 ++++++++++++++++++++++++++--------- 2 files changed, 241 insertions(+), 119 deletions(-) diff --git a/extra/crds.yaml b/extra/crds.yaml index 1824764a2..96d539fed 100644 --- a/extra/crds.yaml +++ b/extra/crds.yaml @@ -2269,76 +2269,78 @@ spec: description: |- A Kafka topic managed by the agent of the referenced Kafka cluster. - The referenced `KafkaCluster` must have `platformAccess` enabled or this resource will - not be reconciled. + The referenced `KafkaCluster` must have `clusterConfig.managementAccess` enabled or this + resource will not be reconciled. properties: - clusterName: - description: Name of a KafkaCluster in the namespace of this KafkaTopic resource. Immutable. - maxLength: 40 - minLength: 1 - type: string + clusterRef: + description: The KafkaCluster managing this topic. Immutable. + properties: + name: + description: Name of a KafkaCluster in the namespace of this KafkaTopic. + maxLength: 40 + minLength: 1 + type: string + required: + - name + type: object x-kubernetes-validations: - - message: clusterName is immutable + - message: clusterRef is immutable rule: self == oldSelf - config: + configOverrides: additionalProperties: type: string default: {} description: |- - Topic configs like `retention.ms`. Keys removed from this map are reset to the broker - default. Keys not listed here are not managed by the agent. - type: object - managementPolicy: - default: - adoption: Refuse - deletion: Retain - description: What an agent may do with a resource inside the product beyond applying its spec. - properties: - adoption: - default: Refuse - description: |- - Whether a resource that already exists in the product, but was not created by the - agent, is taken over. `Refuse` by default. - enum: - - Refuse - - Adopt - type: string - deletion: - default: Retain - description: |- - What happens to the resource in the product when this object is deleted. `Retain` by - default. - enum: - - Retain - - Delete - type: string + Topic configs like `retention.ms`. All other topic-level configs are reset to the broker + defaults, except the replication throttles set by partition reassignments. type: object + deletionPolicy: + default: Retain + description: What happens to a resource in the product when the object managing it is deleted. + enum: + - Retain + - Delete + type: string partitions: - description: Number of partitions. Can only be increased. + description: |- + Number of partitions. Can only be increased. Defaults to the broker setting + `num.partitions` when the topic is created. format: int32 minimum: 1.0 + nullable: true type: integer replicationFactor: - description: Number of replicas of each partition. Changes are not applied to existing topics. + description: |- + Number of replicas of each partition. Changes are not applied to existing topics. + Defaults to the broker setting `default.replication.factor` when the topic is created. format: int16 maximum: 32767.0 minimum: 1.0 + nullable: true type: integer topicName: - description: Name of the topic in Kafka. Immutable. + description: Name of the topic in Kafka. Defaults to `metadata.name`. Immutable. maxLength: 249 minLength: 1 - pattern: ^([a-zA-Z0-9-][a-zA-Z0-9._-]*|_([a-zA-Z0-9.-][a-zA-Z0-9._-]*)?|\.[a-zA-Z0-9_-][a-zA-Z0-9._-]*|\.\.[a-zA-Z0-9._-]+)$ + nullable: true + pattern: ^([a-zA-Z0-9._-]{3,}|[a-zA-Z0-9_-][a-zA-Z0-9._-]?|\.[a-zA-Z0-9_-])$ type: string - x-kubernetes-validations: - - message: topicName is immutable - rule: self == oldSelf + topicOperation: + default: + reconciliationPaused: false + description: Operations on this topic, independent of the `clusterOperation` of the KafkaCluster. + properties: + reconciliationPaused: + default: false + description: Stops the agent from changing the topic in Kafka. The status is still updated. + type: boolean + type: object required: - - clusterName - - partitions - - replicationFactor - - topicName + - clusterRef type: object + x-kubernetes-validations: + - message: topicName is immutable + rule: self.?topicName == oldSelf.?topicName status: nullable: true properties: @@ -2376,36 +2378,54 @@ spec: - type type: object type: array - partitions: - default: [] - description: The partitions of the topic in Kafka. - items: + config: + additionalProperties: properties: - index: - format: int32 - type: integer - replicas: - description: |- - Broker ids holding a replica, the preferred leader first. During a reassignment, this - includes the replicas being added and removed. - items: - format: int32 - type: integer - type: array + source: + description: Where the value comes from, like `DYNAMIC_TOPIC_CONFIG` or `DEFAULT_CONFIG`. + type: string + value: + description: The value, not set for sensitive configs. + nullable: true + type: string required: - - index - - replicas + - source type: object - type: array + default: {} + description: All effective topic configs in Kafka. + type: object + partitions: + description: The number of partitions in Kafka. + format: int32 + nullable: true + type: integer + replicationFactor: + description: |- + The replication factor, not counting replicas that a running reassignment removes. + Not set if the partitions have different replication factors. + format: int16 + maximum: 32767.0 + minimum: -32768.0 + nullable: true + type: integer topicId: description: The topic id in Kafka, as shown by `kafka-topics.sh`. nullable: true type: string + topicName: + description: The name of the topic in Kafka. + maxLength: 249 + minLength: 1 + nullable: true + pattern: ^([a-zA-Z0-9._-]{3,}|[a-zA-Z0-9_-][a-zA-Z0-9._-]?|\.[a-zA-Z0-9_-])$ + type: string type: object required: - spec title: KafkaTopic type: object + selectableFields: + - jsonPath: .spec.clusterRef.name served: true storage: true subresources: diff --git a/rust/shared/src/crd/topic.rs b/rust/shared/src/crd/topic.rs index 3a9ef6681..19bd870de 100644 --- a/rust/shared/src/crd/topic.rs +++ b/rust/shared/src/crd/topic.rs @@ -1,23 +1,23 @@ -use std::collections::BTreeMap; +use std::{collections::BTreeMap, str::FromStr}; use serde::{Deserialize, Serialize}; use stackable_operator::{ attributed_string_type, - crd::platform_access::v1alpha1::ManagementPolicy, + crd::management_access::v1alpha1::DeletionPolicy, k8s_openapi::apimachinery::pkg::apis::meta::v1::Condition, - kube::CustomResource, + kube::{CustomResource, ResourceExt}, schemars::{self, JsonSchema}, - v2::types::operator::ClusterName, + v2::{macros::attributed_string_type::Error as TopicNameError, types::operator::ClusterName}, versioned::versioned, }; attributed_string_type! { TopicName, - "The name of a Kafka topic, not `.` or `..` and not starting with `__`", + "The name of a Kafka topic, not `.` or `..`", "shop.orders", (min_length = 1), (max_length = 249), - (regex = r"^([a-zA-Z0-9-][a-zA-Z0-9._-]*|_([a-zA-Z0-9.-][a-zA-Z0-9._-]*)?|\.[a-zA-Z0-9_-][a-zA-Z0-9._-]*|\.\.[a-zA-Z0-9._-]+)$") + (regex = r"^([a-zA-Z0-9._-]{3,}|[a-zA-Z0-9_-][a-zA-Z0-9._-]?|\.[a-zA-Z0-9_-])$") } #[versioned( @@ -34,47 +34,73 @@ pub mod versioned { /// A Kafka topic managed by the agent of the referenced Kafka cluster. /// /// The referenced `KafkaCluster` must have `clusterConfig.managementAccess` enabled or this - /// resource will - /// not be reconciled. + /// resource will not be reconciled. #[versioned(crd( doc = "A Kafka topic, created and updated by the agent of the referenced Kafka cluster.", group = "kafka.stackable.tech", plural = "kafkatopics", status = "KafkaTopicStatus", + selectable = ".spec.clusterRef.name", namespaced ))] #[derive(Clone, CustomResource, Debug, Deserialize, JsonSchema, PartialEq, Serialize)] #[serde(rename_all = "camelCase")] + #[schemars(extend("x-kubernetes-validations" = [{ + "rule": "self.?topicName == oldSelf.?topicName", + "message": "topicName is immutable" + }]))] pub struct KafkaTopicSpec { - /// Name of a KafkaCluster in the namespace of this KafkaTopic resource. Immutable. + /// The KafkaCluster managing this topic. Immutable. #[schemars(extend("x-kubernetes-validations" = [{ "rule": "self == oldSelf", - "message": "clusterName is immutable" + "message": "clusterRef is immutable" }]))] - pub cluster_name: ClusterName, + pub cluster_ref: v1alpha1::KafkaClusterRef, - #[serde(default)] - pub management_policy: ManagementPolicy, - - /// Name of the topic in Kafka. Immutable. - #[schemars(extend("x-kubernetes-validations" = [{ - "rule": "self == oldSelf", - "message": "topicName is immutable" - }]))] - pub topic_name: TopicName, + /// Name of the topic in Kafka. Defaults to `metadata.name`. Immutable. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub topic_name: Option, - /// Number of partitions. Can only be increased. + /// Number of partitions. Can only be increased. Defaults to the broker setting + /// `num.partitions` when the topic is created. + #[serde(default, skip_serializing_if = "Option::is_none")] #[schemars(range(min = 1))] - pub partitions: i32, + pub partitions: Option, /// Number of replicas of each partition. Changes are not applied to existing topics. + /// Defaults to the broker setting `default.replication.factor` when the topic is created. + #[serde(default, skip_serializing_if = "Option::is_none")] #[schemars(range(min = 1))] - pub replication_factor: i16, + pub replication_factor: Option, + + /// Topic configs like `retention.ms`. All other topic-level configs are reset to the broker + /// defaults, except the replication throttles set by partition reassignments. + #[serde(default)] + pub config_overrides: BTreeMap, - /// Topic configs like `retention.ms`. Keys removed from this map are reset to the broker - /// default. Keys not listed here are not managed by the agent. + // no doc - docs in DeletionPolicy enum. #[serde(default)] - pub config: BTreeMap, + pub deletion_policy: DeletionPolicy, + + // no doc - docs in TopicOperation struct. + #[serde(default)] + pub topic_operation: v1alpha1::TopicOperation, + } + + #[derive(Clone, Debug, Deserialize, JsonSchema, PartialEq, Serialize)] + #[serde(rename_all = "camelCase")] + pub struct KafkaClusterRef { + /// Name of a KafkaCluster in the namespace of this KafkaTopic. + pub name: ClusterName, + } + + /// Operations on this topic, independent of the `clusterOperation` of the KafkaCluster. + #[derive(Clone, Debug, Default, Deserialize, JsonSchema, PartialEq, Serialize)] + #[serde(rename_all = "camelCase")] + pub struct TopicOperation { + /// Stops the agent from changing the topic in Kafka. The status is still updated. + #[serde(default)] + pub reconciliation_paused: bool, } #[derive(Clone, Debug, Default, Deserialize, JsonSchema, PartialEq, Serialize)] @@ -88,58 +114,105 @@ pub mod versioned { #[serde(default, skip_serializing_if = "Option::is_none")] pub topic_id: Option, - /// The partitions of the topic in Kafka. + /// The name of the topic in Kafka. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub topic_name: Option, + + /// The number of partitions in Kafka. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub partitions: Option, + + /// The replication factor, not counting replicas that a running reassignment removes. + /// Not set if the partitions have different replication factors. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub replication_factor: Option, + + /// All effective topic configs in Kafka. #[serde(default)] - pub partitions: Vec, + pub config: BTreeMap, } #[derive(Clone, Debug, Deserialize, JsonSchema, PartialEq, Serialize)] #[serde(rename_all = "camelCase")] - pub struct KafkaTopicPartitionStatus { - pub index: i32, + pub struct KafkaTopicConfigStatus { + /// The value, not set for sensitive configs. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub value: Option, - /// Broker ids holding a replica, the preferred leader first. During a reassignment, this - /// includes the replicas being added and removed. - pub replicas: Vec, + /// Where the value comes from, like `DYNAMIC_TOPIC_CONFIG` or `DEFAULT_CONFIG`. + pub source: String, + } +} + +impl v1alpha1::KafkaTopic { + /// The name of the topic in Kafka: `spec.topicName`, otherwise `metadata.name`. + /// + /// Fails if `metadata.name` is longer than a topic name may be. + pub fn topic_name(&self) -> Result { + match &self.spec.topic_name { + Some(topic_name) => Ok(topic_name.clone()), + None => TopicName::from_str(&self.name_any()), + } } } #[cfg(test)] mod tests { - use std::str::FromStr; - use rstest::rstest; - use stackable_operator::versioned::test_utils::RoundtripTestData; + use stackable_operator::{kube::api::ObjectMeta, versioned::test_utils::RoundtripTestData}; use super::*; impl RoundtripTestData for v1alpha1::KafkaTopicSpec { fn roundtrip_test_data() -> Vec { stackable_operator::utils::yaml_from_str_singleton_map(indoc::indoc! {r#" - - clusterName: simple-kafka - managementPolicy: - adoption: Adopt - deletion: Delete + - clusterRef: + name: simple-kafka topicName: shop.orders partitions: 6 replicationFactor: 3 - config: + configOverrides: retention.ms: "604800000" - - clusterName: simple-kafka - topicName: minimal - partitions: 1 - replicationFactor: 1 + deletionPolicy: Delete + topicOperation: + reconciliationPaused: true + - clusterRef: + name: simple-kafka "#}) .expect("the roundtrip test data should be valid YAML") } } + fn topic(metadata_name: &str, topic_name: Option<&str>) -> v1alpha1::KafkaTopic { + v1alpha1::KafkaTopic { + metadata: ObjectMeta { + name: Some(metadata_name.to_owned()), + ..ObjectMeta::default() + }, + spec: v1alpha1::KafkaTopicSpec { + cluster_ref: v1alpha1::KafkaClusterRef { + name: ClusterName::from_str("simple-kafka") + .expect("the cluster name should be valid"), + }, + topic_name: topic_name.map(TopicName::from_str_unsafe), + partitions: None, + replication_factor: None, + config_overrides: BTreeMap::new(), + deletion_policy: DeletionPolicy::default(), + topic_operation: v1alpha1::TopicOperation::default(), + }, + status: None, + } + } + #[rstest] #[case::dotted("shop.orders")] #[case::uppercase_and_underscore("Shop_Orders-1")] #[case::single_underscore("_internal")] #[case::underscore_only("_")] + #[case::internal_topic("__consumer_offsets")] #[case::leading_dot(".hidden")] + #[case::trailing_dot("a.")] #[case::three_dots("...")] fn accept_topic_name(#[case] name: &str) { assert!( @@ -152,7 +225,6 @@ mod tests { #[case::empty("")] #[case::dot(".")] #[case::two_dots("..")] - #[case::double_underscore_prefix("__consumer_offsets")] #[case::invalid_character("shop/orders")] #[case::too_long(&"a".repeat(250))] fn reject_topic_name(#[case] name: &str) { @@ -162,6 +234,27 @@ mod tests { ); } + #[rstest] + #[case::explicit("my-topic", Some("shop.orders"), "shop.orders")] + #[case::from_metadata_name("shop.orders", None, "shop.orders")] + fn effective_topic_name( + #[case] metadata_name: &str, + #[case] topic_name: Option<&str>, + #[case] expected: &str, + ) { + assert_eq!( + TopicName::from_str_unsafe(expected), + topic(metadata_name, topic_name) + .topic_name() + .expect("the topic name should be valid") + ); + } + + #[test] + fn reject_too_long_metadata_name_as_topic_name() { + assert!(topic(&"a".repeat(250), None).topic_name().is_err()); + } + #[test] fn schema_contains_validations() { let crd = serde_json::to_value( @@ -169,19 +262,28 @@ mod tests { .expect("the CRD should be mergeable"), ) .expect("the CRD should be serializable"); - let spec = &crd["spec"]["versions"][0]["schema"]["openAPIV3Schema"]["properties"]["spec"]["properties"]; + let version = &crd["spec"]["versions"][0]; + let spec = &version["schema"]["openAPIV3Schema"]["properties"]["spec"]; + let properties = &spec["properties"]; assert_eq!( - "self == oldSelf", - spec["clusterName"]["x-kubernetes-validations"][0]["rule"] + ".spec.clusterRef.name", + version["selectableFields"][0]["jsonPath"] ); assert_eq!( "self == oldSelf", - spec["topicName"]["x-kubernetes-validations"][0]["rule"] + properties["clusterRef"]["x-kubernetes-validations"][0]["rule"] + ); + assert_eq!( + "self.?topicName == oldSelf.?topicName", + spec["x-kubernetes-validations"][0]["rule"] + ); + assert_eq!(249, properties["topicName"]["maxLength"]); + assert!(properties["topicName"]["pattern"].is_string()); + assert_eq!(Some(1.0), properties["partitions"]["minimum"].as_f64()); + assert_eq!( + Some(1.0), + properties["replicationFactor"]["minimum"].as_f64() ); - assert_eq!(249, spec["topicName"]["maxLength"]); - assert!(spec["topicName"]["pattern"].is_string()); - assert_eq!(Some(1.0), spec["partitions"]["minimum"].as_f64()); - assert_eq!(Some(1.0), spec["replicationFactor"]["minimum"].as_f64()); } } From d028aa2af4914c0ca20edac5593788bb6780337a Mon Sep 17 00:00:00 2001 From: Benedikt Labrenz Date: Mon, 26 Oct 2026 21:17:26 +0100 Subject: [PATCH 35/93] rewrite the topic agent for the new KafkaTopic CRD --- rust/agent-binary/src/main.rs | 4 +- rust/agent-binary/src/topic/condition.rs | 349 ++++++---- rust/agent-binary/src/topic/controller.rs | 640 ++---------------- rust/agent-binary/src/topic/finalizer.rs | 137 ++++ .../src/topic/managed_config_keys.rs | 69 -- rust/agent-binary/src/topic/mod.rs | 5 +- rust/agent-binary/src/topic/plan.rs | 463 ++++++++----- rust/agent-binary/src/topic/sync.rs | 280 ++++++++ rust/agent-binary/src/topic/test_support.rs | 23 +- rust/agent-binary/src/topic/update_status.rs | 196 ++++++ rust/agent-binary/src/topic/validate.rs | 234 +++++++ 11 files changed, 1450 insertions(+), 950 deletions(-) create mode 100644 rust/agent-binary/src/topic/finalizer.rs delete mode 100644 rust/agent-binary/src/topic/managed_config_keys.rs create mode 100644 rust/agent-binary/src/topic/sync.rs create mode 100644 rust/agent-binary/src/topic/update_status.rs create mode 100644 rust/agent-binary/src/topic/validate.rs diff --git a/rust/agent-binary/src/main.rs b/rust/agent-binary/src/main.rs index b0bbe1aec..5b61c878e 100644 --- a/rust/agent-binary/src/main.rs +++ b/rust/agent-binary/src/main.rs @@ -115,7 +115,7 @@ async fn main() -> anyhow::Result<()> { client.get_api::>(cluster_namespace.as_ref()), watcher::Config::default(), ); - let topic_store = topic_controller.store(); + let kafka_topic_store = topic_controller.store(); topic_controller .graceful_shutdown_on(sigterm_watcher.handle()) .run( @@ -125,7 +125,7 @@ async fn main() -> anyhow::Result<()> { client, cluster_name, security, - topic_store, + kafka_topic_store, )), ) .for_each_concurrent( diff --git a/rust/agent-binary/src/topic/condition.rs b/rust/agent-binary/src/topic/condition.rs index d29c651d3..b8d1eb737 100644 --- a/rust/agent-binary/src/topic/condition.rs +++ b/rust/agent-binary/src/topic/condition.rs @@ -1,22 +1,28 @@ //! The `Ready` and `Synced` conditions of a `KafkaTopic` for the outcome of a sync. use stackable_kafka_admin::client; -use stackable_operator::commons::platform_access::condition::{ - ManagedCondition, ManagedConditionType, +use stackable_operator::status::condition::{ + ClusterConditionStatus, + managed_resource::{ManagedCondition, ManagedConditionType}, }; -const MAX_LISTED_PARTITIONS: usize = 10; +use super::plan::{TopicState, UnsupportedChange}; + +/// The operator writes the bootstrap servers to the discovery ConfigMap only once the +/// listener-operator has assigned addresses to the bootstrap Listeners. +const CLUSTER_NOT_READY_MESSAGE: &str = "the KafkaCluster has no bootstrap servers yet"; #[derive(Debug, Eq, PartialEq)] pub enum TopicSyncOutcome { /// The topic matches the spec Synced { - partitions: usize, - reassigning_partitions: Vec, + reassigning_partition_count: usize, }, + /// The topic matches the spec, apart from the changes the agent does not apply UnsupportedChange { - messages: Vec, + changes: Vec, }, + ReconciliationPaused, ClusterNotReady, InvalidSpec { message: String, @@ -25,11 +31,6 @@ pub enum TopicSyncOutcome { Conflict { message: String, }, - /// The topic exists in Kafka, but this `KafkaTopic` does not own it and is not allowed to adopt - /// it - AlreadyExists { - message: String, - }, KafkaRejected { message: String, }, @@ -49,55 +50,116 @@ impl TopicSyncOutcome { } } -/// The conditions to update. `Ready` says whether the topic is usable through this `KafkaTopic` -/// and is only updated if that is known. -pub fn conditions(outcome: &TopicSyncOutcome) -> Vec { - let available = ready(true, "Available", "the topic exists in Kafka"); - match outcome { - TopicSyncOutcome::Synced { - partitions, - reassigning_partitions, - } => { - let synced = if reassigning_partitions.is_empty() { - synced(true, "Synced", "the topic matches the spec") - } else { - synced( - true, - "ReassignmentInProgress", - &reassignment_message(reassigning_partitions, *partitions), - ) - }; - vec![available, synced] +/// `topic_state` is the topic in Kafka, if it exists and is managed by this `KafkaTopic`. +pub fn conditions_from( + sync_outcome: &TopicSyncOutcome, + topic_state: Option<&TopicState>, +) -> Vec { + vec![ + ready_from(sync_outcome, topic_state), + synced_from(sync_outcome, topic_state), + ] +} + +/// Whether the topic exists in Kafka, `Unknown` if the agent could not find out. +fn ready_from( + sync_outcome: &TopicSyncOutcome, + topic_state: Option<&TopicState>, +) -> ManagedCondition { + use ClusterConditionStatus::{False, True, Unknown}; + + // Also after a failed change, because the topic was observed before. + if let Some(topic_state) = topic_state { + return ready( + True, + "Available", + &format!( + "Topic {} exists with {} partitions", + topic_state.name, topic_state.partition_count + ), + ); + } + match sync_outcome { + TopicSyncOutcome::Synced { .. } + | TopicSyncOutcome::UnsupportedChange { .. } + | TopicSyncOutcome::ReconciliationPaused => { + ready(False, "Unavailable", "the topic does not exist in Kafka") + } + TopicSyncOutcome::Conflict { .. } => { + ready(False, "Conflict", "another KafkaTopic manages the topic") } - TopicSyncOutcome::UnsupportedChange { messages } => vec![ - available, - synced(false, "UnsupportedChange", &messages.join("; ")), - ], TopicSyncOutcome::ClusterNotReady => { - let message = "the KafkaCluster has no bootstrap servers yet"; - vec![ - ready(false, "ClusterNotReady", message), - synced(false, "ClusterNotReady", message), - ] + ready(Unknown, "ClusterNotReady", CLUSTER_NOT_READY_MESSAGE) + } + TopicSyncOutcome::InvalidSpec { message } => ready(Unknown, "InvalidSpec", message), + TopicSyncOutcome::KafkaRejected { message } => ready(Unknown, "KafkaRejected", message), + TopicSyncOutcome::ConnectionFailed { message } => { + ready(Unknown, "ConnectionFailed", message) } - TopicSyncOutcome::Conflict { message } => vec![ - ready(false, "Conflict", "another KafkaTopic manages the topic"), - synced(false, "Conflict", message), - ], - TopicSyncOutcome::AlreadyExists { message } => vec![ - ready( - false, - "AlreadyExists", - "the topic exists in Kafka, but is not owned by this KafkaTopic", + } +} + +/// Whether the last sync applied the spec to the topic. +fn synced_from( + sync_outcome: &TopicSyncOutcome, + topic_state: Option<&TopicState>, +) -> ManagedCondition { + use ClusterConditionStatus::{False, True}; + + match sync_outcome { + TopicSyncOutcome::Synced { + reassigning_partition_count, + } => match (reassigning_partition_count, topic_state) { + (0, _) | (_, None) => synced(True, "Synced", "the topic matches the spec"), + (reassigning, Some(topic_state)) => synced( + True, + "ReassignmentInProgress", + &format!( + "{reassigning} of {} partitions are being reassigned", + topic_state.partition_count + ), ), - synced(false, "AlreadyExists", message), - ], - TopicSyncOutcome::InvalidSpec { message } => vec![synced(false, "InvalidSpec", message)], - TopicSyncOutcome::KafkaRejected { message } => { - vec![synced(false, "KafkaRejected", message)] + }, + TopicSyncOutcome::UnsupportedChange { changes } => { + let messages = changes.iter().map(message_from).collect::>(); + synced(False, "UnsupportedChange", &messages.join("; ")) + } + TopicSyncOutcome::ReconciliationPaused => synced( + False, + "ReconciliationPaused", + "the reconciliation is paused, only the status is updated", + ), + TopicSyncOutcome::Conflict { message } => synced(False, "Conflict", message), + TopicSyncOutcome::ClusterNotReady => { + synced(False, "ClusterNotReady", CLUSTER_NOT_READY_MESSAGE) } + TopicSyncOutcome::InvalidSpec { message } => synced(False, "InvalidSpec", message), + TopicSyncOutcome::KafkaRejected { message } => synced(False, "KafkaRejected", message), TopicSyncOutcome::ConnectionFailed { message } => { - vec![synced(false, "ConnectionFailed", message)] + synced(False, "ConnectionFailed", message) + } + } +} + +fn message_from(unsupported_change: &UnsupportedChange) -> String { + match unsupported_change { + UnsupportedChange::DecreasePartitions { desired, actual } => { + format!("partitions cannot be decreased (spec: {desired}, actual: {actual})") + } + UnsupportedChange::ChangeReplicationFactor { + desired, + actual_min, + actual_max, + } => { + let actual = if actual_min == actual_max { + actual_min.to_string() + } else { + format!("between {actual_min} and {actual_max}") + }; + format!( + "the replication factor cannot be changed (spec: {desired}, actual: {actual}); \ + change it with kafka-reassign-partitions" + ) } } } @@ -112,34 +174,17 @@ fn error_chain(error: &dyn std::error::Error) -> String { .join(": ") } -fn reassignment_message(reassigning_partitions: &[i32], partitions: usize) -> String { - let listed = reassigning_partitions - .iter() - .take(MAX_LISTED_PARTITIONS) - .map(i32::to_string) - .collect::>() - .join(", "); - let more = reassigning_partitions - .len() - .saturating_sub(MAX_LISTED_PARTITIONS); - if more > 0 { - format!("partitions {listed} and {more} more of {partitions} are being reassigned") - } else { - format!("partitions {listed} of {partitions} are being reassigned") - } -} - -fn ready(status: bool, reason: &str, message: &str) -> ManagedCondition { +fn ready(status: ClusterConditionStatus, reason: &str, message: &str) -> ManagedCondition { condition(ManagedConditionType::Ready, status, reason, message) } -fn synced(status: bool, reason: &str, message: &str) -> ManagedCondition { +fn synced(status: ClusterConditionStatus, reason: &str, message: &str) -> ManagedCondition { condition(ManagedConditionType::Synced, status, reason, message) } fn condition( type_: ManagedConditionType, - status: bool, + status: ClusterConditionStatus, reason: &str, message: &str, ) -> ManagedCondition { @@ -153,130 +198,180 @@ fn condition( #[cfg(test)] mod tests { + use std::{collections::BTreeMap, str::FromStr}; + use snafu::Snafu; + use stackable_kafka_shared::crd::topic::TopicName; use super::*; + use crate::topic::condition::ClusterConditionStatus::{False, True, Unknown}; - fn summary(conditions: &[ManagedCondition]) -> Vec<(String, bool, &str)> { + fn summary(conditions: &[ManagedCondition]) -> Vec<(String, ClusterConditionStatus, &str)> { conditions .iter() .map(|condition| { ( condition.type_.to_string(), - condition.status, + condition.status.clone(), condition.reason.as_str(), ) }) .collect() } + fn topic_state(partition_count: i32) -> TopicState { + TopicState { + name: TopicName::from_str("orders").expect("the topic name should be valid"), + id: "UXyU9i5ARn6W00ON2taeWA".to_owned(), + partition_count, + replication_factor: Some(1), + effective_configs: BTreeMap::new(), + } + } + #[test] fn mark_a_synced_topic() { - let conditions = conditions(&TopicSyncOutcome::Synced { - partitions: 6, - reassigning_partitions: vec![], - }); + let conditions = conditions_from( + &TopicSyncOutcome::Synced { + reassigning_partition_count: 0, + }, + Some(&topic_state(6)), + ); assert_eq!( vec![ - ("Ready".to_owned(), true, "Available"), - ("Synced".to_owned(), true, "Synced") + ("Ready".to_owned(), True, "Available"), + ("Synced".to_owned(), True, "Synced") ], summary(&conditions) ); + assert_eq!( + "Topic orders exists with 6 partitions", + conditions[0].message + ); } #[test] fn keep_a_reassigned_topic_synced() { - let conditions = conditions(&TopicSyncOutcome::Synced { - partitions: 6, - reassigning_partitions: vec![0, 2], - }); + let conditions = conditions_from( + &TopicSyncOutcome::Synced { + reassigning_partition_count: 3, + }, + Some(&topic_state(6)), + ); assert_eq!( vec![ - ("Ready".to_owned(), true, "Available"), - ("Synced".to_owned(), true, "ReassignmentInProgress") + ("Ready".to_owned(), True, "Available"), + ("Synced".to_owned(), True, "ReassignmentInProgress") ], summary(&conditions) ); assert_eq!( - "partitions 0, 2 of 6 are being reassigned", + "3 of 6 partitions are being reassigned", conditions[1].message ); } #[test] - fn limit_the_listed_reassigning_partitions() { - let message = reassignment_message(&(0..12).collect::>(), 20); + fn mark_an_unsupported_change() { + let conditions = conditions_from( + &TopicSyncOutcome::UnsupportedChange { + changes: vec![ + UnsupportedChange::DecreasePartitions { + desired: 4, + actual: 6, + }, + UnsupportedChange::ChangeReplicationFactor { + desired: 2, + actual_min: 2, + actual_max: 3, + }, + ], + }, + Some(&topic_state(6)), + ); assert_eq!( - "partitions 0, 1, 2, 3, 4, 5, 6, 7, 8, 9 and 2 more of 20 are being reassigned", - message + vec![ + ("Ready".to_owned(), True, "Available"), + ("Synced".to_owned(), False, "UnsupportedChange") + ], + summary(&conditions) + ); + assert_eq!( + "partitions cannot be decreased (spec: 4, actual: 6); the replication factor cannot \ + be changed (spec: 2, actual: between 2 and 3); change it with \ + kafka-reassign-partitions", + conditions[1].message ); } #[test] - fn only_update_synced_after_an_error() { - let conditions = conditions(&TopicSyncOutcome::ConnectionFailed { - message: "timed out".to_owned(), - }); + fn report_a_missing_topic_while_paused() { + let conditions = conditions_from(&TopicSyncOutcome::ReconciliationPaused, None); assert_eq!( - vec![("Synced".to_owned(), false, "ConnectionFailed")], + vec![ + ("Ready".to_owned(), False, "Unavailable"), + ("Synced".to_owned(), False, "ReconciliationPaused") + ], summary(&conditions) ); } #[test] - fn mark_an_unsupported_change() { - let conditions = conditions(&TopicSyncOutcome::UnsupportedChange { - messages: vec!["first".to_owned(), "second".to_owned()], - }); + fn mark_a_conflict_as_not_ready() { + let conditions = conditions_from( + &TopicSyncOutcome::Conflict { + message: "managed by older".to_owned(), + }, + None, + ); assert_eq!( vec![ - ("Ready".to_owned(), true, "Available"), - ("Synced".to_owned(), false, "UnsupportedChange") + ("Ready".to_owned(), False, "Conflict"), + ("Synced".to_owned(), False, "Conflict") ], summary(&conditions) ); - assert_eq!("first; second", conditions[1].message); } - #[test] - fn leave_ready_untouched_for_an_invalid_spec() { - let conditions = conditions(&TopicSyncOutcome::InvalidSpec { - message: "invalid".to_owned(), - }); - + #[rstest::rstest] + #[case::cluster_not_ready(TopicSyncOutcome::ClusterNotReady, "ClusterNotReady")] + #[case::invalid_spec(TopicSyncOutcome::InvalidSpec { message: "invalid".to_owned() }, "InvalidSpec")] + #[case::kafka_rejected(TopicSyncOutcome::KafkaRejected { message: "denied".to_owned() }, "KafkaRejected")] + #[case::connection_failed(TopicSyncOutcome::ConnectionFailed { message: "timed out".to_owned() }, "ConnectionFailed")] + fn mark_ready_as_unknown_if_the_topic_cannot_be_observed( + #[case] sync_outcome: TopicSyncOutcome, + #[case] reason: &str, + ) { assert_eq!( - vec![("Synced".to_owned(), false, "InvalidSpec")], - summary(&conditions) + vec![ + ("Ready".to_owned(), Unknown, reason), + ("Synced".to_owned(), False, reason) + ], + summary(&conditions_from(&sync_outcome, None)) ); } #[test] - fn keep_the_ready_message_fixed_for_a_foreign_topic() { - let conditions = conditions(&TopicSyncOutcome::AlreadyExists { - message: "set managementPolicy.adoption to Adopt".to_owned(), - }); + fn keep_an_observed_topic_ready_if_a_change_fails() { + let conditions = conditions_from( + &TopicSyncOutcome::KafkaRejected { + message: "denied".to_owned(), + }, + Some(&topic_state(6)), + ); assert_eq!( vec![ - ("Ready".to_owned(), false, "AlreadyExists"), - ("Synced".to_owned(), false, "AlreadyExists") + ("Ready".to_owned(), True, "Available"), + ("Synced".to_owned(), False, "KafkaRejected") ], summary(&conditions) ); - assert_eq!( - "the topic exists in Kafka, but is not owned by this KafkaTopic", - conditions[0].message - ); - assert_eq!( - "set managementPolicy.adoption to Adopt", - conditions[1].message - ); } #[derive(Debug, Snafu)] diff --git a/rust/agent-binary/src/topic/controller.rs b/rust/agent-binary/src/topic/controller.rs index 484bd9e13..105ac03c6 100644 --- a/rust/agent-binary/src/topic/controller.rs +++ b/rust/agent-binary/src/topic/controller.rs @@ -1,26 +1,13 @@ -//! The `KafkaTopic` controller: reconcile, cleanup on deletion and error policy. +//! The `KafkaTopic` controller: reconcile and error policy. -use std::{collections::BTreeSet, path::Path, sync::Arc, time::Duration}; +use std::{sync::Arc, time::Duration}; use const_format::concatcp; -use serde_json::json; use snafu::{ResultExt, Snafu}; -use stackable_kafka_admin::{ - bootstrap::BrokerAddress, - client::{self, KafkaAdminClient, Topic}, - connection::Security, -}; -use stackable_kafka_shared::crd::topic::v1alpha1::{ - KafkaTopic, KafkaTopicPartitionStatus, KafkaTopicStatus, -}; +use stackable_kafka_admin::{client, connection::Security}; +use stackable_kafka_shared::crd::topic::v1alpha1::KafkaTopic; use stackable_operator::{ - commons::platform_access::{ - condition::update_conditions, - ownership::{OWNED_SINCE_ANNOTATION, is_owned}, - }, - k8s_openapi::{apimachinery::pkg::apis::meta::v1::Time, jiff::Timestamp}, kube::{ - ResourceExt, core::{DeserializeGuard, error_boundary}, runtime::{controller::Action, reflector::Store}, }, @@ -32,31 +19,22 @@ use tokio::sync::Mutex; use tracing::info; use super::{ - condition::{TopicSyncOutcome, conditions}, - managed_config_keys::{MANAGED_CONFIG_KEYS_ANNOTATION, managed_config_keys, render}, - plan::{ObservedTopic, TopicPlan, plan}, -}; -use crate::{ - AGENT_NAME, - config::{BOOTSTRAP_SERVERS_FILE, read_bootstrap_servers}, + condition::TopicSyncOutcome, + finalizer::{add_finalizer, reconcile_finalizer}, + sync::{CachedClient, sync}, + update_status::update_status, }; +use crate::AGENT_NAME; pub const TOPIC_CONTROLLER_NAME: &str = "kafkatopic"; pub const TOPIC_FULL_CONTROLLER_NAME: &str = concatcp!(TOPIC_CONTROLLER_NAME, ".", AGENT_NAME); -const FINALIZER: &str = "kafka.stackable.tech/kafkatopic"; -const KAFKA_CLIENT_ID: &str = "stackable-kafka-agent"; - -/// Set by `kafka-reassign-partitions` and Cruise Control for the duration of a reassignment. -const THROTTLE_CONFIG_KEYS: [&str; 2] = [ - "leader.replication.throttled.replicas", - "follower.replication.throttled.replicas", -]; const RESYNC_INTERVAL: Duration = Duration::from_secs(5 * 60); const RETRY_INTERVAL: Duration = Duration::from_secs(10); const ERROR_RETRY_INTERVAL: Duration = Duration::from_secs(30); #[derive(Debug, Snafu, EnumDiscriminants)] +#[snafu(visibility(pub(crate)))] #[strum_discriminants(derive(IntoStaticStr))] pub enum Error { #[snafu(display("the KafkaTopic is invalid"))] @@ -67,14 +45,12 @@ pub enum Error { #[snafu(display("failed to read the bootstrap servers"))] ReadBootstrapServers { source: crate::config::Error }, + #[snafu(display("the KafkaCluster has no bootstrap servers yet"))] + NoBootstrapServers, + #[snafu(display("failed to talk to Kafka"))] Kafka { source: client::Error }, - #[snafu(display("failed to annotate the KafkaTopic"))] - Annotate { - source: stackable_operator::client::Error, - }, - #[snafu(display("failed to update the status of the KafkaTopic"))] UpdateStatus { source: stackable_operator::client::Error, @@ -86,7 +62,7 @@ pub enum Error { }, } -type Result = std::result::Result; +pub(crate) type Result = std::result::Result; impl ReconcilerError for Error { fn category(&self) -> &'static str { @@ -95,113 +71,92 @@ impl ReconcilerError for Error { } pub struct Ctx { - client: stackable_operator::client::Client, - cluster_name: ClusterName, - security: Security, - topics: Store>, + pub(crate) client: stackable_operator::client::Client, + pub(crate) cluster_name: ClusterName, + pub(crate) security: Security, + pub(crate) kafka_topics: Store>, /// One client per agent, rebuilt when the bootstrap servers change - kafka: Mutex>, + pub(crate) client_cache: Mutex>, } impl Ctx { - /// `topics` is the store of the controller, used to detect conflicts between `KafkaTopic`s. + /// `kafka_topics` is the store of the controller, used to detect conflicts between + /// `KafkaTopic`s. pub fn new( client: stackable_operator::client::Client, cluster_name: ClusterName, security: Security, - topics: Store>, + kafka_topics: Store>, ) -> Self { Self { client, cluster_name, security, - topics, - kafka: Mutex::new(None), + kafka_topics, + client_cache: Mutex::new(None), } } } -struct CachedClient { - bootstrap_servers: Vec, - client: KafkaAdminClient, -} - pub async fn reconcile_topic( - topic: Arc>, + kafka_topic: Arc>, ctx: Arc, ) -> Result { info!("Starting reconcile"); // dereference - let topic = topic + let kafka_topic = kafka_topic .0 .as_ref() .map_err(error_boundary::InvalidObject::clone) .context(InvalidKafkaTopicSnafu)?; - if topic.spec.cluster_name != ctx.cluster_name { + if kafka_topic.spec.cluster_ref.name != ctx.cluster_name { return Ok(Action::await_change()); } // finalizer (Kubernetes) - if let Some(action) = reconcile_finalizer(topic, &ctx).await? { + // Left untouched while paused, like operators that only update the status then. A deletion + // waits until the reconciliation is resumed. + let paused = kafka_topic.spec.topic_operation.reconciliation_paused; + if !paused && let Some(action) = reconcile_finalizer(kafka_topic, &ctx).await? { return Ok(action); } - // sync (Kafka + Kubernetes) - let (outcome, observed) = match sync(topic, &ctx).await { - Ok(synced) => synced, - Err(Error::Kafka { source }) => { - let outcome = TopicSyncOutcome::from_client_error(&source); - update_status(topic, &ctx, &outcome, None).await?; - return Err(Error::Kafka { source }); - } - Err(error) => return Err(error), - }; + // sync (Kafka) + let mut topic_state = None; + let sync_result = sync(kafka_topic, &ctx, &mut topic_state).await; - // update status (Kubernetes) - update_status(topic, &ctx, &outcome, observed.as_ref()).await?; - Ok(requeue(&outcome, observed.is_some())) -} - -/// Returns `Some` if the reconcile ends here. -async fn reconcile_finalizer(topic: &KafkaTopic, ctx: &Ctx) -> Result> { - let has_finalizer = topic - .finalizers() - .iter() - .any(|finalizer| finalizer == FINALIZER); - if topic.metadata.deletion_timestamp.is_some() { - if has_finalizer { - cleanup(topic, ctx).await?; - set_finalizer(topic, ctx, false).await?; - } - return Ok(Some(Action::await_change())); + // finalizer (Kubernetes) + // Added before the status, because the status patch changes the resourceVersion. + if !paused { + add_finalizer(kafka_topic, &ctx, topic_state.as_ref()).await?; } - let needs_finalizer = topic.spec.management_policy.deletes_on_cleanup(); - if needs_finalizer != has_finalizer { - // The patch triggers another reconcile with the updated object. - set_finalizer(topic, ctx, needs_finalizer).await?; - return Ok(Some(Action::await_change())); + // update status (Kubernetes) + match sync_result { + Ok(sync_outcome) => { + update_status(kafka_topic, &ctx, &sync_outcome, topic_state.as_ref()).await?; + Ok(requeue(&sync_outcome)) + } + Err(Error::Kafka { source }) => { + let sync_outcome = TopicSyncOutcome::from_client_error(&source); + update_status(kafka_topic, &ctx, &sync_outcome, topic_state.as_ref()).await?; + Err(Error::Kafka { source }) + } + Err(error) => Err(error), } - Ok(None) } -fn requeue(outcome: &TopicSyncOutcome, observed: bool) -> Action { - match outcome { +fn requeue(sync_outcome: &TopicSyncOutcome) -> Action { + match sync_outcome { TopicSyncOutcome::InvalidSpec { .. } => Action::await_change(), - // After a create or a grow, the next reconcile fills in the topic id and partitions. - TopicSyncOutcome::Synced { .. } | TopicSyncOutcome::UnsupportedChange { .. } - if !observed => - { - Action::requeue(RETRY_INTERVAL) - } TopicSyncOutcome::ClusterNotReady => Action::requeue(RETRY_INTERVAL), _ => Action::requeue(RESYNC_INTERVAL), } } pub fn error_policy( - _topic: Arc>, + _kafka_topic: Arc>, error: &Error, _ctx: Arc, ) -> Action { @@ -211,492 +166,19 @@ pub fn error_policy( } } -/// Returns the outcome and, if the `KafkaTopic` owns the topic, the described topic. After a -/// partition grow, the topic is left out, because the description is outdated. -async fn sync(topic: &KafkaTopic, ctx: &Ctx) -> Result<(TopicSyncOutcome, Option)> { - let topic_name = topic.spec.topic_name.as_ref(); - - // validate (no client required) - if let Some(outcome) = validate(topic, &ctx.topics) { - return Ok((outcome, None)); - } - - // observe (Kafka) - let Some(bootstrap_servers) = read_bootstrap_servers(Path::new(BOOTSTRAP_SERVERS_FILE)) - .context(ReadBootstrapServersSnafu)? - else { - return Ok((TopicSyncOutcome::ClusterNotReady, None)); - }; - let mut cache = ctx.kafka.lock().await; - let kafka = client_for(&mut cache, bootstrap_servers, &ctx.security); - let observed = observe(kafka, topic_name).await?; - - // claim ownership (Kubernetes) - let claimed = if is_owned(&topic.metadata) { - false - } else { - if observed.is_some() && !topic.spec.management_policy.adopts_existing() { - let outcome = TopicSyncOutcome::AlreadyExists { - message: format!( - "the topic {topic_name:?} already exists in Kafka; set \ - managementPolicy.adoption to Adopt to take it over" - ), - }; - return Ok((outcome, None)); - } - claim_ownership(topic, ctx).await?; - true - }; - - // plan (no client required) - let recorded_keys = managed_config_keys(topic); - let topic_plan = plan(topic, observed.as_ref(), &recorded_keys); - let grows_partitions = matches!( - &topic_plan, - TopicPlan::Update(update) if update.grow_partitions_to.is_some() - ); - - // execute (Kafka + Kubernetes) - let outcome = match execute(topic, ctx, kafka, topic_plan, recorded_keys).await { - // The topic was created by someone else after it was observed. - Err(Error::Kafka { source }) if claimed && source.is_topic_already_exists() => { - release_ownership(topic, ctx).await?; - return Err(Error::Kafka { source }); - } - result => result?, - }; - - let observed = observed - .filter(|_| !grows_partitions) - .map(|observed| observed.topic); - Ok((outcome, observed)) -} - -/// Returns why the `KafkaTopic` cannot be synced, if it cannot. -fn validate( - topic: &KafkaTopic, - topics: &Store>, -) -> Option { - let throttle_keys = THROTTLE_CONFIG_KEYS - .iter() - .filter(|key| topic.spec.config.contains_key(**key)) - .copied() - .collect::>(); - if !throttle_keys.is_empty() { - return Some(TopicSyncOutcome::InvalidSpec { - message: format!( - "the config keys {throttle_keys:?} are managed by partition reassignments and \ - cannot be set" - ), - }); - } - conflicting_owner(topic, topics).map(|owner| TopicSyncOutcome::Conflict { - message: format!( - "the topic {:?} is managed by the KafkaTopic {owner:?}", - topic.spec.topic_name.as_ref() - ), - }) -} - -async fn observe(kafka: &mut KafkaAdminClient, topic_name: &str) -> Result> { - let Some(topic) = kafka.describe_topic(topic_name).await.context(KafkaSnafu)? else { - return Ok(None); - }; - let partitions = topic - .partitions - .iter() - .map(|partition| partition.index) - .collect::>(); - Ok(Some(ObservedTopic { - configs: kafka - .describe_topic_configs(topic_name) - .await - .context(KafkaSnafu)?, - reassignments: kafka - .list_partition_reassignments(topic_name, &partitions) - .await - .context(KafkaSnafu)?, - topic, - })) -} - -async fn execute( - topic: &KafkaTopic, - ctx: &Ctx, - kafka: &mut KafkaAdminClient, - topic_plan: TopicPlan, - mut recorded_keys: BTreeSet, -) -> Result { - let topic_name = topic.spec.topic_name.as_ref(); - let spec_keys = topic.spec.config.keys().cloned().collect::>(); - - let changes_configs = match &topic_plan { - TopicPlan::Create { .. } => true, - TopicPlan::Update(update) => { - !update.set_configs.is_empty() || !update.delete_configs.is_empty() - } - }; - let intended_keys = &recorded_keys | &spec_keys; - if changes_configs && intended_keys != recorded_keys { - record_managed_config_keys(topic, ctx, &intended_keys).await?; - recorded_keys = intended_keys; - } - - let outcome = match topic_plan { - TopicPlan::Create { - partitions, - replication_factor, - configs, - } => { - kafka - .create_topic(topic_name, partitions, replication_factor, &configs) - .await - .context(KafkaSnafu)?; - info!( - topic = topic_name, - partitions, - replication_factor, - ?configs, - kafka_topic = %topic.name_any(), - "Created the topic" - ); - TopicSyncOutcome::Synced { - partitions: partitions as usize, - reassigning_partitions: Vec::new(), - } - } - TopicPlan::Update(update) => { - if let Some(partitions) = update.grow_partitions_to { - kafka - .create_partitions(topic_name, partitions) - .await - .context(KafkaSnafu)?; - info!( - topic = topic_name, - partitions, - kafka_topic = %topic.name_any(), - "Increased the number of partitions" - ); - } - if changes_configs { - kafka - .incremental_alter_configs( - topic_name, - &update.set_configs, - &update.delete_configs, - ) - .await - .context(KafkaSnafu)?; - info!( - topic = topic_name, - set = ?update.set_configs, - reset = ?update.delete_configs, - kafka_topic = %topic.name_any(), - "Changed the topic configs" - ); - } - if update.unsupported.is_empty() { - TopicSyncOutcome::Synced { - partitions: topic.spec.partitions as usize, - reassigning_partitions: update.reassigning_partitions, - } - } else { - TopicSyncOutcome::UnsupportedChange { - messages: update.unsupported, - } - } - } - }; - - if recorded_keys != spec_keys { - record_managed_config_keys(topic, ctx, &spec_keys).await?; - } - - Ok(outcome) -} - -async fn cleanup(topic: &KafkaTopic, ctx: &Ctx) -> Result<()> { - if topic.spec.management_policy.deletes_on_cleanup() - && is_owned(&topic.metadata) - && conflicting_owner(topic, &ctx.topics).is_none() - { - let bootstrap_servers = read_bootstrap_servers(Path::new(BOOTSTRAP_SERVERS_FILE)) - .context(ReadBootstrapServersSnafu)? - .unwrap_or_default(); - let mut cache = ctx.kafka.lock().await; - client_for(&mut cache, bootstrap_servers, &ctx.security) - .delete_topic(topic.spec.topic_name.as_ref()) - .await - .context(KafkaSnafu)?; - info!( - topic = topic.spec.topic_name.as_ref(), - kafka_topic = %topic.name_any(), - "Deleted the topic" - ); - } - Ok(()) -} - -/// Fails if the `KafkaTopic` was changed since it was read, so concurrent changes to the -/// finalizers are not lost. -async fn set_finalizer(topic: &KafkaTopic, ctx: &Ctx, present: bool) -> Result<()> { - let mut finalizers = topic - .finalizers() - .iter() - .filter(|finalizer| *finalizer != FINALIZER) - .cloned() - .collect::>(); - if present { - finalizers.push(FINALIZER.to_owned()); - } - let patch = json!({ - "metadata": { - "resourceVersion": topic.metadata.resource_version, - "finalizers": finalizers, - } - }); - ctx.client - .merge_patch(topic, patch) - .await - .context(UpdateFinalizersSnafu)?; - Ok(()) -} - -/// Returns the cached client, rebuilt if the bootstrap servers changed. -fn client_for<'a>( - cache: &'a mut Option, - bootstrap_servers: Vec, - security: &Security, -) -> &'a mut KafkaAdminClient { - if cache - .as_ref() - .is_some_and(|cached| cached.bootstrap_servers != bootstrap_servers) - { - *cache = None; - } - let cached = cache.get_or_insert_with(|| { - info!(?bootstrap_servers, "Connecting to new bootstrap servers"); - CachedClient { - bootstrap_servers: bootstrap_servers.clone(), - client: KafkaAdminClient::new(bootstrap_servers, security.clone(), KAFKA_CLIENT_ID), - } - }); - &mut cached.client -} - -/// The oldest other `KafkaTopic` for the same topic of the same cluster, by creation time and -/// name. Checked against all `KafkaTopic`s in the store, so the result does not depend on the -/// reconcile order. -fn conflicting_owner( - topic: &KafkaTopic, - topics: &Store>, -) -> Option { - let key = |topic: &KafkaTopic| (topic.metadata.creation_timestamp.clone(), topic.name_any()); - topics - .state() - .iter() - .filter_map(|other| other.0.as_ref().ok()) - .filter(|other| { - other.spec.cluster_name == topic.spec.cluster_name - && other.spec.topic_name == topic.spec.topic_name - }) - .min_by_key(|other| key(other)) - .filter(|oldest| key(oldest) != key(topic)) - .map(ResourceExt::name_any) -} - -/// Fails if the `KafkaTopic` was changed since it was read, so ownership is never claimed twice. -async fn claim_ownership(topic: &KafkaTopic, ctx: &Ctx) -> Result<()> { - let patch = json!({ - "metadata": { - "resourceVersion": topic.metadata.resource_version, - "annotations": { OWNED_SINCE_ANNOTATION: Timestamp::now().to_string() }, - } - }); - ctx.client - .merge_patch(topic, patch) - .await - .context(AnnotateSnafu)?; - Ok(()) -} - -async fn release_ownership(topic: &KafkaTopic, ctx: &Ctx) -> Result<()> { - let patch = json!({ - "metadata": { - "annotations": { - OWNED_SINCE_ANNOTATION: null, - MANAGED_CONFIG_KEYS_ANNOTATION: null, - }, - } - }); - ctx.client - .merge_patch(topic, patch) - .await - .context(AnnotateSnafu)?; - Ok(()) -} - -async fn record_managed_config_keys( - topic: &KafkaTopic, - ctx: &Ctx, - keys: &BTreeSet, -) -> Result<()> { - let patch = json!({ - "metadata": { "annotations": { MANAGED_CONFIG_KEYS_ANNOTATION: render(keys) } } - }); - ctx.client - .merge_patch(topic, patch) - .await - .context(AnnotateSnafu)?; - Ok(()) -} - -/// Patches the status only if it changed, so the agent does not trigger its own watch. -async fn update_status( - topic: &KafkaTopic, - ctx: &Ctx, - outcome: &TopicSyncOutcome, - observed: Option<&Topic>, -) -> Result<()> { - let current = topic.status.clone().unwrap_or_default(); - let mut status = KafkaTopicStatus { - conditions: update_conditions( - ¤t.conditions, - conditions(outcome), - &Time(Timestamp::now()), - ), - ..current.clone() - }; - match (outcome, observed) { - (_, Some(observed)) => { - status.topic_id = Some(observed.id.clone()); - status.partitions = observed - .partitions - .iter() - .map(|partition| KafkaTopicPartitionStatus { - index: partition.index, - replicas: partition.replicas.clone(), - }) - .collect(); - } - (TopicSyncOutcome::AlreadyExists { .. } | TopicSyncOutcome::Conflict { .. }, None) => { - status.topic_id = None; - status.partitions = Vec::new(); - } - _ => {} - } - - if status != current { - ctx.client - .apply_patch_status(AGENT_NAME, topic, &status) - .await - .context(UpdateStatusSnafu)?; - } - Ok(()) -} - #[cfg(test)] mod tests { - use stackable_operator::{ - k8s_openapi::apimachinery::pkg::apis::meta::v1::Time, kube::runtime::watcher, - }; - use super::*; - use crate::topic::test_support::kafka_topic; - - fn created_topic( - name: &str, - created_at: i64, - topic_name: &str, - ) -> DeserializeGuard { - let mut topic = kafka_topic(name, topic_name); - topic.metadata.creation_timestamp = Some(Time( - Timestamp::from_second(created_at).expect("the timestamp should be valid"), - )); - DeserializeGuard(Ok(topic)) - } - - fn store(topics: Vec>) -> Store> { - let (store, mut writer) = stackable_operator::kube::runtime::reflector::store(); - for topic in topics { - writer.apply_watcher_event(&watcher::Event::Apply(topic)); - } - store - } - - fn inner(topic: &DeserializeGuard) -> &KafkaTopic { - topic.0.as_ref().expect("the topic should be valid") - } - - #[test] - fn the_oldest_kafka_topic_owns_the_topic() { - let older = created_topic("older", 10, "orders"); - let newer = created_topic("newer", 20, "orders"); - let other_topic = created_topic("other", 5, "payments"); - let topics = store(vec![older.clone(), newer.clone(), other_topic]); - - assert_eq!(None, conflicting_owner(inner(&older), &topics)); - assert_eq!( - Some("older".to_owned()), - conflicting_owner(inner(&newer), &topics) - ); - } - - #[test] - fn the_name_decides_between_kafka_topics_of_the_same_age() { - let first = created_topic("a", 10, "orders"); - let second = created_topic("b", 10, "orders"); - let topics = store(vec![second.clone(), first.clone()]); - - assert_eq!(None, conflicting_owner(inner(&first), &topics)); - assert_eq!( - Some("a".to_owned()), - conflicting_owner(inner(&second), &topics) - ); - } - - #[rstest::rstest] - #[case::no_throttle(&[("retention.ms", "1000")], false)] - #[case::throttle(&[("follower.replication.throttled.replicas", "*")], true)] - fn reject_throttle_config_keys(#[case] config: &[(&str, &str)], #[case] invalid: bool) { - let mut topic = kafka_topic("orders", "orders"); - topic.spec.config = config - .iter() - .map(|(key, value)| (key.to_string(), value.to_string())) - .collect(); - - assert_eq!( - invalid, - matches!( - validate(&topic, &store(vec![])), - Some(TopicSyncOutcome::InvalidSpec { .. }) - ) - ); - } - - fn synced() -> TopicSyncOutcome { - TopicSyncOutcome::Synced { - partitions: 1, - reassigning_partitions: vec![], - } - } #[rstest::rstest] - #[case::invalid_spec(TopicSyncOutcome::InvalidSpec { message: String::new() }, false, Action::await_change())] - #[case::created(synced(), false, Action::requeue(RETRY_INTERVAL))] - #[case::cluster_not_ready( - TopicSyncOutcome::ClusterNotReady, - false, - Action::requeue(RETRY_INTERVAL) + #[case::invalid_spec(TopicSyncOutcome::InvalidSpec { message: String::new() }, Action::await_change())] + #[case::cluster_not_ready(TopicSyncOutcome::ClusterNotReady, Action::requeue(RETRY_INTERVAL))] + #[case::synced(TopicSyncOutcome::Synced { reassigning_partition_count: 0 }, Action::requeue(RESYNC_INTERVAL))] + #[case::paused( + TopicSyncOutcome::ReconciliationPaused, + Action::requeue(RESYNC_INTERVAL) )] - #[case::synced(synced(), true, Action::requeue(RESYNC_INTERVAL))] - #[case::grown_with_unsupported_change(TopicSyncOutcome::UnsupportedChange { messages: vec![] }, false, Action::requeue(RETRY_INTERVAL))] - #[case::unsupported_change(TopicSyncOutcome::UnsupportedChange { messages: vec![] }, true, Action::requeue(RESYNC_INTERVAL))] - fn requeue_by_outcome( - #[case] outcome: TopicSyncOutcome, - #[case] observed: bool, - #[case] expected: Action, - ) { - assert_eq!(expected, requeue(&outcome, observed)); + fn requeue_by_outcome(#[case] sync_outcome: TopicSyncOutcome, #[case] expected: Action) { + assert_eq!(expected, requeue(&sync_outcome)); } } diff --git a/rust/agent-binary/src/topic/finalizer.rs b/rust/agent-binary/src/topic/finalizer.rs new file mode 100644 index 000000000..78c4d4d36 --- /dev/null +++ b/rust/agent-binary/src/topic/finalizer.rs @@ -0,0 +1,137 @@ +//! The finalizer of a `KafkaTopic` and the cleanup in Kafka when it is deleted. + +use std::path::Path; + +use serde_json::json; +use snafu::{OptionExt, ResultExt}; +use stackable_kafka_shared::crd::topic::v1alpha1::KafkaTopic; +use stackable_operator::{ + crd::management_access::v1alpha1::DeletionPolicy, + kube::{ResourceExt, runtime::controller::Action}, +}; +use tracing::info; + +use super::{ + controller::{ + Ctx, KafkaSnafu, NoBootstrapServersSnafu, ReadBootstrapServersSnafu, Result, + UpdateFinalizersSnafu, + }, + plan::TopicState, + sync::client_for, + validate::has_other_kafka_topic, +}; +use crate::config::{BOOTSTRAP_SERVERS_FILE, read_bootstrap_servers}; + +const FINALIZER: &str = "kafka.stackable.tech/kafkatopic"; + +/// Handles a deletion and removes a finalizer that is no longer needed. Returns `Some` if the +/// reconcile ends here. +pub(crate) async fn reconcile_finalizer( + kafka_topic: &KafkaTopic, + ctx: &Ctx, +) -> Result> { + let has_finalizer = has_finalizer(kafka_topic); + + if kafka_topic.metadata.deletion_timestamp.is_some() { + if has_finalizer { + cleanup(kafka_topic, ctx).await?; + set_finalizer(kafka_topic, ctx, false).await?; + } + return Ok(Some(Action::await_change())); + } + + if has_finalizer && kafka_topic.spec.deletion_policy != DeletionPolicy::Delete { + // The patch triggers another reconcile with the updated object. + set_finalizer(kafka_topic, ctx, false).await?; + return Ok(Some(Action::await_change())); + } + Ok(None) +} + +/// Adds the finalizer only once the topic exists and is managed by this `KafkaTopic`, so deleting +/// a `KafkaTopic` that never managed the topic, e.g. because of an invalid spec or a conflict, +/// does not delete it. +pub(crate) async fn add_finalizer( + kafka_topic: &KafkaTopic, + ctx: &Ctx, + topic_state: Option<&TopicState>, +) -> Result<()> { + if kafka_topic.spec.deletion_policy == DeletionPolicy::Delete + && topic_state.is_some() + && !has_finalizer(kafka_topic) + { + set_finalizer(kafka_topic, ctx, true).await?; + } + Ok(()) +} + +fn has_finalizer(kafka_topic: &KafkaTopic) -> bool { + kafka_topic + .finalizers() + .iter() + .any(|finalizer| finalizer == FINALIZER) +} + +/// Deletes the topic in Kafka if the `deletionPolicy` says so and no other `KafkaTopic` declares +/// it. +/// +/// The policy is read from the `KafkaTopic` being deleted, so switching it to `Retain` releases a +/// deletion that is stuck because Kafka cannot be reached. +async fn cleanup(kafka_topic: &KafkaTopic, ctx: &Ctx) -> Result<()> { + if kafka_topic.spec.deletion_policy != DeletionPolicy::Delete { + return Ok(()); + } + // A topic with an invalid name was never created. + let Ok(topic_name) = kafka_topic.topic_name() else { + return Ok(()); + }; + if has_other_kafka_topic(kafka_topic, &topic_name, &ctx.kafka_topics) { + return Ok(()); + } + + let bootstrap_servers = read_bootstrap_servers(Path::new(BOOTSTRAP_SERVERS_FILE)) + .context(ReadBootstrapServersSnafu)? + .context(NoBootstrapServersSnafu)?; + let mut client_cache = ctx.client_cache.lock().await; + client_for(&mut client_cache, bootstrap_servers, &ctx.security) + .delete_topic(topic_name.as_ref()) + .await + .context(KafkaSnafu)?; + + info!( + topic = %topic_name, + kafka_topic = %kafka_topic.name_any(), + "Deleted the topic" + ); + + Ok(()) +} + +/// Fails if the `KafkaTopic` was changed since it was read, so concurrent changes to the +/// finalizers are not lost. +async fn set_finalizer( + kafka_topic: &KafkaTopic, + ctx: &Ctx, + should_have_finalizer: bool, +) -> Result<()> { + let mut finalizers = kafka_topic + .finalizers() + .iter() + .filter(|finalizer| *finalizer != FINALIZER) + .cloned() + .collect::>(); + if should_have_finalizer { + finalizers.push(FINALIZER.to_owned()); + } + let patch = json!({ + "metadata": { + "resourceVersion": kafka_topic.metadata.resource_version, + "finalizers": finalizers, + } + }); + ctx.client + .merge_patch(kafka_topic, patch) + .await + .context(UpdateFinalizersSnafu)?; + Ok(()) +} diff --git a/rust/agent-binary/src/topic/managed_config_keys.rs b/rust/agent-binary/src/topic/managed_config_keys.rs deleted file mode 100644 index 069d3230d..000000000 --- a/rust/agent-binary/src/topic/managed_config_keys.rs +++ /dev/null @@ -1,69 +0,0 @@ -//! The topic config keys the agent manages, recorded on the `KafkaTopic`. -//! -//! Keys removed from the spec are reset in Kafka, but only if the agent set them before. Kafka -//! only accepts its known topic config names, none of which contain a comma, so the keys are -//! stored comma-separated. - -use std::collections::BTreeSet; - -use stackable_kafka_shared::crd::topic::v1alpha1::KafkaTopic; -use stackable_operator::kube::ResourceExt; - -pub const MANAGED_CONFIG_KEYS_ANNOTATION: &str = "kafka.stackable.tech/managed-config-keys"; - -pub fn managed_config_keys(topic: &KafkaTopic) -> BTreeSet { - topic - .annotations() - .get(MANAGED_CONFIG_KEYS_ANNOTATION) - .map(|value| { - value - .split(',') - .map(str::trim) - .filter(|key| !key.is_empty()) - .map(str::to_owned) - .collect() - }) - .unwrap_or_default() -} - -pub fn render(keys: &BTreeSet) -> String { - keys.iter() - .map(String::as_str) - .collect::>() - .join(",") -} - -#[cfg(test)] -mod tests { - use std::collections::BTreeMap; - - use rstest::rstest; - - use super::*; - use crate::topic::test_support::kafka_topic; - - #[rstest] - #[case::no_annotation(None, &[])] - #[case::empty(Some(""), &[])] - #[case::keys(Some("retention.ms,cleanup.policy"), &["cleanup.policy", "retention.ms"])] - #[case::spaces_and_empty_entries(Some(" retention.ms, ,"), &["retention.ms"])] - fn read_managed_config_keys(#[case] annotation: Option<&str>, #[case] expected: &[&str]) { - let mut topic = kafka_topic("orders", "orders"); - topic.metadata.annotations = annotation.map(|value| { - BTreeMap::from([(MANAGED_CONFIG_KEYS_ANNOTATION.to_owned(), value.to_owned())]) - }); - - let expected = expected - .iter() - .map(|key| key.to_string()) - .collect::>(); - assert_eq!(expected, managed_config_keys(&topic)); - } - - #[test] - fn render_sorted_keys() { - let keys = BTreeSet::from(["retention.ms".to_owned(), "cleanup.policy".to_owned()]); - - assert_eq!("cleanup.policy,retention.ms", render(&keys)); - } -} diff --git a/rust/agent-binary/src/topic/mod.rs b/rust/agent-binary/src/topic/mod.rs index a1ef9ffd9..062a1e7aa 100644 --- a/rust/agent-binary/src/topic/mod.rs +++ b/rust/agent-binary/src/topic/mod.rs @@ -2,7 +2,10 @@ mod condition; pub mod controller; -mod managed_config_keys; +mod finalizer; mod plan; +mod sync; #[cfg(test)] mod test_support; +mod update_status; +mod validate; diff --git a/rust/agent-binary/src/topic/plan.rs b/rust/agent-binary/src/topic/plan.rs index 44e5f71ff..81e4ce022 100644 --- a/rust/agent-binary/src/topic/plan.rs +++ b/rust/agent-binary/src/topic/plan.rs @@ -2,95 +2,201 @@ use std::collections::{BTreeMap, BTreeSet}; -use stackable_kafka_admin::client::{PartitionReassignment, Topic}; -use stackable_kafka_shared::crd::topic::v1alpha1::KafkaTopic; +use stackable_kafka_admin::client::{ + ConfigSource, CreatedTopic, PartitionReassignment, Topic, TopicConfig, +}; +use stackable_kafka_shared::crd::topic::{TopicName, v1alpha1::KafkaTopicSpec}; + +/// Set by `kafka-reassign-partitions` and Cruise Control for the duration of a reassignment. +/// Resetting them would lift the throttle of a running reassignment. +pub const THROTTLE_CONFIG_KEYS: [&str; 2] = [ + "leader.replication.throttled.replicas", + "follower.replication.throttled.replicas", +]; /// The state of an existing topic in Kafka. pub struct ObservedTopic { pub topic: Topic, - pub configs: BTreeMap, + pub effective_configs: BTreeMap, pub reassignments: Vec, } +/// The state of a topic in Kafka as shown in the status. +#[derive(Debug, Eq, PartialEq)] +pub struct TopicState { + pub name: TopicName, + pub id: String, + pub partition_count: i32, + pub replication_factor: Option, + pub effective_configs: BTreeMap, +} + +impl TopicState { + pub fn observed(name: TopicName, observed: ObservedTopic) -> Self { + let replication_factor = replication_factor(&observed); + Self { + name, + id: observed.topic.id, + partition_count: observed.topic.partitions.len() as i32, + replication_factor, + effective_configs: observed.effective_configs, + } + } + + pub fn created(name: TopicName, created: CreatedTopic) -> Self { + Self { + name, + id: created.id, + partition_count: created.partitions, + replication_factor: Some(created.replication_factor), + effective_configs: created.configs, + } + } +} + #[derive(Debug, Eq, PartialEq)] pub enum TopicPlan { Create { - partitions: i32, - replication_factor: i16, - configs: BTreeMap, + partition_count: Option, + replication_factor: Option, + config_overrides: BTreeMap, + }, + Update { + topic_update: TopicUpdate, + /// Applied are only the changes in `topic_update` + unsupported_changes: Vec, }, - Update(TopicUpdate), } -/// Changes to an existing topic. No grow, configs or unsupported differences means the topic -/// matches the spec. +/// Changes to an existing topic. No changes means the topic matches the spec, apart from +/// [`UnsupportedChange`]s. #[derive(Debug, Default, Eq, PartialEq)] pub struct TopicUpdate { - pub grow_partitions_to: Option, - pub set_configs: BTreeMap, - pub delete_configs: BTreeSet, - /// Differences the agent does not apply, as messages for the status - pub unsupported: Vec, - pub reassigning_partitions: Vec, + /// Only increased, see [`UnsupportedChange::DecreasePartitions`] + pub new_partition_count: Option, + pub configs_to_set: BTreeMap, + /// Topic-level configs to remove, so that the broker configs apply + pub configs_to_reset: BTreeSet, } -pub fn plan( - topic: &KafkaTopic, - observed: Option<&ObservedTopic>, - managed_config_keys: &BTreeSet, -) -> TopicPlan { - let spec = &topic.spec; +/// A difference to the spec the agent does not apply. The other changes are still applied. +#[derive(Debug, Eq, PartialEq)] +pub enum UnsupportedChange { + /// Kafka cannot remove partitions. + DecreasePartitions { desired: i32, actual: i32 }, + /// The partitions have between `actual_min` and `actual_max` replicas, not counting replicas + /// that a running reassignment removes. + /// + /// Kafka can only change the replicas of existing partitions with + /// `AlterPartitionReassignments` and explicit replica lists. Choosing the brokers is left to + /// `kafka-reassign-partitions` or Cruise Control. + ChangeReplicationFactor { + desired: i16, + actual_min: usize, + actual_max: usize, + }, +} + +pub fn plan(spec: &KafkaTopicSpec, observed: Option<&ObservedTopic>) -> TopicPlan { let Some(observed) = observed else { return TopicPlan::Create { - partitions: spec.partitions, + partition_count: spec.partitions, replication_factor: spec.replication_factor, - configs: spec.config.clone(), + config_overrides: spec.config_overrides.clone(), }; }; - let mut update = TopicUpdate::default(); - - let observed_partitions = observed.topic.partitions.len() as i32; - if spec.partitions > observed_partitions { - update.grow_partitions_to = Some(spec.partitions); - } else if spec.partitions < observed_partitions { - update.unsupported.push(format!( - "the topic has {observed_partitions} partitions, but the spec requests {}; partitions cannot be \ - removed", - spec.partitions - )); + let actual_partition_count = observed.topic.partitions.len() as i32; + let (configs_to_set, configs_to_reset) = + config_changes(&spec.config_overrides, &observed.effective_configs); + + TopicPlan::Update { + topic_update: TopicUpdate { + new_partition_count: spec + .partitions + .filter(|desired| *desired > actual_partition_count), + configs_to_set, + configs_to_reset, + }, + unsupported_changes: unsupported_changes(spec, observed), + } +} + +fn unsupported_changes(spec: &KafkaTopicSpec, observed: &ObservedTopic) -> Vec { + let mut changes = Vec::new(); + + let actual_partition_count = observed.topic.partitions.len() as i32; + if let Some(desired) = spec.partitions + && desired < actual_partition_count + { + changes.push(UnsupportedChange::DecreasePartitions { + desired, + actual: actual_partition_count, + }); } - if let Some(message) = replication_factor_difference( - spec.replication_factor, - &target_replication_factors(observed), - ) { - update.unsupported.push(message); + let replication_factors = target_replication_factors(observed); + if let Some(desired) = spec.replication_factor + && let (Some(&actual_min), Some(&actual_max)) = + (replication_factors.first(), replication_factors.last()) + && (actual_min != actual_max || actual_min != desired as usize) + { + changes.push(UnsupportedChange::ChangeReplicationFactor { + desired, + actual_min, + actual_max, + }); } - update.set_configs = spec - .config + changes +} + +/// The configs to set and the configs to reset, so that the topic-level configs are exactly the +/// desired ones. The throttles of partition reassignments are left alone. +fn config_changes( + desired: &BTreeMap, + effective_configs: &BTreeMap, +) -> (BTreeMap, BTreeSet) { + // Topic configs have no PASSWORD type, so Kafka does not redact their topic-level values. + let topic_level_configs = effective_configs + .iter() + .filter(|(_, config)| config.source == ConfigSource::DynamicTopic) + .map(|(key, config)| (key, config.value.as_deref())) + .collect::>(); + + // Compared with the topic-level value, not the effective one: a desired value that equals the + // broker config is still pinned on the topic. Otherwise a later change of the broker config + // would apply to the topic until the next resync. Unchanged values are skipped, so a resync + // without drift does not write to Kafka. + let configs_to_set = desired .iter() - .filter(|(key, value)| observed.configs.get(*key) != Some(*value)) + .filter(|(key, value)| topic_level_configs.get(key) != Some(&Some(value.as_str()))) .map(|(key, value)| (key.clone(), value.clone())) .collect(); - update.delete_configs = managed_config_keys - .iter() - .filter(|key| !spec.config.contains_key(*key) && observed.configs.contains_key(*key)) - .cloned() - .collect(); - - update.reassigning_partitions = observed - .reassignments - .iter() - .map(|reassignment| reassignment.index) + let configs_to_reset = topic_level_configs + .keys() + .filter(|key| !desired.contains_key(**key) && !THROTTLE_CONFIG_KEYS.contains(&key.as_str())) + .map(|key| key.to_string()) .collect(); + (configs_to_set, configs_to_reset) +} - TopicPlan::Update(update) +/// The replication factor of the partitions, excluding replicas a reassignment removes. `None` if +/// the partitions have different replication factors. +fn replication_factor(observed: &ObservedTopic) -> Option { + match target_replication_factors(observed).as_slice() { + [replication_factor] => i16::try_from(*replication_factor).ok(), + _ => None, + } } -/// The replication factor of each partition, excluding replicas a reassignment removes. -fn target_replication_factors(observed: &ObservedTopic) -> BTreeSet { +/// The distinct replication factors of the partitions in ascending order, excluding replicas a +/// reassignment removes. +/// +/// Kafka stores no replication factor per topic. During a reassignment, the replicas of a +/// partition include the ones being added and removed. `kafka-topics.sh --describe` shows the +/// replicas minus the ones being added of the first partition instead. +fn target_replication_factors(observed: &ObservedTopic) -> Vec { observed .topic .partitions @@ -100,35 +206,14 @@ fn target_replication_factors(observed: &ObservedTopic) -> BTreeSet { .reassignments .iter() .find(|reassignment| reassignment.index == partition.index) - .map_or(0, |reassignment| reassignment.removing_replicas.len()); - partition.replicas.len().saturating_sub(removing) + .map_or(0, |reassignment| reassignment.removing_replicas); + partition.replicas.saturating_sub(removing) }) + .collect::>() + .into_iter() .collect() } -fn replication_factor_difference( - desired: i16, - replication_factors: &BTreeSet, -) -> Option { - let (Some(min), Some(max)) = (replication_factors.first(), replication_factors.last()) else { - return None; - }; - - let hint = "change it with kafka-reassign-partitions"; - if min != max { - Some(format!( - "the partitions have between {min} and {max} replicas, but the spec requests a \ - replication factor of {desired}; {hint}" - )) - } else if *min != desired as usize { - Some(format!( - "the topic has a replication factor of {min}, but the spec requests {desired}; {hint}" - )) - } else { - None - } -} - #[cfg(test)] mod tests { use stackable_kafka_admin::client::Partition; @@ -136,12 +221,16 @@ mod tests { use super::*; use crate::topic::test_support::kafka_topic; - fn topic(partitions: i32, replication_factor: i16, config: &[(&str, &str)]) -> KafkaTopic { - let mut topic = kafka_topic("orders", "orders"); - topic.spec.partitions = partitions; - topic.spec.replication_factor = replication_factor; - topic.spec.config = configs(config); - topic + fn spec( + partitions: Option, + replication_factor: Option, + config: &[(&str, &str)], + ) -> KafkaTopicSpec { + let mut spec = kafka_topic("orders", Some("orders")).spec; + spec.partitions = partitions; + spec.replication_factor = replication_factor; + spec.config_overrides = configs(config); + spec } fn configs(entries: &[(&str, &str)]) -> BTreeMap { @@ -155,8 +244,30 @@ mod tests { keys.iter().map(|key| key.to_string()).collect() } - /// A topic whose partitions have the given replicas - fn observed(replicas: &[&[i32]], config: &[(&str, &str)]) -> ObservedTopic { + fn topic_configs(entries: &[(&str, &str, ConfigSource)]) -> BTreeMap { + entries + .iter() + .map(|(key, value, source)| { + ( + key.to_string(), + TopicConfig { + value: Some(value.to_string()), + source: *source, + }, + ) + }) + .collect() + } + + fn update(topic_update: TopicUpdate, unsupported_changes: Vec) -> TopicPlan { + TopicPlan::Update { + topic_update, + unsupported_changes, + } + } + + /// A topic whose partitions have the given number of replicas + fn observed(replicas: &[usize], configs: &[(&str, &str, ConfigSource)]) -> ObservedTopic { ObservedTopic { topic: Topic { id: "UXyU9i5ARn6W00ON2taeWA".to_owned(), @@ -165,24 +276,24 @@ mod tests { .zip(0..) .map(|(replicas, index)| Partition { index, - replicas: replicas.to_vec(), + replicas: *replicas, }) .collect(), }, - configs: configs(config), + effective_configs: topic_configs(configs), reassignments: vec![], } } #[test] fn create_missing_topic() { - let plan = plan(&topic(3, 2, &[("retention.ms", "1000")]), None, &keys(&[])); + let plan = plan(&spec(Some(3), None, &[("retention.ms", "1000")]), None); assert_eq!( TopicPlan::Create { - partitions: 3, - replication_factor: 2, - configs: configs(&[("retention.ms", "1000")]), + partition_count: Some(3), + replication_factor: None, + config_overrides: configs(&[("retention.ms", "1000")]), }, plan ); @@ -190,135 +301,161 @@ mod tests { #[test] fn do_nothing_for_a_matching_topic() { - let observed = observed(&[&[1, 2], &[2, 3]], &[("retention.ms", "1000")]); + let observed = observed( + &[2, 2], + &[("retention.ms", "1000", ConfigSource::DynamicTopic)], + ); + let spec = spec(Some(2), Some(2), &[("retention.ms", "1000")]); - let plan = plan( - &topic(2, 2, &[("retention.ms", "1000")]), - Some(&observed), - &keys(&["retention.ms"]), + assert_eq!( + update(TopicUpdate::default(), vec![]), + plan(&spec, Some(&observed)) ); + } - assert_eq!(TopicPlan::Update(TopicUpdate::default()), plan); + #[test] + fn leave_partitions_and_replication_factor_alone_if_not_set() { + let observed = observed(&[3, 3, 3], &[]); + let spec = spec(None, None, &[]); + + assert_eq!( + update(TopicUpdate::default(), vec![]), + plan(&spec, Some(&observed)) + ); } #[test] fn grow_partitions() { - let observed = observed(&[&[1], &[2]], &[]); + let observed = observed(&[1, 1], &[]); - let plan = plan(&topic(4, 1, &[]), Some(&observed), &keys(&[])); + let plan = plan(&spec(Some(4), Some(1), &[]), Some(&observed)); assert_eq!( - TopicPlan::Update(TopicUpdate { - grow_partitions_to: Some(4), - ..TopicUpdate::default() - }), + update( + TopicUpdate { + new_partition_count: Some(4), + ..TopicUpdate::default() + }, + vec![] + ), plan ); } #[test] fn report_fewer_partitions_as_unsupported() { - let observed = observed(&[&[1], &[2], &[3]], &[]); + let observed = observed(&[1, 1, 1], &[]); - let TopicPlan::Update(update) = plan(&topic(2, 1, &[]), Some(&observed), &keys(&[])) else { - panic!("the topic exists"); - }; + let plan = plan(&spec(Some(2), None, &[]), Some(&observed)); - assert_eq!(None, update.grow_partitions_to); - assert_eq!(1, update.unsupported.len()); - assert!(update.unsupported[0].contains("cannot be removed")); + assert_eq!( + update( + TopicUpdate::default(), + vec![UnsupportedChange::DecreasePartitions { + desired: 2, + actual: 3 + }] + ), + plan + ); } #[test] - fn set_changed_and_reset_removed_configs() { + fn replace_the_topic_level_configs() { let observed = observed( - &[&[1]], + &[1], &[ - ("retention.ms", "1000"), - ("cleanup.policy", "compact"), - ("segment.bytes", "1024"), - ("leader.replication.throttled.replicas", "0:1"), + ("retention.ms", "1000", ConfigSource::DynamicTopic), + ("cleanup.policy", "compact", ConfigSource::DynamicTopic), + ("segment.bytes", "1024", ConfigSource::Default), + ("max.message.bytes", "10", ConfigSource::StaticBroker), + ( + "leader.replication.throttled.replicas", + "0:1", + ConfigSource::DynamicTopic, + ), ], ); let plan = plan( - &topic( - 1, - 1, + &spec( + None, + None, &[("retention.ms", "2000"), ("max.message.bytes", "10")], ), Some(&observed), - &keys(&["retention.ms", "cleanup.policy", "min.insync.replicas"]), ); assert_eq!( - TopicPlan::Update(TopicUpdate { - set_configs: configs(&[("max.message.bytes", "10"), ("retention.ms", "2000")]), - // `segment.bytes` and the throttle were set by others, `min.insync.replicas` is - // already gone - delete_configs: keys(&["cleanup.policy"]), - ..TopicUpdate::default() - }), + update( + TopicUpdate { + // `max.message.bytes` already has the value, but not on the topic + configs_to_set: configs(&[ + ("max.message.bytes", "10"), + ("retention.ms", "2000") + ]), + // `segment.bytes` is not set on the topic, the throttle belongs to a + // reassignment + configs_to_reset: keys(&["cleanup.policy"]), + ..TopicUpdate::default() + }, + vec![] + ), plan ); } #[test] fn report_a_different_replication_factor_as_unsupported() { - let observed = observed(&[&[1, 2, 3], &[2, 3, 4]], &[]); + let observed = observed(&[3, 3], &[]); - let TopicPlan::Update(update) = plan(&topic(2, 2, &[]), Some(&observed), &keys(&[])) else { - panic!("the topic exists"); - }; - - assert_eq!(1, update.unsupported.len()); - assert!(update.unsupported[0].contains("replication factor of 3")); + assert_eq!( + vec![UnsupportedChange::ChangeReplicationFactor { + desired: 2, + actual_min: 3, + actual_max: 3 + }], + unsupported_changes(&spec(None, Some(2), &[]), &observed) + ); } #[test] fn report_uneven_replica_counts_as_unsupported() { - let observed = observed(&[&[1, 2], &[2, 3, 4]], &[]); + let observed = observed(&[2, 3], &[]); - let TopicPlan::Update(update) = plan(&topic(2, 2, &[]), Some(&observed), &keys(&[])) else { - panic!("the topic exists"); - }; - - assert_eq!(1, update.unsupported.len()); - assert!(update.unsupported[0].contains("between 2 and 3 replicas")); + assert_eq!( + vec![UnsupportedChange::ChangeReplicationFactor { + desired: 2, + actual_min: 2, + actual_max: 3 + }], + unsupported_changes(&spec(None, Some(2), &[]), &observed) + ); } #[test] fn accept_a_reassignment_towards_the_desired_replication_factor() { - let mut observed = observed(&[&[4, 5, 1, 2], &[2, 3]], &[]); + let mut observed = observed(&[4, 2], &[]); observed.reassignments = vec![PartitionReassignment { index: 0, - adding_replicas: vec![4, 5], - removing_replicas: vec![1, 2], + removing_replicas: 2, }]; - let plan = plan(&topic(2, 2, &[]), Some(&observed), &keys(&[])); - assert_eq!( - TopicPlan::Update(TopicUpdate { - reassigning_partitions: vec![0], - ..TopicUpdate::default() - }), - plan + Vec::::new(), + unsupported_changes(&spec(None, Some(2), &[]), &observed) ); } #[test] - fn exclude_removed_replicas_from_the_replication_factor() { - let mut observed = observed(&[&[4, 5, 1, 2], &[2, 3, 4]], &[]); - observed.reassignments = vec![PartitionReassignment { + fn report_the_target_replication_factor() { + let mut reassigning = observed(&[4, 2], &[]); + reassigning.reassignments = vec![PartitionReassignment { index: 0, - adding_replicas: vec![4, 5], - removing_replicas: vec![1, 2], + removing_replicas: 2, }]; - assert_eq!( - BTreeSet::from([2, 3]), - target_replication_factors(&observed) - ); + assert_eq!(Some(2), replication_factor(&reassigning)); + assert_eq!(None, replication_factor(&observed(&[2, 3], &[]))); } } diff --git a/rust/agent-binary/src/topic/sync.rs b/rust/agent-binary/src/topic/sync.rs new file mode 100644 index 000000000..7608efe9a --- /dev/null +++ b/rust/agent-binary/src/topic/sync.rs @@ -0,0 +1,280 @@ +//! Brings the topic in Kafka to the state of its `KafkaTopic` spec. + +use std::{ + collections::{BTreeMap, BTreeSet}, + path::Path, +}; + +use snafu::ResultExt; +use stackable_kafka_admin::{ + bootstrap::BrokerAddress, + client::{KafkaAdminClient, TopicConfig}, + connection::Security, +}; +use stackable_kafka_shared::crd::topic::{TopicName, v1alpha1::KafkaTopic}; +use stackable_operator::kube::ResourceExt; +use tracing::info; + +use super::{ + condition::TopicSyncOutcome, + controller::{Ctx, KafkaSnafu, ReadBootstrapServersSnafu, Result}, + plan::{ObservedTopic, TopicPlan, TopicState, plan}, + validate::validate, +}; +use crate::config::{BOOTSTRAP_SERVERS_FILE, read_bootstrap_servers}; + +const KAFKA_CLIENT_ID: &str = "stackable-kafka-agent"; + +pub(crate) struct CachedClient { + bootstrap_servers: Vec, + admin_client: KafkaAdminClient, +} + +/// Sets `topic_state` to the topic in Kafka, if it exists and is managed by this `KafkaTopic`. +/// It is kept up to date with each applied change, also if a later step fails. +pub(crate) async fn sync( + kafka_topic: &KafkaTopic, + ctx: &Ctx, + topic_state: &mut Option, +) -> Result { + // validate (no client required) + let topic_name = match validate(kafka_topic, &ctx.kafka_topics) { + Ok(topic_name) => topic_name, + Err(sync_outcome) => return Ok(sync_outcome), + }; + + // observe (Kafka) + let Some(bootstrap_servers) = read_bootstrap_servers(Path::new(BOOTSTRAP_SERVERS_FILE)) + .context(ReadBootstrapServersSnafu)? + else { + return Ok(TopicSyncOutcome::ClusterNotReady); + }; + let mut client_cache = ctx.client_cache.lock().await; + let admin_client = client_for(&mut client_cache, bootstrap_servers, &ctx.security); + let observed = observe(admin_client, &topic_name).await?; + + // plan (no client required) + let topic_plan = plan(&kafka_topic.spec, observed.as_ref()); + let reassigning_partition_count = observed + .as_ref() + .map_or(0, |observed| observed.reassignments.len()); + *topic_state = observed.map(|observed| TopicState::observed(topic_name.clone(), observed)); + if kafka_topic.spec.topic_operation.reconciliation_paused { + return Ok(TopicSyncOutcome::ReconciliationPaused); + } + + // execute (Kafka) + execute( + kafka_topic, + admin_client, + topic_name, + &topic_plan, + topic_state, + ) + .await?; + + let sync_outcome = match topic_plan { + TopicPlan::Update { + unsupported_changes, + .. + } if !unsupported_changes.is_empty() => TopicSyncOutcome::UnsupportedChange { + changes: unsupported_changes, + }, + _ => TopicSyncOutcome::Synced { + reassigning_partition_count, + }, + }; + Ok(sync_outcome) +} + +async fn observe( + admin_client: &mut KafkaAdminClient, + topic_name: &TopicName, +) -> Result> { + let topic_name = topic_name.as_ref(); + let Some(topic) = admin_client + .describe_topic(topic_name) + .await + .context(KafkaSnafu)? + else { + return Ok(None); + }; + let partitions = topic + .partitions + .iter() + .map(|partition| partition.index) + .collect::>(); + Ok(Some(ObservedTopic { + effective_configs: admin_client + .describe_topic_configs(topic_name) + .await + .context(KafkaSnafu)?, + reassignments: admin_client + .list_partition_reassignments(topic_name, &partitions) + .await + .context(KafkaSnafu)?, + topic, + })) +} + +/// Applies the changes to `topic_state`, the topic in Kafka before them. +async fn execute( + kafka_topic: &KafkaTopic, + admin_client: &mut KafkaAdminClient, + topic_name: TopicName, + topic_plan: &TopicPlan, + topic_state: &mut Option, +) -> Result<()> { + match topic_plan { + TopicPlan::Create { + partition_count, + replication_factor, + config_overrides, + } => { + let created_state = create( + kafka_topic, + admin_client, + topic_name, + *partition_count, + *replication_factor, + config_overrides, + ) + .await?; + *topic_state = Some(created_state); + } + TopicPlan::Update { topic_update, .. } => { + // Partitions + if let Some(new_partition_count) = topic_update.new_partition_count { + grow_partitions(kafka_topic, admin_client, &topic_name, new_partition_count) + .await?; + if let Some(topic_state) = topic_state { + topic_state.partition_count = new_partition_count; + } + } + + // Config + if !topic_update.configs_to_set.is_empty() || !topic_update.configs_to_reset.is_empty() + { + let effective_configs = replace_configs( + kafka_topic, + admin_client, + &topic_name, + &topic_update.configs_to_set, + &topic_update.configs_to_reset, + ) + .await?; + if let Some(topic_state) = topic_state { + topic_state.effective_configs = effective_configs; + } + } + } + } + Ok(()) +} + +/// The CreateTopics response contains the topic id, the counts and the effective configs, so no +/// describe is needed afterwards. Without the `DESCRIBE_CONFIGS` permission, Kafka creates the +/// topic but the request fails, and so does every later sync, because observing the topic needs +/// that permission as well. +async fn create( + kafka_topic: &KafkaTopic, + admin_client: &mut KafkaAdminClient, + topic_name: TopicName, + partition_count: Option, + replication_factor: Option, + config_overrides: &BTreeMap, +) -> Result { + let created = admin_client + .create_topic( + topic_name.as_ref(), + partition_count, + replication_factor, + config_overrides, + ) + .await + .context(KafkaSnafu)?; + info!( + topic = %topic_name, + partitions = created.partitions, + replication_factor = created.replication_factor, + configs = ?config_overrides, + kafka_topic = %kafka_topic.name_any(), + "Created the topic" + ); + Ok(TopicState::created(topic_name, created)) +} + +async fn grow_partitions( + kafka_topic: &KafkaTopic, + admin_client: &mut KafkaAdminClient, + topic_name: &TopicName, + partition_count: i32, +) -> Result<()> { + admin_client + .create_partitions(topic_name.as_ref(), partition_count) + .await + .context(KafkaSnafu)?; + info!( + topic = %topic_name, + partitions = partition_count, + kafka_topic = %kafka_topic.name_any(), + "Increased the number of partitions" + ); + Ok(()) +} + +/// Returns the effective configs of the topic after the change. They are described again, +/// because IncrementalAlterConfigs does not return them and a reset key falls back to a broker +/// config the agent does not know. +async fn replace_configs( + kafka_topic: &KafkaTopic, + admin_client: &mut KafkaAdminClient, + topic_name: &TopicName, + configs_to_set: &BTreeMap, + configs_to_reset: &BTreeSet, +) -> Result> { + admin_client + .incremental_alter_configs(topic_name.as_ref(), configs_to_set, configs_to_reset) + .await + .context(KafkaSnafu)?; + info!( + topic = %topic_name, + set = ?configs_to_set, + reset = ?configs_to_reset, + kafka_topic = %kafka_topic.name_any(), + "Changed the topic configs" + ); + admin_client + .describe_topic_configs(topic_name.as_ref()) + .await + .context(KafkaSnafu) +} + +/// Returns the cached client, rebuilt if the bootstrap servers changed. +/// +/// The client reads the TLS files on every new connection and reconnects after a failed request, +/// so rotated certificates are picked up without restarting the agent. +pub(crate) fn client_for<'a>( + client_cache: &'a mut Option, + bootstrap_servers: Vec, + security: &Security, +) -> &'a mut KafkaAdminClient { + if client_cache + .as_ref() + .is_some_and(|cached| cached.bootstrap_servers != bootstrap_servers) + { + *client_cache = None; + } + let cached = client_cache.get_or_insert_with(|| { + info!(?bootstrap_servers, "Connecting to new bootstrap servers"); + CachedClient { + bootstrap_servers: bootstrap_servers.clone(), + admin_client: KafkaAdminClient::new( + bootstrap_servers, + security.clone(), + KAFKA_CLIENT_ID, + ), + } + }); + &mut cached.admin_client +} diff --git a/rust/agent-binary/src/topic/test_support.rs b/rust/agent-binary/src/topic/test_support.rs index 44830af98..bba58c0ba 100644 --- a/rust/agent-binary/src/topic/test_support.rs +++ b/rust/agent-binary/src/topic/test_support.rs @@ -2,22 +2,27 @@ use std::str::FromStr; use stackable_kafka_shared::crd::topic::{ TopicName, - v1alpha1::{KafkaTopic, KafkaTopicSpec}, + v1alpha1::{KafkaClusterRef, KafkaTopic, KafkaTopicSpec}, }; use stackable_operator::v2::types::operator::ClusterName; /// A `KafkaTopic` of the cluster `simple-kafka` with one partition, one replica and no configs. -pub fn kafka_topic(name: &str, topic_name: &str) -> KafkaTopic { +pub fn kafka_topic(name: &str, topic_name: Option<&str>) -> KafkaTopic { KafkaTopic::new( name, KafkaTopicSpec { - cluster_name: ClusterName::from_str("simple-kafka") - .expect("the cluster name should be valid"), - management_policy: Default::default(), - topic_name: TopicName::from_str(topic_name).expect("the topic name should be valid"), - partitions: 1, - replication_factor: 1, - config: Default::default(), + cluster_ref: KafkaClusterRef { + name: ClusterName::from_str("simple-kafka") + .expect("the cluster name should be valid"), + }, + topic_name: topic_name.map(|topic_name| { + TopicName::from_str(topic_name).expect("the topic name should be valid") + }), + partitions: Some(1), + replication_factor: Some(1), + config_overrides: Default::default(), + deletion_policy: Default::default(), + topic_operation: Default::default(), }, ) } diff --git a/rust/agent-binary/src/topic/update_status.rs b/rust/agent-binary/src/topic/update_status.rs new file mode 100644 index 000000000..3c6ffd07c --- /dev/null +++ b/rust/agent-binary/src/topic/update_status.rs @@ -0,0 +1,196 @@ +//! The status of a `KafkaTopic` for the outcome of a sync. + +use snafu::ResultExt; +use stackable_kafka_shared::crd::topic::v1alpha1::{ + KafkaTopic, KafkaTopicConfigStatus, KafkaTopicStatus, +}; +use stackable_operator::{ + k8s_openapi::{apimachinery::pkg::apis::meta::v1::Time, jiff::Timestamp}, + status::condition::managed_resource::update_conditions, +}; + +use super::{ + condition::{TopicSyncOutcome, conditions_from}, + controller::{Ctx, Result, UpdateStatusSnafu}, + plan::TopicState, +}; +use crate::AGENT_NAME; + +/// Patches the status only if it changed, so the agent does not trigger its own watch. +pub(crate) async fn update_status( + kafka_topic: &KafkaTopic, + ctx: &Ctx, + sync_outcome: &TopicSyncOutcome, + topic_state: Option<&TopicState>, +) -> Result<()> { + let current_status = kafka_topic.status.clone().unwrap_or_default(); + let new_status = status_from( + ¤t_status, + sync_outcome, + topic_state, + &Time(Timestamp::now()), + kafka_topic.metadata.generation, + ); + + if new_status != current_status { + ctx.client + .apply_patch_status(AGENT_NAME, kafka_topic, &new_status) + .await + .context(UpdateStatusSnafu)?; + } + Ok(()) +} + +/// Fields not known after the outcome are kept from `current_status`. +fn status_from( + current_status: &KafkaTopicStatus, + sync_outcome: &TopicSyncOutcome, + topic_state: Option<&TopicState>, + now: &Time, + observed_generation: Option, +) -> KafkaTopicStatus { + let conditions = update_conditions( + ¤t_status.conditions, + conditions_from(sync_outcome, topic_state), + now, + observed_generation, + ); + match (sync_outcome, topic_state) { + (_, Some(topic_state)) => KafkaTopicStatus { + conditions, + topic_id: Some(topic_state.id.clone()), + topic_name: Some(topic_state.name.clone()), + partitions: Some(topic_state.partition_count), + replication_factor: topic_state.replication_factor, + config: topic_state + .effective_configs + .iter() + .map(|(key, config)| { + let config = KafkaTopicConfigStatus { + value: config.value.clone(), + source: config.source.to_string(), + }; + (key.clone(), config) + }) + .collect(), + }, + // The topic does not exist or is managed by another KafkaTopic. + (TopicSyncOutcome::Conflict { .. } | TopicSyncOutcome::ReconciliationPaused, None) => { + KafkaTopicStatus { + conditions, + ..KafkaTopicStatus::default() + } + } + _ => KafkaTopicStatus { + conditions, + ..current_status.clone() + }, + } +} + +#[cfg(test)] +mod tests { + use std::{collections::BTreeMap, str::FromStr}; + + use stackable_kafka_admin::client::{ConfigSource, TopicConfig}; + use stackable_kafka_shared::crd::topic::TopicName; + + use super::*; + + fn now() -> Time { + Time(Timestamp::from_second(10).expect("the timestamp should be valid")) + } + + fn topic_state() -> TopicState { + TopicState { + name: TopicName::from_str("orders").expect("the topic name should be valid"), + id: "UXyU9i5ARn6W00ON2taeWA".to_owned(), + partition_count: 6, + replication_factor: Some(3), + effective_configs: BTreeMap::from([( + "retention.ms".to_owned(), + TopicConfig { + value: Some("1000".to_owned()), + source: ConfigSource::DynamicTopic, + }, + )]), + } + } + + fn synced() -> TopicSyncOutcome { + TopicSyncOutcome::Synced { + reassigning_partition_count: 0, + } + } + + #[test] + fn show_the_topic_in_kafka() { + let status = status_from( + &KafkaTopicStatus::default(), + &synced(), + Some(&topic_state()), + &now(), + Some(2), + ); + + assert_eq!(Some("UXyU9i5ARn6W00ON2taeWA".to_owned()), status.topic_id); + assert_eq!(Some(6), status.partitions); + assert_eq!(Some(3), status.replication_factor); + assert_eq!( + Some(&KafkaTopicConfigStatus { + value: Some("1000".to_owned()), + source: "DYNAMIC_TOPIC_CONFIG".to_owned(), + }), + status.config.get("retention.ms") + ); + assert!( + status + .conditions + .iter() + .all(|condition| condition.observed_generation == Some(2)) + ); + } + + #[test] + fn keep_the_topic_if_kafka_cannot_be_reached() { + let current = status_from( + &KafkaTopicStatus::default(), + &synced(), + Some(&topic_state()), + &now(), + Some(1), + ); + + let status = status_from( + ¤t, + &TopicSyncOutcome::ConnectionFailed { + message: "timed out".to_owned(), + }, + None, + &now(), + Some(1), + ); + + assert_eq!(current.topic_id, status.topic_id); + assert_eq!(current.config, status.config); + } + + #[rstest::rstest] + #[case::conflict(TopicSyncOutcome::Conflict { message: String::new() })] + #[case::paused_without_topic(TopicSyncOutcome::ReconciliationPaused)] + fn clear_the_topic_if_it_is_not_managed(#[case] sync_outcome: TopicSyncOutcome) { + let current = status_from( + &KafkaTopicStatus::default(), + &synced(), + Some(&topic_state()), + &now(), + Some(1), + ); + + let status = status_from(¤t, &sync_outcome, None, &now(), Some(1)); + + assert_eq!(None, status.topic_id); + assert!(status.config.is_empty()); + assert_eq!(2, status.conditions.len()); + } +} diff --git a/rust/agent-binary/src/topic/validate.rs b/rust/agent-binary/src/topic/validate.rs new file mode 100644 index 000000000..d5055d4dc --- /dev/null +++ b/rust/agent-binary/src/topic/validate.rs @@ -0,0 +1,234 @@ +//! Whether a `KafkaTopic` can be synced, without talking to Kafka. + +use stackable_kafka_shared::crd::topic::{TopicName, v1alpha1::KafkaTopic}; +use stackable_operator::kube::{ResourceExt, core::DeserializeGuard, runtime::reflector::Store}; + +use super::{condition::TopicSyncOutcome, plan::THROTTLE_CONFIG_KEYS}; + +/// Returns the name of the topic in Kafka, or why the `KafkaTopic` cannot be synced. +pub(crate) fn validate( + kafka_topic: &KafkaTopic, + kafka_topics: &Store>, +) -> Result { + let topic_name = kafka_topic + .topic_name() + .map_err(|error| TopicSyncOutcome::InvalidSpec { + message: format!( + "metadata.name is not a valid topic name, recreate the KafkaTopic with topicName \ + set: {error}" + ), + })?; + + let throttle_keys = THROTTLE_CONFIG_KEYS + .iter() + .filter(|key| kafka_topic.spec.config_overrides.contains_key(**key)) + .copied() + .collect::>(); + if !throttle_keys.is_empty() { + return Err(TopicSyncOutcome::InvalidSpec { + message: format!( + "the config keys {throttle_keys:?} are managed by partition reassignments and \ + cannot be set" + ), + }); + } + + if let Some(owner) = conflicting_owner(kafka_topic, &topic_name, kafka_topics) { + return Err(TopicSyncOutcome::Conflict { + message: format!("the topic {topic_name} is managed by the KafkaTopic {owner:?}"), + }); + } + + Ok(topic_name) +} + +/// The oldest other `KafkaTopic` for the same topic of the same cluster, by creation time and +/// name. Checked against all `KafkaTopic`s in the store, so the result does not depend on the +/// reconcile order. +pub(crate) fn conflicting_owner( + kafka_topic: &KafkaTopic, + topic_name: &TopicName, + kafka_topics: &Store>, +) -> Option { + let key = |kafka_topic: &KafkaTopic| { + ( + kafka_topic.metadata.creation_timestamp.clone(), + kafka_topic.name_any(), + ) + }; + kafka_topics + .state() + .iter() + .filter_map(|other_kafka_topic| other_kafka_topic.0.as_ref().ok()) + .filter(|other_kafka_topic| is_same_topic(kafka_topic, other_kafka_topic, topic_name)) + .min_by_key(|other_kafka_topic| key(other_kafka_topic)) + .filter(|oldest_kafka_topic| key(oldest_kafka_topic) != key(kafka_topic)) + .map(ResourceExt::name_any) +} + +/// Whether another `KafkaTopic` that is not being deleted declares the same topic of the same +/// cluster, regardless of its age. +pub(crate) fn has_other_kafka_topic( + kafka_topic: &KafkaTopic, + topic_name: &TopicName, + kafka_topics: &Store>, +) -> bool { + kafka_topics + .state() + .iter() + .filter_map(|other_kafka_topic| other_kafka_topic.0.as_ref().ok()) + .any(|other_kafka_topic| { + other_kafka_topic.name_any() != kafka_topic.name_any() + && other_kafka_topic.metadata.deletion_timestamp.is_none() + && is_same_topic(kafka_topic, other_kafka_topic, topic_name) + }) +} + +fn is_same_topic( + kafka_topic: &KafkaTopic, + other_kafka_topic: &KafkaTopic, + topic_name: &TopicName, +) -> bool { + other_kafka_topic.spec.cluster_ref.name == kafka_topic.spec.cluster_ref.name + && other_kafka_topic + .topic_name() + .is_ok_and(|other_topic_name| &other_topic_name == topic_name) +} + +#[cfg(test)] +mod tests { + use std::str::FromStr; + + use stackable_operator::{ + k8s_openapi::{apimachinery::pkg::apis::meta::v1::Time, jiff::Timestamp}, + kube::runtime::watcher, + }; + + use super::*; + use crate::topic::test_support::kafka_topic; + + fn created_topic( + name: &str, + created_at: i64, + topic_name: Option<&str>, + ) -> DeserializeGuard { + let mut topic = kafka_topic(name, topic_name); + topic.metadata.creation_timestamp = Some(Time( + Timestamp::from_second(created_at).expect("the timestamp should be valid"), + )); + DeserializeGuard(Ok(topic)) + } + + fn store(topics: Vec>) -> Store> { + let (store, mut writer) = stackable_operator::kube::runtime::reflector::store(); + for topic in topics { + writer.apply_watcher_event(&watcher::Event::Apply(topic)); + } + store + } + + fn inner(topic: &DeserializeGuard) -> &KafkaTopic { + topic.0.as_ref().expect("the topic should be valid") + } + + fn topic_name(name: &str) -> TopicName { + TopicName::from_str(name).expect("the topic name should be valid") + } + + #[test] + fn the_oldest_kafka_topic_owns_the_topic() { + let older = created_topic("older", 10, Some("orders")); + let newer = created_topic("newer", 20, Some("orders")); + let other_topic = created_topic("other", 5, Some("payments")); + let topics = store(vec![older.clone(), newer.clone(), other_topic]); + + assert_eq!( + None, + conflicting_owner(inner(&older), &topic_name("orders"), &topics) + ); + assert_eq!( + Some("older".to_owned()), + conflicting_owner(inner(&newer), &topic_name("orders"), &topics) + ); + } + + #[test] + fn the_name_decides_between_kafka_topics_of_the_same_age() { + let first = created_topic("a", 10, Some("orders")); + let second = created_topic("b", 10, Some("orders")); + let topics = store(vec![second.clone(), first.clone()]); + + assert_eq!( + None, + conflicting_owner(inner(&first), &topic_name("orders"), &topics) + ); + assert_eq!( + Some("a".to_owned()), + conflicting_owner(inner(&second), &topic_name("orders"), &topics) + ); + } + + #[test] + fn compare_the_effective_topic_names() { + let explicit = created_topic("explicit", 10, Some("orders")); + let from_metadata_name = created_topic("orders", 20, None); + let topics = store(vec![explicit.clone(), from_metadata_name.clone()]); + + assert_eq!( + Some("explicit".to_owned()), + conflicting_owner(inner(&from_metadata_name), &topic_name("orders"), &topics) + ); + } + + #[test] + fn find_other_kafka_topics_regardless_of_their_age() { + let older = created_topic("older", 10, Some("orders")); + let newer = created_topic("newer", 20, Some("orders")); + let mut deleted = created_topic("deleted", 30, Some("payments")); + if let Ok(deleted) = &mut deleted.0 { + deleted.metadata.deletion_timestamp = deleted.metadata.creation_timestamp.clone(); + } + let payments = created_topic("payments", 40, None); + let topics = store(vec![older.clone(), newer, deleted, payments.clone()]); + + assert!(has_other_kafka_topic( + inner(&older), + &topic_name("orders"), + &topics + )); + assert!(!has_other_kafka_topic( + inner(&payments), + &topic_name("payments"), + &topics + )); + } + + #[rstest::rstest] + #[case::no_throttle(&[("retention.ms", "1000")], false)] + #[case::throttle(&[("follower.replication.throttled.replicas", "*")], true)] + fn reject_throttle_config_keys(#[case] config: &[(&str, &str)], #[case] invalid: bool) { + let mut topic = kafka_topic("orders", Some("orders")); + topic.spec.config_overrides = config + .iter() + .map(|(key, value)| (key.to_string(), value.to_string())) + .collect(); + + assert_eq!( + invalid, + matches!( + validate(&topic, &store(vec![])), + Err(TopicSyncOutcome::InvalidSpec { .. }) + ) + ); + } + + #[test] + fn reject_a_too_long_metadata_name_without_topic_name() { + let topic = kafka_topic(&"a".repeat(250), None); + + assert!(matches!( + validate(&topic, &store(vec![])), + Err(TopicSyncOutcome::InvalidSpec { .. }) + )); + } +} From 61b1005ae663fcfd0c74d72394ac38bf284dba02 Mon Sep 17 00:00:00 2001 From: Benedikt Labrenz Date: Tue, 27 Oct 2026 09:01:23 +0100 Subject: [PATCH 36/93] update operator-rs --- Cargo.lock | 31 +++++++++++----- Cargo.nix | 95 +++++++++++++++++++++++++++++++++++------------ Cargo.toml | 2 +- crate-hashes.json | 19 +++++----- 4 files changed, 105 insertions(+), 42 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 12e294d61..1c2030e7a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1712,7 +1712,7 @@ dependencies = [ [[package]] name = "k8s-version" version = "0.1.4" -source = "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#079f7413186c44b0bf9723c1361ea955613c39b1" +source = "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#b815f203fda98c109be46aa67e7cf3921c808603" dependencies = [ "darling 0.24.1", "regex", @@ -3165,10 +3165,22 @@ version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596" +[[package]] +name = "stackable-agent" +version = "0.1.0" +source = "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#b815f203fda98c109be46aa67e7cf3921c808603" +dependencies = [ + "k8s-openapi", + "schemars", + "serde", + "stackable-versioned", + "strum", +] + [[package]] name = "stackable-certs" version = "0.4.1" -source = "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#079f7413186c44b0bf9723c1361ea955613c39b1" +source = "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#b815f203fda98c109be46aa67e7cf3921c808603" dependencies = [ "const-oid", "ecdsa", @@ -3235,7 +3247,7 @@ dependencies = [ [[package]] name = "stackable-operator" version = "0.121.0" -source = "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#079f7413186c44b0bf9723c1361ea955613c39b1" +source = "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#b815f203fda98c109be46aa67e7cf3921c808603" dependencies = [ "base64 0.23.1", "clap", @@ -3260,6 +3272,7 @@ dependencies = [ "serde_json", "sha2", "snafu 0.9.2", + "stackable-agent", "stackable-operator-derive", "stackable-shared", "stackable-telemetry", @@ -3279,7 +3292,7 @@ dependencies = [ [[package]] name = "stackable-operator-derive" version = "0.3.1" -source = "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#079f7413186c44b0bf9723c1361ea955613c39b1" +source = "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#b815f203fda98c109be46aa67e7cf3921c808603" dependencies = [ "darling 0.24.1", "proc-macro2", @@ -3290,7 +3303,7 @@ dependencies = [ [[package]] name = "stackable-shared" version = "0.2.0" -source = "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#079f7413186c44b0bf9723c1361ea955613c39b1" +source = "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#b815f203fda98c109be46aa67e7cf3921c808603" dependencies = [ "jiff", "k8s-openapi", @@ -3307,7 +3320,7 @@ dependencies = [ [[package]] name = "stackable-telemetry" version = "0.6.5" -source = "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#079f7413186c44b0bf9723c1361ea955613c39b1" +source = "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#b815f203fda98c109be46aa67e7cf3921c808603" dependencies = [ "axum", "clap", @@ -3331,7 +3344,7 @@ dependencies = [ [[package]] name = "stackable-versioned" version = "0.13.0" -source = "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#079f7413186c44b0bf9723c1361ea955613c39b1" +source = "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#b815f203fda98c109be46aa67e7cf3921c808603" dependencies = [ "kube", "schemars", @@ -3344,7 +3357,7 @@ dependencies = [ [[package]] name = "stackable-versioned-macros" version = "0.13.0" -source = "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#079f7413186c44b0bf9723c1361ea955613c39b1" +source = "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#b815f203fda98c109be46aa67e7cf3921c808603" dependencies = [ "convert_case", "darling 0.24.1", @@ -3361,7 +3374,7 @@ dependencies = [ [[package]] name = "stackable-webhook" version = "0.10.0" -source = "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#079f7413186c44b0bf9723c1361ea955613c39b1" +source = "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#b815f203fda98c109be46aa67e7cf3921c808603" dependencies = [ "arc-swap", "async-trait", diff --git a/Cargo.nix b/Cargo.nix index 3494b58fe..416da7837 100644 --- a/Cargo.nix +++ b/Cargo.nix @@ -5455,8 +5455,8 @@ rec { workspace_member = null; src = pkgs.fetchgit { url = "https://github.com/stackabletech/operator-rs.git"; - rev = "079f7413186c44b0bf9723c1361ea955613c39b1"; - sha256 = "0cgziqra8097hp05ynib1qpw4c95n972f4w2rk9l3llyp8r1vmci"; + rev = "b815f203fda98c109be46aa67e7cf3921c808603"; + sha256 = "0vdnhvfn85qnm5845mad24smhdp2y6nznlgagbp3g42pi042g9pm"; }; libName = "k8s_version"; authors = [ @@ -10381,6 +10381,49 @@ rec { "std" = [ "alloc" ]; }; }; + "stackable-agent" = rec { + crateName = "stackable-agent"; + version = "0.1.0"; + edition = "2024"; + workspace_member = null; + src = pkgs.fetchgit { + url = "https://github.com/stackabletech/operator-rs.git"; + rev = "b815f203fda98c109be46aa67e7cf3921c808603"; + sha256 = "0vdnhvfn85qnm5845mad24smhdp2y6nznlgagbp3g42pi042g9pm"; + }; + libName = "stackable_agent"; + authors = [ + "Stackable GmbH " + ]; + dependencies = [ + { + name = "k8s-openapi"; + packageId = "k8s-openapi"; + usesDefaultFeatures = false; + features = [ "schemars" "v1_36" ]; + } + { + name = "schemars"; + packageId = "schemars"; + features = [ "url2" ]; + } + { + name = "serde"; + packageId = "serde"; + features = [ "derive" ]; + } + { + name = "stackable-versioned"; + packageId = "stackable-versioned"; + } + { + name = "strum"; + packageId = "strum"; + features = [ "derive" ]; + } + ]; + + }; "stackable-certs" = rec { crateName = "stackable-certs"; version = "0.4.1"; @@ -10388,8 +10431,8 @@ rec { workspace_member = null; src = pkgs.fetchgit { url = "https://github.com/stackabletech/operator-rs.git"; - rev = "079f7413186c44b0bf9723c1361ea955613c39b1"; - sha256 = "0cgziqra8097hp05ynib1qpw4c95n972f4w2rk9l3llyp8r1vmci"; + rev = "b815f203fda98c109be46aa67e7cf3921c808603"; + sha256 = "0vdnhvfn85qnm5845mad24smhdp2y6nznlgagbp3g42pi042g9pm"; }; libName = "stackable_certs"; authors = [ @@ -10511,7 +10554,7 @@ rec { { name = "stackable-operator"; packageId = "stackable-operator"; - features = [ "crds" "webhook" ]; + features = [ "agent" "crds" "webhook" ]; } { name = "tokio"; @@ -10592,7 +10635,7 @@ rec { { name = "stackable-operator"; packageId = "stackable-operator"; - features = [ "crds" "webhook" ]; + features = [ "agent" "crds" "webhook" ]; } { name = "strum"; @@ -10637,7 +10680,7 @@ rec { { name = "stackable-operator"; packageId = "stackable-operator"; - features = [ "crds" "webhook" ]; + features = [ "agent" "crds" "webhook" ]; } { name = "strum"; @@ -10660,8 +10703,8 @@ rec { workspace_member = null; src = pkgs.fetchgit { url = "https://github.com/stackabletech/operator-rs.git"; - rev = "079f7413186c44b0bf9723c1361ea955613c39b1"; - sha256 = "0cgziqra8097hp05ynib1qpw4c95n972f4w2rk9l3llyp8r1vmci"; + rev = "b815f203fda98c109be46aa67e7cf3921c808603"; + sha256 = "0vdnhvfn85qnm5845mad24smhdp2y6nznlgagbp3g42pi042g9pm"; }; libName = "stackable_operator"; authors = [ @@ -10772,6 +10815,11 @@ rec { name = "snafu"; packageId = "snafu 0.9.2"; } + { + name = "stackable-agent"; + packageId = "stackable-agent"; + optional = true; + } { name = "stackable-operator-derive"; packageId = "stackable-operator-derive"; @@ -10838,11 +10886,12 @@ rec { } ]; features = { + "agent" = [ "dep:stackable-agent" ]; "certs" = [ "dep:stackable-certs" ]; "client-feature-gates" = [ "dep:winnow" ]; "crds" = [ "dep:stackable-versioned" ]; "default" = [ "crds" ]; - "full" = [ "client-feature-gates" "crds" "certs" "test-support" "time" "webhook" "kube-ws" "kube-cel" "kube-http-proxy" "kube-socks5" ]; + "full" = [ "agent" "client-feature-gates" "crds" "certs" "test-support" "time" "webhook" "kube-ws" "kube-cel" "kube-http-proxy" "kube-socks5" ]; "kube-cel" = [ "kube/cel" ]; "kube-http-proxy" = [ "kube/http-proxy" ]; "kube-socks5" = [ "kube/socks5" ]; @@ -10850,7 +10899,7 @@ rec { "time" = [ "stackable-shared/time" ]; "webhook" = [ "dep:stackable-webhook" ]; }; - resolvedDefaultFeatures = [ "crds" "default" "webhook" ]; + resolvedDefaultFeatures = [ "agent" "crds" "default" "webhook" ]; }; "stackable-operator-derive" = rec { crateName = "stackable-operator-derive"; @@ -10859,8 +10908,8 @@ rec { workspace_member = null; src = pkgs.fetchgit { url = "https://github.com/stackabletech/operator-rs.git"; - rev = "079f7413186c44b0bf9723c1361ea955613c39b1"; - sha256 = "0cgziqra8097hp05ynib1qpw4c95n972f4w2rk9l3llyp8r1vmci"; + rev = "b815f203fda98c109be46aa67e7cf3921c808603"; + sha256 = "0vdnhvfn85qnm5845mad24smhdp2y6nznlgagbp3g42pi042g9pm"; }; procMacro = true; libName = "stackable_operator_derive"; @@ -10894,8 +10943,8 @@ rec { workspace_member = null; src = pkgs.fetchgit { url = "https://github.com/stackabletech/operator-rs.git"; - rev = "079f7413186c44b0bf9723c1361ea955613c39b1"; - sha256 = "0cgziqra8097hp05ynib1qpw4c95n972f4w2rk9l3llyp8r1vmci"; + rev = "b815f203fda98c109be46aa67e7cf3921c808603"; + sha256 = "0vdnhvfn85qnm5845mad24smhdp2y6nznlgagbp3g42pi042g9pm"; }; libName = "stackable_shared"; authors = [ @@ -10975,8 +11024,8 @@ rec { workspace_member = null; src = pkgs.fetchgit { url = "https://github.com/stackabletech/operator-rs.git"; - rev = "079f7413186c44b0bf9723c1361ea955613c39b1"; - sha256 = "0cgziqra8097hp05ynib1qpw4c95n972f4w2rk9l3llyp8r1vmci"; + rev = "b815f203fda98c109be46aa67e7cf3921c808603"; + sha256 = "0vdnhvfn85qnm5845mad24smhdp2y6nznlgagbp3g42pi042g9pm"; }; libName = "stackable_telemetry"; authors = [ @@ -11085,8 +11134,8 @@ rec { workspace_member = null; src = pkgs.fetchgit { url = "https://github.com/stackabletech/operator-rs.git"; - rev = "079f7413186c44b0bf9723c1361ea955613c39b1"; - sha256 = "0cgziqra8097hp05ynib1qpw4c95n972f4w2rk9l3llyp8r1vmci"; + rev = "b815f203fda98c109be46aa67e7cf3921c808603"; + sha256 = "0vdnhvfn85qnm5845mad24smhdp2y6nznlgagbp3g42pi042g9pm"; }; libName = "stackable_versioned"; authors = [ @@ -11131,8 +11180,8 @@ rec { workspace_member = null; src = pkgs.fetchgit { url = "https://github.com/stackabletech/operator-rs.git"; - rev = "079f7413186c44b0bf9723c1361ea955613c39b1"; - sha256 = "0cgziqra8097hp05ynib1qpw4c95n972f4w2rk9l3llyp8r1vmci"; + rev = "b815f203fda98c109be46aa67e7cf3921c808603"; + sha256 = "0vdnhvfn85qnm5845mad24smhdp2y6nznlgagbp3g42pi042g9pm"; }; procMacro = true; libName = "stackable_versioned_macros"; @@ -11195,8 +11244,8 @@ rec { workspace_member = null; src = pkgs.fetchgit { url = "https://github.com/stackabletech/operator-rs.git"; - rev = "079f7413186c44b0bf9723c1361ea955613c39b1"; - sha256 = "0cgziqra8097hp05ynib1qpw4c95n972f4w2rk9l3llyp8r1vmci"; + rev = "b815f203fda98c109be46aa67e7cf3921c808603"; + sha256 = "0vdnhvfn85qnm5845mad24smhdp2y6nznlgagbp3g42pi042g9pm"; }; libName = "stackable_webhook"; authors = [ diff --git a/Cargo.toml b/Cargo.toml index 81beda9ec..ae973805a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -10,7 +10,7 @@ edition = "2024" repository = "https://github.com/stackabletech/kafka-operator" [workspace.dependencies] -stackable-operator = { git = "https://github.com/stackabletech/operator-rs.git", branch = "feat/platform-access", features = ["crds", "webhook"] } +stackable-operator = { git = "https://github.com/stackabletech/operator-rs.git", branch = "feat/platform-access", features = ["agent", "crds", "webhook"] } stackable-kafka-shared = { path = "rust/shared" } anyhow = "1.0" diff --git a/crate-hashes.json b/crate-hashes.json index a5ee97020..017105d5a 100644 --- a/crate-hashes.json +++ b/crate-hashes.json @@ -1,11 +1,12 @@ { - "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#k8s-version@0.1.4": "0cgziqra8097hp05ynib1qpw4c95n972f4w2rk9l3llyp8r1vmci", - "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#stackable-certs@0.4.1": "0cgziqra8097hp05ynib1qpw4c95n972f4w2rk9l3llyp8r1vmci", - "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#stackable-operator-derive@0.3.1": "0cgziqra8097hp05ynib1qpw4c95n972f4w2rk9l3llyp8r1vmci", - "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#stackable-operator@0.121.0": "0cgziqra8097hp05ynib1qpw4c95n972f4w2rk9l3llyp8r1vmci", - "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#stackable-shared@0.2.0": "0cgziqra8097hp05ynib1qpw4c95n972f4w2rk9l3llyp8r1vmci", - "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#stackable-telemetry@0.6.5": "0cgziqra8097hp05ynib1qpw4c95n972f4w2rk9l3llyp8r1vmci", - "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#stackable-versioned-macros@0.13.0": "0cgziqra8097hp05ynib1qpw4c95n972f4w2rk9l3llyp8r1vmci", - "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#stackable-versioned@0.13.0": "0cgziqra8097hp05ynib1qpw4c95n972f4w2rk9l3llyp8r1vmci", - "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#stackable-webhook@0.10.0": "0cgziqra8097hp05ynib1qpw4c95n972f4w2rk9l3llyp8r1vmci" + "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#k8s-version@0.1.4": "0vdnhvfn85qnm5845mad24smhdp2y6nznlgagbp3g42pi042g9pm", + "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#stackable-agent@0.1.0": "0vdnhvfn85qnm5845mad24smhdp2y6nznlgagbp3g42pi042g9pm", + "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#stackable-certs@0.4.1": "0vdnhvfn85qnm5845mad24smhdp2y6nznlgagbp3g42pi042g9pm", + "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#stackable-operator-derive@0.3.1": "0vdnhvfn85qnm5845mad24smhdp2y6nznlgagbp3g42pi042g9pm", + "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#stackable-operator@0.121.0": "0vdnhvfn85qnm5845mad24smhdp2y6nznlgagbp3g42pi042g9pm", + "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#stackable-shared@0.2.0": "0vdnhvfn85qnm5845mad24smhdp2y6nznlgagbp3g42pi042g9pm", + "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#stackable-telemetry@0.6.5": "0vdnhvfn85qnm5845mad24smhdp2y6nznlgagbp3g42pi042g9pm", + "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#stackable-versioned-macros@0.13.0": "0vdnhvfn85qnm5845mad24smhdp2y6nznlgagbp3g42pi042g9pm", + "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#stackable-versioned@0.13.0": "0vdnhvfn85qnm5845mad24smhdp2y6nznlgagbp3g42pi042g9pm", + "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#stackable-webhook@0.10.0": "0vdnhvfn85qnm5845mad24smhdp2y6nznlgagbp3g42pi042g9pm" } \ No newline at end of file From 66bf00b3c793146f015618205db72f828b356eee Mon Sep 17 00:00:00 2001 From: Benedikt Labrenz Date: Tue, 27 Oct 2026 11:32:09 +0100 Subject: [PATCH 37/93] replace hard-coded selectors on agent deployment with util from operator-rs --- Cargo.lock | 20 ++++++++++---------- Cargo.nix | 20 ++++++++++---------- rust/operator-binary/src/controller.rs | 8 +++++++- rust/operator-binary/src/main.rs | 6 ++---- 4 files changed, 29 insertions(+), 25 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 1c2030e7a..288c23de1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1712,7 +1712,7 @@ dependencies = [ [[package]] name = "k8s-version" version = "0.1.4" -source = "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#b815f203fda98c109be46aa67e7cf3921c808603" +source = "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#ccfff5810f9b5affae4d144b3cf04ca88677e23e" dependencies = [ "darling 0.24.1", "regex", @@ -3168,7 +3168,7 @@ checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596" [[package]] name = "stackable-agent" version = "0.1.0" -source = "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#b815f203fda98c109be46aa67e7cf3921c808603" +source = "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#ccfff5810f9b5affae4d144b3cf04ca88677e23e" dependencies = [ "k8s-openapi", "schemars", @@ -3180,7 +3180,7 @@ dependencies = [ [[package]] name = "stackable-certs" version = "0.4.1" -source = "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#b815f203fda98c109be46aa67e7cf3921c808603" +source = "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#ccfff5810f9b5affae4d144b3cf04ca88677e23e" dependencies = [ "const-oid", "ecdsa", @@ -3247,7 +3247,7 @@ dependencies = [ [[package]] name = "stackable-operator" version = "0.121.0" -source = "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#b815f203fda98c109be46aa67e7cf3921c808603" +source = "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#ccfff5810f9b5affae4d144b3cf04ca88677e23e" dependencies = [ "base64 0.23.1", "clap", @@ -3292,7 +3292,7 @@ dependencies = [ [[package]] name = "stackable-operator-derive" version = "0.3.1" -source = "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#b815f203fda98c109be46aa67e7cf3921c808603" +source = "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#ccfff5810f9b5affae4d144b3cf04ca88677e23e" dependencies = [ "darling 0.24.1", "proc-macro2", @@ -3303,7 +3303,7 @@ dependencies = [ [[package]] name = "stackable-shared" version = "0.2.0" -source = "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#b815f203fda98c109be46aa67e7cf3921c808603" +source = "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#ccfff5810f9b5affae4d144b3cf04ca88677e23e" dependencies = [ "jiff", "k8s-openapi", @@ -3320,7 +3320,7 @@ dependencies = [ [[package]] name = "stackable-telemetry" version = "0.6.5" -source = "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#b815f203fda98c109be46aa67e7cf3921c808603" +source = "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#ccfff5810f9b5affae4d144b3cf04ca88677e23e" dependencies = [ "axum", "clap", @@ -3344,7 +3344,7 @@ dependencies = [ [[package]] name = "stackable-versioned" version = "0.13.0" -source = "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#b815f203fda98c109be46aa67e7cf3921c808603" +source = "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#ccfff5810f9b5affae4d144b3cf04ca88677e23e" dependencies = [ "kube", "schemars", @@ -3357,7 +3357,7 @@ dependencies = [ [[package]] name = "stackable-versioned-macros" version = "0.13.0" -source = "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#b815f203fda98c109be46aa67e7cf3921c808603" +source = "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#ccfff5810f9b5affae4d144b3cf04ca88677e23e" dependencies = [ "convert_case", "darling 0.24.1", @@ -3374,7 +3374,7 @@ dependencies = [ [[package]] name = "stackable-webhook" version = "0.10.0" -source = "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#b815f203fda98c109be46aa67e7cf3921c808603" +source = "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#ccfff5810f9b5affae4d144b3cf04ca88677e23e" dependencies = [ "arc-swap", "async-trait", diff --git a/Cargo.nix b/Cargo.nix index 416da7837..0529432e5 100644 --- a/Cargo.nix +++ b/Cargo.nix @@ -5455,7 +5455,7 @@ rec { workspace_member = null; src = pkgs.fetchgit { url = "https://github.com/stackabletech/operator-rs.git"; - rev = "b815f203fda98c109be46aa67e7cf3921c808603"; + rev = "ccfff5810f9b5affae4d144b3cf04ca88677e23e"; sha256 = "0vdnhvfn85qnm5845mad24smhdp2y6nznlgagbp3g42pi042g9pm"; }; libName = "k8s_version"; @@ -10388,7 +10388,7 @@ rec { workspace_member = null; src = pkgs.fetchgit { url = "https://github.com/stackabletech/operator-rs.git"; - rev = "b815f203fda98c109be46aa67e7cf3921c808603"; + rev = "ccfff5810f9b5affae4d144b3cf04ca88677e23e"; sha256 = "0vdnhvfn85qnm5845mad24smhdp2y6nznlgagbp3g42pi042g9pm"; }; libName = "stackable_agent"; @@ -10431,7 +10431,7 @@ rec { workspace_member = null; src = pkgs.fetchgit { url = "https://github.com/stackabletech/operator-rs.git"; - rev = "b815f203fda98c109be46aa67e7cf3921c808603"; + rev = "ccfff5810f9b5affae4d144b3cf04ca88677e23e"; sha256 = "0vdnhvfn85qnm5845mad24smhdp2y6nznlgagbp3g42pi042g9pm"; }; libName = "stackable_certs"; @@ -10703,7 +10703,7 @@ rec { workspace_member = null; src = pkgs.fetchgit { url = "https://github.com/stackabletech/operator-rs.git"; - rev = "b815f203fda98c109be46aa67e7cf3921c808603"; + rev = "ccfff5810f9b5affae4d144b3cf04ca88677e23e"; sha256 = "0vdnhvfn85qnm5845mad24smhdp2y6nznlgagbp3g42pi042g9pm"; }; libName = "stackable_operator"; @@ -10908,7 +10908,7 @@ rec { workspace_member = null; src = pkgs.fetchgit { url = "https://github.com/stackabletech/operator-rs.git"; - rev = "b815f203fda98c109be46aa67e7cf3921c808603"; + rev = "ccfff5810f9b5affae4d144b3cf04ca88677e23e"; sha256 = "0vdnhvfn85qnm5845mad24smhdp2y6nznlgagbp3g42pi042g9pm"; }; procMacro = true; @@ -10943,7 +10943,7 @@ rec { workspace_member = null; src = pkgs.fetchgit { url = "https://github.com/stackabletech/operator-rs.git"; - rev = "b815f203fda98c109be46aa67e7cf3921c808603"; + rev = "ccfff5810f9b5affae4d144b3cf04ca88677e23e"; sha256 = "0vdnhvfn85qnm5845mad24smhdp2y6nznlgagbp3g42pi042g9pm"; }; libName = "stackable_shared"; @@ -11024,7 +11024,7 @@ rec { workspace_member = null; src = pkgs.fetchgit { url = "https://github.com/stackabletech/operator-rs.git"; - rev = "b815f203fda98c109be46aa67e7cf3921c808603"; + rev = "ccfff5810f9b5affae4d144b3cf04ca88677e23e"; sha256 = "0vdnhvfn85qnm5845mad24smhdp2y6nznlgagbp3g42pi042g9pm"; }; libName = "stackable_telemetry"; @@ -11134,7 +11134,7 @@ rec { workspace_member = null; src = pkgs.fetchgit { url = "https://github.com/stackabletech/operator-rs.git"; - rev = "b815f203fda98c109be46aa67e7cf3921c808603"; + rev = "ccfff5810f9b5affae4d144b3cf04ca88677e23e"; sha256 = "0vdnhvfn85qnm5845mad24smhdp2y6nznlgagbp3g42pi042g9pm"; }; libName = "stackable_versioned"; @@ -11180,7 +11180,7 @@ rec { workspace_member = null; src = pkgs.fetchgit { url = "https://github.com/stackabletech/operator-rs.git"; - rev = "b815f203fda98c109be46aa67e7cf3921c808603"; + rev = "ccfff5810f9b5affae4d144b3cf04ca88677e23e"; sha256 = "0vdnhvfn85qnm5845mad24smhdp2y6nznlgagbp3g42pi042g9pm"; }; procMacro = true; @@ -11244,7 +11244,7 @@ rec { workspace_member = null; src = pkgs.fetchgit { url = "https://github.com/stackabletech/operator-rs.git"; - rev = "b815f203fda98c109be46aa67e7cf3921c808603"; + rev = "ccfff5810f9b5affae4d144b3cf04ca88677e23e"; sha256 = "0vdnhvfn85qnm5845mad24smhdp2y6nznlgagbp3g42pi042g9pm"; }; libName = "stackable_webhook"; diff --git a/rust/operator-binary/src/controller.rs b/rust/operator-binary/src/controller.rs index 82ff790a2..b9618ede8 100644 --- a/rust/operator-binary/src/controller.rs +++ b/rust/operator-binary/src/controller.rs @@ -31,7 +31,7 @@ use stackable_operator::{ kube::{ Resource, api::{DynamicObject, ObjectMeta}, - core::{DeserializeGuard, error_boundary}, + core::{DeserializeGuard, Selector, error_boundary}, runtime::{controller::Action, reflector::ObjectRef}, }, logging::controller::ReconcilerError, @@ -39,6 +39,7 @@ use stackable_operator::{ v2::{ HasName, HasUid, NameIsValidLabelValue, agent_utils::AgentResourceNames, + kvp::label, role_group_utils::ResourceNames, role_utils, types::{ @@ -82,6 +83,11 @@ constant!(PRODUCT_NAME: ProductName = APP_NAME); constant!(OPERATOR_NAME: OperatorName = KAFKA_OPERATOR_NAME); constant!(CONTROLLER_NAME: ControllerName = KAFKA_CONTROLLER_NAME); +/// Selector matching the agents of all KafkaClusters, e.g. to watch them. +pub fn all_agents_selector() -> Selector { + Selector::from_iter(BTreeMap::from(label::all_agents_selector(&PRODUCT_NAME))) +} + #[derive(Snafu, Debug)] pub enum PodDescriptorsError { #[snafu(display( diff --git a/rust/operator-binary/src/main.rs b/rust/operator-binary/src/main.rs index 73367def6..7a0a9c5f9 100644 --- a/rust/operator-binary/src/main.rs +++ b/rust/operator-binary/src/main.rs @@ -37,7 +37,7 @@ use stackable_operator::{ }; use crate::{ - controller::KAFKA_FULL_CONTROLLER_NAME, + controller::{KAFKA_FULL_CONTROLLER_NAME, all_agents_selector}, crd::{KAFKA_OPERATOR_NAME, KafkaCluster, KafkaClusterVersion, v1alpha1}, webhooks::conversion::create_webhook_server, }; @@ -187,9 +187,7 @@ async fn main() -> anyhow::Result<()> { .owns( watch_namespace.get_api::>(&client), // The agent is the only Deployment - watcher::Config::default().labels( - "app.kubernetes.io/name=kafka,app.kubernetes.io/component=kafka-agent", - ), + watcher::Config::default().labels_from(&all_agents_selector()), ) .watches( watch_namespace.get_api::>(&client), From 624bebd61133324d2dc3980f0e28e015826bae4d Mon Sep 17 00:00:00 2001 From: Benedikt Labrenz Date: Tue, 27 Oct 2026 13:54:24 +0100 Subject: [PATCH 38/93] add topic_controller to operator for cleaning up after agents --- .../templates/clusterrole-agent.yaml | 2 +- .../templates/clusterrole-operator.yaml | 18 + rust/agent-binary/src/topic/finalizer.rs | 10 +- rust/operator-binary/src/controller.rs | 13 +- rust/operator-binary/src/main.rs | 87 +++- rust/operator-binary/src/topic_controller.rs | 385 ++++++++++++++++++ rust/shared/src/crd/topic.rs | 4 + 7 files changed, 505 insertions(+), 14 deletions(-) create mode 100644 rust/operator-binary/src/topic_controller.rs diff --git a/deploy/helm/kafka-operator/templates/clusterrole-agent.yaml b/deploy/helm/kafka-operator/templates/clusterrole-agent.yaml index 652fd210f..0de7e9b8c 100644 --- a/deploy/helm/kafka-operator/templates/clusterrole-agent.yaml +++ b/deploy/helm/kafka-operator/templates/clusterrole-agent.yaml @@ -8,7 +8,7 @@ metadata: labels: {{- include "kafka-operator.labels" . | nindent 4 }} rules: - # KafkaTopics of the agent's cluster: watched, annotated (ownership, managed config keys) and finalizers. + # KafkaTopics of the agent's cluster: watched, and their finalizer added and removed. - apiGroups: - {{ include "kafka-operator.name" . }}.stackable.tech resources: diff --git a/deploy/helm/kafka-operator/templates/clusterrole-operator.yaml b/deploy/helm/kafka-operator/templates/clusterrole-operator.yaml index aafed503c..f8f8193a1 100644 --- a/deploy/helm/kafka-operator/templates/clusterrole-operator.yaml +++ b/deploy/helm/kafka-operator/templates/clusterrole-operator.yaml @@ -128,6 +128,24 @@ rules: - {{ include "kafka-operator.name" . }}clusters/status verbs: - patch + # KafkaTopics: watched to take over while no agent is available, and their finalizer removed + # once the KafkaCluster is gone. + - apiGroups: + - {{ include "kafka-operator.name" . }}.stackable.tech + resources: + - kafkatopics + verbs: + - get + - list + - watch + - patch + # Status subresource: conditions set while the agent is unavailable. + - apiGroups: + - {{ include "kafka-operator.name" . }}.stackable.tech + resources: + - kafkatopics/status + verbs: + - patch # Read authentication class configuration referenced in the KafkaCluster spec. - apiGroups: - authentication.stackable.tech diff --git a/rust/agent-binary/src/topic/finalizer.rs b/rust/agent-binary/src/topic/finalizer.rs index a26d8a470..95a657785 100644 --- a/rust/agent-binary/src/topic/finalizer.rs +++ b/rust/agent-binary/src/topic/finalizer.rs @@ -4,7 +4,7 @@ use std::path::Path; use serde_json::json; use snafu::{OptionExt, ResultExt}; -use stackable_kafka_shared::crd::topic::v1alpha1::KafkaTopic; +use stackable_kafka_shared::crd::topic::{KAFKA_TOPIC_FINALIZER, v1alpha1::KafkaTopic}; use stackable_operator::{ agent::deletion::v1alpha1::DeletionPolicy, kube::{ResourceExt, runtime::controller::Action}, @@ -22,8 +22,6 @@ use super::{ }; use crate::config::{BOOTSTRAP_SERVERS_FILE, read_bootstrap_servers}; -const FINALIZER: &str = "kafka.stackable.tech/kafkatopic"; - /// Handles a deletion and removes a finalizer that is no longer needed. Returns `Some` if the /// reconcile ends here. pub(crate) async fn reconcile_finalizer( @@ -69,7 +67,7 @@ fn has_finalizer(kafka_topic: &KafkaTopic) -> bool { kafka_topic .finalizers() .iter() - .any(|finalizer| finalizer == FINALIZER) + .any(|finalizer| finalizer == KAFKA_TOPIC_FINALIZER) } /// Deletes the topic in Kafka if the `deletionPolicy` says so and no other `KafkaTopic` declares @@ -117,11 +115,11 @@ async fn set_finalizer( let mut finalizers = kafka_topic .finalizers() .iter() - .filter(|finalizer| *finalizer != FINALIZER) + .filter(|finalizer| *finalizer != KAFKA_TOPIC_FINALIZER) .cloned() .collect::>(); if should_have_finalizer { - finalizers.push(FINALIZER.to_owned()); + finalizers.push(KAFKA_TOPIC_FINALIZER.to_owned()); } let patch = json!({ "metadata": { diff --git a/rust/operator-binary/src/controller.rs b/rust/operator-binary/src/controller.rs index b9618ede8..0561030cc 100644 --- a/rust/operator-binary/src/controller.rs +++ b/rust/operator-binary/src/controller.rs @@ -88,6 +88,14 @@ pub fn all_agents_selector() -> Selector { Selector::from_iter(BTreeMap::from(label::all_agents_selector(&PRODUCT_NAME))) } +/// Type-safe names for the resources of the agent of the given cluster. +pub fn agent_resource_names_of(cluster_name: &ClusterName) -> AgentResourceNames { + AgentResourceNames { + cluster_name: cluster_name.clone(), + product_name: PRODUCT_NAME.clone(), + } +} + #[derive(Snafu, Debug)] pub enum PodDescriptorsError { #[snafu(display( @@ -272,10 +280,7 @@ impl ValidatedCluster { /// Type-safe names for the resources of the agent. pub fn agent_resource_names(&self) -> AgentResourceNames { - AgentResourceNames { - cluster_name: self.name.clone(), - product_name: PRODUCT_NAME.clone(), - } + agent_resource_names_of(&self.name) } /// Type-safe names for the resources of a given role group. diff --git a/rust/operator-binary/src/main.rs b/rust/operator-binary/src/main.rs index 07d454027..7df6ab26e 100644 --- a/rust/operator-binary/src/main.rs +++ b/rust/operator-binary/src/main.rs @@ -1,13 +1,15 @@ // TODO: Look into how to properly resolve `clippy::large_enum_variant`. // This will need changes in our and upstream error types. -#![allow(clippy::result_large_err)] +#![allow(clippy::large_enum_variant, clippy::result_large_err)] use std::sync::Arc; use anyhow::anyhow; use clap::Parser; use futures::{FutureExt, StreamExt, TryFutureExt}; -use stackable_kafka_shared::crd::topic::{KafkaTopic, KafkaTopicVersion}; +use stackable_kafka_shared::crd::topic::{ + KafkaTopic, KafkaTopicVersion, v1alpha1 as topic_v1alpha1, +}; use stackable_operator::{ YamlSchema, cli::{Command, RunArguments}, @@ -40,11 +42,13 @@ use stackable_operator::{ use crate::{ controller::{KAFKA_FULL_CONTROLLER_NAME, all_agents_selector}, crd::{KAFKA_OPERATOR_NAME, KafkaCluster, KafkaClusterVersion, v1alpha1}, + topic_controller::{TOPIC_FULL_CONTROLLER_NAME, kafka_topics_of_cluster}, webhooks::conversion::create_webhook_server, }; mod controller; mod crd; +mod topic_controller; mod webhooks; mod built_info { @@ -133,6 +137,10 @@ async fn main() -> anyhow::Result<()> { "CRD {crd} established", crd = v1alpha1::KafkaCluster::crd_name() )); + let kafka_topic_crd_check = readiness_checks.register(format!( + "CRD {crd} established", + crd = topic_v1alpha1::KafkaTopic::crd_name() + )); let webhook_server = create_webhook_server( &operator_environment, @@ -234,13 +242,86 @@ async fn main() -> anyhow::Result<()> { ) .map(anyhow::Ok); + // Topic controller to clean up after deleted or unavailable agents + let topic_event_recorder = Arc::new(Recorder::new( + client.as_kube_client(), + Reporter { + controller: TOPIC_FULL_CONTROLLER_NAME.to_string(), + instance: None, + }, + )); + + let topic_controller = Controller::new( + watch_namespace.get_api::>(&client), + watcher::Config::default(), + ); + let kafka_topic_store_for_clusters = topic_controller.store(); + let kafka_topic_store_for_agents = topic_controller.store(); + let topic_controller = topic_controller + .watches( + watch_namespace.get_api::>(&client), + watcher::Config::default(), + move |kafka_cluster| { + kafka_topics_of_cluster( + &kafka_topic_store_for_clusters, + kafka_cluster.namespace(), + Some(&kafka_cluster.name_any()), + ) + }, + ) + .watches( + watch_namespace.get_api::>(&client), + watcher::Config::default().labels_from(&all_agents_selector()), + move |agent_deployment| { + kafka_topics_of_cluster( + &kafka_topic_store_for_agents, + agent_deployment.namespace(), + agent_deployment + .labels() + .get("app.kubernetes.io/instance") + .map(String::as_str), + ) + }, + ) + .graceful_shutdown_on(sigterm_watcher.handle()) + .run( + topic_controller::reconcile_topic, + topic_controller::error_policy, + Arc::new(topic_controller::Ctx { + client: client.clone(), + }), + ) + .for_each_concurrent(16, move |result| { + let topic_event_recorder = topic_event_recorder.clone(); + async move { + report_controller_reconciled( + &topic_event_recorder, + TOPIC_FULL_CONTROLLER_NAME, + &result, + ) + .await; + } + }) + .map(anyhow::Ok); + let delayed_kafka_controller = async { signal::crd_established(&client, v1alpha1::KafkaCluster::crd_name()).await?; kafka_cluster_crd_check.mark_passed(); kafka_controller.await }; - futures::try_join!(delayed_kafka_controller, eos_checker, webhook_server)?; + let delayed_topic_controller = async { + signal::crd_established(&client, topic_v1alpha1::KafkaTopic::crd_name()).await?; + kafka_topic_crd_check.mark_passed(); + topic_controller.await + }; + + futures::try_join!( + delayed_kafka_controller, + delayed_topic_controller, + eos_checker, + webhook_server + )?; } }; diff --git a/rust/operator-binary/src/topic_controller.rs b/rust/operator-binary/src/topic_controller.rs new file mode 100644 index 000000000..25414026c --- /dev/null +++ b/rust/operator-binary/src/topic_controller.rs @@ -0,0 +1,385 @@ +//! Takes over `KafkaTopic`s while no agent is available: reports it in their status and removes +//! their finalizer once the KafkaCluster is gone, so deleting the KafkaCluster or the namespace +//! does not block. + +use std::sync::Arc; + +use const_format::concatcp; +use serde_json::json; +use snafu::{ResultExt, Snafu}; +use stackable_kafka_shared::crd::topic::{KAFKA_TOPIC_FINALIZER, v1alpha1::KafkaTopic}; +use stackable_operator::{ + agent::condition::{ + ConditionStatus, ManagedCondition, ManagedConditionType, update_conditions, + }, + k8s_openapi::{ + api::apps::v1::Deployment, apimachinery::pkg::apis::meta::v1::Time, jiff::Timestamp, + }, + kube::{ + ResourceExt, + core::{DeserializeGuard, error_boundary}, + runtime::{ + controller::Action, + reflector::{ObjectRef, Store}, + }, + }, + logging::controller::ReconcilerError, + shared::time::Duration, +}; +use strum::{EnumDiscriminants, IntoStaticStr}; + +use crate::{ + controller::agent_resource_names_of, + crd::{KAFKA_OPERATOR_NAME, v1alpha1::KafkaCluster}, +}; + +pub const TOPIC_CONTROLLER_NAME: &str = "kafkatopic"; +pub const TOPIC_FULL_CONTROLLER_NAME: &str = + concatcp!(TOPIC_CONTROLLER_NAME, '.', KAFKA_OPERATOR_NAME); + +const AGENT_UNAVAILABLE_REASON: &str = "AgentUnavailable"; + +#[derive(Debug, Snafu, EnumDiscriminants)] +#[strum_discriminants(derive(IntoStaticStr))] +pub enum Error { + #[snafu(display("the KafkaTopic is invalid"))] + InvalidKafkaTopic { + source: error_boundary::InvalidObject, + }, + + #[snafu(display("failed to get the KafkaCluster"))] + GetKafkaCluster { + source: stackable_operator::client::Error, + }, + + #[snafu(display("failed to get the agent Deployment"))] + GetAgentDeployment { + source: stackable_operator::client::Error, + }, + + #[snafu(display("failed to update the status of the KafkaTopic"))] + UpdateStatus { + source: stackable_operator::client::Error, + }, + + #[snafu(display("failed to remove the finalizer of the KafkaTopic"))] + RemoveFinalizer { + source: stackable_operator::client::Error, + }, +} + +type Result = std::result::Result; + +impl ReconcilerError for Error { + fn category(&self) -> &'static str { + ErrorDiscriminants::from(self).into() + } +} + +pub struct Ctx { + pub client: stackable_operator::client::Client, +} + +/// The KafkaCluster of a `KafkaTopic`, as far as its agent is concerned. +#[derive(Debug, Eq, PartialEq)] +enum ClusterState { + /// Does not exist or has a `deletionTimestamp` + MissingOrDeleting, + ManagementAccessDisabled, + ManagementAccessEnabled, +} + +#[derive(Debug, Eq, PartialEq)] +enum OperatorAction { + LeaveToAgent, + RemoveFinalizer, + SetAgentUnavailableConditions { message: &'static str }, +} + +pub async fn reconcile_topic( + kafka_topic: Arc>, + ctx: Arc, +) -> Result { + let kafka_topic = kafka_topic + .0 + .as_ref() + .map_err(error_boundary::InvalidObject::clone) + .context(InvalidKafkaTopicSnafu)?; + let namespace = kafka_topic.namespace().unwrap_or_default(); + let cluster_name = &kafka_topic.spec.cluster_ref.name; + + let kafka_cluster = ctx + .client + .get_opt::>(cluster_name.as_ref(), &namespace) + .await + .context(GetKafkaClusterSnafu)?; + let agent_deployment_name = agent_resource_names_of(cluster_name).deployment_name(); + let agent_deployment = ctx + .client + .get_opt::(agent_deployment_name.as_ref(), &namespace) + .await + .context(GetAgentDeploymentSnafu)?; + + match operator_action_for( + kafka_topic, + &cluster_state_from(kafka_cluster.as_ref()), + agent_deployment + .as_ref() + .is_some_and(is_agent_deployment_available), + ) { + OperatorAction::LeaveToAgent => {} + OperatorAction::RemoveFinalizer => remove_finalizer(kafka_topic, &ctx).await?, + OperatorAction::SetAgentUnavailableConditions { message } => { + set_agent_unavailable_conditions(kafka_topic, &ctx, message).await? + } + } + Ok(Action::await_change()) +} + +pub fn error_policy( + _kafka_topic: Arc>, + error: &Error, + _ctx: Arc, +) -> Action { + match error { + Error::InvalidKafkaTopic { .. } => Action::await_change(), + _ => Action::requeue(*Duration::from_secs(30)), + } +} + +/// The `KafkaTopic`s of the KafkaCluster `cluster_name`, to reconcile them when the KafkaCluster +/// or its agent changes. +pub fn kafka_topics_of_cluster( + kafka_topics: &Store>, + namespace: Option, + cluster_name: Option<&str>, +) -> Vec>> { + kafka_topics + .state() + .into_iter() + .filter(|kafka_topic| { + kafka_topic.0.as_ref().is_ok_and(|kafka_topic| { + kafka_topic.namespace() == namespace + && Some(kafka_topic.spec.cluster_ref.name.as_ref()) == cluster_name + }) + }) + .map(|kafka_topic| ObjectRef::from_obj(&*kafka_topic)) + .collect() +} + +fn cluster_state_from(kafka_cluster: Option<&DeserializeGuard>) -> ClusterState { + match kafka_cluster.map(|kafka_cluster| &kafka_cluster.0) { + None => ClusterState::MissingOrDeleting, + Some(Ok(kafka_cluster)) if kafka_cluster.metadata.deletion_timestamp.is_some() => { + ClusterState::MissingOrDeleting + } + Some(Ok(kafka_cluster)) + if !kafka_cluster + .spec + .cluster_config + .management_access + .as_ref() + .is_some_and(|management_access| management_access.enabled) => + { + ClusterState::ManagementAccessDisabled + } + // An invalid KafkaCluster keeps its agent, so the state of the agent decides. + Some(_) => ClusterState::ManagementAccessEnabled, + } +} + +fn operator_action_for( + kafka_topic: &KafkaTopic, + cluster_state: &ClusterState, + agent_available: bool, +) -> OperatorAction { + match cluster_state { + ClusterState::MissingOrDeleting if kafka_topic.metadata.deletion_timestamp.is_some() => { + if kafka_topic + .finalizers() + .iter() + .any(|finalizer| finalizer == KAFKA_TOPIC_FINALIZER) + { + OperatorAction::RemoveFinalizer + } else { + OperatorAction::LeaveToAgent + } + } + ClusterState::MissingOrDeleting => OperatorAction::SetAgentUnavailableConditions { + message: "the KafkaCluster does not exist", + }, + ClusterState::ManagementAccessDisabled => OperatorAction::SetAgentUnavailableConditions { + message: "the KafkaCluster has no agent, because clusterConfig.managementAccess is \ + not enabled", + }, + ClusterState::ManagementAccessEnabled if agent_available => OperatorAction::LeaveToAgent, + ClusterState::ManagementAccessEnabled => OperatorAction::SetAgentUnavailableConditions { + message: "the agent of the KafkaCluster is not available", + }, + } +} + +fn is_agent_deployment_available(agent_deployment: &Deployment) -> bool { + agent_deployment + .status + .as_ref() + .and_then(|status| status.conditions.as_ref()) + .is_some_and(|conditions| { + conditions + .iter() + .any(|condition| condition.type_ == "Available" && condition.status == "True") + }) +} + +/// Only the conditions are applied, so the other fields stay owned by the agent and keep the +/// last state it observed. +async fn set_agent_unavailable_conditions( + kafka_topic: &KafkaTopic, + ctx: &Ctx, + message: &str, +) -> Result<()> { + let current_conditions = kafka_topic + .status + .as_ref() + .map(|status| status.conditions.as_slice()) + .unwrap_or_default(); + let new_conditions = update_conditions( + current_conditions, + vec![ + agent_unavailable_condition( + ManagedConditionType::Ready, + ConditionStatus::Unknown, + message, + ), + agent_unavailable_condition( + ManagedConditionType::Synced, + ConditionStatus::False, + message, + ), + ], + &Time(Timestamp::now()), + kafka_topic.metadata.generation, + ); + + if new_conditions != current_conditions { + ctx.client + .apply_patch_status( + TOPIC_CONTROLLER_NAME, + kafka_topic, + &json!({ "conditions": new_conditions }), + ) + .await + .context(UpdateStatusSnafu)?; + } + Ok(()) +} + +fn agent_unavailable_condition( + type_: ManagedConditionType, + status: ConditionStatus, + message: &str, +) -> ManagedCondition { + ManagedCondition { + type_, + status, + reason: AGENT_UNAVAILABLE_REASON.to_owned(), + message: message.to_owned(), + } +} + +/// Fails if the `KafkaTopic` was changed since it was read, so concurrent changes to the +/// finalizers, e.g. by an agent that is still shutting down, are not lost. +async fn remove_finalizer(kafka_topic: &KafkaTopic, ctx: &Ctx) -> Result<()> { + let finalizers = kafka_topic + .finalizers() + .iter() + .filter(|finalizer| *finalizer != KAFKA_TOPIC_FINALIZER) + .cloned() + .collect::>(); + let patch = json!({ + "metadata": { + "resourceVersion": kafka_topic.metadata.resource_version, + "finalizers": finalizers, + } + }); + ctx.client + .merge_patch(kafka_topic, patch) + .await + .context(RemoveFinalizerSnafu)?; + Ok(()) +} + +#[cfg(test)] +mod tests { + use rstest::rstest; + use stackable_operator::kube::api::ObjectMeta; + + use super::*; + + fn kafka_topic(deleting: bool, with_finalizer: bool) -> KafkaTopic { + let kafka_topic: KafkaTopic = serde_json::from_value(json!({ + "apiVersion": "kafka.stackable.tech/v1alpha1", + "kind": "KafkaTopic", + "metadata": { "name": "orders", "namespace": "default" }, + "spec": { "clusterRef": { "name": "simple-kafka" } }, + })) + .expect("the KafkaTopic should be valid"); + KafkaTopic { + metadata: ObjectMeta { + deletion_timestamp: deleting.then_some(Time(Timestamp::UNIX_EPOCH)), + finalizers: with_finalizer.then(|| vec![KAFKA_TOPIC_FINALIZER.to_owned()]), + ..kafka_topic.metadata + }, + ..kafka_topic + } + } + + #[rstest] + #[case::cluster_missing(ClusterState::MissingOrDeleting, true, true)] + #[case::cluster_missing_without_agent(ClusterState::MissingOrDeleting, false, true)] + #[case::management_access_disabled(ClusterState::ManagementAccessDisabled, false, false)] + #[case::agent_available(ClusterState::ManagementAccessEnabled, true, false)] + #[case::agent_unavailable(ClusterState::ManagementAccessEnabled, false, false)] + fn remove_the_finalizer_only_if_the_cluster_is_missing_or_deleting( + #[case] cluster_state: ClusterState, + #[case] agent_available: bool, + #[case] removes_finalizer: bool, + ) { + assert_eq!( + removes_finalizer, + operator_action_for(&kafka_topic(true, true), &cluster_state, agent_available) + == OperatorAction::RemoveFinalizer + ); + } + + #[rstest] + #[case::cluster_missing(ClusterState::MissingOrDeleting, true, true)] + #[case::management_access_disabled(ClusterState::ManagementAccessDisabled, true, true)] + #[case::agent_available(ClusterState::ManagementAccessEnabled, true, false)] + #[case::agent_unavailable(ClusterState::ManagementAccessEnabled, false, true)] + fn set_agent_unavailable_conditions_without_an_available_agent( + #[case] cluster_state: ClusterState, + #[case] agent_available: bool, + #[case] sets_conditions: bool, + ) { + assert_eq!( + sets_conditions, + matches!( + operator_action_for(&kafka_topic(false, false), &cluster_state, agent_available), + OperatorAction::SetAgentUnavailableConditions { .. } + ) + ); + } + + #[test] + fn leave_a_deleting_kafka_topic_without_finalizer_alone() { + assert_eq!( + OperatorAction::LeaveToAgent, + operator_action_for( + &kafka_topic(true, false), + &ClusterState::MissingOrDeleting, + false + ) + ); + } +} diff --git a/rust/shared/src/crd/topic.rs b/rust/shared/src/crd/topic.rs index 01600daae..3818f032f 100644 --- a/rust/shared/src/crd/topic.rs +++ b/rust/shared/src/crd/topic.rs @@ -11,6 +11,10 @@ use stackable_operator::{ versioned::versioned, }; +/// Set by the agent on a `KafkaTopic` that deletes its topic. Removed by the agent after the +/// deletion, or by the operator once the KafkaCluster is gone, so no agent can remove it anymore. +pub const KAFKA_TOPIC_FINALIZER: &str = "kafka.stackable.tech/kafkatopic"; + attributed_string_type! { TopicName, "The name of a Kafka topic, not `.` or `..`", From 14f9ee81698d3b305006b0e5672d82dfcdc25b9e Mon Sep 17 00:00:00 2001 From: Benedikt Labrenz Date: Tue, 27 Oct 2026 14:21:16 +0100 Subject: [PATCH 39/93] use headless service to connect to bootstrap servers --- .../controller/build/properties/listener.rs | 19 ++++- .../src/controller/build/properties/mod.rs | 11 ++- .../src/controller/build/resource/agent.rs | 80 +++++++++++-------- rust/shared/src/agent.rs | 4 +- 4 files changed, 71 insertions(+), 43 deletions(-) diff --git a/rust/operator-binary/src/controller/build/properties/listener.rs b/rust/operator-binary/src/controller/build/properties/listener.rs index acc632558..2cb533769 100644 --- a/rust/operator-binary/src/controller/build/properties/listener.rs +++ b/rust/operator-binary/src/controller/build/properties/listener.rs @@ -6,7 +6,13 @@ use std::collections::BTreeMap; -use stackable_operator::{commons::networking::DomainName, v2::types::kubernetes::NamespaceName}; +use stackable_operator::{ + commons::networking::DomainName, + v2::types::{ + common::Port, + kubernetes::{NamespaceName, ServiceName}, + }, +}; use crate::{ controller::{RoleGroupName, ValidatedCluster, security::ValidatedKafkaSecurity}, @@ -151,6 +157,17 @@ pub(crate) fn pod_fqdn( format!("${{env:POD_NAME}}.{sts_service_name}.{namespace}.svc.{cluster_domain}") } +/// `..svc.:`, which resolves to the Pods of a headless +/// Service. +pub(crate) fn headless_service_address( + headless_service_name: &ServiceName, + namespace: &NamespaceName, + cluster_domain: &DomainName, + port: &Port, +) -> String { + format!("{headless_service_name}.{namespace}.svc.{cluster_domain}:{port}") +} + #[cfg(test)] mod tests { use stackable_operator::{ diff --git a/rust/operator-binary/src/controller/build/properties/mod.rs b/rust/operator-binary/src/controller/build/properties/mod.rs index e751613aa..708c121ee 100644 --- a/rust/operator-binary/src/controller/build/properties/mod.rs +++ b/rust/operator-binary/src/controller/build/properties/mod.rs @@ -103,12 +103,11 @@ pub(crate) fn kraft_controllers( client_port = desc.client_port, ) } else { - format!( - "{service}.{namespace}.svc.{cluster_domain}:{client_port}", - service = desc.role_group_service_name, - namespace = desc.namespace, - cluster_domain = desc.cluster_domain, - client_port = desc.client_port, + listener::headless_service_address( + &desc.role_group_service_name, + &desc.namespace, + &desc.cluster_domain, + &desc.client_port, ) } }) diff --git a/rust/operator-binary/src/controller/build/resource/agent.rs b/rust/operator-binary/src/controller/build/resource/agent.rs index 0f288f964..00cc65c65 100644 --- a/rust/operator-binary/src/controller/build/resource/agent.rs +++ b/rust/operator-binary/src/controller/build/resource/agent.rs @@ -3,8 +3,8 @@ use std::str::FromStr; use stackable_kafka_shared::agent::{ - ClientSecurity, DISCOVERY_DIR, ENV_CLIENT_SECURITY, ENV_CLUSTER_NAME, ENV_CLUSTER_NAMESPACE, - KAFKA_TLS_DIR, + ClientSecurity, ENV_BOOTSTRAP_SERVERS, ENV_CLIENT_SECURITY, ENV_CLUSTER_NAME, + ENV_CLUSTER_NAMESPACE, KAFKA_TLS_DIR, }; use stackable_operator::{ builder::{ @@ -37,13 +37,18 @@ use stackable_operator::{ }, }; -use crate::controller::{ - AgentConnection, ValidatedAgentConfig, ValidatedCluster, - build::{agent_selector, recommended_labels_for_agent_resources}, +use crate::{ + controller::{ + AgentConnection, ValidatedAgentConfig, ValidatedCluster, + build::{ + agent_selector, properties::listener::headless_service_address, + recommended_labels_for_agent_resources, + }, + }, + crd::role::KafkaRole, }; constant!(AGENT_CONTAINER_NAME: ContainerName = "agent"); -constant!(DISCOVERY_VOLUME_NAME: VolumeName = "discovery"); constant!(KAFKA_TLS_VOLUME_NAME: VolumeName = "tls-kafka"); constant!(KUBERNETES_NODE_NAME: EnvVarName = "KUBERNETES_NODE_NAME"); constant!(KUBERNETES_CLUSTER_DOMAIN: EnvVarName = "KUBERNETES_CLUSTER_DOMAIN"); @@ -56,7 +61,6 @@ pub fn build_agent_deployment( let mut cb_agent = new_container_builder(&AGENT_CONTAINER_NAME); let mut pod_builder = PodBuilder::new(); - add_discovery_volume_and_mount(cluster, &mut pod_builder, &mut cb_agent); match &agent_config.connection { AgentConnection::Plaintext => {} AgentConnection::Tls { @@ -162,6 +166,7 @@ fn agent_env_vars(cluster: &ValidatedCluster, agent_config: &ValidatedAgentConfi }; EnvVarSet::new() + .with_value(&ENV_BOOTSTRAP_SERVERS, bootstrap_servers(cluster)) .with_value(&ENV_CLUSTER_NAME, cluster.name.to_string()) .with_value(&ENV_CLUSTER_NAMESPACE, cluster.namespace.to_string()) .with_value( @@ -177,21 +182,30 @@ fn agent_env_vars(cluster: &ValidatedCluster, agent_config: &ValidatedAgentConfi .expect("the env var name is a valid EnvVarName") } -fn add_discovery_volume_and_mount( - cluster: &ValidatedCluster, - pod_builder: &mut PodBuilder, - container_builder: &mut ContainerBuilder, -) { - pod_builder - .add_volume( - VolumeBuilder::new(&*DISCOVERY_VOLUME_NAME) - .with_config_map(cluster.name.to_string()) - .build(), - ) - .expect("the volume name is statically defined and must not collide"); - container_builder - .add_volume_mount(&*DISCOVERY_VOLUME_NAME, DISCOVERY_DIR) - .expect("the mount path is statically defined and must not collide"); +/// The headless Services of the broker role groups, which resolve to the broker Pods. +/// +/// They keep the traffic within the Kubernetes cluster regardless of the ListenerClass. The pod +/// scope of the broker certificates puts them into the SANs, so hostname verification passes. The +/// agent stays on the bootstrap connection, so it never follows the advertised addresses. +fn bootstrap_servers(cluster: &ValidatedCluster) -> String { + let client_port = cluster.cluster_config.kafka_security.client_port(); + cluster + .role_group_configs + .get(&KafkaRole::Broker) + .into_iter() + .flat_map(|role_group_configs| role_group_configs.keys()) + .map(|role_group_name| { + headless_service_address( + &cluster + .role_group_resource_names(&KafkaRole::Broker, role_group_name) + .headless_service_name(), + &cluster.namespace, + &cluster.cluster_domain, + &client_port, + ) + }) + .collect::>() + .join(",") } fn add_kafka_tls_volume_and_mount( @@ -333,28 +347,27 @@ mod tests { assert_eq!( pod_spec["containers"][0]["volumeMounts"], json!([ - {"mountPath": "/stackable/discovery", "name": "discovery"}, {"mountPath": "/stackable/secrets/tls-kafka", "name": "tls-kafka"}, {"mountPath": "/stackable/secrets/tls-management-access", "name": "tls-management-access"} ]) ); assert_eq!( - pod_spec["volumes"][0]["configMap"], - json!({"name": "simple-kafka"}) - ); - assert_eq!( - pod_spec["volumes"][1]["ephemeral"]["volumeClaimTemplate"]["metadata"]["annotations"], + pod_spec["volumes"][0]["ephemeral"]["volumeClaimTemplate"]["metadata"]["annotations"], json!({ "secrets.stackable.tech/class": "tls", "secrets.stackable.tech/format": "tls-pem", "secrets.stackable.tech/provision-parts": "public" }) ); - assert_eq!(pod_spec["volumes"][2]["name"], "tls-management-access"); + assert_eq!(pod_spec["volumes"][1]["name"], "tls-management-access"); assert_eq!( pod_spec["containers"][0]["env"], json!([ + { + "name": "KAFKA_BOOTSTRAP_SERVERS", + "value": "simple-kafka-broker-default-headless.default.svc.cluster.local:9093" + }, {"name": "KAFKA_CLIENT_SECURITY", "value": "mutual-tls"}, {"name": "KAFKA_CLUSTER_NAME", "value": "simple-kafka"}, {"name": "KAFKA_CLUSTER_NAMESPACE", "value": "default"}, @@ -374,7 +387,7 @@ mod tests { )); assert_eq!( - deployment["spec"]["template"]["spec"]["volumes"][2], + deployment["spec"]["template"]["spec"]["volumes"][1], json!({"name": "tls-management-access", "secret": {"optional": false, "secretName": "my-cert"}}) ); } @@ -386,11 +399,10 @@ mod tests { )); let pod_spec = &deployment["spec"]["template"]["spec"]; - assert_eq!(pod_spec["volumes"][0]["name"], "discovery"); - assert_eq!(pod_spec["volumes"][1]["name"], "tls-kafka"); - assert_eq!(pod_spec["volumes"][2], Value::Null); + assert_eq!(pod_spec["volumes"][0]["name"], "tls-kafka"); + assert_eq!(pod_spec["volumes"][1], Value::Null); assert_eq!( - pod_spec["containers"][0]["env"][0], + pod_spec["containers"][0]["env"][1], json!({"name": "KAFKA_CLIENT_SECURITY", "value": "tls"}) ); } diff --git a/rust/shared/src/agent.rs b/rust/shared/src/agent.rs index 617360172..c50170e06 100644 --- a/rust/shared/src/agent.rs +++ b/rust/shared/src/agent.rs @@ -5,14 +5,14 @@ use std::str::FromStr; use stackable_operator::{constant, v2::builder::pod::container::EnvVarName}; use strum::{Display, EnumString}; -pub const DISCOVERY_DIR: &str = "/stackable/discovery"; - /// Directory of the CA certificate the Kafka brokers are verified with. pub const KAFKA_TLS_DIR: &str = "/stackable/secrets/tls-kafka"; constant!(pub ENV_CLUSTER_NAME: EnvVarName = "KAFKA_CLUSTER_NAME"); constant!(pub ENV_CLUSTER_NAMESPACE: EnvVarName = "KAFKA_CLUSTER_NAMESPACE"); constant!(pub ENV_CLIENT_SECURITY: EnvVarName = "KAFKA_CLIENT_SECURITY"); +// Comma-separated `host:port` list +constant!(pub ENV_BOOTSTRAP_SERVERS: EnvVarName = "KAFKA_BOOTSTRAP_SERVERS"); /// How the agent connects to the brokers. #[derive(Clone, Copy, Debug, Display, EnumString, Eq, PartialEq)] From cbde9565eb0718858c3d32be38e8952fccda4097 Mon Sep 17 00:00:00 2001 From: Benedikt Labrenz Date: Tue, 27 Oct 2026 14:31:16 +0100 Subject: [PATCH 40/93] rename cb_agent to container_builder_agent --- .../src/controller/build/resource/agent.rs | 22 ++++++++++++------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/rust/operator-binary/src/controller/build/resource/agent.rs b/rust/operator-binary/src/controller/build/resource/agent.rs index 00cc65c65..2de45253a 100644 --- a/rust/operator-binary/src/controller/build/resource/agent.rs +++ b/rust/operator-binary/src/controller/build/resource/agent.rs @@ -58,30 +58,36 @@ pub fn build_agent_deployment( agent_config: &ValidatedAgentConfig, ) -> Deployment { let resource_names = cluster.agent_resource_names(); - let mut cb_agent = new_container_builder(&AGENT_CONTAINER_NAME); + let mut container_builder_agent = new_container_builder(&AGENT_CONTAINER_NAME); let mut pod_builder = PodBuilder::new(); match &agent_config.connection { AgentConnection::Plaintext => {} AgentConnection::Tls { server_ca_secret_class, - } => { - add_kafka_tls_volume_and_mount(server_ca_secret_class, &mut pod_builder, &mut cb_agent) - } + } => add_kafka_tls_volume_and_mount( + server_ca_secret_class, + &mut pod_builder, + &mut container_builder_agent, + ), AgentConnection::MutualTls { server_ca_secret_class, client_credential, } => { - add_kafka_tls_volume_and_mount(server_ca_secret_class, &mut pod_builder, &mut cb_agent); + add_kafka_tls_volume_and_mount( + server_ca_secret_class, + &mut pod_builder, + &mut container_builder_agent, + ); client_credential - .add_volumes_and_mounts(&mut pod_builder, vec![&mut cb_agent]) + .add_volumes_and_mounts(&mut pod_builder, vec![&mut container_builder_agent]) .expect( "the agent Pod has no other volume or mount for the TLS client certificate", ); } } - cb_agent + container_builder_agent .image(&agent_config.image) .command(vec!["stackable-kafka-agent".to_owned()]) .args(vec!["run".to_owned()]) @@ -102,7 +108,7 @@ pub fn build_agent_deployment( .build(), ) .image_pull_secrets_from_product_image(&cluster.image) - .add_container(cb_agent.build()) + .add_container(container_builder_agent.build()) .service_account_name(resource_names.service_account_name().to_string()) .security_context( PodSecurityContextBuilder::with_stackable_defaults() From e5e54fd896921cc48e0275962dc06b2ee3d932aa Mon Sep 17 00:00:00 2001 From: Benedikt Labrenz Date: Tue, 27 Oct 2026 14:54:47 +0100 Subject: [PATCH 41/93] add deref tests for new constants --- .../src/controller/build/resource/agent.rs | 9 +++++++++ rust/shared/src/agent.rs | 9 +++++++++ 2 files changed, 18 insertions(+) diff --git a/rust/operator-binary/src/controller/build/resource/agent.rs b/rust/operator-binary/src/controller/build/resource/agent.rs index 2de45253a..966b62fff 100644 --- a/rust/operator-binary/src/controller/build/resource/agent.rs +++ b/rust/operator-binary/src/controller/build/resource/agent.rs @@ -294,6 +294,15 @@ mod tests { .expect("must be serializable") } + #[test] + fn test_constants() { + // Test that dereferencing the constants does not panic. + let _ = *AGENT_CONTAINER_NAME; + let _ = *KAFKA_TLS_VOLUME_NAME; + let _ = *KUBERNETES_NODE_NAME; + let _ = *KUBERNETES_CLUSTER_DOMAIN; + } + #[test] fn agent_config_is_only_resolved_if_management_access_is_enabled() { assert!( diff --git a/rust/shared/src/agent.rs b/rust/shared/src/agent.rs index c50170e06..c481cbc47 100644 --- a/rust/shared/src/agent.rs +++ b/rust/shared/src/agent.rs @@ -31,6 +31,15 @@ mod tests { use super::*; + #[test] + fn test_constants() { + // Test that dereferencing the constants does not panic. + let _ = *ENV_CLUSTER_NAME; + let _ = *ENV_CLUSTER_NAMESPACE; + let _ = *ENV_CLIENT_SECURITY; + let _ = *ENV_BOOTSTRAP_SERVERS; + } + #[rstest] #[case::plaintext(ClientSecurity::Plaintext, "plaintext")] #[case::tls(ClientSecurity::Tls, "tls")] From ac90b9f9a771ab688aee5e2518d8c6a9166ff530 Mon Sep 17 00:00:00 2001 From: Benedikt Labrenz Date: Tue, 27 Oct 2026 15:14:33 +0100 Subject: [PATCH 42/93] read bootstrap servers from KAFKA_BOOTSTRAP_SERVERS in the agent and create client only once --- rust/agent-binary/src/config.rs | 96 +------------------ rust/agent-binary/src/main.rs | 18 +++- rust/agent-binary/src/topic/condition.rs | 12 --- rust/agent-binary/src/topic/controller.rs | 24 ++--- rust/agent-binary/src/topic/finalizer.rs | 19 +--- rust/agent-binary/src/topic/sync.rs | 62 ++---------- .../controller/build/properties/listener.rs | 2 - rust/operator-binary/src/topic_controller.rs | 3 - 8 files changed, 36 insertions(+), 200 deletions(-) diff --git a/rust/agent-binary/src/config.rs b/rust/agent-binary/src/config.rs index 0f3319e97..9c2dcc47d 100644 --- a/rust/agent-binary/src/config.rs +++ b/rust/agent-binary/src/config.rs @@ -1,38 +1,14 @@ //! The agent's view of the files and settings the operator provides. -use std::{ - fs, io, - path::{Path, PathBuf}, -}; - use const_format::concatcp; -use snafu::{ResultExt, Snafu}; -use stackable_kafka_admin::{ - bootstrap::{self, BrokerAddress, parse_bootstrap_servers}, - connection::Security, -}; -use stackable_kafka_shared::agent::{ClientSecurity, DISCOVERY_DIR, KAFKA_TLS_DIR}; +use stackable_kafka_admin::connection::Security; +use stackable_kafka_shared::agent::{ClientSecurity, KAFKA_TLS_DIR}; use stackable_operator::crd::management_access::tls::MOUNT_PATH; -/// The key `KAFKA` of the discovery ConfigMap, holding `host:port,host:port` -pub const BOOTSTRAP_SERVERS_FILE: &str = concatcp!(DISCOVERY_DIR, "/KAFKA"); - const CA_FILE: &str = concatcp!(KAFKA_TLS_DIR, "/ca.crt"); const CERTIFICATE_FILE: &str = concatcp!(MOUNT_PATH, "/tls.crt"); const KEY_FILE: &str = concatcp!(MOUNT_PATH, "/tls.key"); -#[derive(Debug, Snafu)] -pub enum Error { - #[snafu(display("failed to read the bootstrap servers from {path:?}"))] - ReadBootstrapServers { source: io::Error, path: PathBuf }, - - #[snafu(display("the bootstrap servers in {path:?} are invalid"))] - ParseBootstrapServers { - source: bootstrap::Error, - path: PathBuf, - }, -} - pub fn security(client_security: ClientSecurity) -> Security { match client_security { ClientSecurity::Plaintext => Security::Plaintext, @@ -45,17 +21,6 @@ pub fn security(client_security: ClientSecurity) -> Security { } } -/// Returns `None` while the operator has written no bootstrap servers to the discovery ConfigMap yet. -pub fn read_bootstrap_servers(path: &Path) -> Result>, Error> { - let value = fs::read_to_string(path).context(ReadBootstrapServersSnafu { path })?; - if value.trim().is_empty() { - return Ok(None); - } - parse_bootstrap_servers(&value) - .map(Some) - .context(ParseBootstrapServersSnafu { path }) -} - #[cfg(test)] mod tests { use rstest::rstest; @@ -81,61 +46,4 @@ mod tests { fn map_client_security(#[case] client_security: ClientSecurity, #[case] expected: Security) { assert_eq!(expected, security(client_security)); } - - #[rstest] - #[case::empty("", None)] - #[case::blank(" \n", None)] - #[case::servers( - "kafka-0:9093,kafka-1:9093", - Some(vec![ - BrokerAddress { host: "kafka-0".to_owned(), port: 9093 }, - BrokerAddress { host: "kafka-1".to_owned(), port: 9093 }, - ]) - )] - fn read_written_bootstrap_servers( - #[case] content: &str, - #[case] expected: Option>, - ) { - let path = temp_file(content); - - let bootstrap_servers = read_bootstrap_servers(&path); - fs::remove_file(&path).expect("the file should be removable"); - - assert_eq!( - expected, - bootstrap_servers.expect("the bootstrap servers should be readable") - ); - } - - #[test] - fn reject_missing_bootstrap_servers_file() { - let path = std::env::temp_dir().join("missing-bootstrap-servers"); - - assert!(matches!( - read_bootstrap_servers(&path).expect_err("a missing file should be an error"), - Error::ReadBootstrapServers { .. } - )); - } - - #[test] - fn reject_invalid_bootstrap_servers() { - let path = temp_file("kafka"); - - let error = read_bootstrap_servers(&path); - fs::remove_file(&path).expect("the file should be removable"); - - assert!(matches!( - error.expect_err("the bootstrap servers should be invalid"), - Error::ParseBootstrapServers { .. } - )); - } - - /// Writes the content to a file unique to this test thread. - fn temp_file(content: &str) -> PathBuf { - let thread = std::thread::current(); - let name = thread.name().unwrap_or("test").replace("::", "-"); - let path = std::env::temp_dir().join(format!("bootstrap-{}-{name}", std::process::id())); - fs::write(&path, content).expect("the file should be writable"); - path - } } diff --git a/rust/agent-binary/src/main.rs b/rust/agent-binary/src/main.rs index 5b61c878e..2c950f168 100644 --- a/rust/agent-binary/src/main.rs +++ b/rust/agent-binary/src/main.rs @@ -6,8 +6,12 @@ use std::sync::Arc; use clap::Parser; use futures::StreamExt; +use stackable_kafka_admin::{bootstrap::parse_bootstrap_servers, client::KafkaAdminClient}; use stackable_kafka_shared::{ - agent::{ClientSecurity, ENV_CLIENT_SECURITY, ENV_CLUSTER_NAME, ENV_CLUSTER_NAMESPACE}, + agent::{ + ClientSecurity, ENV_BOOTSTRAP_SERVERS, ENV_CLIENT_SECURITY, ENV_CLUSTER_NAME, + ENV_CLUSTER_NAMESPACE, + }, crd::topic::v1alpha1::KafkaTopic, }; use stackable_operator::{ @@ -34,6 +38,8 @@ mod topic; pub const AGENT_NAME: &str = "kafka-agent.stackable.tech"; +const KAFKA_CLIENT_ID: &str = "stackable-kafka-agent"; + mod built_info { include!(concat!(env!("OUT_DIR"), "/built.rs")); } @@ -59,6 +65,10 @@ struct KafkaAgentRun { #[arg(long, env = AsRef::::as_ref(&*ENV_CLIENT_SECURITY))] client_security: ClientSecurity, + /// The brokers to connect to, as `host:port,host:port`. + #[arg(long, env = AsRef::::as_ref(&*ENV_BOOTSTRAP_SERVERS))] + bootstrap_servers: String, + #[clap(flatten)] common: CommonOptions, } @@ -71,6 +81,7 @@ async fn main() -> anyhow::Result<()> { cluster_name, cluster_namespace, client_security, + bootstrap_servers, common: CommonOptions { telemetry, @@ -90,12 +101,15 @@ async fn main() -> anyhow::Result<()> { ); let security = config::security(client_security); + let bootstrap_servers = parse_bootstrap_servers(&bootstrap_servers)?; tracing::info!( %cluster_name, %cluster_namespace, ?security, + ?bootstrap_servers, "Configured the agent" ); + let admin_client = KafkaAdminClient::new(bootstrap_servers, security, KAFKA_CLIENT_ID); // Watches for the SIGTERM signal and gracefully shuts down the controller. let sigterm_watcher = SignalWatcher::sigterm()?; @@ -124,7 +138,7 @@ async fn main() -> anyhow::Result<()> { Arc::new(controller::Ctx::new( client, cluster_name, - security, + admin_client, kafka_topic_store, )), ) diff --git a/rust/agent-binary/src/topic/condition.rs b/rust/agent-binary/src/topic/condition.rs index d1c6fb8bc..46ed4f803 100644 --- a/rust/agent-binary/src/topic/condition.rs +++ b/rust/agent-binary/src/topic/condition.rs @@ -7,10 +7,6 @@ use stackable_operator::agent::condition::{ use super::plan::{TopicState, UnsupportedChange}; -/// The operator writes the bootstrap servers to the discovery ConfigMap only once the -/// listener-operator has assigned addresses to the bootstrap Listeners. -const CLUSTER_NOT_READY_MESSAGE: &str = "the KafkaCluster has no bootstrap servers yet"; - #[derive(Debug, Eq, PartialEq)] pub enum TopicSyncOutcome { /// The topic matches the spec @@ -22,7 +18,6 @@ pub enum TopicSyncOutcome { changes: Vec, }, ReconciliationPaused, - ClusterNotReady, InvalidSpec { message: String, }, @@ -87,9 +82,6 @@ fn ready_from( TopicSyncOutcome::Conflict { .. } => { ready(False, "Conflict", "another KafkaTopic manages the topic") } - TopicSyncOutcome::ClusterNotReady => { - ready(Unknown, "ClusterNotReady", CLUSTER_NOT_READY_MESSAGE) - } TopicSyncOutcome::InvalidSpec { message } => ready(Unknown, "InvalidSpec", message), TopicSyncOutcome::KafkaRejected { message } => ready(Unknown, "KafkaRejected", message), TopicSyncOutcome::ConnectionFailed { message } => { @@ -129,9 +121,6 @@ fn synced_from( "the reconciliation is paused, only the status is updated", ), TopicSyncOutcome::Conflict { message } => synced(False, "Conflict", message), - TopicSyncOutcome::ClusterNotReady => { - synced(False, "ClusterNotReady", CLUSTER_NOT_READY_MESSAGE) - } TopicSyncOutcome::InvalidSpec { message } => synced(False, "InvalidSpec", message), TopicSyncOutcome::KafkaRejected { message } => synced(False, "KafkaRejected", message), TopicSyncOutcome::ConnectionFailed { message } => { @@ -338,7 +327,6 @@ mod tests { } #[rstest::rstest] - #[case::cluster_not_ready(TopicSyncOutcome::ClusterNotReady, "ClusterNotReady")] #[case::invalid_spec(TopicSyncOutcome::InvalidSpec { message: "invalid".to_owned() }, "InvalidSpec")] #[case::kafka_rejected(TopicSyncOutcome::KafkaRejected { message: "denied".to_owned() }, "KafkaRejected")] #[case::connection_failed(TopicSyncOutcome::ConnectionFailed { message: "timed out".to_owned() }, "ConnectionFailed")] diff --git a/rust/agent-binary/src/topic/controller.rs b/rust/agent-binary/src/topic/controller.rs index 105ac03c6..8d50d046a 100644 --- a/rust/agent-binary/src/topic/controller.rs +++ b/rust/agent-binary/src/topic/controller.rs @@ -4,7 +4,7 @@ use std::{sync::Arc, time::Duration}; use const_format::concatcp; use snafu::{ResultExt, Snafu}; -use stackable_kafka_admin::{client, connection::Security}; +use stackable_kafka_admin::client::{self, KafkaAdminClient}; use stackable_kafka_shared::crd::topic::v1alpha1::KafkaTopic; use stackable_operator::{ kube::{ @@ -21,7 +21,7 @@ use tracing::info; use super::{ condition::TopicSyncOutcome, finalizer::{add_finalizer, reconcile_finalizer}, - sync::{CachedClient, sync}, + sync::sync, update_status::update_status, }; use crate::AGENT_NAME; @@ -30,7 +30,6 @@ pub const TOPIC_CONTROLLER_NAME: &str = "kafkatopic"; pub const TOPIC_FULL_CONTROLLER_NAME: &str = concatcp!(TOPIC_CONTROLLER_NAME, ".", AGENT_NAME); const RESYNC_INTERVAL: Duration = Duration::from_secs(5 * 60); -const RETRY_INTERVAL: Duration = Duration::from_secs(10); const ERROR_RETRY_INTERVAL: Duration = Duration::from_secs(30); #[derive(Debug, Snafu, EnumDiscriminants)] @@ -42,12 +41,6 @@ pub enum Error { source: error_boundary::InvalidObject, }, - #[snafu(display("failed to read the bootstrap servers"))] - ReadBootstrapServers { source: crate::config::Error }, - - #[snafu(display("the KafkaCluster has no bootstrap servers yet"))] - NoBootstrapServers, - #[snafu(display("failed to talk to Kafka"))] Kafka { source: client::Error }, @@ -73,10 +66,10 @@ impl ReconcilerError for Error { pub struct Ctx { pub(crate) client: stackable_operator::client::Client, pub(crate) cluster_name: ClusterName, - pub(crate) security: Security, pub(crate) kafka_topics: Store>, - /// One client per agent, rebuilt when the bootstrap servers change - pub(crate) client_cache: Mutex>, + /// Reads the TLS files on every new connection and reconnects after a failed request, so + /// rotated certificates are picked up without restarting the agent. + pub(crate) admin_client: Mutex, } impl Ctx { @@ -85,15 +78,14 @@ impl Ctx { pub fn new( client: stackable_operator::client::Client, cluster_name: ClusterName, - security: Security, + admin_client: KafkaAdminClient, kafka_topics: Store>, ) -> Self { Self { client, cluster_name, - security, kafka_topics, - client_cache: Mutex::new(None), + admin_client: Mutex::new(admin_client), } } } @@ -150,7 +142,6 @@ pub async fn reconcile_topic( fn requeue(sync_outcome: &TopicSyncOutcome) -> Action { match sync_outcome { TopicSyncOutcome::InvalidSpec { .. } => Action::await_change(), - TopicSyncOutcome::ClusterNotReady => Action::requeue(RETRY_INTERVAL), _ => Action::requeue(RESYNC_INTERVAL), } } @@ -172,7 +163,6 @@ mod tests { #[rstest::rstest] #[case::invalid_spec(TopicSyncOutcome::InvalidSpec { message: String::new() }, Action::await_change())] - #[case::cluster_not_ready(TopicSyncOutcome::ClusterNotReady, Action::requeue(RETRY_INTERVAL))] #[case::synced(TopicSyncOutcome::Synced { reassigning_partition_count: 0 }, Action::requeue(RESYNC_INTERVAL))] #[case::paused( TopicSyncOutcome::ReconciliationPaused, diff --git a/rust/agent-binary/src/topic/finalizer.rs b/rust/agent-binary/src/topic/finalizer.rs index 95a657785..1eb9fc39d 100644 --- a/rust/agent-binary/src/topic/finalizer.rs +++ b/rust/agent-binary/src/topic/finalizer.rs @@ -1,9 +1,7 @@ //! The finalizer of a `KafkaTopic` and the cleanup in Kafka when it is deleted. -use std::path::Path; - use serde_json::json; -use snafu::{OptionExt, ResultExt}; +use snafu::ResultExt; use stackable_kafka_shared::crd::topic::{KAFKA_TOPIC_FINALIZER, v1alpha1::KafkaTopic}; use stackable_operator::{ agent::deletion::v1alpha1::DeletionPolicy, @@ -12,15 +10,10 @@ use stackable_operator::{ use tracing::info; use super::{ - controller::{ - Ctx, KafkaSnafu, NoBootstrapServersSnafu, ReadBootstrapServersSnafu, Result, - UpdateFinalizersSnafu, - }, + controller::{Ctx, KafkaSnafu, Result, UpdateFinalizersSnafu}, plan::TopicState, - sync::client_for, validate::has_other_kafka_topic, }; -use crate::config::{BOOTSTRAP_SERVERS_FILE, read_bootstrap_servers}; /// Handles a deletion and removes a finalizer that is no longer needed. Returns `Some` if the /// reconcile ends here. @@ -87,11 +80,9 @@ async fn cleanup(kafka_topic: &KafkaTopic, ctx: &Ctx) -> Result<()> { return Ok(()); } - let bootstrap_servers = read_bootstrap_servers(Path::new(BOOTSTRAP_SERVERS_FILE)) - .context(ReadBootstrapServersSnafu)? - .context(NoBootstrapServersSnafu)?; - let mut client_cache = ctx.client_cache.lock().await; - client_for(&mut client_cache, bootstrap_servers, &ctx.security) + ctx.admin_client + .lock() + .await .delete_topic(topic_name.as_ref()) .await .context(KafkaSnafu)?; diff --git a/rust/agent-binary/src/topic/sync.rs b/rust/agent-binary/src/topic/sync.rs index 7608efe9a..4acd0d27f 100644 --- a/rust/agent-binary/src/topic/sync.rs +++ b/rust/agent-binary/src/topic/sync.rs @@ -1,34 +1,19 @@ //! Brings the topic in Kafka to the state of its `KafkaTopic` spec. -use std::{ - collections::{BTreeMap, BTreeSet}, - path::Path, -}; +use std::collections::{BTreeMap, BTreeSet}; use snafu::ResultExt; -use stackable_kafka_admin::{ - bootstrap::BrokerAddress, - client::{KafkaAdminClient, TopicConfig}, - connection::Security, -}; +use stackable_kafka_admin::client::{KafkaAdminClient, TopicConfig}; use stackable_kafka_shared::crd::topic::{TopicName, v1alpha1::KafkaTopic}; use stackable_operator::kube::ResourceExt; use tracing::info; use super::{ condition::TopicSyncOutcome, - controller::{Ctx, KafkaSnafu, ReadBootstrapServersSnafu, Result}, + controller::{Ctx, KafkaSnafu, Result}, plan::{ObservedTopic, TopicPlan, TopicState, plan}, validate::validate, }; -use crate::config::{BOOTSTRAP_SERVERS_FILE, read_bootstrap_servers}; - -const KAFKA_CLIENT_ID: &str = "stackable-kafka-agent"; - -pub(crate) struct CachedClient { - bootstrap_servers: Vec, - admin_client: KafkaAdminClient, -} /// Sets `topic_state` to the topic in Kafka, if it exists and is managed by this `KafkaTopic`. /// It is kept up to date with each applied change, also if a later step fails. @@ -44,14 +29,8 @@ pub(crate) async fn sync( }; // observe (Kafka) - let Some(bootstrap_servers) = read_bootstrap_servers(Path::new(BOOTSTRAP_SERVERS_FILE)) - .context(ReadBootstrapServersSnafu)? - else { - return Ok(TopicSyncOutcome::ClusterNotReady); - }; - let mut client_cache = ctx.client_cache.lock().await; - let admin_client = client_for(&mut client_cache, bootstrap_servers, &ctx.security); - let observed = observe(admin_client, &topic_name).await?; + let mut admin_client = ctx.admin_client.lock().await; + let observed = observe(&mut admin_client, &topic_name).await?; // plan (no client required) let topic_plan = plan(&kafka_topic.spec, observed.as_ref()); @@ -66,7 +45,7 @@ pub(crate) async fn sync( // execute (Kafka) execute( kafka_topic, - admin_client, + &mut admin_client, topic_name, &topic_plan, topic_state, @@ -249,32 +228,3 @@ async fn replace_configs( .await .context(KafkaSnafu) } - -/// Returns the cached client, rebuilt if the bootstrap servers changed. -/// -/// The client reads the TLS files on every new connection and reconnects after a failed request, -/// so rotated certificates are picked up without restarting the agent. -pub(crate) fn client_for<'a>( - client_cache: &'a mut Option, - bootstrap_servers: Vec, - security: &Security, -) -> &'a mut KafkaAdminClient { - if client_cache - .as_ref() - .is_some_and(|cached| cached.bootstrap_servers != bootstrap_servers) - { - *client_cache = None; - } - let cached = client_cache.get_or_insert_with(|| { - info!(?bootstrap_servers, "Connecting to new bootstrap servers"); - CachedClient { - bootstrap_servers: bootstrap_servers.clone(), - admin_client: KafkaAdminClient::new( - bootstrap_servers, - security.clone(), - KAFKA_CLIENT_ID, - ), - } - }); - &mut cached.admin_client -} diff --git a/rust/operator-binary/src/controller/build/properties/listener.rs b/rust/operator-binary/src/controller/build/properties/listener.rs index 2cb533769..e7a460b7c 100644 --- a/rust/operator-binary/src/controller/build/properties/listener.rs +++ b/rust/operator-binary/src/controller/build/properties/listener.rs @@ -157,8 +157,6 @@ pub(crate) fn pod_fqdn( format!("${{env:POD_NAME}}.{sts_service_name}.{namespace}.svc.{cluster_domain}") } -/// `..svc.:`, which resolves to the Pods of a headless -/// Service. pub(crate) fn headless_service_address( headless_service_name: &ServiceName, namespace: &NamespaceName, diff --git a/rust/operator-binary/src/topic_controller.rs b/rust/operator-binary/src/topic_controller.rs index 25414026c..e724e2a95 100644 --- a/rust/operator-binary/src/topic_controller.rs +++ b/rust/operator-binary/src/topic_controller.rs @@ -80,7 +80,6 @@ pub struct Ctx { pub client: stackable_operator::client::Client, } -/// The KafkaCluster of a `KafkaTopic`, as far as its agent is concerned. #[derive(Debug, Eq, PartialEq)] enum ClusterState { /// Does not exist or has a `deletionTimestamp` @@ -147,8 +146,6 @@ pub fn error_policy( } } -/// The `KafkaTopic`s of the KafkaCluster `cluster_name`, to reconcile them when the KafkaCluster -/// or its agent changes. pub fn kafka_topics_of_cluster( kafka_topics: &Store>, namespace: Option, From 3fb16debcfdc67d52fefca1984614abf208ed662 Mon Sep 17 00:00:00 2001 From: Benedikt Labrenz Date: Tue, 27 Oct 2026 15:26:50 +0100 Subject: [PATCH 43/93] skip planning while the KafkaTopic reconciliation is paused --- rust/agent-binary/src/topic/plan.rs | 9 ++++----- rust/agent-binary/src/topic/sync.rs | 10 ++++++---- 2 files changed, 10 insertions(+), 9 deletions(-) diff --git a/rust/agent-binary/src/topic/plan.rs b/rust/agent-binary/src/topic/plan.rs index 81e4ce022..a52b21ec6 100644 --- a/rust/agent-binary/src/topic/plan.rs +++ b/rust/agent-binary/src/topic/plan.rs @@ -32,14 +32,13 @@ pub struct TopicState { } impl TopicState { - pub fn observed(name: TopicName, observed: ObservedTopic) -> Self { - let replication_factor = replication_factor(&observed); + pub fn observed(name: TopicName, observed: &ObservedTopic) -> Self { Self { name, - id: observed.topic.id, + id: observed.topic.id.clone(), partition_count: observed.topic.partitions.len() as i32, - replication_factor, - effective_configs: observed.effective_configs, + replication_factor: replication_factor(observed), + effective_configs: observed.effective_configs.clone(), } } diff --git a/rust/agent-binary/src/topic/sync.rs b/rust/agent-binary/src/topic/sync.rs index 4acd0d27f..fd34efb0d 100644 --- a/rust/agent-binary/src/topic/sync.rs +++ b/rust/agent-binary/src/topic/sync.rs @@ -31,16 +31,18 @@ pub(crate) async fn sync( // observe (Kafka) let mut admin_client = ctx.admin_client.lock().await; let observed = observe(&mut admin_client, &topic_name).await?; + *topic_state = observed + .as_ref() + .map(|observed| TopicState::observed(topic_name.clone(), observed)); + if kafka_topic.spec.topic_operation.reconciliation_paused { + return Ok(TopicSyncOutcome::ReconciliationPaused); + } // plan (no client required) let topic_plan = plan(&kafka_topic.spec, observed.as_ref()); let reassigning_partition_count = observed .as_ref() .map_or(0, |observed| observed.reassignments.len()); - *topic_state = observed.map(|observed| TopicState::observed(topic_name.clone(), observed)); - if kafka_topic.spec.topic_operation.reconciliation_paused { - return Ok(TopicSyncOutcome::ReconciliationPaused); - } // execute (Kafka) execute( From b3e247d1badb2aa88db5ee3c4a2a7bc5900ca6af Mon Sep 17 00:00:00 2001 From: Benedikt Labrenz Date: Tue, 27 Oct 2026 15:30:28 +0100 Subject: [PATCH 44/93] watch only the KafkaTopics of the agent's cluster via a field selector --- rust/agent-binary/src/main.rs | 4 ++-- rust/agent-binary/src/topic/controller.rs | 7 ------- 2 files changed, 2 insertions(+), 9 deletions(-) diff --git a/rust/agent-binary/src/main.rs b/rust/agent-binary/src/main.rs index 2c950f168..48efbb2d3 100644 --- a/rust/agent-binary/src/main.rs +++ b/rust/agent-binary/src/main.rs @@ -127,7 +127,8 @@ async fn main() -> anyhow::Result<()> { let topic_controller = Controller::new( client.get_api::>(cluster_namespace.as_ref()), - watcher::Config::default(), + // Works because the CRD declares the field as a selectableField. + watcher::Config::default().fields(&format!("spec.clusterRef.name={cluster_name}")), ); let kafka_topic_store = topic_controller.store(); topic_controller @@ -137,7 +138,6 @@ async fn main() -> anyhow::Result<()> { controller::error_policy, Arc::new(controller::Ctx::new( client, - cluster_name, admin_client, kafka_topic_store, )), diff --git a/rust/agent-binary/src/topic/controller.rs b/rust/agent-binary/src/topic/controller.rs index 8d50d046a..cdc9a4966 100644 --- a/rust/agent-binary/src/topic/controller.rs +++ b/rust/agent-binary/src/topic/controller.rs @@ -12,7 +12,6 @@ use stackable_operator::{ runtime::{controller::Action, reflector::Store}, }, logging::controller::ReconcilerError, - v2::types::operator::ClusterName, }; use strum::{EnumDiscriminants, IntoStaticStr}; use tokio::sync::Mutex; @@ -65,7 +64,6 @@ impl ReconcilerError for Error { pub struct Ctx { pub(crate) client: stackable_operator::client::Client, - pub(crate) cluster_name: ClusterName, pub(crate) kafka_topics: Store>, /// Reads the TLS files on every new connection and reconnects after a failed request, so /// rotated certificates are picked up without restarting the agent. @@ -77,13 +75,11 @@ impl Ctx { /// `KafkaTopic`s. pub fn new( client: stackable_operator::client::Client, - cluster_name: ClusterName, admin_client: KafkaAdminClient, kafka_topics: Store>, ) -> Self { Self { client, - cluster_name, kafka_topics, admin_client: Mutex::new(admin_client), } @@ -102,9 +98,6 @@ pub async fn reconcile_topic( .as_ref() .map_err(error_boundary::InvalidObject::clone) .context(InvalidKafkaTopicSnafu)?; - if kafka_topic.spec.cluster_ref.name != ctx.cluster_name { - return Ok(Action::await_change()); - } // finalizer (Kubernetes) // Left untouched while paused, like operators that only update the status then. A deletion From 6b9300431e29f69baed8fc16175c5f30af53206b Mon Sep 17 00:00:00 2001 From: Benedikt Labrenz Date: Tue, 27 Oct 2026 15:52:42 +0100 Subject: [PATCH 45/93] reject topic names starting with __ (internal topics) --- extra/crds.yaml | 3 +++ rust/shared/src/crd/topic.rs | 8 ++++++++ 2 files changed, 11 insertions(+) diff --git a/extra/crds.yaml b/extra/crds.yaml index d8d32e1cd..d7b522841 100644 --- a/extra/crds.yaml +++ b/extra/crds.yaml @@ -2325,6 +2325,9 @@ spec: nullable: true pattern: ^([a-zA-Z0-9._-]{3,}|[a-zA-Z0-9_-][a-zA-Z0-9._-]?|\.[a-zA-Z0-9_-])$ type: string + x-kubernetes-validations: + - message: topics starting with __ are internal to Kafka + rule: '!self.startsWith(''__'')' topicOperation: default: reconciliationPaused: false diff --git a/rust/shared/src/crd/topic.rs b/rust/shared/src/crd/topic.rs index 3818f032f..d5ca60339 100644 --- a/rust/shared/src/crd/topic.rs +++ b/rust/shared/src/crd/topic.rs @@ -63,6 +63,10 @@ pub mod versioned { /// Name of the topic in Kafka. Defaults to `metadata.name`. Immutable. #[serde(default, skip_serializing_if = "Option::is_none")] + #[schemars(extend("x-kubernetes-validations" = [{ + "rule": "!self.startsWith('__')", + "message": "topics starting with __ are internal to Kafka" + }]))] pub topic_name: Option, /// Number of partitions. Can only be increased. Defaults to the broker setting @@ -274,6 +278,10 @@ mod tests { spec["x-kubernetes-validations"][0]["rule"] ); assert_eq!(249, properties["topicName"]["maxLength"]); + assert_eq!( + "!self.startsWith('__')", + properties["topicName"]["x-kubernetes-validations"][0]["rule"] + ); assert!(properties["topicName"]["pattern"].is_string()); assert_eq!(Some(1.0), properties["partitions"]["minimum"].as_f64()); assert_eq!( From 57b853bb4ea3af7871f1a3a4ced0743961f3ea0d Mon Sep 17 00:00:00 2001 From: Benedikt Labrenz Date: Tue, 27 Oct 2026 18:10:40 +0100 Subject: [PATCH 46/93] fix topic name immutability validation --- Cargo.nix | 20 ++++----- crate-hashes.json | 20 ++++----- extra/crds.yaml | 2 +- rust/shared/src/crd/topic.rs | 5 ++- .../kuttl/topic-authentication/10-assert.yaml | 1 + .../topic-authentication/11-assert.yaml.j2 | 6 ++- .../kuttl/topic-authentication/20-assert.yaml | 19 ++++++-- .../topic-authentication/20-create-topic.yaml | 3 +- .../topic-authentication/topic-condition.sh | 11 ----- tests/templates/kuttl/topic/10-assert.yaml | 1 + tests/templates/kuttl/topic/20-assert.yaml | 40 +++++++++++++---- .../kuttl/topic/20-create-topic.yaml | 5 ++- tests/templates/kuttl/topic/21-assert.yaml | 21 +++++++++ .../kuttl/topic/21-create-defaults-topic.yaml | 10 +++++ .../kuttl/topic/22-reject-internal-topic.yaml | 8 ++++ .../kuttl/topic/22_internal_topic.yaml | 9 ++++ tests/templates/kuttl/topic/30-assert.yaml | 24 +++++++--- .../kuttl/topic/30-change-topic.yaml | 5 ++- .../topic/31-reject-topic-name-change.yaml | 22 ++++++++++ tests/templates/kuttl/topic/41-assert.yaml | 31 +++++++++---- ...ey.yaml => 41-remove-config-override.yaml} | 2 +- tests/templates/kuttl/topic/50-assert.yaml | 19 ++++++-- .../kuttl/topic/50-unsupported-change.yaml | 5 ++- tests/templates/kuttl/topic/51-assert.yaml | 6 --- tests/templates/kuttl/topic/60-assert.yaml | 17 +++++-- .../kuttl/topic/60-invalid-spec.yaml | 5 ++- .../kuttl/topic/70-create-foreign-topic.yaml | 4 +- ...topic.yaml => 71-adopt-foreign-topic.yaml} | 4 +- tests/templates/kuttl/topic/71-assert.yaml | 31 +++++++++++-- .../kuttl/topic/72-adopt-foreign-topic.yaml | 12 ----- tests/templates/kuttl/topic/72-assert.yaml | 8 ---- tests/templates/kuttl/topic/75-assert.yaml | 22 ++++++++++ .../templates/kuttl/topic/75-pause-topic.yaml | 16 +++++++ tests/templates/kuttl/topic/76-assert.yaml | 22 ++++++++++ ...orted-change.yaml => 76-resume-topic.yaml} | 9 ++-- tests/templates/kuttl/topic/77-assert.yaml | 21 +++++++++ .../kuttl/topic/77-invalid-config-value.yaml | 11 +++++ tests/templates/kuttl/topic/80-assert.yaml | 44 ++++++++++++++++--- .../topic/80-create-conflicting-topic.yaml | 4 +- tests/templates/kuttl/topic/81-assert.yaml | 1 + tests/templates/kuttl/topic/90-assert.yaml | 8 ---- .../90-delete-legacy-topic-in-kafka.yaml | 13 ------ .../kuttl/topic/91-delete-topics.yaml | 3 ++ tests/templates/kuttl/topic/92-assert.yaml | 20 +++++++++ .../kuttl/topic/92-create-cleanup-topic.yaml | 13 ++++++ tests/templates/kuttl/topic/95-assert.yaml | 18 ++++++++ tests/templates/kuttl/topic/96-assert.yaml | 15 +++++++ .../topic/96-delete-topic-without-agent.yaml | 6 +++ .../kuttl/topic/97-delete-kafka-cluster.yaml | 7 +++ tests/templates/kuttl/topic/97-errors.yaml | 6 +++ .../templates/kuttl/topic/topic-condition.sh | 11 ----- tests/test-definition.yaml | 2 +- 52 files changed, 496 insertions(+), 152 deletions(-) delete mode 100644 tests/templates/kuttl/topic-authentication/topic-condition.sh create mode 100644 tests/templates/kuttl/topic/21-assert.yaml create mode 100644 tests/templates/kuttl/topic/21-create-defaults-topic.yaml create mode 100644 tests/templates/kuttl/topic/22-reject-internal-topic.yaml create mode 100644 tests/templates/kuttl/topic/22_internal_topic.yaml create mode 100644 tests/templates/kuttl/topic/31-reject-topic-name-change.yaml rename tests/templates/kuttl/topic/{41-remove-config-key.yaml => 41-remove-config-override.yaml} (74%) delete mode 100644 tests/templates/kuttl/topic/51-assert.yaml rename tests/templates/kuttl/topic/{71-refuse-foreign-topic.yaml => 71-adopt-foreign-topic.yaml} (71%) delete mode 100644 tests/templates/kuttl/topic/72-adopt-foreign-topic.yaml delete mode 100644 tests/templates/kuttl/topic/72-assert.yaml create mode 100644 tests/templates/kuttl/topic/75-assert.yaml create mode 100644 tests/templates/kuttl/topic/75-pause-topic.yaml create mode 100644 tests/templates/kuttl/topic/76-assert.yaml rename tests/templates/kuttl/topic/{51-revert-unsupported-change.yaml => 76-resume-topic.yaml} (53%) create mode 100644 tests/templates/kuttl/topic/77-assert.yaml create mode 100644 tests/templates/kuttl/topic/77-invalid-config-value.yaml delete mode 100644 tests/templates/kuttl/topic/90-assert.yaml delete mode 100644 tests/templates/kuttl/topic/90-delete-legacy-topic-in-kafka.yaml create mode 100644 tests/templates/kuttl/topic/92-assert.yaml create mode 100644 tests/templates/kuttl/topic/92-create-cleanup-topic.yaml create mode 100644 tests/templates/kuttl/topic/95-assert.yaml create mode 100644 tests/templates/kuttl/topic/96-assert.yaml create mode 100644 tests/templates/kuttl/topic/96-delete-topic-without-agent.yaml create mode 100644 tests/templates/kuttl/topic/97-delete-kafka-cluster.yaml create mode 100644 tests/templates/kuttl/topic/97-errors.yaml delete mode 100644 tests/templates/kuttl/topic/topic-condition.sh diff --git a/Cargo.nix b/Cargo.nix index 573392067..c4e4c8ec6 100644 --- a/Cargo.nix +++ b/Cargo.nix @@ -5510,7 +5510,7 @@ rec { src = pkgs.fetchgit { url = "https://github.com/stackabletech/operator-rs.git"; rev = "ccfff5810f9b5affae4d144b3cf04ca88677e23e"; - sha256 = "0vdnhvfn85qnm5845mad24smhdp2y6nznlgagbp3g42pi042g9pm"; + sha256 = "052r8sbdgl89fy4jrkl1jjzjp5fj731pips5a95jd5hm51ymwi4s"; }; libName = "k8s_version"; authors = [ @@ -10489,7 +10489,7 @@ rec { src = pkgs.fetchgit { url = "https://github.com/stackabletech/operator-rs.git"; rev = "ccfff5810f9b5affae4d144b3cf04ca88677e23e"; - sha256 = "0vdnhvfn85qnm5845mad24smhdp2y6nznlgagbp3g42pi042g9pm"; + sha256 = "052r8sbdgl89fy4jrkl1jjzjp5fj731pips5a95jd5hm51ymwi4s"; }; libName = "stackable_agent"; authors = [ @@ -10532,7 +10532,7 @@ rec { src = pkgs.fetchgit { url = "https://github.com/stackabletech/operator-rs.git"; rev = "ccfff5810f9b5affae4d144b3cf04ca88677e23e"; - sha256 = "0vdnhvfn85qnm5845mad24smhdp2y6nznlgagbp3g42pi042g9pm"; + sha256 = "052r8sbdgl89fy4jrkl1jjzjp5fj731pips5a95jd5hm51ymwi4s"; }; libName = "stackable_certs"; authors = [ @@ -10934,7 +10934,7 @@ rec { src = pkgs.fetchgit { url = "https://github.com/stackabletech/operator-rs.git"; rev = "ccfff5810f9b5affae4d144b3cf04ca88677e23e"; - sha256 = "0vdnhvfn85qnm5845mad24smhdp2y6nznlgagbp3g42pi042g9pm"; + sha256 = "052r8sbdgl89fy4jrkl1jjzjp5fj731pips5a95jd5hm51ymwi4s"; }; libName = "stackable_operator"; authors = [ @@ -11139,7 +11139,7 @@ rec { src = pkgs.fetchgit { url = "https://github.com/stackabletech/operator-rs.git"; rev = "ccfff5810f9b5affae4d144b3cf04ca88677e23e"; - sha256 = "0vdnhvfn85qnm5845mad24smhdp2y6nznlgagbp3g42pi042g9pm"; + sha256 = "052r8sbdgl89fy4jrkl1jjzjp5fj731pips5a95jd5hm51ymwi4s"; }; procMacro = true; libName = "stackable_operator_derive"; @@ -11174,7 +11174,7 @@ rec { src = pkgs.fetchgit { url = "https://github.com/stackabletech/operator-rs.git"; rev = "ccfff5810f9b5affae4d144b3cf04ca88677e23e"; - sha256 = "0vdnhvfn85qnm5845mad24smhdp2y6nznlgagbp3g42pi042g9pm"; + sha256 = "052r8sbdgl89fy4jrkl1jjzjp5fj731pips5a95jd5hm51ymwi4s"; }; libName = "stackable_shared"; authors = [ @@ -11255,7 +11255,7 @@ rec { src = pkgs.fetchgit { url = "https://github.com/stackabletech/operator-rs.git"; rev = "ccfff5810f9b5affae4d144b3cf04ca88677e23e"; - sha256 = "0vdnhvfn85qnm5845mad24smhdp2y6nznlgagbp3g42pi042g9pm"; + sha256 = "052r8sbdgl89fy4jrkl1jjzjp5fj731pips5a95jd5hm51ymwi4s"; }; libName = "stackable_telemetry"; authors = [ @@ -11365,7 +11365,7 @@ rec { src = pkgs.fetchgit { url = "https://github.com/stackabletech/operator-rs.git"; rev = "ccfff5810f9b5affae4d144b3cf04ca88677e23e"; - sha256 = "0vdnhvfn85qnm5845mad24smhdp2y6nznlgagbp3g42pi042g9pm"; + sha256 = "052r8sbdgl89fy4jrkl1jjzjp5fj731pips5a95jd5hm51ymwi4s"; }; libName = "stackable_versioned"; authors = [ @@ -11411,7 +11411,7 @@ rec { src = pkgs.fetchgit { url = "https://github.com/stackabletech/operator-rs.git"; rev = "ccfff5810f9b5affae4d144b3cf04ca88677e23e"; - sha256 = "0vdnhvfn85qnm5845mad24smhdp2y6nznlgagbp3g42pi042g9pm"; + sha256 = "052r8sbdgl89fy4jrkl1jjzjp5fj731pips5a95jd5hm51ymwi4s"; }; procMacro = true; libName = "stackable_versioned_macros"; @@ -11475,7 +11475,7 @@ rec { src = pkgs.fetchgit { url = "https://github.com/stackabletech/operator-rs.git"; rev = "ccfff5810f9b5affae4d144b3cf04ca88677e23e"; - sha256 = "0vdnhvfn85qnm5845mad24smhdp2y6nznlgagbp3g42pi042g9pm"; + sha256 = "052r8sbdgl89fy4jrkl1jjzjp5fj731pips5a95jd5hm51ymwi4s"; }; libName = "stackable_webhook"; authors = [ diff --git a/crate-hashes.json b/crate-hashes.json index 017105d5a..405952ac3 100644 --- a/crate-hashes.json +++ b/crate-hashes.json @@ -1,12 +1,12 @@ { - "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#k8s-version@0.1.4": "0vdnhvfn85qnm5845mad24smhdp2y6nznlgagbp3g42pi042g9pm", - "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#stackable-agent@0.1.0": "0vdnhvfn85qnm5845mad24smhdp2y6nznlgagbp3g42pi042g9pm", - "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#stackable-certs@0.4.1": "0vdnhvfn85qnm5845mad24smhdp2y6nznlgagbp3g42pi042g9pm", - "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#stackable-operator-derive@0.3.1": "0vdnhvfn85qnm5845mad24smhdp2y6nznlgagbp3g42pi042g9pm", - "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#stackable-operator@0.121.0": "0vdnhvfn85qnm5845mad24smhdp2y6nznlgagbp3g42pi042g9pm", - "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#stackable-shared@0.2.0": "0vdnhvfn85qnm5845mad24smhdp2y6nznlgagbp3g42pi042g9pm", - "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#stackable-telemetry@0.6.5": "0vdnhvfn85qnm5845mad24smhdp2y6nznlgagbp3g42pi042g9pm", - "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#stackable-versioned-macros@0.13.0": "0vdnhvfn85qnm5845mad24smhdp2y6nznlgagbp3g42pi042g9pm", - "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#stackable-versioned@0.13.0": "0vdnhvfn85qnm5845mad24smhdp2y6nznlgagbp3g42pi042g9pm", - "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#stackable-webhook@0.10.0": "0vdnhvfn85qnm5845mad24smhdp2y6nznlgagbp3g42pi042g9pm" + "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#k8s-version@0.1.4": "052r8sbdgl89fy4jrkl1jjzjp5fj731pips5a95jd5hm51ymwi4s", + "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#stackable-agent@0.1.0": "052r8sbdgl89fy4jrkl1jjzjp5fj731pips5a95jd5hm51ymwi4s", + "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#stackable-certs@0.4.1": "052r8sbdgl89fy4jrkl1jjzjp5fj731pips5a95jd5hm51ymwi4s", + "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#stackable-operator-derive@0.3.1": "052r8sbdgl89fy4jrkl1jjzjp5fj731pips5a95jd5hm51ymwi4s", + "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#stackable-operator@0.121.0": "052r8sbdgl89fy4jrkl1jjzjp5fj731pips5a95jd5hm51ymwi4s", + "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#stackable-shared@0.2.0": "052r8sbdgl89fy4jrkl1jjzjp5fj731pips5a95jd5hm51ymwi4s", + "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#stackable-telemetry@0.6.5": "052r8sbdgl89fy4jrkl1jjzjp5fj731pips5a95jd5hm51ymwi4s", + "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#stackable-versioned-macros@0.13.0": "052r8sbdgl89fy4jrkl1jjzjp5fj731pips5a95jd5hm51ymwi4s", + "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#stackable-versioned@0.13.0": "052r8sbdgl89fy4jrkl1jjzjp5fj731pips5a95jd5hm51ymwi4s", + "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#stackable-webhook@0.10.0": "052r8sbdgl89fy4jrkl1jjzjp5fj731pips5a95jd5hm51ymwi4s" } \ No newline at end of file diff --git a/extra/crds.yaml b/extra/crds.yaml index d7b522841..92407addb 100644 --- a/extra/crds.yaml +++ b/extra/crds.yaml @@ -2343,7 +2343,7 @@ spec: type: object x-kubernetes-validations: - message: topicName is immutable - rule: self.?topicName == oldSelf.?topicName + rule: has(self.topicName) == has(oldSelf.topicName) && (!has(self.topicName) || self.topicName == oldSelf.topicName) status: nullable: true properties: diff --git a/rust/shared/src/crd/topic.rs b/rust/shared/src/crd/topic.rs index d5ca60339..4bf75071d 100644 --- a/rust/shared/src/crd/topic.rs +++ b/rust/shared/src/crd/topic.rs @@ -50,7 +50,7 @@ pub mod versioned { #[derive(Clone, CustomResource, Debug, Deserialize, JsonSchema, PartialEq, Serialize)] #[serde(rename_all = "camelCase")] #[schemars(extend("x-kubernetes-validations" = [{ - "rule": "self.?topicName == oldSelf.?topicName", + "rule": "has(self.topicName) == has(oldSelf.topicName) && (!has(self.topicName) || self.topicName == oldSelf.topicName)", "message": "topicName is immutable" }]))] pub struct KafkaTopicSpec { @@ -274,7 +274,8 @@ mod tests { properties["clusterRef"]["x-kubernetes-validations"][0]["rule"] ); assert_eq!( - "self.?topicName == oldSelf.?topicName", + "has(self.topicName) == has(oldSelf.topicName) && (!has(self.topicName) || \ + self.topicName == oldSelf.topicName)", spec["x-kubernetes-validations"][0]["rule"] ); assert_eq!(249, properties["topicName"]["maxLength"]); diff --git a/tests/templates/kuttl/topic-authentication/10-assert.yaml b/tests/templates/kuttl/topic-authentication/10-assert.yaml index 5b1be639f..bc622bead 100644 --- a/tests/templates/kuttl/topic-authentication/10-assert.yaml +++ b/tests/templates/kuttl/topic-authentication/10-assert.yaml @@ -1,4 +1,5 @@ --- +# The KafkaCluster and its agent are running. apiVersion: kuttl.dev/v1beta1 kind: TestAssert timeout: 600 diff --git a/tests/templates/kuttl/topic-authentication/11-assert.yaml.j2 b/tests/templates/kuttl/topic-authentication/11-assert.yaml.j2 index c7c299818..1853fe47e 100644 --- a/tests/templates/kuttl/topic-authentication/11-assert.yaml.j2 +++ b/tests/templates/kuttl/topic-authentication/11-assert.yaml.j2 @@ -6,9 +6,11 @@ kind: TestAssert timeout: 60 commands: {% if authentication == 'anonymous' %} - - script: test "$(kubectl get deployment --namespace $NAMESPACE test-kafka-agent --output jsonpath='{.spec.template.spec.volumes[*].name}')" = "discovery tls-kafka" + - script: test "$(kubectl get deployment --namespace $NAMESPACE test-kafka-agent --output jsonpath='{.spec.template.spec.volumes[*].name}')" = "tls-kafka" - script: test "$(kubectl get deployment --namespace $NAMESPACE test-kafka-agent --output jsonpath='{.spec.template.spec.containers[0].env[?(@.name=="KAFKA_CLIENT_SECURITY")].value}')" = tls {% else %} - - script: test "$(kubectl get deployment --namespace $NAMESPACE test-kafka-agent --output jsonpath='{.spec.template.spec.volumes[*].name}')" = "discovery tls-kafka tls-management-access" + - script: test "$(kubectl get deployment --namespace $NAMESPACE test-kafka-agent --output jsonpath='{.spec.template.spec.volumes[*].name}')" = "tls-kafka tls-management-access" - script: test "$(kubectl get deployment --namespace $NAMESPACE test-kafka-agent --output jsonpath='{.spec.template.spec.containers[0].env[?(@.name=="KAFKA_CLIENT_SECURITY")].value}')" = mutual-tls {% endif %} + # The headless Service keeps the traffic within the Kubernetes cluster. + - script: test "$(kubectl get deployment --namespace $NAMESPACE test-kafka-agent --output jsonpath='{.spec.template.spec.containers[0].env[?(@.name=="KAFKA_BOOTSTRAP_SERVERS")].value}')" = "test-kafka-broker-default-headless.$NAMESPACE.svc.cluster.local:9093" diff --git a/tests/templates/kuttl/topic-authentication/20-assert.yaml b/tests/templates/kuttl/topic-authentication/20-assert.yaml index fcdd3b419..7d87800ab 100644 --- a/tests/templates/kuttl/topic-authentication/20-assert.yaml +++ b/tests/templates/kuttl/topic-authentication/20-assert.yaml @@ -1,8 +1,19 @@ --- +# The agent authenticates to Kafka and creates the topic. apiVersion: kuttl.dev/v1beta1 kind: TestAssert timeout: 300 -commands: - - script: bash topic-condition.sh orders Ready True Available - - script: bash topic-condition.sh orders Synced True Synced - - script: 'bash kafka-cli.sh kafka-topics.sh --describe --topic orders | grep "PartitionCount: 3"' +--- +apiVersion: kafka.stackable.tech/v1alpha1 +kind: KafkaTopic +metadata: + name: orders +status: + conditions: + - type: Ready + status: "True" + reason: Available + - type: Synced + status: "True" + reason: Synced + partitions: 3 diff --git a/tests/templates/kuttl/topic-authentication/20-create-topic.yaml b/tests/templates/kuttl/topic-authentication/20-create-topic.yaml index a0020413f..8a57a0e5d 100644 --- a/tests/templates/kuttl/topic-authentication/20-create-topic.yaml +++ b/tests/templates/kuttl/topic-authentication/20-create-topic.yaml @@ -4,7 +4,8 @@ kind: KafkaTopic metadata: name: orders spec: - clusterName: test-kafka + clusterRef: + name: test-kafka topicName: orders partitions: 3 replicationFactor: 1 diff --git a/tests/templates/kuttl/topic-authentication/topic-condition.sh b/tests/templates/kuttl/topic-authentication/topic-condition.sh deleted file mode 100644 index 59a05a0d7..000000000 --- a/tests/templates/kuttl/topic-authentication/topic-condition.sh +++ /dev/null @@ -1,11 +0,0 @@ -#!/usr/bin/env bash -# Usage: topic-condition.sh -# Fails unless the condition of the KafkaTopic has the given status and reason. -set -euo pipefail - -actual=$(kubectl get kafkatopic --namespace "$NAMESPACE" "$1" \ - --output "jsonpath={.status.conditions[?(@.type==\"$2\")].status}/{.status.conditions[?(@.type==\"$2\")].reason}") -if [ "$actual" != "$3/$4" ]; then - echo "$1: expected the condition $2 to be $3/$4, but it is \"$actual\"" - exit 1 -fi diff --git a/tests/templates/kuttl/topic/10-assert.yaml b/tests/templates/kuttl/topic/10-assert.yaml index dcb6e2619..bd9419b21 100644 --- a/tests/templates/kuttl/topic/10-assert.yaml +++ b/tests/templates/kuttl/topic/10-assert.yaml @@ -1,4 +1,5 @@ --- +# The KafkaCluster and its agent are running. apiVersion: kuttl.dev/v1beta1 kind: TestAssert timeout: 600 diff --git a/tests/templates/kuttl/topic/20-assert.yaml b/tests/templates/kuttl/topic/20-assert.yaml index fcc58d2c5..d358950b7 100644 --- a/tests/templates/kuttl/topic/20-assert.yaml +++ b/tests/templates/kuttl/topic/20-assert.yaml @@ -1,12 +1,36 @@ --- +# With the default deletionPolicy Retain, the KafkaTopic has no finalizer. apiVersion: kuttl.dev/v1beta1 kind: TestAssert timeout: 300 -commands: - - script: bash topic-condition.sh orders Ready True Available - - script: bash topic-condition.sh orders Synced True Synced - - script: test -n "$(kubectl get kafkatopic --namespace $NAMESPACE orders --output jsonpath='{.status.topicId}')" - - script: test "$(kubectl get kafkatopic --namespace $NAMESPACE orders --output jsonpath='{.status.partitions[*].index}')" = "0 1 2" - - script: 'bash kafka-cli.sh kafka-topics.sh --describe --topic orders | grep "PartitionCount: 3"' - - script: bash kafka-cli.sh kafka-configs.sh --describe --entity-type topics --entity-name orders | grep "retention.ms=3600000" - - script: bash kafka-cli.sh kafka-configs.sh --describe --entity-type topics --entity-name orders | grep "max.message.bytes=2097152" +resourceRefs: + - apiVersion: kafka.stackable.tech/v1alpha1 + kind: KafkaTopic + name: orders + ref: orders +assertAll: + - celExpr: orders.status.topicId != '' + - celExpr: "!has(orders.metadata.finalizers)" +--- +apiVersion: kafka.stackable.tech/v1alpha1 +kind: KafkaTopic +metadata: + name: orders +status: + conditions: + - type: Ready + status: "True" + reason: Available + - type: Synced + status: "True" + reason: Synced + topicName: orders + partitions: 3 + replicationFactor: 1 + config: + retention.ms: + value: "3600000" + source: DYNAMIC_TOPIC_CONFIG + max.message.bytes: + value: "2097152" + source: DYNAMIC_TOPIC_CONFIG diff --git a/tests/templates/kuttl/topic/20-create-topic.yaml b/tests/templates/kuttl/topic/20-create-topic.yaml index e91eecfb7..d64165b49 100644 --- a/tests/templates/kuttl/topic/20-create-topic.yaml +++ b/tests/templates/kuttl/topic/20-create-topic.yaml @@ -4,10 +4,11 @@ kind: KafkaTopic metadata: name: orders spec: - clusterName: test-kafka + clusterRef: + name: test-kafka topicName: orders partitions: 3 replicationFactor: 1 - config: + configOverrides: retention.ms: "3600000" max.message.bytes: "2097152" diff --git a/tests/templates/kuttl/topic/21-assert.yaml b/tests/templates/kuttl/topic/21-assert.yaml new file mode 100644 index 000000000..fa8996d99 --- /dev/null +++ b/tests/templates/kuttl/topic/21-assert.yaml @@ -0,0 +1,21 @@ +--- +# The broker defaults num.partitions and default.replication.factor are both 1. +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +timeout: 300 +--- +apiVersion: kafka.stackable.tech/v1alpha1 +kind: KafkaTopic +metadata: + name: defaults +status: + conditions: + - type: Ready + status: "True" + reason: Available + - type: Synced + status: "True" + reason: Synced + topicName: defaults + partitions: 1 + replicationFactor: 1 diff --git a/tests/templates/kuttl/topic/21-create-defaults-topic.yaml b/tests/templates/kuttl/topic/21-create-defaults-topic.yaml new file mode 100644 index 000000000..bfac2b989 --- /dev/null +++ b/tests/templates/kuttl/topic/21-create-defaults-topic.yaml @@ -0,0 +1,10 @@ +--- +# Without topicName, partitions and replicationFactor: the topic is named after the KafkaTopic and +# gets the broker defaults. +apiVersion: kafka.stackable.tech/v1alpha1 +kind: KafkaTopic +metadata: + name: defaults +spec: + clusterRef: + name: test-kafka diff --git a/tests/templates/kuttl/topic/22-reject-internal-topic.yaml b/tests/templates/kuttl/topic/22-reject-internal-topic.yaml new file mode 100644 index 000000000..a2518122b --- /dev/null +++ b/tests/templates/kuttl/topic/22-reject-internal-topic.yaml @@ -0,0 +1,8 @@ +--- +# The API server rejects topics starting with __, which are internal to Kafka. +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +commands: + - script: >- + kubectl apply --namespace $NAMESPACE --filename 22_internal_topic.yaml 2>&1 + | grep "topics starting with __ are internal to Kafka" diff --git a/tests/templates/kuttl/topic/22_internal_topic.yaml b/tests/templates/kuttl/topic/22_internal_topic.yaml new file mode 100644 index 000000000..8003b8509 --- /dev/null +++ b/tests/templates/kuttl/topic/22_internal_topic.yaml @@ -0,0 +1,9 @@ +--- +apiVersion: kafka.stackable.tech/v1alpha1 +kind: KafkaTopic +metadata: + name: consumer-offsets +spec: + clusterRef: + name: test-kafka + topicName: __consumer_offsets diff --git a/tests/templates/kuttl/topic/30-assert.yaml b/tests/templates/kuttl/topic/30-assert.yaml index f8eaa1371..0f98e197c 100644 --- a/tests/templates/kuttl/topic/30-assert.yaml +++ b/tests/templates/kuttl/topic/30-assert.yaml @@ -1,9 +1,23 @@ --- +# The agent adds the partitions and applies the changed config. apiVersion: kuttl.dev/v1beta1 kind: TestAssert timeout: 300 -commands: - - script: bash topic-condition.sh orders Synced True Synced - - script: test "$(kubectl get kafkatopic --namespace $NAMESPACE orders --output jsonpath='{.status.partitions[*].index}')" = "0 1 2 3 4" - - script: 'bash kafka-cli.sh kafka-topics.sh --describe --topic orders | grep "PartitionCount: 5"' - - script: bash kafka-cli.sh kafka-configs.sh --describe --entity-type topics --entity-name orders | grep "retention.ms=7200000" +--- +apiVersion: kafka.stackable.tech/v1alpha1 +kind: KafkaTopic +metadata: + name: orders +status: + conditions: + - type: Ready + status: "True" + reason: Available + - type: Synced + status: "True" + reason: Synced + partitions: 5 + config: + retention.ms: + value: "7200000" + source: DYNAMIC_TOPIC_CONFIG diff --git a/tests/templates/kuttl/topic/30-change-topic.yaml b/tests/templates/kuttl/topic/30-change-topic.yaml index 9b4b26679..ef186c7b8 100644 --- a/tests/templates/kuttl/topic/30-change-topic.yaml +++ b/tests/templates/kuttl/topic/30-change-topic.yaml @@ -4,10 +4,11 @@ kind: KafkaTopic metadata: name: orders spec: - clusterName: test-kafka + clusterRef: + name: test-kafka topicName: orders partitions: 5 replicationFactor: 1 - config: + configOverrides: retention.ms: "7200000" max.message.bytes: "2097152" diff --git a/tests/templates/kuttl/topic/31-reject-topic-name-change.yaml b/tests/templates/kuttl/topic/31-reject-topic-name-change.yaml new file mode 100644 index 000000000..14d5e3af7 --- /dev/null +++ b/tests/templates/kuttl/topic/31-reject-topic-name-change.yaml @@ -0,0 +1,22 @@ +--- +# The API server rejects changing, removing or adding topicName. +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +commands: + - script: >- + kubectl patch kafkatopic orders --namespace $NAMESPACE --type merge + --patch '{"spec": {"topicName": "renamed"}}' 2>&1 + | grep "topicName is immutable" + - script: >- + kubectl patch kafkatopic orders --namespace $NAMESPACE --type json + --patch '[{"op": "remove", "path": "/spec/topicName"}]' 2>&1 + | grep "topicName is immutable" + - script: >- + kubectl patch kafkatopic defaults --namespace $NAMESPACE --type merge + --patch '{"spec": {"topicName": "defaults"}}' 2>&1 + | grep "topicName is immutable" + - script: test "$(kubectl get kafkatopic --namespace $NAMESPACE orders --output jsonpath='{.spec.topicName}')" = orders + # An explicit null means the same as no topicName. + - script: >- + kubectl patch kafkatopic defaults --namespace $NAMESPACE --type json + --patch '[{"op": "add", "path": "/spec/topicName", "value": null}]' diff --git a/tests/templates/kuttl/topic/41-assert.yaml b/tests/templates/kuttl/topic/41-assert.yaml index 796e09f85..2a5e4b777 100644 --- a/tests/templates/kuttl/topic/41-assert.yaml +++ b/tests/templates/kuttl/topic/41-assert.yaml @@ -1,12 +1,27 @@ --- -# The key removed from the spec is reset, the key set out of band stays. +# The configOverrides replace all topic-level configs: the key removed from the spec and the key +# set out of band are both reset to the default. apiVersion: kuttl.dev/v1beta1 kind: TestAssert timeout: 300 -commands: - - script: bash topic-condition.sh orders Synced True Synced - - script: | - configs=$(bash kafka-cli.sh kafka-configs.sh --describe --entity-type topics --entity-name orders) - ! echo "$configs" | grep max.message.bytes - - script: bash kafka-cli.sh kafka-configs.sh --describe --entity-type topics --entity-name orders | grep "segment.ms=86400000" - - script: bash kafka-cli.sh kafka-configs.sh --describe --entity-type topics --entity-name orders | grep "retention.ms=7200000" +--- +apiVersion: kafka.stackable.tech/v1alpha1 +kind: KafkaTopic +metadata: + name: orders +status: + conditions: + - type: Ready + status: "True" + reason: Available + - type: Synced + status: "True" + reason: Synced + config: + retention.ms: + value: "7200000" + source: DYNAMIC_TOPIC_CONFIG + max.message.bytes: + source: DEFAULT_CONFIG + segment.ms: + source: DEFAULT_CONFIG diff --git a/tests/templates/kuttl/topic/41-remove-config-key.yaml b/tests/templates/kuttl/topic/41-remove-config-override.yaml similarity index 74% rename from tests/templates/kuttl/topic/41-remove-config-key.yaml rename to tests/templates/kuttl/topic/41-remove-config-override.yaml index 880129971..ca1786493 100644 --- a/tests/templates/kuttl/topic/41-remove-config-key.yaml +++ b/tests/templates/kuttl/topic/41-remove-config-override.yaml @@ -5,4 +5,4 @@ kind: TestStep commands: - script: >- kubectl patch kafkatopic orders --namespace $NAMESPACE --type json - --patch '[{"op": "remove", "path": "/spec/config/max.message.bytes"}]' + --patch '[{"op": "remove", "path": "/spec/configOverrides/max.message.bytes"}]' diff --git a/tests/templates/kuttl/topic/50-assert.yaml b/tests/templates/kuttl/topic/50-assert.yaml index 443a7e5a8..b7945043f 100644 --- a/tests/templates/kuttl/topic/50-assert.yaml +++ b/tests/templates/kuttl/topic/50-assert.yaml @@ -3,7 +3,18 @@ apiVersion: kuttl.dev/v1beta1 kind: TestAssert timeout: 300 -commands: - - script: bash topic-condition.sh orders Synced False UnsupportedChange - - script: bash topic-condition.sh orders Ready True Available - - script: 'bash kafka-cli.sh kafka-topics.sh --describe --topic orders | grep "PartitionCount: 5"' +--- +apiVersion: kafka.stackable.tech/v1alpha1 +kind: KafkaTopic +metadata: + name: orders +status: + conditions: + - type: Ready + status: "True" + reason: Available + - type: Synced + status: "False" + reason: UnsupportedChange + partitions: 5 + replicationFactor: 1 diff --git a/tests/templates/kuttl/topic/50-unsupported-change.yaml b/tests/templates/kuttl/topic/50-unsupported-change.yaml index 72f427bc8..9424a96d4 100644 --- a/tests/templates/kuttl/topic/50-unsupported-change.yaml +++ b/tests/templates/kuttl/topic/50-unsupported-change.yaml @@ -5,9 +5,10 @@ kind: KafkaTopic metadata: name: orders spec: - clusterName: test-kafka + clusterRef: + name: test-kafka topicName: orders partitions: 2 replicationFactor: 3 - config: + configOverrides: retention.ms: "7200000" diff --git a/tests/templates/kuttl/topic/51-assert.yaml b/tests/templates/kuttl/topic/51-assert.yaml deleted file mode 100644 index 3f2cfc72e..000000000 --- a/tests/templates/kuttl/topic/51-assert.yaml +++ /dev/null @@ -1,6 +0,0 @@ ---- -apiVersion: kuttl.dev/v1beta1 -kind: TestAssert -timeout: 300 -commands: - - script: bash topic-condition.sh orders Synced True Synced diff --git a/tests/templates/kuttl/topic/60-assert.yaml b/tests/templates/kuttl/topic/60-assert.yaml index d212ade8a..a2ae21b5c 100644 --- a/tests/templates/kuttl/topic/60-assert.yaml +++ b/tests/templates/kuttl/topic/60-assert.yaml @@ -1,11 +1,22 @@ --- -# Ready stays unknown, because the agent did not look at the topic in Kafka. +# Ready is unknown, because the agent did not look at the topic in Kafka. apiVersion: kuttl.dev/v1beta1 kind: TestAssert timeout: 300 commands: - - script: bash topic-condition.sh throttled Synced False InvalidSpec - - script: bash topic-condition.sh throttled Ready "" "" - script: | topics=$(bash kafka-cli.sh kafka-topics.sh --list) ! echo "$topics" | grep -x throttled +--- +apiVersion: kafka.stackable.tech/v1alpha1 +kind: KafkaTopic +metadata: + name: throttled +status: + conditions: + - type: Ready + status: Unknown + reason: InvalidSpec + - type: Synced + status: "False" + reason: InvalidSpec diff --git a/tests/templates/kuttl/topic/60-invalid-spec.yaml b/tests/templates/kuttl/topic/60-invalid-spec.yaml index effcf8bc3..e331d3661 100644 --- a/tests/templates/kuttl/topic/60-invalid-spec.yaml +++ b/tests/templates/kuttl/topic/60-invalid-spec.yaml @@ -5,9 +5,10 @@ kind: KafkaTopic metadata: name: throttled spec: - clusterName: test-kafka + clusterRef: + name: test-kafka topicName: throttled partitions: 1 replicationFactor: 1 - config: + configOverrides: leader.replication.throttled.replicas: "*" diff --git a/tests/templates/kuttl/topic/70-create-foreign-topic.yaml b/tests/templates/kuttl/topic/70-create-foreign-topic.yaml index da72f46ac..9591c9ca4 100644 --- a/tests/templates/kuttl/topic/70-create-foreign-topic.yaml +++ b/tests/templates/kuttl/topic/70-create-foreign-topic.yaml @@ -3,4 +3,6 @@ apiVersion: kuttl.dev/v1beta1 kind: TestStep commands: - - script: bash kafka-cli.sh kafka-topics.sh --create --topic legacy --partitions 1 --replication-factor 1 + - script: >- + bash kafka-cli.sh kafka-topics.sh --create --topic legacy --partitions 1 --replication-factor 1 + --config cleanup.policy=compact diff --git a/tests/templates/kuttl/topic/71-refuse-foreign-topic.yaml b/tests/templates/kuttl/topic/71-adopt-foreign-topic.yaml similarity index 71% rename from tests/templates/kuttl/topic/71-refuse-foreign-topic.yaml rename to tests/templates/kuttl/topic/71-adopt-foreign-topic.yaml index fe7eef33e..71b6bedf9 100644 --- a/tests/templates/kuttl/topic/71-refuse-foreign-topic.yaml +++ b/tests/templates/kuttl/topic/71-adopt-foreign-topic.yaml @@ -4,7 +4,9 @@ kind: KafkaTopic metadata: name: legacy spec: - clusterName: test-kafka + clusterRef: + name: test-kafka topicName: legacy partitions: 1 replicationFactor: 1 + deletionPolicy: Delete diff --git a/tests/templates/kuttl/topic/71-assert.yaml b/tests/templates/kuttl/topic/71-assert.yaml index 3ed15c319..2a3198895 100644 --- a/tests/templates/kuttl/topic/71-assert.yaml +++ b/tests/templates/kuttl/topic/71-assert.yaml @@ -1,8 +1,31 @@ --- -# The adoption policy defaults to Refuse. +# Existing topics are adopted, and their topic-level configs are replaced. With the deletionPolicy +# Delete, the adopted topic gets a finalizer. apiVersion: kuttl.dev/v1beta1 kind: TestAssert timeout: 300 -commands: - - script: bash topic-condition.sh legacy Ready False AlreadyExists - - script: test -z "$(kubectl get kafkatopic --namespace $NAMESPACE legacy --output jsonpath='{.status.topicId}')" +resourceRefs: + - apiVersion: kafka.stackable.tech/v1alpha1 + kind: KafkaTopic + name: legacy + ref: legacy +assertAll: + - celExpr: legacy.status.topicId != '' +--- +apiVersion: kafka.stackable.tech/v1alpha1 +kind: KafkaTopic +metadata: + name: legacy + finalizers: + - kafka.stackable.tech/kafkatopic +status: + conditions: + - type: Ready + status: "True" + reason: Available + - type: Synced + status: "True" + reason: Synced + config: + cleanup.policy: + source: DEFAULT_CONFIG diff --git a/tests/templates/kuttl/topic/72-adopt-foreign-topic.yaml b/tests/templates/kuttl/topic/72-adopt-foreign-topic.yaml deleted file mode 100644 index ef3a851da..000000000 --- a/tests/templates/kuttl/topic/72-adopt-foreign-topic.yaml +++ /dev/null @@ -1,12 +0,0 @@ ---- -apiVersion: kafka.stackable.tech/v1alpha1 -kind: KafkaTopic -metadata: - name: legacy -spec: - clusterName: test-kafka - topicName: legacy - partitions: 1 - replicationFactor: 1 - managementPolicy: - adoption: Adopt diff --git a/tests/templates/kuttl/topic/72-assert.yaml b/tests/templates/kuttl/topic/72-assert.yaml deleted file mode 100644 index 02e00c9f0..000000000 --- a/tests/templates/kuttl/topic/72-assert.yaml +++ /dev/null @@ -1,8 +0,0 @@ ---- -apiVersion: kuttl.dev/v1beta1 -kind: TestAssert -timeout: 300 -commands: - - script: bash topic-condition.sh legacy Ready True Available - - script: bash topic-condition.sh legacy Synced True Synced - - script: test -n "$(kubectl get kafkatopic --namespace $NAMESPACE legacy --output jsonpath='{.metadata.annotations.stackable\.tech/owned-since}')" diff --git a/tests/templates/kuttl/topic/75-assert.yaml b/tests/templates/kuttl/topic/75-assert.yaml new file mode 100644 index 000000000..04bb2d1fc --- /dev/null +++ b/tests/templates/kuttl/topic/75-assert.yaml @@ -0,0 +1,22 @@ +--- +# The status still shows the topic in Kafka, with the old config. +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +timeout: 300 +--- +apiVersion: kafka.stackable.tech/v1alpha1 +kind: KafkaTopic +metadata: + name: orders +status: + conditions: + - type: Ready + status: "True" + reason: Available + - type: Synced + status: "False" + reason: ReconciliationPaused + config: + retention.ms: + value: "7200000" + source: DYNAMIC_TOPIC_CONFIG diff --git a/tests/templates/kuttl/topic/75-pause-topic.yaml b/tests/templates/kuttl/topic/75-pause-topic.yaml new file mode 100644 index 000000000..ea35cd524 --- /dev/null +++ b/tests/templates/kuttl/topic/75-pause-topic.yaml @@ -0,0 +1,16 @@ +--- +# The agent only observes a paused topic, so the changed config is not applied. +apiVersion: kafka.stackable.tech/v1alpha1 +kind: KafkaTopic +metadata: + name: orders +spec: + clusterRef: + name: test-kafka + topicName: orders + partitions: 5 + replicationFactor: 1 + configOverrides: + retention.ms: "10800000" + topicOperation: + reconciliationPaused: true diff --git a/tests/templates/kuttl/topic/76-assert.yaml b/tests/templates/kuttl/topic/76-assert.yaml new file mode 100644 index 000000000..08177dc20 --- /dev/null +++ b/tests/templates/kuttl/topic/76-assert.yaml @@ -0,0 +1,22 @@ +--- +# After resuming, the agent applies the config changed while paused. +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +timeout: 300 +--- +apiVersion: kafka.stackable.tech/v1alpha1 +kind: KafkaTopic +metadata: + name: orders +status: + conditions: + - type: Ready + status: "True" + reason: Available + - type: Synced + status: "True" + reason: Synced + config: + retention.ms: + value: "10800000" + source: DYNAMIC_TOPIC_CONFIG diff --git a/tests/templates/kuttl/topic/51-revert-unsupported-change.yaml b/tests/templates/kuttl/topic/76-resume-topic.yaml similarity index 53% rename from tests/templates/kuttl/topic/51-revert-unsupported-change.yaml rename to tests/templates/kuttl/topic/76-resume-topic.yaml index 4a1fa1998..8cc896dc1 100644 --- a/tests/templates/kuttl/topic/51-revert-unsupported-change.yaml +++ b/tests/templates/kuttl/topic/76-resume-topic.yaml @@ -4,9 +4,12 @@ kind: KafkaTopic metadata: name: orders spec: - clusterName: test-kafka + clusterRef: + name: test-kafka topicName: orders partitions: 5 replicationFactor: 1 - config: - retention.ms: "7200000" + configOverrides: + retention.ms: "10800000" + topicOperation: + reconciliationPaused: false diff --git a/tests/templates/kuttl/topic/77-assert.yaml b/tests/templates/kuttl/topic/77-assert.yaml new file mode 100644 index 000000000..26c91f18b --- /dev/null +++ b/tests/templates/kuttl/topic/77-assert.yaml @@ -0,0 +1,21 @@ +--- +# The topic stays available, and the status keeps the config observed in Kafka. +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +timeout: 300 +--- +apiVersion: kafka.stackable.tech/v1alpha1 +kind: KafkaTopic +metadata: + name: defaults +status: + conditions: + - type: Ready + status: "True" + reason: Available + - type: Synced + status: "False" + reason: KafkaRejected + config: + retention.ms: + source: DEFAULT_CONFIG diff --git a/tests/templates/kuttl/topic/77-invalid-config-value.yaml b/tests/templates/kuttl/topic/77-invalid-config-value.yaml new file mode 100644 index 000000000..a230cce9c --- /dev/null +++ b/tests/templates/kuttl/topic/77-invalid-config-value.yaml @@ -0,0 +1,11 @@ +--- +# Kafka rejects the config value, which only it can validate. +apiVersion: kafka.stackable.tech/v1alpha1 +kind: KafkaTopic +metadata: + name: defaults +spec: + clusterRef: + name: test-kafka + configOverrides: + retention.ms: "abc" diff --git a/tests/templates/kuttl/topic/80-assert.yaml b/tests/templates/kuttl/topic/80-assert.yaml index ab8423f0a..20b4780b3 100644 --- a/tests/templates/kuttl/topic/80-assert.yaml +++ b/tests/templates/kuttl/topic/80-assert.yaml @@ -1,9 +1,43 @@ --- -# The older KafkaTopic keeps managing the topic. +# The older KafkaTopic keeps managing the topic. The newer one gets no finalizer, so deleting it +# does not delete the topic. apiVersion: kuttl.dev/v1beta1 kind: TestAssert timeout: 300 -commands: - - script: bash topic-condition.sh orders-copy Ready False Conflict - - script: bash topic-condition.sh orders Ready True Available - - script: bash kafka-cli.sh kafka-configs.sh --describe --entity-type topics --entity-name orders | grep "retention.ms=7200000" +resourceRefs: + - apiVersion: kafka.stackable.tech/v1alpha1 + kind: KafkaTopic + name: orders-copy + ref: orders_copy +assertAll: + - celExpr: "!has(orders_copy.metadata.finalizers)" +--- +apiVersion: kafka.stackable.tech/v1alpha1 +kind: KafkaTopic +metadata: + name: orders-copy +status: + conditions: + - type: Ready + status: "False" + reason: Conflict + - type: Synced + status: "False" + reason: Conflict +--- +apiVersion: kafka.stackable.tech/v1alpha1 +kind: KafkaTopic +metadata: + name: orders +status: + conditions: + - type: Ready + status: "True" + reason: Available + - type: Synced + status: "True" + reason: Synced + config: + retention.ms: + value: "10800000" + source: DYNAMIC_TOPIC_CONFIG diff --git a/tests/templates/kuttl/topic/80-create-conflicting-topic.yaml b/tests/templates/kuttl/topic/80-create-conflicting-topic.yaml index 3a2b28c54..7314dcc8e 100644 --- a/tests/templates/kuttl/topic/80-create-conflicting-topic.yaml +++ b/tests/templates/kuttl/topic/80-create-conflicting-topic.yaml @@ -5,7 +5,9 @@ kind: KafkaTopic metadata: name: orders-copy spec: - clusterName: test-kafka + clusterRef: + name: test-kafka topicName: orders partitions: 5 replicationFactor: 1 + deletionPolicy: Delete diff --git a/tests/templates/kuttl/topic/81-assert.yaml b/tests/templates/kuttl/topic/81-assert.yaml index 091073c7d..1e7d88dc6 100644 --- a/tests/templates/kuttl/topic/81-assert.yaml +++ b/tests/templates/kuttl/topic/81-assert.yaml @@ -1,4 +1,5 @@ --- +# Deleting the conflicting KafkaTopic leaves the topic in Kafka. apiVersion: kuttl.dev/v1beta1 kind: TestAssert timeout: 300 diff --git a/tests/templates/kuttl/topic/90-assert.yaml b/tests/templates/kuttl/topic/90-assert.yaml deleted file mode 100644 index b2bebc87c..000000000 --- a/tests/templates/kuttl/topic/90-assert.yaml +++ /dev/null @@ -1,8 +0,0 @@ ---- -# Only KafkaTopics that delete their topic in Kafka have a finalizer. -apiVersion: kuttl.dev/v1beta1 -kind: TestAssert -timeout: 300 -commands: - - script: test "$(kubectl get kafkatopic --namespace $NAMESPACE legacy --output jsonpath='{.metadata.finalizers}')" = '["kafka.stackable.tech/kafkatopic"]' - - script: test -z "$(kubectl get kafkatopic --namespace $NAMESPACE orders --output jsonpath='{.metadata.finalizers}')" diff --git a/tests/templates/kuttl/topic/90-delete-legacy-topic-in-kafka.yaml b/tests/templates/kuttl/topic/90-delete-legacy-topic-in-kafka.yaml deleted file mode 100644 index 2ea201fe0..000000000 --- a/tests/templates/kuttl/topic/90-delete-legacy-topic-in-kafka.yaml +++ /dev/null @@ -1,13 +0,0 @@ ---- -apiVersion: kafka.stackable.tech/v1alpha1 -kind: KafkaTopic -metadata: - name: legacy -spec: - clusterName: test-kafka - topicName: legacy - partitions: 1 - replicationFactor: 1 - managementPolicy: - adoption: Adopt - deletion: Delete diff --git a/tests/templates/kuttl/topic/91-delete-topics.yaml b/tests/templates/kuttl/topic/91-delete-topics.yaml index 430f58415..5de81d40e 100644 --- a/tests/templates/kuttl/topic/91-delete-topics.yaml +++ b/tests/templates/kuttl/topic/91-delete-topics.yaml @@ -11,3 +11,6 @@ delete: - apiVersion: kafka.stackable.tech/v1alpha1 kind: KafkaTopic name: throttled + - apiVersion: kafka.stackable.tech/v1alpha1 + kind: KafkaTopic + name: defaults diff --git a/tests/templates/kuttl/topic/92-assert.yaml b/tests/templates/kuttl/topic/92-assert.yaml new file mode 100644 index 000000000..64caeabbd --- /dev/null +++ b/tests/templates/kuttl/topic/92-assert.yaml @@ -0,0 +1,20 @@ +--- +# While the agent is still running, it creates the topic and adds the finalizer. +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +timeout: 300 +--- +apiVersion: kafka.stackable.tech/v1alpha1 +kind: KafkaTopic +metadata: + name: cleanup + finalizers: + - kafka.stackable.tech/kafkatopic +status: + conditions: + - type: Ready + status: "True" + reason: Available + - type: Synced + status: "True" + reason: Synced diff --git a/tests/templates/kuttl/topic/92-create-cleanup-topic.yaml b/tests/templates/kuttl/topic/92-create-cleanup-topic.yaml new file mode 100644 index 000000000..f14412ee9 --- /dev/null +++ b/tests/templates/kuttl/topic/92-create-cleanup-topic.yaml @@ -0,0 +1,13 @@ +--- +# From here on, the operator takes over the KafkaTopic, because the agent is removed in the +# following steps. +apiVersion: kafka.stackable.tech/v1alpha1 +kind: KafkaTopic +metadata: + name: cleanup +spec: + clusterRef: + name: test-kafka + partitions: 1 + replicationFactor: 1 + deletionPolicy: Delete diff --git a/tests/templates/kuttl/topic/95-assert.yaml b/tests/templates/kuttl/topic/95-assert.yaml new file mode 100644 index 000000000..9df1148ed --- /dev/null +++ b/tests/templates/kuttl/topic/95-assert.yaml @@ -0,0 +1,18 @@ +--- +# Without an agent, the operator sets the conditions of the remaining KafkaTopics. +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +timeout: 300 +--- +apiVersion: kafka.stackable.tech/v1alpha1 +kind: KafkaTopic +metadata: + name: cleanup +status: + conditions: + - type: Ready + status: Unknown + reason: AgentUnavailable + - type: Synced + status: "False" + reason: AgentUnavailable diff --git a/tests/templates/kuttl/topic/96-assert.yaml b/tests/templates/kuttl/topic/96-assert.yaml new file mode 100644 index 000000000..d99243f83 --- /dev/null +++ b/tests/templates/kuttl/topic/96-assert.yaml @@ -0,0 +1,15 @@ +--- +# The operator keeps the finalizer while the KafkaCluster exists, so the deletion waits for an +# agent. +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +timeout: 300 +commands: + - script: sleep 30 +resourceRefs: + - apiVersion: kafka.stackable.tech/v1alpha1 + kind: KafkaTopic + name: cleanup + ref: cleanup +assertAll: + - celExpr: has(cleanup.metadata.deletionTimestamp) diff --git a/tests/templates/kuttl/topic/96-delete-topic-without-agent.yaml b/tests/templates/kuttl/topic/96-delete-topic-without-agent.yaml new file mode 100644 index 000000000..57614fa46 --- /dev/null +++ b/tests/templates/kuttl/topic/96-delete-topic-without-agent.yaml @@ -0,0 +1,6 @@ +--- +# kuttl would wait for the deletion, which only finishes with the KafkaCluster. +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +commands: + - script: kubectl delete kafkatopic cleanup --namespace $NAMESPACE --wait=false diff --git a/tests/templates/kuttl/topic/97-delete-kafka-cluster.yaml b/tests/templates/kuttl/topic/97-delete-kafka-cluster.yaml new file mode 100644 index 000000000..039bc529d --- /dev/null +++ b/tests/templates/kuttl/topic/97-delete-kafka-cluster.yaml @@ -0,0 +1,7 @@ +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +delete: + - apiVersion: kafka.stackable.tech/v1alpha1 + kind: KafkaCluster + name: test-kafka diff --git a/tests/templates/kuttl/topic/97-errors.yaml b/tests/templates/kuttl/topic/97-errors.yaml new file mode 100644 index 000000000..4e62a73a6 --- /dev/null +++ b/tests/templates/kuttl/topic/97-errors.yaml @@ -0,0 +1,6 @@ +--- +# Without the KafkaCluster, the operator removes the finalizer, so the deletion finishes. +apiVersion: kafka.stackable.tech/v1alpha1 +kind: KafkaTopic +metadata: + name: cleanup diff --git a/tests/templates/kuttl/topic/topic-condition.sh b/tests/templates/kuttl/topic/topic-condition.sh deleted file mode 100644 index 59a05a0d7..000000000 --- a/tests/templates/kuttl/topic/topic-condition.sh +++ /dev/null @@ -1,11 +0,0 @@ -#!/usr/bin/env bash -# Usage: topic-condition.sh -# Fails unless the condition of the KafkaTopic has the given status and reason. -set -euo pipefail - -actual=$(kubectl get kafkatopic --namespace "$NAMESPACE" "$1" \ - --output "jsonpath={.status.conditions[?(@.type==\"$2\")].status}/{.status.conditions[?(@.type==\"$2\")].reason}") -if [ "$actual" != "$3/$4" ]; then - echo "$1: expected the condition $2 to be $3/$4, but it is \"$actual\"" - exit 1 -fi diff --git a/tests/test-definition.yaml b/tests/test-definition.yaml index 5c9116acf..85ac91363 100644 --- a/tests/test-definition.yaml +++ b/tests/test-definition.yaml @@ -7,7 +7,7 @@ dimensions: - name: kafka-kraft values: - 3.9.2 - # - 4.3.1 + - 4.3.1 - name: kafka values: - 3.9.2 From 42d02bfe8fd19d7b4712ae8ca3cb7372431412cb Mon Sep 17 00:00:00 2001 From: Benedikt Labrenz Date: Tue, 27 Oct 2026 18:52:16 +0100 Subject: [PATCH 47/93] regenerate-nix --- Cargo.nix | 20 ++++++++++---------- crate-hashes.json | 20 ++++++++++---------- 2 files changed, 20 insertions(+), 20 deletions(-) diff --git a/Cargo.nix b/Cargo.nix index 0529432e5..691a18e6b 100644 --- a/Cargo.nix +++ b/Cargo.nix @@ -5456,7 +5456,7 @@ rec { src = pkgs.fetchgit { url = "https://github.com/stackabletech/operator-rs.git"; rev = "ccfff5810f9b5affae4d144b3cf04ca88677e23e"; - sha256 = "0vdnhvfn85qnm5845mad24smhdp2y6nznlgagbp3g42pi042g9pm"; + sha256 = "052r8sbdgl89fy4jrkl1jjzjp5fj731pips5a95jd5hm51ymwi4s"; }; libName = "k8s_version"; authors = [ @@ -10389,7 +10389,7 @@ rec { src = pkgs.fetchgit { url = "https://github.com/stackabletech/operator-rs.git"; rev = "ccfff5810f9b5affae4d144b3cf04ca88677e23e"; - sha256 = "0vdnhvfn85qnm5845mad24smhdp2y6nznlgagbp3g42pi042g9pm"; + sha256 = "052r8sbdgl89fy4jrkl1jjzjp5fj731pips5a95jd5hm51ymwi4s"; }; libName = "stackable_agent"; authors = [ @@ -10432,7 +10432,7 @@ rec { src = pkgs.fetchgit { url = "https://github.com/stackabletech/operator-rs.git"; rev = "ccfff5810f9b5affae4d144b3cf04ca88677e23e"; - sha256 = "0vdnhvfn85qnm5845mad24smhdp2y6nznlgagbp3g42pi042g9pm"; + sha256 = "052r8sbdgl89fy4jrkl1jjzjp5fj731pips5a95jd5hm51ymwi4s"; }; libName = "stackable_certs"; authors = [ @@ -10704,7 +10704,7 @@ rec { src = pkgs.fetchgit { url = "https://github.com/stackabletech/operator-rs.git"; rev = "ccfff5810f9b5affae4d144b3cf04ca88677e23e"; - sha256 = "0vdnhvfn85qnm5845mad24smhdp2y6nznlgagbp3g42pi042g9pm"; + sha256 = "052r8sbdgl89fy4jrkl1jjzjp5fj731pips5a95jd5hm51ymwi4s"; }; libName = "stackable_operator"; authors = [ @@ -10909,7 +10909,7 @@ rec { src = pkgs.fetchgit { url = "https://github.com/stackabletech/operator-rs.git"; rev = "ccfff5810f9b5affae4d144b3cf04ca88677e23e"; - sha256 = "0vdnhvfn85qnm5845mad24smhdp2y6nznlgagbp3g42pi042g9pm"; + sha256 = "052r8sbdgl89fy4jrkl1jjzjp5fj731pips5a95jd5hm51ymwi4s"; }; procMacro = true; libName = "stackable_operator_derive"; @@ -10944,7 +10944,7 @@ rec { src = pkgs.fetchgit { url = "https://github.com/stackabletech/operator-rs.git"; rev = "ccfff5810f9b5affae4d144b3cf04ca88677e23e"; - sha256 = "0vdnhvfn85qnm5845mad24smhdp2y6nznlgagbp3g42pi042g9pm"; + sha256 = "052r8sbdgl89fy4jrkl1jjzjp5fj731pips5a95jd5hm51ymwi4s"; }; libName = "stackable_shared"; authors = [ @@ -11025,7 +11025,7 @@ rec { src = pkgs.fetchgit { url = "https://github.com/stackabletech/operator-rs.git"; rev = "ccfff5810f9b5affae4d144b3cf04ca88677e23e"; - sha256 = "0vdnhvfn85qnm5845mad24smhdp2y6nznlgagbp3g42pi042g9pm"; + sha256 = "052r8sbdgl89fy4jrkl1jjzjp5fj731pips5a95jd5hm51ymwi4s"; }; libName = "stackable_telemetry"; authors = [ @@ -11135,7 +11135,7 @@ rec { src = pkgs.fetchgit { url = "https://github.com/stackabletech/operator-rs.git"; rev = "ccfff5810f9b5affae4d144b3cf04ca88677e23e"; - sha256 = "0vdnhvfn85qnm5845mad24smhdp2y6nznlgagbp3g42pi042g9pm"; + sha256 = "052r8sbdgl89fy4jrkl1jjzjp5fj731pips5a95jd5hm51ymwi4s"; }; libName = "stackable_versioned"; authors = [ @@ -11181,7 +11181,7 @@ rec { src = pkgs.fetchgit { url = "https://github.com/stackabletech/operator-rs.git"; rev = "ccfff5810f9b5affae4d144b3cf04ca88677e23e"; - sha256 = "0vdnhvfn85qnm5845mad24smhdp2y6nznlgagbp3g42pi042g9pm"; + sha256 = "052r8sbdgl89fy4jrkl1jjzjp5fj731pips5a95jd5hm51ymwi4s"; }; procMacro = true; libName = "stackable_versioned_macros"; @@ -11245,7 +11245,7 @@ rec { src = pkgs.fetchgit { url = "https://github.com/stackabletech/operator-rs.git"; rev = "ccfff5810f9b5affae4d144b3cf04ca88677e23e"; - sha256 = "0vdnhvfn85qnm5845mad24smhdp2y6nznlgagbp3g42pi042g9pm"; + sha256 = "052r8sbdgl89fy4jrkl1jjzjp5fj731pips5a95jd5hm51ymwi4s"; }; libName = "stackable_webhook"; authors = [ diff --git a/crate-hashes.json b/crate-hashes.json index 017105d5a..405952ac3 100644 --- a/crate-hashes.json +++ b/crate-hashes.json @@ -1,12 +1,12 @@ { - "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#k8s-version@0.1.4": "0vdnhvfn85qnm5845mad24smhdp2y6nznlgagbp3g42pi042g9pm", - "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#stackable-agent@0.1.0": "0vdnhvfn85qnm5845mad24smhdp2y6nznlgagbp3g42pi042g9pm", - "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#stackable-certs@0.4.1": "0vdnhvfn85qnm5845mad24smhdp2y6nznlgagbp3g42pi042g9pm", - "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#stackable-operator-derive@0.3.1": "0vdnhvfn85qnm5845mad24smhdp2y6nznlgagbp3g42pi042g9pm", - "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#stackable-operator@0.121.0": "0vdnhvfn85qnm5845mad24smhdp2y6nznlgagbp3g42pi042g9pm", - "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#stackable-shared@0.2.0": "0vdnhvfn85qnm5845mad24smhdp2y6nznlgagbp3g42pi042g9pm", - "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#stackable-telemetry@0.6.5": "0vdnhvfn85qnm5845mad24smhdp2y6nznlgagbp3g42pi042g9pm", - "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#stackable-versioned-macros@0.13.0": "0vdnhvfn85qnm5845mad24smhdp2y6nznlgagbp3g42pi042g9pm", - "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#stackable-versioned@0.13.0": "0vdnhvfn85qnm5845mad24smhdp2y6nznlgagbp3g42pi042g9pm", - "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#stackable-webhook@0.10.0": "0vdnhvfn85qnm5845mad24smhdp2y6nznlgagbp3g42pi042g9pm" + "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#k8s-version@0.1.4": "052r8sbdgl89fy4jrkl1jjzjp5fj731pips5a95jd5hm51ymwi4s", + "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#stackable-agent@0.1.0": "052r8sbdgl89fy4jrkl1jjzjp5fj731pips5a95jd5hm51ymwi4s", + "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#stackable-certs@0.4.1": "052r8sbdgl89fy4jrkl1jjzjp5fj731pips5a95jd5hm51ymwi4s", + "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#stackable-operator-derive@0.3.1": "052r8sbdgl89fy4jrkl1jjzjp5fj731pips5a95jd5hm51ymwi4s", + "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#stackable-operator@0.121.0": "052r8sbdgl89fy4jrkl1jjzjp5fj731pips5a95jd5hm51ymwi4s", + "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#stackable-shared@0.2.0": "052r8sbdgl89fy4jrkl1jjzjp5fj731pips5a95jd5hm51ymwi4s", + "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#stackable-telemetry@0.6.5": "052r8sbdgl89fy4jrkl1jjzjp5fj731pips5a95jd5hm51ymwi4s", + "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#stackable-versioned-macros@0.13.0": "052r8sbdgl89fy4jrkl1jjzjp5fj731pips5a95jd5hm51ymwi4s", + "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#stackable-versioned@0.13.0": "052r8sbdgl89fy4jrkl1jjzjp5fj731pips5a95jd5hm51ymwi4s", + "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#stackable-webhook@0.10.0": "052r8sbdgl89fy4jrkl1jjzjp5fj731pips5a95jd5hm51ymwi4s" } \ No newline at end of file From 150886f43a90c073369eafff340f7fbeec9329f7 Mon Sep 17 00:00:00 2001 From: Benedikt Labrenz Date: Tue, 27 Oct 2026 19:02:09 +0100 Subject: [PATCH 48/93] return only topic-level config and no source --- rust/kafka-admin/src/client.rs | 160 ++++++++------------------------- 1 file changed, 37 insertions(+), 123 deletions(-) diff --git a/rust/kafka-admin/src/client.rs b/rust/kafka-admin/src/client.rs index f5145ff54..5fe2712ae 100644 --- a/rust/kafka-admin/src/client.rs +++ b/rust/kafka-admin/src/client.rs @@ -2,7 +2,6 @@ use std::{ collections::{BTreeMap, BTreeSet}, - fmt, time::Duration, }; @@ -52,6 +51,7 @@ const CONNECTION_MAX_IDLE: Duration = Duration::from_secs(5 * 60); const BROKER_DEFAULT: i32 = -1; const RESOURCE_TYPE_TOPIC: i8 = 2; +const CONFIG_SOURCE_DYNAMIC_TOPIC: i8 = 1; const CONFIG_OPERATION_SET: i8 = 0; const CONFIG_OPERATION_DELETE: i8 = 1; @@ -102,53 +102,8 @@ pub struct CreatedTopic { pub id: String, pub partitions: i32, pub replication_factor: i16, - /// All configs of the topic, as [`KafkaAdminClient::describe_topic_configs`] returns them - pub configs: BTreeMap, -} - -#[derive(Clone, Debug, Eq, PartialEq)] -pub struct TopicConfig { - /// `None` for sensitive configs - pub value: Option, - pub source: ConfigSource, -} - -/// Where the value of a topic config comes from. -#[derive(Clone, Copy, Debug, Eq, PartialEq)] -pub enum ConfigSource { - DynamicTopic, - DynamicBroker, - DynamicDefaultBroker, - StaticBroker, - Default, - Unknown, -} - -impl ConfigSource { - fn from_code(code: i8) -> Self { - match code { - 1 => Self::DynamicTopic, - 2 => Self::DynamicBroker, - 3 => Self::DynamicDefaultBroker, - 4 => Self::StaticBroker, - 5 => Self::Default, - _ => Self::Unknown, - } - } -} - -/// The names used by Kafka's admin client and `kafka-configs.sh`. -impl fmt::Display for ConfigSource { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - f.write_str(match self { - Self::DynamicTopic => "DYNAMIC_TOPIC_CONFIG", - Self::DynamicBroker => "DYNAMIC_BROKER_CONFIG", - Self::DynamicDefaultBroker => "DYNAMIC_DEFAULT_BROKER_CONFIG", - Self::StaticBroker => "STATIC_BROKER_CONFIG", - Self::Default => "DEFAULT_CONFIG", - Self::Unknown => "UNKNOWN", - }) - } + /// The configs set on the topic, as [`KafkaAdminClient::describe_topic_configs`] returns them + pub configs: BTreeMap, } #[derive(Clone, Debug, Eq, PartialEq)] @@ -202,11 +157,11 @@ impl KafkaAdminClient { describe_topic_result(response, topic) } - /// Returns all configs of the topic, including broker and default values. + /// Returns the configs set on the topic itself, without broker and default values. pub async fn describe_topic_configs( &mut self, topic: &str, - ) -> Result> { + ) -> Result> { let response = self .send( describe_topic_configs_request(topic), @@ -381,7 +336,7 @@ fn describe_topic_configs_request(topic: &str) -> DescribeConfigsRequest { fn describe_topic_configs_result( response: DescribeConfigsResponse, topic: &str, -) -> Result> { +) -> Result> { let result = first(response.results, DescribeConfigsRequest::KEY, topic)?; check( DescribeConfigsRequest::KEY, @@ -390,23 +345,20 @@ fn describe_topic_configs_result( result.error_message, )?; - Ok(result - .configs - .into_iter() - .map(|config| { - ( - config.name.to_string(), - topic_config(config.value, config.config_source), - ) - }) - .collect()) + Ok(topic_level_configs(result.configs.into_iter().map( + |config| (config.name, config.value, config.config_source), + ))) } -fn topic_config(value: Option, source: i8) -> TopicConfig { - TopicConfig { - value: value.map(|value| value.to_string()), - source: ConfigSource::from_code(source), - } +/// Topic configs have no PASSWORD type, so Kafka does not redact their topic-level values. +fn topic_level_configs( + configs: impl IntoIterator, i8)>, +) -> BTreeMap { + configs + .into_iter() + .filter(|(_, _, source)| *source == CONFIG_SOURCE_DYNAMIC_TOPIC) + .filter_map(|(name, value, _)| Some((name.to_string(), value?.to_string()))) + .collect() } fn create_topic_request( @@ -453,17 +405,13 @@ fn create_topic_result(response: CreateTopicsResponse, topic: &str) -> Result, source: i8| { DescribeConfigsResourceResult::default() .with_name(StrBytes::from_static_str(name)) @@ -738,54 +686,21 @@ mod tests { }; let response = DescribeConfigsResponse::default().with_results(vec![ DescribeConfigsResult::default().with_configs(vec![ - config("retention.ms", Some("1000"), 1), + config("retention.ms", Some("1000"), CONFIG_SOURCE_DYNAMIC_TOPIC), + // DEFAULT_CONFIG config("cleanup.policy", Some("delete"), 5), + // STATIC_BROKER_CONFIG config("sasl.secret", None, 4), ]), ]); assert_eq!( - BTreeMap::from([ - ( - "retention.ms".to_owned(), - TopicConfig { - value: Some("1000".to_owned()), - source: ConfigSource::DynamicTopic, - } - ), - ( - "cleanup.policy".to_owned(), - TopicConfig { - value: Some("delete".to_owned()), - source: ConfigSource::Default, - } - ), - ( - "sasl.secret".to_owned(), - TopicConfig { - value: None, - source: ConfigSource::StaticBroker, - } - ), - ]), + BTreeMap::from([("retention.ms".to_owned(), "1000".to_owned())]), describe_topic_configs_result(response, TOPIC) .expect("the configs should be described") ); } - #[rstest] - #[case::dynamic_topic(1, ConfigSource::DynamicTopic, "DYNAMIC_TOPIC_CONFIG")] - #[case::default(5, ConfigSource::Default, "DEFAULT_CONFIG")] - #[case::not_for_topics(8, ConfigSource::Unknown, "UNKNOWN")] - fn name_config_sources_like_kafka( - #[case] code: i8, - #[case] source: ConfigSource, - #[case] name: &str, - ) { - assert_eq!(source, ConfigSource::from_code(code)); - assert_eq!(name, source.to_string()); - } - #[test] fn create_topic_with_configs() { let configs = BTreeMap::from([("retention.ms".to_owned(), "1000".to_owned())]); @@ -829,7 +744,12 @@ mod tests { CreatableTopicConfigs::default() .with_name(StrBytes::from_static_str("retention.ms")) .with_value(Some(StrBytes::from_static_str("1000"))) - .with_config_source(1), + .with_config_source(CONFIG_SOURCE_DYNAMIC_TOPIC), + // DEFAULT_CONFIG + CreatableTopicConfigs::default() + .with_name(StrBytes::from_static_str("cleanup.policy")) + .with_value(Some(StrBytes::from_static_str("delete"))) + .with_config_source(5), ])), ]); @@ -838,13 +758,7 @@ mod tests { id: "UXyU9i5ARn6W00ON2taeWA".to_owned(), partitions: 6, replication_factor: 3, - configs: BTreeMap::from([( - "retention.ms".to_owned(), - TopicConfig { - value: Some("1000".to_owned()), - source: ConfigSource::DynamicTopic, - } - )]), + configs: BTreeMap::from([("retention.ms".to_owned(), "1000".to_owned())]), }, create_topic_result(response, TOPIC).expect("the topic should be created") ); From 56b9894724ab488e2fc63731312263319949f328 Mon Sep 17 00:00:00 2001 From: Benedikt Labrenz Date: Tue, 27 Oct 2026 19:12:03 +0100 Subject: [PATCH 49/93] document source of magic kafka numbers --- rust/kafka-admin/src/client.rs | 26 ++++++++++++++++++-------- rust/kafka-admin/src/connection.rs | 9 +++++++++ 2 files changed, 27 insertions(+), 8 deletions(-) diff --git a/rust/kafka-admin/src/client.rs b/rust/kafka-admin/src/client.rs index 5fe2712ae..70e418004 100644 --- a/rust/kafka-admin/src/client.rs +++ b/rust/kafka-admin/src/client.rs @@ -32,6 +32,8 @@ use crate::{ connection::{self, Connection, Security}, }; +// The highest versions that Kafka 3.9.2 supports and 4.3.1 still supports, see `validVersions` in +// `clients/src/main/resources/common/message/Request.json` of the Kafka repository. const METADATA_VERSION: i16 = 12; const CREATE_TOPICS_VERSION: i16 = 7; const DELETE_TOPICS_VERSION: i16 = 6; @@ -44,12 +46,20 @@ const LIST_PARTITION_REASSIGNMENTS_VERSION: i16 = 0; const BROKER_TIMEOUT_MS: i32 = 25_000; const _: () = assert!((BROKER_TIMEOUT_MS as u128) < connection::REQUEST_TIMEOUT.as_millis()); -/// Must stay below the broker's `connections.max.idle.ms` (10 minutes by default). +/// Must stay below the broker's `connections.max.idle.ms`, 10 minutes by default, see +/// `CONNECTIONS_MAX_IDLE_MS_DEFAULT` in +/// `server/src/main/java/org/apache/kafka/network/SocketServerConfigs.java`. const CONNECTION_MAX_IDLE: Duration = Duration::from_secs(5 * 60); -/// Lets the broker apply `num.partitions` or `default.replication.factor` when creating a topic. +/// Lets the broker apply `num.partitions` or `default.replication.factor` when creating a topic, +/// see `NumPartitions` and `ReplicationFactor` in +/// `clients/src/main/resources/common/message/CreateTopicsRequest.json`. const BROKER_DEFAULT: i32 = -1; +// The codes of the enums in Kafka's Java client, in `clients/src/main/java/org/apache/kafka/`: +// - `ConfigResource.Type` in `common/config/ConfigResource.java` +// - `ConfigSource` in `common/requests/DescribeConfigsResponse.java` +// - `AlterConfigOp.OpType` in `clients/admin/AlterConfigOp.java` const RESOURCE_TYPE_TOPIC: i8 = 2; const CONFIG_SOURCE_DYNAMIC_TOPIC: i8 = 1; const CONFIG_OPERATION_SET: i8 = 0; @@ -559,6 +569,9 @@ mod tests { const TOPIC: &str = "orders"; const APIS: &[(i16, i16, i16)] = &[(DeleteTopicsRequest::KEY, 0, DELETE_TOPICS_VERSION)]; + // `ConfigSource`, like `CONFIG_SOURCE_DYNAMIC_TOPIC` + const CONFIG_SOURCE_STATIC_BROKER: i8 = 4; + const CONFIG_SOURCE_DEFAULT: i8 = 5; #[tokio::test] async fn try_the_next_bootstrap_server() { @@ -687,10 +700,8 @@ mod tests { let response = DescribeConfigsResponse::default().with_results(vec![ DescribeConfigsResult::default().with_configs(vec![ config("retention.ms", Some("1000"), CONFIG_SOURCE_DYNAMIC_TOPIC), - // DEFAULT_CONFIG - config("cleanup.policy", Some("delete"), 5), - // STATIC_BROKER_CONFIG - config("sasl.secret", None, 4), + config("cleanup.policy", Some("delete"), CONFIG_SOURCE_DEFAULT), + config("sasl.secret", None, CONFIG_SOURCE_STATIC_BROKER), ]), ]); @@ -745,11 +756,10 @@ mod tests { .with_name(StrBytes::from_static_str("retention.ms")) .with_value(Some(StrBytes::from_static_str("1000"))) .with_config_source(CONFIG_SOURCE_DYNAMIC_TOPIC), - // DEFAULT_CONFIG CreatableTopicConfigs::default() .with_name(StrBytes::from_static_str("cleanup.policy")) .with_value(Some(StrBytes::from_static_str("delete"))) - .with_config_source(5), + .with_config_source(CONFIG_SOURCE_DEFAULT), ])), ]); diff --git a/rust/kafka-admin/src/connection.rs b/rust/kafka-admin/src/connection.rs index 13fe0d3a3..85274ca8e 100644 --- a/rust/kafka-admin/src/connection.rs +++ b/rust/kafka-admin/src/connection.rs @@ -27,9 +27,18 @@ use crate::{ tls, }; +// The defaults of Kafka's Java admin client, in `clients/src/main/java/org/apache/kafka/clients/`: +// - `socket.connection.setup.timeout.ms`, `DEFAULT_SOCKET_CONNECTION_SETUP_TIMEOUT_MS` in +// `CommonClientConfigs.java` +// - `request.timeout.ms` in `admin/AdminClientConfig.java` const CONNECT_TIMEOUT: Duration = Duration::from_secs(10); pub(crate) const REQUEST_TIMEOUT: Duration = Duration::from_secs(30); + +/// Admin responses are far smaller, so a larger frame means a broken stream. const MAX_FRAME_SIZE: usize = 64 * 1024 * 1024; + +/// The first version with `ClientSoftwareName` and `ClientSoftwareVersion`, see +/// `clients/src/main/resources/common/message/ApiVersionsRequest.json` in the Kafka repository. const API_VERSIONS_VERSION: i16 = 3; #[derive(Debug, Snafu)] From 1aee2b0dbabffaa8bd12439ebdd5b2374c5dfe88 Mon Sep 17 00:00:00 2001 From: Benedikt Labrenz Date: Tue, 27 Oct 2026 19:33:20 +0100 Subject: [PATCH 50/93] use ApiKey enum --- rust/kafka-admin/src/client.rs | 60 +++++++++++++++--------------- rust/kafka-admin/src/codec.rs | 43 +++++++++++++++------ rust/kafka-admin/src/connection.rs | 48 ++++++++++++++---------- 3 files changed, 91 insertions(+), 60 deletions(-) diff --git a/rust/kafka-admin/src/client.rs b/rust/kafka-admin/src/client.rs index 70e418004..0620aaeff 100644 --- a/rust/kafka-admin/src/client.rs +++ b/rust/kafka-admin/src/client.rs @@ -10,8 +10,8 @@ use kafka_protocol::{ ResponseError, error::ParseResponseErrorCode, messages::{ - CreatePartitionsRequest, CreateTopicsRequest, CreateTopicsResponse, DeleteTopicsRequest, - DeleteTopicsResponse, DescribeConfigsRequest, DescribeConfigsResponse, + ApiKey, CreatePartitionsRequest, CreateTopicsRequest, CreateTopicsResponse, + DeleteTopicsRequest, DeleteTopicsResponse, DescribeConfigsRequest, DescribeConfigsResponse, IncrementalAlterConfigsRequest, ListPartitionReassignmentsRequest, ListPartitionReassignmentsResponse, MetadataRequest, MetadataResponse, TopicName, create_partitions_request::CreatePartitionsTopic, @@ -80,16 +80,16 @@ pub enum Error { Send { source: connection::Error }, #[snafu(display( - "the response to the request with API key {api_key} contains no result for the topic {topic:?}" + "the response to the {api_key:?} request contains no result for the topic {topic:?}" ))] - MissingResult { api_key: i16, topic: String }, + MissingResult { api_key: ApiKey, topic: String }, #[snafu(display( - "the broker rejected the request with API key {api_key} for the topic {topic:?} with {error}{}", + "the broker rejected the {api_key:?} request for the topic {topic:?} with {error}{}", message.as_deref().map(|message| format!(": {message}")).unwrap_or_default() ))] Kafka { - api_key: i16, + api_key: ApiKey, topic: String, error: ResponseError, message: Option, @@ -207,9 +207,9 @@ impl KafkaAdminClient { ) .await?; - let result = first(response.results, CreatePartitionsRequest::KEY, topic)?; + let result = first(response.results, ApiKey::CreatePartitions, topic)?; check( - CreatePartitionsRequest::KEY, + ApiKey::CreatePartitions, topic, result.error_code, result.error_message, @@ -231,13 +231,9 @@ impl KafkaAdminClient { ) .await?; - let result = first( - response.responses, - IncrementalAlterConfigsRequest::KEY, - topic, - )?; + let result = first(response.responses, ApiKey::IncrementalAlterConfigs, topic)?; check( - IncrementalAlterConfigsRequest::KEY, + ApiKey::IncrementalAlterConfigs, topic, result.error_code, result.error_message, @@ -310,11 +306,11 @@ fn describe_topic_request(topic: &str) -> MetadataRequest { } fn describe_topic_result(response: MetadataResponse, topic: &str) -> Result> { - let result = first(response.topics, MetadataRequest::KEY, topic)?; + let result = first(response.topics, ApiKey::Metadata, topic)?; if result.error_code.err() == Some(ResponseError::UnknownTopicOrPartition) { return Ok(None); } - check(MetadataRequest::KEY, topic, result.error_code, None)?; + check(ApiKey::Metadata, topic, result.error_code, None)?; let partitions = result .partitions @@ -347,9 +343,9 @@ fn describe_topic_configs_result( response: DescribeConfigsResponse, topic: &str, ) -> Result> { - let result = first(response.results, DescribeConfigsRequest::KEY, topic)?; + let result = first(response.results, ApiKey::DescribeConfigs, topic)?; check( - DescribeConfigsRequest::KEY, + ApiKey::DescribeConfigs, topic, result.error_code, result.error_message, @@ -397,16 +393,16 @@ fn create_topic_request( } fn create_topic_result(response: CreateTopicsResponse, topic: &str) -> Result { - let result = first(response.topics, CreateTopicsRequest::KEY, topic)?; + let result = first(response.topics, ApiKey::CreateTopics, topic)?; check( - CreateTopicsRequest::KEY, + ApiKey::CreateTopics, topic, result.error_code, result.error_message, )?; // Without the details, e.g. if `DESCRIBE_CONFIGS` is not allowed, the topic was still created. check( - CreateTopicsRequest::KEY, + ApiKey::CreateTopics, topic, result.topic_config_error_code, None, @@ -471,7 +467,7 @@ fn delete_topic_request(topic: &str) -> DeleteTopicsRequest { } fn delete_topic_result(response: DeleteTopicsResponse, topic: &str) -> Result<()> { - let result = first(response.responses, DeleteTopicsRequest::KEY, topic)?; + let result = first(response.responses, ApiKey::DeleteTopics, topic)?; // It's ok if topic or partition are already being deleted if matches!( result.error_code.err(), @@ -480,7 +476,7 @@ fn delete_topic_result(response: DeleteTopicsResponse, topic: &str) -> Result<() return Ok(()); } check( - DeleteTopicsRequest::KEY, + ApiKey::DeleteTopics, topic, result.error_code, result.error_message, @@ -507,7 +503,7 @@ fn list_partition_reassignments_result( topic: &str, ) -> Result> { check( - ListPartitionReassignmentsRequest::KEY, + ApiKey::ListPartitionReassignments, topic, response.error_code, response.error_message, @@ -530,14 +526,14 @@ fn topic_name(topic: &str) -> TopicName { } // Used instead of [0] to correctly handle empty results & take ownership -fn first(results: Vec, api_key: i16, topic: &str) -> Result { +fn first(results: Vec, api_key: ApiKey, topic: &str) -> Result { results .into_iter() .next() .context(MissingResultSnafu { api_key, topic }) } -fn check(api_key: i16, topic: &str, error_code: i16, message: Option) -> Result<()> { +fn check(api_key: ApiKey, topic: &str, error_code: i16, message: Option) -> Result<()> { match error_code.err() { None => Ok(()), Some(error) => KafkaSnafu { @@ -928,7 +924,7 @@ mod tests { #[test] fn report_rejected_request() { let error = check( - CreateTopicsRequest::KEY, + ApiKey::CreateTopics, TOPIC, ResponseError::TopicAlreadyExists.code(), Some(StrBytes::from_static_str("Topic 'orders' already exists.")), @@ -950,12 +946,18 @@ mod tests { fn reject_response_without_result() { let error = first( Vec::::new(), - DeleteTopicsRequest::KEY, + ApiKey::DeleteTopics, TOPIC, ) .expect_err("the result should be missing"); - assert!(matches!(error, Error::MissingResult { .. })); + assert!(matches!( + error, + Error::MissingResult { + api_key: ApiKey::DeleteTopics, + .. + } + )); } fn kafka_admin_client(ports: &[u16]) -> KafkaAdminClient { diff --git a/rust/kafka-admin/src/codec.rs b/rust/kafka-admin/src/codec.rs index 0c69347df..fe19943fd 100644 --- a/rust/kafka-admin/src/codec.rs +++ b/rust/kafka-admin/src/codec.rs @@ -3,23 +3,33 @@ use bytes::{Bytes, BytesMut}; use kafka_protocol::{ - messages::{RequestHeader, ResponseHeader}, + messages::{ApiKey, RequestHeader, ResponseHeader}, protocol::{Decodable, Encodable, HeaderVersion, Request, StrBytes}, }; use snafu::{ResultExt, Snafu, ensure}; #[derive(Debug, Snafu)] pub enum Error { - #[snafu(display("failed to encode the request with API key {api_key}"))] - EncodeRequest { source: anyhow::Error, api_key: i16 }, - - #[snafu(display("failed to decode the response with API key {api_key}"))] - DecodeResponse { source: anyhow::Error, api_key: i16 }, + #[snafu(display("failed to encode the {api_key:?} request"))] + EncodeRequest { + source: anyhow::Error, + api_key: ApiKey, + }, + + #[snafu(display("failed to decode the {api_key:?} response"))] + DecodeResponse { + source: anyhow::Error, + api_key: ApiKey, + }, #[snafu(display("expected correlation id {expected}, received {received}"))] CorrelationIdMismatch { expected: i32, received: i32 }, } +pub fn api_key() -> ApiKey { + ApiKey::try_from(R::KEY).expect("kafka-protocol generates the requests and their API keys") +} + pub fn encode_request( request: &R, version: i16, @@ -35,10 +45,14 @@ pub fn encode_request( let mut frame = BytesMut::new(); header .encode(&mut frame, R::header_version(version)) - .context(EncodeRequestSnafu { api_key: R::KEY })?; + .context(EncodeRequestSnafu { + api_key: api_key::(), + })?; request .encode(&mut frame, version) - .context(EncodeRequestSnafu { api_key: R::KEY })?; + .context(EncodeRequestSnafu { + api_key: api_key::(), + })?; Ok(frame.freeze()) } @@ -47,8 +61,11 @@ pub fn decode_response( version: i16, correlation_id: i32, ) -> Result { - let header = ResponseHeader::decode(&mut frame, R::Response::header_version(version)) - .context(DecodeResponseSnafu { api_key: R::KEY })?; + let header = ResponseHeader::decode(&mut frame, R::Response::header_version(version)).context( + DecodeResponseSnafu { + api_key: api_key::(), + }, + )?; ensure!( header.correlation_id == correlation_id, CorrelationIdMismatchSnafu { @@ -56,7 +73,9 @@ pub fn decode_response( received: header.correlation_id, } ); - R::Response::decode(&mut frame, version).context(DecodeResponseSnafu { api_key: R::KEY }) + R::Response::decode(&mut frame, version).context(DecodeResponseSnafu { + api_key: api_key::(), + }) } #[cfg(test)] @@ -75,7 +94,7 @@ mod tests { let header = decode_request_header_from_buffer(&mut frame).expect("the header should be decodable"); - assert_eq!(3, header.request_api_key); + assert_eq!(ApiKey::Metadata as i16, header.request_api_key); assert_eq!(12, header.request_api_version); assert_eq!(7, header.correlation_id); assert_eq!(Some("agent"), header.client_id.as_deref()); diff --git a/rust/kafka-admin/src/connection.rs b/rust/kafka-admin/src/connection.rs index 85274ca8e..efb6a606f 100644 --- a/rust/kafka-admin/src/connection.rs +++ b/rust/kafka-admin/src/connection.rs @@ -6,7 +6,7 @@ use futures::{SinkExt, StreamExt}; use kafka_protocol::{ ResponseError, error::ParseResponseErrorCode, - messages::{ApiVersionsRequest, ApiVersionsResponse}, + messages::{ApiKey, ApiVersionsRequest, ApiVersionsResponse}, protocol::{Request, StrBytes, VersionRange}, }; use rustls::{ClientConfig, pki_types::ServerName}; @@ -23,7 +23,7 @@ use tokio_util::{ use crate::{ bootstrap::BrokerAddress, - codec::{self, decode_response, encode_request}, + codec::{self, api_key, decode_response, encode_request}, tls, }; @@ -76,23 +76,26 @@ pub enum Error { #[snafu(display("the broker closed the connection"))] ConnectionClosed, - #[snafu(display("timed out waiting for the response to the request with API key {api_key}"))] - RequestTimeout { source: Elapsed, api_key: i16 }, + #[snafu(display("timed out waiting for the response to the {api_key:?} request"))] + RequestTimeout { source: Elapsed, api_key: ApiKey }, #[snafu(display("failed to encode or decode a message"))] Codec { source: codec::Error }, - #[snafu(display("the broker returned {error:?} for the request with API key {api_key}"))] - Kafka { error: ResponseError, api_key: i16 }, + #[snafu(display("the broker returned {error:?} for the {api_key:?} request"))] + Kafka { + error: ResponseError, + api_key: ApiKey, + }, - #[snafu(display("the broker does not support the API key {api_key}"))] - UnsupportedApi { api_key: i16 }, + #[snafu(display("the broker does not support the {api_key:?} API"))] + UnsupportedApi { api_key: ApiKey }, #[snafu(display( - "the broker supports the versions {broker_versions} of the API key {api_key}, but not {version}" + "the broker supports the versions {broker_versions} of the {api_key:?} API, but not {version}" ))] UnsupportedVersion { - api_key: i16, + api_key: ApiKey, version: i16, broker_versions: VersionRange, }, @@ -165,7 +168,7 @@ impl Connection { // Each API that is relevant for us has a version that is supported by all of our Kafka versions. // We use these versions and pin them in client.rs. // We will have to add API version negotiation at some point in the future. - check_version(&self.broker_api_versions, R::KEY, version)?; + check_version(&self.broker_api_versions, api_key::(), version)?; self.send_with_version(request, version).await } @@ -187,7 +190,9 @@ impl Connection { .context(ReadSnafu) }) .await - .context(RequestTimeoutSnafu { api_key: R::KEY })??; + .context(RequestTimeoutSnafu { + api_key: api_key::(), + })??; decode_response::(response.freeze(), version, self.correlation_id).context(CodecSnafu) } @@ -217,7 +222,7 @@ fn api_versions_result(response: ApiVersionsResponse) -> Result Result, - api_key: i16, + api_key: ApiKey, version: i16, ) -> Result<()> { let broker_versions = broker_api_versions - .get(&api_key) + .get(&(api_key as i16)) .context(UnsupportedApiSnafu { api_key })?; ensure!( (broker_versions.min..=broker_versions.max).contains(&version), @@ -445,16 +450,21 @@ mod tests { let broker_api_versions = BTreeMap::from([(MetadataRequest::KEY, VersionRange { min: 0, max: 13 })]); - check_version(&broker_api_versions, MetadataRequest::KEY, 12) + check_version(&broker_api_versions, ApiKey::Metadata, 12) .expect("the version should be supported"); } #[test] fn reject_api_unknown_to_the_broker() { - let error = check_version(&BTreeMap::new(), MetadataRequest::KEY, 12) + let error = check_version(&BTreeMap::new(), ApiKey::Metadata, 12) .expect_err("the API should be unsupported"); - assert!(matches!(error, Error::UnsupportedApi { .. })); + assert!(matches!( + error, + Error::UnsupportedApi { + api_key: ApiKey::Metadata + } + )); } #[test] @@ -462,7 +472,7 @@ mod tests { let broker_api_versions = BTreeMap::from([(MetadataRequest::KEY, VersionRange { min: 0, max: 9 })]); - let error = check_version(&broker_api_versions, MetadataRequest::KEY, 12) + let error = check_version(&broker_api_versions, ApiKey::Metadata, 12) .expect_err("the version should be unsupported"); assert!(matches!( From 41f8b3708848d01b0d00b2d2655df914169583db Mon Sep 17 00:00:00 2001 From: Benedikt Labrenz Date: Wed, 28 Oct 2026 10:15:35 +0100 Subject: [PATCH 51/93] change controller and CRD to use status.config without source --- extra/crds.yaml | 14 +-- rust/agent-binary/src/topic/condition.rs | 2 +- rust/agent-binary/src/topic/plan.rs | 95 +++++++------------ rust/agent-binary/src/topic/sync.rs | 15 ++- rust/agent-binary/src/topic/update_status.rs | 32 +------ rust/shared/src/crd/topic.rs | 15 +-- .../00-patch-ns.yaml.j2 | 0 .../05-create-agent-credential.yaml.j2 | 0 .../10-assert.yaml | 0 .../10-install-kafka.yaml | 0 .../10_kafka.yaml.j2 | 0 .../11-assert.yaml.j2 | 0 .../20-assert.yaml | 0 .../20-create-topic.yaml | 0 .../kafka-cli.sh | 0 tests/templates/kuttl/topic/20-assert.yaml | 8 +- tests/templates/kuttl/topic/30-assert.yaml | 4 +- tests/templates/kuttl/topic/41-assert.yaml | 16 ++-- tests/templates/kuttl/topic/71-assert.yaml | 4 +- tests/templates/kuttl/topic/75-assert.yaml | 4 +- tests/templates/kuttl/topic/76-assert.yaml | 4 +- tests/templates/kuttl/topic/77-assert.yaml | 12 ++- tests/templates/kuttl/topic/80-assert.yaml | 4 +- tests/test-definition.yaml | 18 ++-- 24 files changed, 85 insertions(+), 162 deletions(-) rename tests/templates/kuttl/{topic-authentication => management-access}/00-patch-ns.yaml.j2 (100%) rename tests/templates/kuttl/{topic-authentication => management-access}/05-create-agent-credential.yaml.j2 (100%) rename tests/templates/kuttl/{topic-authentication => management-access}/10-assert.yaml (100%) rename tests/templates/kuttl/{topic-authentication => management-access}/10-install-kafka.yaml (100%) rename tests/templates/kuttl/{topic-authentication => management-access}/10_kafka.yaml.j2 (100%) rename tests/templates/kuttl/{topic-authentication => management-access}/11-assert.yaml.j2 (100%) rename tests/templates/kuttl/{topic-authentication => management-access}/20-assert.yaml (100%) rename tests/templates/kuttl/{topic-authentication => management-access}/20-create-topic.yaml (100%) rename tests/templates/kuttl/{topic-authentication => management-access}/kafka-cli.sh (100%) diff --git a/extra/crds.yaml b/extra/crds.yaml index 92407addb..8e6f437c8 100644 --- a/extra/crds.yaml +++ b/extra/crds.yaml @@ -2383,19 +2383,9 @@ spec: type: array config: additionalProperties: - properties: - source: - description: Where the value comes from, like `DYNAMIC_TOPIC_CONFIG` or `DEFAULT_CONFIG`. - type: string - value: - description: The value, not set for sensitive configs. - nullable: true - type: string - required: - - source - type: object + type: string default: {} - description: All effective topic configs in Kafka. + description: The configs set on the topic in Kafka, without broker and default values. type: object partitions: description: The number of partitions in Kafka. diff --git a/rust/agent-binary/src/topic/condition.rs b/rust/agent-binary/src/topic/condition.rs index 46ed4f803..ccdba66be 100644 --- a/rust/agent-binary/src/topic/condition.rs +++ b/rust/agent-binary/src/topic/condition.rs @@ -213,7 +213,7 @@ mod tests { id: "UXyU9i5ARn6W00ON2taeWA".to_owned(), partition_count, replication_factor: Some(1), - effective_configs: BTreeMap::new(), + topic_configs: BTreeMap::new(), } } diff --git a/rust/agent-binary/src/topic/plan.rs b/rust/agent-binary/src/topic/plan.rs index a52b21ec6..ca70fca83 100644 --- a/rust/agent-binary/src/topic/plan.rs +++ b/rust/agent-binary/src/topic/plan.rs @@ -2,9 +2,7 @@ use std::collections::{BTreeMap, BTreeSet}; -use stackable_kafka_admin::client::{ - ConfigSource, CreatedTopic, PartitionReassignment, Topic, TopicConfig, -}; +use stackable_kafka_admin::client::{CreatedTopic, PartitionReassignment, Topic}; use stackable_kafka_shared::crd::topic::{TopicName, v1alpha1::KafkaTopicSpec}; /// Set by `kafka-reassign-partitions` and Cruise Control for the duration of a reassignment. @@ -17,7 +15,7 @@ pub const THROTTLE_CONFIG_KEYS: [&str; 2] = [ /// The state of an existing topic in Kafka. pub struct ObservedTopic { pub topic: Topic, - pub effective_configs: BTreeMap, + pub topic_configs: BTreeMap, pub reassignments: Vec, } @@ -28,7 +26,7 @@ pub struct TopicState { pub id: String, pub partition_count: i32, pub replication_factor: Option, - pub effective_configs: BTreeMap, + pub topic_configs: BTreeMap, } impl TopicState { @@ -38,7 +36,7 @@ impl TopicState { id: observed.topic.id.clone(), partition_count: observed.topic.partitions.len() as i32, replication_factor: replication_factor(observed), - effective_configs: observed.effective_configs.clone(), + topic_configs: observed.topic_configs.clone(), } } @@ -48,7 +46,7 @@ impl TopicState { id: created.id, partition_count: created.partitions, replication_factor: Some(created.replication_factor), - effective_configs: created.configs, + topic_configs: created.configs, } } } @@ -107,7 +105,7 @@ pub fn plan(spec: &KafkaTopicSpec, observed: Option<&ObservedTopic>) -> TopicPla let actual_partition_count = observed.topic.partitions.len() as i32; let (configs_to_set, configs_to_reset) = - config_changes(&spec.config_overrides, &observed.effective_configs); + config_changes(&spec.config_overrides, &observed.topic_configs); TopicPlan::Update { topic_update: TopicUpdate { @@ -153,29 +151,27 @@ fn unsupported_changes(spec: &KafkaTopicSpec, observed: &ObservedTopic) -> Vec, - effective_configs: &BTreeMap, + desired_topic_configs: &BTreeMap, + current_topic_configs: &BTreeMap, ) -> (BTreeMap, BTreeSet) { - // Topic configs have no PASSWORD type, so Kafka does not redact their topic-level values. - let topic_level_configs = effective_configs - .iter() - .filter(|(_, config)| config.source == ConfigSource::DynamicTopic) - .map(|(key, config)| (key, config.value.as_deref())) - .collect::>(); - - // Compared with the topic-level value, not the effective one: a desired value that equals the - // broker config is still pinned on the topic. Otherwise a later change of the broker config - // would apply to the topic until the next resync. Unchanged values are skipped, so a resync + // A desired value that equals the broker config is still set on the topic, so a later change of + // the broker config does not apply to the topic. Unchanged values are skipped, so a resync // without drift does not write to Kafka. - let configs_to_set = desired + let configs_to_set = desired_topic_configs .iter() - .filter(|(key, value)| topic_level_configs.get(key) != Some(&Some(value.as_str()))) + .filter(|(key, value)| { + current_topic_configs.get(*key) != Some(value) + && !THROTTLE_CONFIG_KEYS.contains(&key.as_str()) + }) .map(|(key, value)| (key.clone(), value.clone())) .collect(); - let configs_to_reset = topic_level_configs + let configs_to_reset = current_topic_configs .keys() - .filter(|key| !desired.contains_key(**key) && !THROTTLE_CONFIG_KEYS.contains(&key.as_str())) - .map(|key| key.to_string()) + .filter(|key| { + !desired_topic_configs.contains_key(*key) + && !THROTTLE_CONFIG_KEYS.contains(&key.as_str()) + }) + .cloned() .collect(); (configs_to_set, configs_to_reset) } @@ -243,21 +239,6 @@ mod tests { keys.iter().map(|key| key.to_string()).collect() } - fn topic_configs(entries: &[(&str, &str, ConfigSource)]) -> BTreeMap { - entries - .iter() - .map(|(key, value, source)| { - ( - key.to_string(), - TopicConfig { - value: Some(value.to_string()), - source: *source, - }, - ) - }) - .collect() - } - fn update(topic_update: TopicUpdate, unsupported_changes: Vec) -> TopicPlan { TopicPlan::Update { topic_update, @@ -265,8 +246,9 @@ mod tests { } } - /// A topic whose partitions have the given number of replicas - fn observed(replicas: &[usize], configs: &[(&str, &str, ConfigSource)]) -> ObservedTopic { + /// A topic whose partitions have the given number of replicas, with the given topic-level + /// configs + fn observed(replicas: &[usize], topic_configs: &[(&str, &str)]) -> ObservedTopic { ObservedTopic { topic: Topic { id: "UXyU9i5ARn6W00ON2taeWA".to_owned(), @@ -279,7 +261,7 @@ mod tests { }) .collect(), }, - effective_configs: topic_configs(configs), + topic_configs: configs(topic_configs), reassignments: vec![], } } @@ -300,10 +282,7 @@ mod tests { #[test] fn do_nothing_for_a_matching_topic() { - let observed = observed( - &[2, 2], - &[("retention.ms", "1000", ConfigSource::DynamicTopic)], - ); + let observed = observed(&[2, 2], &[("retention.ms", "1000")]); let spec = spec(Some(2), Some(2), &[("retention.ms", "1000")]); assert_eq!( @@ -364,15 +343,9 @@ mod tests { let observed = observed( &[1], &[ - ("retention.ms", "1000", ConfigSource::DynamicTopic), - ("cleanup.policy", "compact", ConfigSource::DynamicTopic), - ("segment.bytes", "1024", ConfigSource::Default), - ("max.message.bytes", "10", ConfigSource::StaticBroker), - ( - "leader.replication.throttled.replicas", - "0:1", - ConfigSource::DynamicTopic, - ), + ("retention.ms", "1000"), + ("cleanup.policy", "compact"), + ("leader.replication.throttled.replicas", "0:1"), ], ); @@ -380,7 +353,11 @@ mod tests { &spec( None, None, - &[("retention.ms", "2000"), ("max.message.bytes", "10")], + &[ + ("retention.ms", "2000"), + ("max.message.bytes", "10"), + ("follower.replication.throttled.replicas", "*"), + ], ), Some(&observed), ); @@ -388,13 +365,11 @@ mod tests { assert_eq!( update( TopicUpdate { - // `max.message.bytes` already has the value, but not on the topic configs_to_set: configs(&[ ("max.message.bytes", "10"), ("retention.ms", "2000") ]), - // `segment.bytes` is not set on the topic, the throttle belongs to a - // reassignment + // The throttles belong to a reassignment configs_to_reset: keys(&["cleanup.policy"]), ..TopicUpdate::default() }, diff --git a/rust/agent-binary/src/topic/sync.rs b/rust/agent-binary/src/topic/sync.rs index fd34efb0d..97f980288 100644 --- a/rust/agent-binary/src/topic/sync.rs +++ b/rust/agent-binary/src/topic/sync.rs @@ -3,7 +3,7 @@ use std::collections::{BTreeMap, BTreeSet}; use snafu::ResultExt; -use stackable_kafka_admin::client::{KafkaAdminClient, TopicConfig}; +use stackable_kafka_admin::client::KafkaAdminClient; use stackable_kafka_shared::crd::topic::{TopicName, v1alpha1::KafkaTopic}; use stackable_operator::kube::ResourceExt; use tracing::info; @@ -86,7 +86,7 @@ async fn observe( .map(|partition| partition.index) .collect::>(); Ok(Some(ObservedTopic { - effective_configs: admin_client + topic_configs: admin_client .describe_topic_configs(topic_name) .await .context(KafkaSnafu)?, @@ -136,7 +136,7 @@ async fn execute( // Config if !topic_update.configs_to_set.is_empty() || !topic_update.configs_to_reset.is_empty() { - let effective_configs = replace_configs( + let topic_configs = replace_configs( kafka_topic, admin_client, &topic_name, @@ -145,7 +145,7 @@ async fn execute( ) .await?; if let Some(topic_state) = topic_state { - topic_state.effective_configs = effective_configs; + topic_state.topic_configs = topic_configs; } } } @@ -204,16 +204,15 @@ async fn grow_partitions( Ok(()) } -/// Returns the effective configs of the topic after the change. They are described again, -/// because IncrementalAlterConfigs does not return them and a reset key falls back to a broker -/// config the agent does not know. +/// Returns the topic-level configs after the change, described again because +/// IncrementalAlterConfigs does not return them. async fn replace_configs( kafka_topic: &KafkaTopic, admin_client: &mut KafkaAdminClient, topic_name: &TopicName, configs_to_set: &BTreeMap, configs_to_reset: &BTreeSet, -) -> Result> { +) -> Result> { admin_client .incremental_alter_configs(topic_name.as_ref(), configs_to_set, configs_to_reset) .await diff --git a/rust/agent-binary/src/topic/update_status.rs b/rust/agent-binary/src/topic/update_status.rs index bff07684b..b7fdcfae6 100644 --- a/rust/agent-binary/src/topic/update_status.rs +++ b/rust/agent-binary/src/topic/update_status.rs @@ -1,9 +1,7 @@ //! The status of a `KafkaTopic` for the outcome of a sync. use snafu::ResultExt; -use stackable_kafka_shared::crd::topic::v1alpha1::{ - KafkaTopic, KafkaTopicConfigStatus, KafkaTopicStatus, -}; +use stackable_kafka_shared::crd::topic::v1alpha1::{KafkaTopic, KafkaTopicStatus}; use stackable_operator::{ agent::condition::update_conditions, k8s_openapi::{apimachinery::pkg::apis::meta::v1::Time, jiff::Timestamp}, @@ -62,17 +60,7 @@ fn status_from( topic_name: Some(topic_state.name.clone()), partitions: Some(topic_state.partition_count), replication_factor: topic_state.replication_factor, - config: topic_state - .effective_configs - .iter() - .map(|(key, config)| { - let config = KafkaTopicConfigStatus { - value: config.value.clone(), - source: config.source.to_string(), - }; - (key.clone(), config) - }) - .collect(), + config: topic_state.topic_configs.clone(), }, // The topic does not exist or is managed by another KafkaTopic. (TopicSyncOutcome::Conflict { .. } | TopicSyncOutcome::ReconciliationPaused, None) => { @@ -92,7 +80,6 @@ fn status_from( mod tests { use std::{collections::BTreeMap, str::FromStr}; - use stackable_kafka_admin::client::{ConfigSource, TopicConfig}; use stackable_kafka_shared::crd::topic::TopicName; use super::*; @@ -107,13 +94,7 @@ mod tests { id: "UXyU9i5ARn6W00ON2taeWA".to_owned(), partition_count: 6, replication_factor: Some(3), - effective_configs: BTreeMap::from([( - "retention.ms".to_owned(), - TopicConfig { - value: Some("1000".to_owned()), - source: ConfigSource::DynamicTopic, - }, - )]), + topic_configs: BTreeMap::from([("retention.ms".to_owned(), "1000".to_owned())]), } } @@ -137,11 +118,8 @@ mod tests { assert_eq!(Some(6), status.partitions); assert_eq!(Some(3), status.replication_factor); assert_eq!( - Some(&KafkaTopicConfigStatus { - value: Some("1000".to_owned()), - source: "DYNAMIC_TOPIC_CONFIG".to_owned(), - }), - status.config.get("retention.ms") + Some("1000"), + status.config.get("retention.ms").map(String::as_str) ); assert!( status diff --git a/rust/shared/src/crd/topic.rs b/rust/shared/src/crd/topic.rs index 4bf75071d..d2816c979 100644 --- a/rust/shared/src/crd/topic.rs +++ b/rust/shared/src/crd/topic.rs @@ -126,20 +126,9 @@ pub mod versioned { #[serde(default, skip_serializing_if = "Option::is_none")] pub replication_factor: Option, - /// All effective topic configs in Kafka. + /// The configs set on the topic in Kafka, without broker and default values. #[serde(default)] - pub config: BTreeMap, - } - - #[derive(Clone, Debug, Deserialize, JsonSchema, PartialEq, Serialize)] - #[serde(rename_all = "camelCase")] - pub struct KafkaTopicConfigStatus { - /// The value, not set for sensitive configs. - #[serde(default, skip_serializing_if = "Option::is_none")] - pub value: Option, - - /// Where the value comes from, like `DYNAMIC_TOPIC_CONFIG` or `DEFAULT_CONFIG`. - pub source: String, + pub config: BTreeMap, } } diff --git a/tests/templates/kuttl/topic-authentication/00-patch-ns.yaml.j2 b/tests/templates/kuttl/management-access/00-patch-ns.yaml.j2 similarity index 100% rename from tests/templates/kuttl/topic-authentication/00-patch-ns.yaml.j2 rename to tests/templates/kuttl/management-access/00-patch-ns.yaml.j2 diff --git a/tests/templates/kuttl/topic-authentication/05-create-agent-credential.yaml.j2 b/tests/templates/kuttl/management-access/05-create-agent-credential.yaml.j2 similarity index 100% rename from tests/templates/kuttl/topic-authentication/05-create-agent-credential.yaml.j2 rename to tests/templates/kuttl/management-access/05-create-agent-credential.yaml.j2 diff --git a/tests/templates/kuttl/topic-authentication/10-assert.yaml b/tests/templates/kuttl/management-access/10-assert.yaml similarity index 100% rename from tests/templates/kuttl/topic-authentication/10-assert.yaml rename to tests/templates/kuttl/management-access/10-assert.yaml diff --git a/tests/templates/kuttl/topic-authentication/10-install-kafka.yaml b/tests/templates/kuttl/management-access/10-install-kafka.yaml similarity index 100% rename from tests/templates/kuttl/topic-authentication/10-install-kafka.yaml rename to tests/templates/kuttl/management-access/10-install-kafka.yaml diff --git a/tests/templates/kuttl/topic-authentication/10_kafka.yaml.j2 b/tests/templates/kuttl/management-access/10_kafka.yaml.j2 similarity index 100% rename from tests/templates/kuttl/topic-authentication/10_kafka.yaml.j2 rename to tests/templates/kuttl/management-access/10_kafka.yaml.j2 diff --git a/tests/templates/kuttl/topic-authentication/11-assert.yaml.j2 b/tests/templates/kuttl/management-access/11-assert.yaml.j2 similarity index 100% rename from tests/templates/kuttl/topic-authentication/11-assert.yaml.j2 rename to tests/templates/kuttl/management-access/11-assert.yaml.j2 diff --git a/tests/templates/kuttl/topic-authentication/20-assert.yaml b/tests/templates/kuttl/management-access/20-assert.yaml similarity index 100% rename from tests/templates/kuttl/topic-authentication/20-assert.yaml rename to tests/templates/kuttl/management-access/20-assert.yaml diff --git a/tests/templates/kuttl/topic-authentication/20-create-topic.yaml b/tests/templates/kuttl/management-access/20-create-topic.yaml similarity index 100% rename from tests/templates/kuttl/topic-authentication/20-create-topic.yaml rename to tests/templates/kuttl/management-access/20-create-topic.yaml diff --git a/tests/templates/kuttl/topic-authentication/kafka-cli.sh b/tests/templates/kuttl/management-access/kafka-cli.sh similarity index 100% rename from tests/templates/kuttl/topic-authentication/kafka-cli.sh rename to tests/templates/kuttl/management-access/kafka-cli.sh diff --git a/tests/templates/kuttl/topic/20-assert.yaml b/tests/templates/kuttl/topic/20-assert.yaml index d358950b7..7290dfc03 100644 --- a/tests/templates/kuttl/topic/20-assert.yaml +++ b/tests/templates/kuttl/topic/20-assert.yaml @@ -28,9 +28,5 @@ status: partitions: 3 replicationFactor: 1 config: - retention.ms: - value: "3600000" - source: DYNAMIC_TOPIC_CONFIG - max.message.bytes: - value: "2097152" - source: DYNAMIC_TOPIC_CONFIG + retention.ms: "3600000" + max.message.bytes: "2097152" diff --git a/tests/templates/kuttl/topic/30-assert.yaml b/tests/templates/kuttl/topic/30-assert.yaml index 0f98e197c..7d407115b 100644 --- a/tests/templates/kuttl/topic/30-assert.yaml +++ b/tests/templates/kuttl/topic/30-assert.yaml @@ -18,6 +18,4 @@ status: reason: Synced partitions: 5 config: - retention.ms: - value: "7200000" - source: DYNAMIC_TOPIC_CONFIG + retention.ms: "7200000" diff --git a/tests/templates/kuttl/topic/41-assert.yaml b/tests/templates/kuttl/topic/41-assert.yaml index 2a5e4b777..5544c145f 100644 --- a/tests/templates/kuttl/topic/41-assert.yaml +++ b/tests/templates/kuttl/topic/41-assert.yaml @@ -4,6 +4,14 @@ apiVersion: kuttl.dev/v1beta1 kind: TestAssert timeout: 300 +resourceRefs: + - apiVersion: kafka.stackable.tech/v1alpha1 + kind: KafkaTopic + name: orders + ref: orders +assertAll: + - celExpr: "!('max.message.bytes' in orders.status.config)" + - celExpr: "!('segment.ms' in orders.status.config)" --- apiVersion: kafka.stackable.tech/v1alpha1 kind: KafkaTopic @@ -18,10 +26,4 @@ status: status: "True" reason: Synced config: - retention.ms: - value: "7200000" - source: DYNAMIC_TOPIC_CONFIG - max.message.bytes: - source: DEFAULT_CONFIG - segment.ms: - source: DEFAULT_CONFIG + retention.ms: "7200000" diff --git a/tests/templates/kuttl/topic/71-assert.yaml b/tests/templates/kuttl/topic/71-assert.yaml index 2a3198895..6da767ba8 100644 --- a/tests/templates/kuttl/topic/71-assert.yaml +++ b/tests/templates/kuttl/topic/71-assert.yaml @@ -11,6 +11,7 @@ resourceRefs: ref: legacy assertAll: - celExpr: legacy.status.topicId != '' + - celExpr: "!has(legacy.status.config) || !('cleanup.policy' in legacy.status.config)" --- apiVersion: kafka.stackable.tech/v1alpha1 kind: KafkaTopic @@ -26,6 +27,3 @@ status: - type: Synced status: "True" reason: Synced - config: - cleanup.policy: - source: DEFAULT_CONFIG diff --git a/tests/templates/kuttl/topic/75-assert.yaml b/tests/templates/kuttl/topic/75-assert.yaml index 04bb2d1fc..6a856dcf0 100644 --- a/tests/templates/kuttl/topic/75-assert.yaml +++ b/tests/templates/kuttl/topic/75-assert.yaml @@ -17,6 +17,4 @@ status: status: "False" reason: ReconciliationPaused config: - retention.ms: - value: "7200000" - source: DYNAMIC_TOPIC_CONFIG + retention.ms: "7200000" diff --git a/tests/templates/kuttl/topic/76-assert.yaml b/tests/templates/kuttl/topic/76-assert.yaml index 08177dc20..b82d73e2d 100644 --- a/tests/templates/kuttl/topic/76-assert.yaml +++ b/tests/templates/kuttl/topic/76-assert.yaml @@ -17,6 +17,4 @@ status: status: "True" reason: Synced config: - retention.ms: - value: "10800000" - source: DYNAMIC_TOPIC_CONFIG + retention.ms: "10800000" diff --git a/tests/templates/kuttl/topic/77-assert.yaml b/tests/templates/kuttl/topic/77-assert.yaml index 26c91f18b..e60dcb2cd 100644 --- a/tests/templates/kuttl/topic/77-assert.yaml +++ b/tests/templates/kuttl/topic/77-assert.yaml @@ -1,8 +1,15 @@ --- -# The topic stays available, and the status keeps the config observed in Kafka. +# The topic stays available, and the rejected value is not set on the topic. apiVersion: kuttl.dev/v1beta1 kind: TestAssert timeout: 300 +resourceRefs: + - apiVersion: kafka.stackable.tech/v1alpha1 + kind: KafkaTopic + name: defaults + ref: defaults +assertAll: + - celExpr: "!has(defaults.status.config) || !('retention.ms' in defaults.status.config)" --- apiVersion: kafka.stackable.tech/v1alpha1 kind: KafkaTopic @@ -16,6 +23,3 @@ status: - type: Synced status: "False" reason: KafkaRejected - config: - retention.ms: - source: DEFAULT_CONFIG diff --git a/tests/templates/kuttl/topic/80-assert.yaml b/tests/templates/kuttl/topic/80-assert.yaml index 20b4780b3..c22dd5c05 100644 --- a/tests/templates/kuttl/topic/80-assert.yaml +++ b/tests/templates/kuttl/topic/80-assert.yaml @@ -38,6 +38,4 @@ status: status: "True" reason: Synced config: - retention.ms: - value: "10800000" - source: DYNAMIC_TOPIC_CONFIG + retention.ms: "10800000" diff --git a/tests/test-definition.yaml b/tests/test-definition.yaml index 85ac91363..c3873d452 100644 --- a/tests/test-definition.yaml +++ b/tests/test-definition.yaml @@ -87,15 +87,6 @@ tests: dimensions: - kafka-kraft - openshift - - name: topic - dimensions: - - kafka-kraft - - openshift - - name: topic-authentication - dimensions: - - kafka-kraft - - management-access-authentication - - openshift - name: smoke dimensions: - kafka @@ -162,6 +153,15 @@ tests: - openshift - broker-listener-class - bootstrap-listener-class + - name: topic + dimensions: + - kafka-kraft + - openshift + - name: management-access + dimensions: + - kafka-kraft + - management-access-authentication + - openshift suites: - name: nightly From f1946f7b87188256453459b459bdca7eb91ce0b2 Mon Sep 17 00:00:00 2001 From: Benedikt Labrenz Date: Wed, 28 Oct 2026 11:59:24 +0100 Subject: [PATCH 52/93] add grace period before operator takes over topic status --- rust/operator-binary/src/topic_controller.rs | 190 ++++++++++++++++--- 1 file changed, 162 insertions(+), 28 deletions(-) diff --git a/rust/operator-binary/src/topic_controller.rs b/rust/operator-binary/src/topic_controller.rs index e724e2a95..b7492c94d 100644 --- a/rust/operator-binary/src/topic_controller.rs +++ b/rust/operator-binary/src/topic_controller.rs @@ -13,7 +13,9 @@ use stackable_operator::{ ConditionStatus, ManagedCondition, ManagedConditionType, update_conditions, }, k8s_openapi::{ - api::apps::v1::Deployment, apimachinery::pkg::apis::meta::v1::Time, jiff::Timestamp, + api::apps::v1::Deployment, + apimachinery::pkg::apis::meta::v1::Time, + jiff::{SignedDuration, Timestamp}, }, kube::{ ResourceExt, @@ -39,6 +41,12 @@ pub const TOPIC_FULL_CONTROLLER_NAME: &str = const AGENT_UNAVAILABLE_REASON: &str = "AgentUnavailable"; +/// `DeploymentConditionType` of `apps/v1`, true once the minimum number of replicas is available +const DEPLOYMENT_CONDITION_AVAILABLE: &str = "Available"; + +/// Shorter outages of the agent are not reported, such as a rollout that pulls a new image. +const AGENT_UNAVAILABLE_GRACE_PERIOD: SignedDuration = SignedDuration::from_secs(60); + #[derive(Debug, Snafu, EnumDiscriminants)] #[strum_discriminants(derive(IntoStaticStr))] pub enum Error { @@ -88,9 +96,17 @@ enum ClusterState { ManagementAccessEnabled, } +#[derive(Debug, Eq, PartialEq)] +enum AgentState { + Available, + /// `None` if the agent Deployment does not exist + UnavailableSince(Option), +} + #[derive(Debug, Eq, PartialEq)] enum OperatorAction { LeaveToAgent, + WaitForAgent { remaining: std::time::Duration }, RemoveFinalizer, SetAgentUnavailableConditions { message: &'static str }, } @@ -122,11 +138,11 @@ pub async fn reconcile_topic( match operator_action_for( kafka_topic, &cluster_state_from(kafka_cluster.as_ref()), - agent_deployment - .as_ref() - .is_some_and(is_agent_deployment_available), + &agent_state_from(agent_deployment.as_ref()), + Timestamp::now(), ) { OperatorAction::LeaveToAgent => {} + OperatorAction::WaitForAgent { remaining } => return Ok(Action::requeue(remaining)), OperatorAction::RemoveFinalizer => remove_finalizer(kafka_topic, &ctx).await?, OperatorAction::SetAgentUnavailableConditions { message } => { set_agent_unavailable_conditions(kafka_topic, &ctx, message).await? @@ -188,7 +204,8 @@ fn cluster_state_from(kafka_cluster: Option<&DeserializeGuard>) -> fn operator_action_for( kafka_topic: &KafkaTopic, cluster_state: &ClusterState, - agent_available: bool, + agent_state: &AgentState, + now: Timestamp, ) -> OperatorAction { match cluster_state { ClusterState::MissingOrDeleting if kafka_topic.metadata.deletion_timestamp.is_some() => { @@ -209,23 +226,50 @@ fn operator_action_for( message: "the KafkaCluster has no agent, because clusterConfig.managementAccess is \ not enabled", }, - ClusterState::ManagementAccessEnabled if agent_available => OperatorAction::LeaveToAgent, - ClusterState::ManagementAccessEnabled => OperatorAction::SetAgentUnavailableConditions { - message: "the agent of the KafkaCluster is not available", + ClusterState::ManagementAccessEnabled => match agent_state { + AgentState::Available => OperatorAction::LeaveToAgent, + AgentState::UnavailableSince(Some(since)) + if now.duration_since(*since) < AGENT_UNAVAILABLE_GRACE_PERIOD => + { + OperatorAction::WaitForAgent { + remaining: (AGENT_UNAVAILABLE_GRACE_PERIOD - now.duration_since(*since)) + .unsigned_abs(), + } + } + AgentState::UnavailableSince(_) => OperatorAction::SetAgentUnavailableConditions { + message: "the agent of the KafkaCluster is not available", + }, }, } } -fn is_agent_deployment_available(agent_deployment: &Deployment) -> bool { - agent_deployment +/// A Deployment that has no `Available` condition yet is unavailable since its creation. +fn agent_state_from(agent_deployment: Option<&Deployment>) -> AgentState { + let Some(agent_deployment) = agent_deployment else { + return AgentState::UnavailableSince(None); + }; + let available_condition = agent_deployment .status .as_ref() .and_then(|status| status.conditions.as_ref()) - .is_some_and(|conditions| { + .and_then(|conditions| { conditions .iter() - .any(|condition| condition.type_ == "Available" && condition.status == "True") - }) + .find(|condition| condition.type_ == DEPLOYMENT_CONDITION_AVAILABLE) + }); + match available_condition { + Some(condition) if condition.status == "True" => AgentState::Available, + Some(condition) => { + AgentState::UnavailableSince(condition.last_transition_time.as_ref().map(|time| time.0)) + } + None => AgentState::UnavailableSince( + agent_deployment + .metadata + .creation_timestamp + .as_ref() + .map(|time| time.0), + ), + } } /// Only the conditions are applied, so the other fields stay owned by the agent and keep the @@ -331,43 +375,100 @@ mod tests { } } + fn now() -> Timestamp { + Timestamp::from_second(1_000).expect("the timestamp should be valid") + } + + fn seconds_ago(seconds: i64) -> Timestamp { + now() - SignedDuration::from_secs(seconds) + } + #[rstest] - #[case::cluster_missing(ClusterState::MissingOrDeleting, true, true)] - #[case::cluster_missing_without_agent(ClusterState::MissingOrDeleting, false, true)] - #[case::management_access_disabled(ClusterState::ManagementAccessDisabled, false, false)] - #[case::agent_available(ClusterState::ManagementAccessEnabled, true, false)] - #[case::agent_unavailable(ClusterState::ManagementAccessEnabled, false, false)] + #[case::cluster_missing(ClusterState::MissingOrDeleting, AgentState::Available, true)] + #[case::cluster_missing_without_agent( + ClusterState::MissingOrDeleting, + AgentState::UnavailableSince(None), + true + )] + #[case::management_access_disabled( + ClusterState::ManagementAccessDisabled, + AgentState::UnavailableSince(None), + false + )] + #[case::agent_available(ClusterState::ManagementAccessEnabled, AgentState::Available, false)] + #[case::agent_unavailable( + ClusterState::ManagementAccessEnabled, + AgentState::UnavailableSince(None), + false + )] fn remove_the_finalizer_only_if_the_cluster_is_missing_or_deleting( #[case] cluster_state: ClusterState, - #[case] agent_available: bool, + #[case] agent_state: AgentState, #[case] removes_finalizer: bool, ) { assert_eq!( removes_finalizer, - operator_action_for(&kafka_topic(true, true), &cluster_state, agent_available) - == OperatorAction::RemoveFinalizer + operator_action_for( + &kafka_topic(true, true), + &cluster_state, + &agent_state, + now() + ) == OperatorAction::RemoveFinalizer ); } #[rstest] - #[case::cluster_missing(ClusterState::MissingOrDeleting, true, true)] - #[case::management_access_disabled(ClusterState::ManagementAccessDisabled, true, true)] - #[case::agent_available(ClusterState::ManagementAccessEnabled, true, false)] - #[case::agent_unavailable(ClusterState::ManagementAccessEnabled, false, true)] + #[case::cluster_missing(ClusterState::MissingOrDeleting, AgentState::Available, true)] + #[case::management_access_disabled( + ClusterState::ManagementAccessDisabled, + AgentState::Available, + true + )] + #[case::agent_available(ClusterState::ManagementAccessEnabled, AgentState::Available, false)] + #[case::agent_deployment_missing( + ClusterState::ManagementAccessEnabled, + AgentState::UnavailableSince(None), + true + )] + #[case::agent_unavailable_for_longer_than_the_grace_period( + ClusterState::ManagementAccessEnabled, + AgentState::UnavailableSince(Some(seconds_ago(61))), + true + )] fn set_agent_unavailable_conditions_without_an_available_agent( #[case] cluster_state: ClusterState, - #[case] agent_available: bool, + #[case] agent_state: AgentState, #[case] sets_conditions: bool, ) { assert_eq!( sets_conditions, matches!( - operator_action_for(&kafka_topic(false, false), &cluster_state, agent_available), + operator_action_for( + &kafka_topic(false, false), + &cluster_state, + &agent_state, + now() + ), OperatorAction::SetAgentUnavailableConditions { .. } ) ); } + #[test] + fn wait_for_an_agent_that_is_unavailable_within_the_grace_period() { + assert_eq!( + OperatorAction::WaitForAgent { + remaining: std::time::Duration::from_secs(50) + }, + operator_action_for( + &kafka_topic(false, false), + &ClusterState::ManagementAccessEnabled, + &AgentState::UnavailableSince(Some(seconds_ago(10))), + now() + ) + ); + } + #[test] fn leave_a_deleting_kafka_topic_without_finalizer_alone() { assert_eq!( @@ -375,8 +476,41 @@ mod tests { operator_action_for( &kafka_topic(true, false), &ClusterState::MissingOrDeleting, - false + &AgentState::UnavailableSince(None), + now() ) ); } + + fn agent_deployment(available: Option<&str>) -> Deployment { + serde_json::from_value(json!({ + "metadata": { "creationTimestamp": "1970-01-01T00:10:00Z" }, + "status": { + "conditions": available.map(|status| vec![json!({ + "type": "Available", + "status": status, + "lastTransitionTime": "1970-01-01T00:15:00Z", + })]), + }, + })) + .expect("the Deployment should be valid") + } + + #[rstest] + #[case::missing(None, AgentState::UnavailableSince(None))] + #[case::available(Some(agent_deployment(Some("True"))), AgentState::Available)] + #[case::unavailable_since_the_transition( + Some(agent_deployment(Some("False"))), + AgentState::UnavailableSince(Some(Timestamp::from_second(900).expect("valid"))) + )] + #[case::unavailable_since_the_creation( + Some(agent_deployment(None)), + AgentState::UnavailableSince(Some(Timestamp::from_second(600).expect("valid"))) + )] + fn derive_the_agent_state_from_the_deployment( + #[case] agent_deployment: Option, + #[case] expected: AgentState, + ) { + assert_eq!(expected, agent_state_from(agent_deployment.as_ref())); + } } From 8b8934ce44fcd1e10140009793c4563ae3ddf35c Mon Sep 17 00:00:00 2001 From: Benedikt Labrenz Date: Wed, 28 Oct 2026 15:10:20 +0100 Subject: [PATCH 53/93] add docs and changelog entry --- CHANGELOG.md | 11 ++ docs/modules/kafka/pages/index.adoc | 2 + .../pages/usage-guide/management-access.adoc | 123 ++++++++++++++++ .../kafka/pages/usage-guide/topics.adoc | 135 ++++++++++++++++++ docs/modules/kafka/partials/nav.adoc | 3 + 5 files changed, 274 insertions(+) create mode 100644 docs/modules/kafka/pages/usage-guide/management-access.adoc create mode 100644 docs/modules/kafka/pages/usage-guide/topics.adoc diff --git a/CHANGELOG.md b/CHANGELOG.md index 4fd40a4b0..58d95c218 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,12 @@ All notable changes to this project will be documented in this file. - Support for Kafka 4.3.1 ([#1022]). - Add `/ready` endpoint to the operator Deployment, which reports the CRD installation status ([#1029]). - Brokers now have a default affinity to the OPA Pods when OPA authorization is configured ([#1035]). +- Add management access via `spec.clusterConfig.managementAccess`. When enabled, the operator + deploys an agent per KafkaCluster, which authenticates anonymously or with a TLS client + certificate ([#1028]). +- Add the KafkaTopic CRD to manage topics declaratively. The agent creates and adopts topics, + replaces their configs with `configOverrides`, increases their partitions and optionally deletes + them. The operator removes the finalizers of KafkaTopics whose KafkaCluster is deleted ([#1050]). ### Changed @@ -53,6 +59,9 @@ All notable changes to this project will be documented in this file. - Support Vector 0.58.0 ([#1039], [#1044], [#1046]). - test: Update OPA to 1.20.2 in integration tests and docs example ([#1042]). - The operator Deployment runs with a restricted security context and a read-only root filesystem, so it can be deployed into namespaces that enforce the `restricted` Pod Security Standard ([#1049]). +- The `/ready` endpoint of the operator now also waits for the KafkaTopic CRD. If the CRDs are not + maintained by the operator (`maintenance.customResourceDefinitions.maintain: false`), install the + KafkaTopic CRD together with the KafkaCluster CRD ([#1050]). ### Fixed @@ -89,6 +98,7 @@ All notable changes to this project will be documented in this file. [#1021]: https://github.com/stackabletech/kafka-operator/pull/1021 [#1022]: https://github.com/stackabletech/kafka-operator/pull/1022 [#1024]: https://github.com/stackabletech/kafka-operator/pull/1024 +[#1028]: https://github.com/stackabletech/kafka-operator/pull/1028 [#1029]: https://github.com/stackabletech/kafka-operator/pull/1029 [#1034]: https://github.com/stackabletech/kafka-operator/pull/1034 [#1035]: https://github.com/stackabletech/kafka-operator/pull/1035 @@ -99,6 +109,7 @@ All notable changes to this project will be documented in this file. [#1046]: https://github.com/stackabletech/kafka-operator/pull/1046 [#1047]: https://github.com/stackabletech/kafka-operator/pull/1047 [#1049]: https://github.com/stackabletech/kafka-operator/pull/1049 +[#1050]: https://github.com/stackabletech/kafka-operator/pull/1050 ## [26.7.0] - 2026-07-21 diff --git a/docs/modules/kafka/pages/index.adoc b/docs/modules/kafka/pages/index.adoc index 13d1ac30f..5b0c25a5c 100644 --- a/docs/modules/kafka/pages/index.adoc +++ b/docs/modules/kafka/pages/index.adoc @@ -5,6 +5,7 @@ :github: https://github.com/stackabletech/kafka-operator/ :crd: {crd-ref}{crd-ref-release}&operator=kafka-operator :crd-kafkacluster: {crd-ref}/kafka.stackable.tech/v1alpha1/kafkacluster{crd-ref-release} +:crd-kafkatopic: {crd-ref}/kafka.stackable.tech/v1alpha1/kafkatopic{crd-ref-release} :feature-tracker: https://features.stackable.tech/unified :metadata-quorum: https://cwiki.apache.org/confluence/display/KAFKA/KIP-500%3A+Replace+ZooKeeper+with+a+Self-Managed+Metadata+Quorum :pegelonline: https://www.pegelonline.wsv.de/webservice/ueberblick @@ -75,3 +76,4 @@ include::partial$supported-versions.adoc[] * The {github}[kafka-operator {external-link-icon}^] GitHub repository * The operator feature overview in the {feature-tracker}[feature tracker {external-link-icon}^] * The {crd-kafkacluster}[KafkaCluster {external-link-icon}^] CRD documentation +* The {crd-kafkatopic}[KafkaTopic {external-link-icon}^] CRD documentation diff --git a/docs/modules/kafka/pages/usage-guide/management-access.adoc b/docs/modules/kafka/pages/usage-guide/management-access.adoc new file mode 100644 index 000000000..eebeb13e1 --- /dev/null +++ b/docs/modules/kafka/pages/usage-guide/management-access.adoc @@ -0,0 +1,123 @@ += Management access +:description: Let the Stackable Operator for Apache Kafka manage resources inside a Kafka cluster, such as topics, through a per-cluster agent. + +Management access allows the Stackable Data Platform (SDP) to manage resources inside an Apache Kafka cluster, such as xref:usage-guide/topics.adoc[topics]. + +== Agents + +The agent is an optional Deployment next to the Kafka cluster that manages resources inside Kafka, such as topics. +The operator deploys it when management access is enabled. + +== Limitations + +Management access is not available for all clusters: + +* Clusters that use ZooKeeper are not supported, only clusters in xref:usage-guide/kraft-controller.adoc[KRaft mode]. +* Clusters with Kerberos authentication are not supported. + +== Configuration + +Enable management access with `spec.clusterConfig.managementAccess` and choose how the agent authenticates to Kafka: + +[source,yaml] +---- +apiVersion: kafka.stackable.tech/v1alpha1 +kind: KafkaCluster +metadata: + name: simple-kafka +spec: + image: + productVersion: 4.3.1 + clusterConfig: + metadataManager: KRaft + tls: + serverSecretClass: tls + authentication: + - authenticationClass: kafka-client-tls + managementAccess: + enabled: true # <1> + authentication: + userProvided: + tls: + secretClassName: kafka-client-auth-secret # <2> + controllers: + roleGroups: + default: + replicas: 3 + brokers: + roleGroups: + default: + replicas: 3 +---- +<1> Deploys the agent. Setting it to `false` removes the agent again. +<2> The SecretClass that provisions the client certificate of the agent, here the same SecretClass that the brokers use to verify client certificates, see xref:usage-guide/security.adoc[Security]. + +The agent supports the following authentication methods: + +`anonymous: {}`:: The agent connects without a client certificate. +This is only possible if the cluster does not require client authentication. +`tls.secretClassName`:: A client certificate provisioned by the xref:secret-operator:index.adoc[Secret Operator] from the given SecretClass. +`tls.secretName`:: A static Secret holding the client certificate in the keys `tls.crt` and `tls.key` (PEM), for example a Secret of type `kubernetes.io/tls`. + +The agent verifies the brokers with the CA of the SecretClass used by the client listener, that is `spec.clusterConfig.tls.serverSecretClass` or the `clientCertSecretClass` of the TLS AuthenticationClass. + +== Authorization + +The principal of the agent must be allowed to manage topics in Kafka, for example by xref:usage-guide/security.adoc#authorization[OPA authorization]. +The principal is derived from the subject of the client certificate. +The agent needs the following permissions: + +[cols="2,3", options="header"] +|=== +| Kafka API | Operation + +| Metadata | `DESCRIBE` on the topic +| CreateTopics | `CREATE` on the cluster or the topic, and `DESCRIBE_CONFIGS` on the topic +| CreatePartitions | `ALTER` on the topic +| DescribeConfigs | `DESCRIBE_CONFIGS` on the topic +| IncrementalAlterConfigs | `ALTER_CONFIGS` on the topic +| DeleteTopics | `DELETE` on the topic +| ListPartitionReassignments | `DESCRIBE` on the cluster +|=== + +NOTE: Without `DESCRIBE_CONFIGS`, Kafka still creates a topic, but the agent cannot observe it. +The KafkaTopic then reports `KafkaRejected`. + +== Network + +The agent connects to the headless Services of the broker role groups, which resolve to the broker Pods. +The traffic of the agent therefore stays within the Kubernetes cluster, regardless of the xref:usage-guide/listenerclass.adoc[ListenerClasses] of the cluster. + +== Agent resources + +The operator creates the following resources for the agent, named after the KafkaCluster: + +* A Deployment `-agent` with a single replica, limited to 500m CPU and 128Mi memory by default. +* A ServiceAccount `-agent`. +* A RoleBinding `-agent`, which binds the ClusterRole `kafka-agent-clusterrole` deployed by the Helm chart. + +The Deployment can be adjusted with xref:concepts:overrides.adoc#object-overrides[object overrides], for example to change the resources of the agent: + +[source,yaml] +---- +spec: + objectOverrides: + - apiVersion: apps/v1 + kind: Deployment + metadata: + name: simple-kafka-agent + spec: + template: + spec: + containers: + - name: agent + resources: + limits: + memory: 256Mi +---- + +== Disabling management access + +Setting `enabled` to `false` or removing `managementAccess` removes the agent. +The KafkaTopics and the topics in Kafka stay as they are. +While no agent is available, the operator reports `AgentUnavailable` in the conditions of the KafkaTopics of the cluster, see xref:usage-guide/topics.adoc#deletion[Deletion] for the effect on deleted KafkaTopics. diff --git a/docs/modules/kafka/pages/usage-guide/topics.adoc b/docs/modules/kafka/pages/usage-guide/topics.adoc new file mode 100644 index 000000000..67d27d876 --- /dev/null +++ b/docs/modules/kafka/pages/usage-guide/topics.adoc @@ -0,0 +1,135 @@ += Topics +:description: Manage Apache Kafka topics declaratively with KafkaTopic resources and the Stackable Operator for Apache Kafka. + +A KafkaTopic declares the desired state of a topic in an Apache Kafka cluster: its name, partitions, replication factor and configs. +The xref:usage-guide/management-access.adoc#_agents[agent] of the cluster creates the topic and keeps it in this state. +This requires xref:usage-guide/management-access.adoc[management access] to be enabled on the KafkaCluster. + +The KafkaTopic is the source of truth for its topic: changes made to the topic outside of the KafkaTopic, for example with `kafka-configs.sh`, are reverted. +The agent checks every topic at least every five minutes. + +NOTE: Other operators, such as Strimzi, also provide a resource of the kind `KafkaTopic`. +If both are installed, use the full resource name with `kubectl`, for example `kubectl get kafkatopics.kafka.stackable.tech`. + +== Configuration + +[source,yaml] +---- +apiVersion: kafka.stackable.tech/v1alpha1 +kind: KafkaTopic +metadata: + name: orders +spec: + clusterRef: + name: simple-kafka # <1> + topicName: shop.orders # <2> + partitions: 6 # <3> + replicationFactor: 3 # <4> + configOverrides: # <5> + retention.ms: "604800000" + cleanup.policy: delete + deletionPolicy: Retain # <6> +---- +<1> The KafkaCluster in the same namespace that manages the topic. It cannot be changed. +<2> The name of the topic in Kafka. Defaults to `metadata.name`. It cannot be changed. +<3> The number of partitions. Defaults to the broker setting `num.partitions` when the topic is created. +<4> The number of replicas of each partition. Defaults to the broker setting `default.replication.factor` when the topic is created. +<5> The configs of the topic, see <>. +<6> What happens to the topic when the KafkaTopic is deleted, see <>. + +=== Topic name + +A topic name consists of up to 249 characters out of `a-z`, `A-Z`, `0-9`, `.`, `_` and `-`, and cannot be `.` or `..`. +Kafka allows names that are not valid Kubernetes object names, for example with uppercase letters or underscores. +Such topics need `topicName`. + +Topics starting with `+__+` are internal to Kafka, such as `+__consumer_offsets+`, and are rejected. + +The topic name cannot be changed, and `topicName` can neither be added nor removed later. +To rename a topic, create a new KafkaTopic. + +=== Partitions and replication factor + +The number of partitions can only be increased, because Kafka cannot remove partitions. + +Changing the replication factor of an existing topic is not supported, because Kafka requires an explicit assignment of the replicas to brokers. +Use `kafka-reassign-partitions.sh` or Cruise Control for this. + +An unsupported change is reported in the `Synced` condition, and all other changes are still applied. + +[[configs]] +=== Configs + +`configOverrides` replaces all configs set on the topic. +Configs that are set on the topic but missing in `configOverrides` are reset to the broker defaults. + +The throttles of partition reassignments, `leader.replication.throttled.replicas` and `follower.replication.throttled.replicas`, are left untouched and cannot be set in `configOverrides`. + +== Existing topics + +A KafkaTopic for a topic that already exists in Kafka takes over the topic. + +WARNING: Taking over a topic is destructive. +The configs of the topic are replaced by `configOverrides`, which can delete data, for example with a shorter `retention.ms` or another `cleanup.policy`. +With `deletionPolicy: Delete`, deleting the KafkaTopic also deletes a topic that existed before the KafkaTopic. + +A KafkaTopic refers to its topic by name. +If the topic is deleted and created again outside of the KafkaTopic, the KafkaTopic takes over the new topic as well. + +If several KafkaTopics declare the same topic in the same cluster, the oldest KafkaTopic manages the topic, and the others report a `Conflict`. +When the oldest KafkaTopic is deleted, the next one takes over the topic within five minutes. + +== Status + +The status shows the topic in Kafka: + +[source,yaml] +---- +status: + conditions: ... + topicId: UXyU9i5ARn6W00ON2taeWA # <1> + topicName: shop.orders # <2> + partitions: 6 + replicationFactor: 3 # <3> + config: # <4> + cleanup.policy: delete + retention.ms: "604800000" +---- +<1> The topic ID as shown by `kafka-topics.sh`. +<2> The name of the topic in Kafka. +<3> The replication factor, excluding replicas that a running reassignment removes. It is not set if the partitions have different replication factors. +<4> The configs set on the topic, without broker and default values. + +The `Ready` condition shows whether the topic exists in Kafka. +The `Synced` condition shows whether the topic matches the KafkaTopic. +If it does not, the reason and message of the condition explain why, for example an unsupported change, a request rejected by Kafka or a paused reconciliation. + +== Pausing + +The reconciliation of a single KafkaTopic can be paused, independently of the xref:usage-guide/operations/cluster-operations.adoc[cluster operations] of the KafkaCluster: + +[source,yaml] +---- +spec: + topicOperation: + reconciliationPaused: true +---- + +While paused, the agent does not change the topic in Kafka, but still updates the status. + +[[deletion]] +== Deletion + +With the default `deletionPolicy: Retain`, deleting a KafkaTopic leaves the topic in Kafka. + +With `deletionPolicy: Delete`, the agent deletes the topic in Kafka when the KafkaTopic is deleted, unless another KafkaTopic declares the same topic. +The KafkaTopic gets a finalizer once the agent manages the topic, so the deletion waits until the topic is deleted. +If the reconciliation of the KafkaTopic is paused, the deletion waits until it is resumed. +If the brokers cannot be reached, for example while the KafkaCluster is xref:usage-guide/operations/cluster-operations.adoc[stopped], switching the KafkaTopic to `deletionPolicy: Retain` releases the deletion. + +When the KafkaCluster or its namespace is deleted, the operator removes the finalizers of its KafkaTopics, so the deletion does not block. +The topics are not deleted in Kafka in this case. +If the KafkaCluster is created again with the same PersistentVolumeClaims, its topics still exist, but without KafkaTopics. + +If the KafkaCluster exists but has no agent, for example because management access is disabled, the deletion waits for the agent. +In this case, the finalizer `kafka.stackable.tech/kafkatopic` can be removed manually, which leaves the topic in Kafka. diff --git a/docs/modules/kafka/partials/nav.adoc b/docs/modules/kafka/partials/nav.adoc index 7849e974a..07064ed47 100644 --- a/docs/modules/kafka/partials/nav.adoc +++ b/docs/modules/kafka/partials/nav.adoc @@ -6,6 +6,8 @@ ** xref:kafka:usage-guide/listenerclass.adoc[] ** xref:kafka:usage-guide/storage-resources.adoc[] ** xref:kafka:usage-guide/security.adoc[] +** xref:kafka:usage-guide/management-access.adoc[] +** xref:kafka:usage-guide/topics.adoc[] ** xref:kafka:usage-guide/monitoring.adoc[] ** xref:kafka:usage-guide/logging.adoc[] ** xref:kafka:usage-guide/overrides.adoc[] @@ -18,6 +20,7 @@ * xref:kafka:reference/index.adoc[] ** xref:kafka:reference/crds.adoc[] *** {crd-ref}/kafka.stackable.tech/v1alpha1/kafkacluster{crd-ref-release}[KafkaCluster {external-link-icon}^] +*** {crd-ref}/kafka.stackable.tech/v1alpha1/kafkatopic{crd-ref-release}[KafkaTopic {external-link-icon}^] ** xref:kafka:reference/discovery.adoc[] ** xref:kafka:reference/commandline-parameters.adoc[] ** xref:kafka:reference/environment-variables.adoc[] From bdda9d08b1bdcf5b044f74c24ea119c182cfb3ac Mon Sep 17 00:00:00 2001 From: Benedikt Labrenz Date: Wed, 28 Oct 2026 15:53:58 +0100 Subject: [PATCH 54/93] bump operator-rs --- Cargo.lock | 20 ++++++++++---------- Cargo.nix | 20 ++++++++++---------- 2 files changed, 20 insertions(+), 20 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 288c23de1..e612f604b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1712,7 +1712,7 @@ dependencies = [ [[package]] name = "k8s-version" version = "0.1.4" -source = "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#ccfff5810f9b5affae4d144b3cf04ca88677e23e" +source = "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#63760c6a00a9b6c0fa564201c2b442307b5e7069" dependencies = [ "darling 0.24.1", "regex", @@ -3168,7 +3168,7 @@ checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596" [[package]] name = "stackable-agent" version = "0.1.0" -source = "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#ccfff5810f9b5affae4d144b3cf04ca88677e23e" +source = "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#63760c6a00a9b6c0fa564201c2b442307b5e7069" dependencies = [ "k8s-openapi", "schemars", @@ -3180,7 +3180,7 @@ dependencies = [ [[package]] name = "stackable-certs" version = "0.4.1" -source = "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#ccfff5810f9b5affae4d144b3cf04ca88677e23e" +source = "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#63760c6a00a9b6c0fa564201c2b442307b5e7069" dependencies = [ "const-oid", "ecdsa", @@ -3247,7 +3247,7 @@ dependencies = [ [[package]] name = "stackable-operator" version = "0.121.0" -source = "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#ccfff5810f9b5affae4d144b3cf04ca88677e23e" +source = "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#63760c6a00a9b6c0fa564201c2b442307b5e7069" dependencies = [ "base64 0.23.1", "clap", @@ -3292,7 +3292,7 @@ dependencies = [ [[package]] name = "stackable-operator-derive" version = "0.3.1" -source = "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#ccfff5810f9b5affae4d144b3cf04ca88677e23e" +source = "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#63760c6a00a9b6c0fa564201c2b442307b5e7069" dependencies = [ "darling 0.24.1", "proc-macro2", @@ -3303,7 +3303,7 @@ dependencies = [ [[package]] name = "stackable-shared" version = "0.2.0" -source = "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#ccfff5810f9b5affae4d144b3cf04ca88677e23e" +source = "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#63760c6a00a9b6c0fa564201c2b442307b5e7069" dependencies = [ "jiff", "k8s-openapi", @@ -3320,7 +3320,7 @@ dependencies = [ [[package]] name = "stackable-telemetry" version = "0.6.5" -source = "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#ccfff5810f9b5affae4d144b3cf04ca88677e23e" +source = "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#63760c6a00a9b6c0fa564201c2b442307b5e7069" dependencies = [ "axum", "clap", @@ -3344,7 +3344,7 @@ dependencies = [ [[package]] name = "stackable-versioned" version = "0.13.0" -source = "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#ccfff5810f9b5affae4d144b3cf04ca88677e23e" +source = "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#63760c6a00a9b6c0fa564201c2b442307b5e7069" dependencies = [ "kube", "schemars", @@ -3357,7 +3357,7 @@ dependencies = [ [[package]] name = "stackable-versioned-macros" version = "0.13.0" -source = "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#ccfff5810f9b5affae4d144b3cf04ca88677e23e" +source = "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#63760c6a00a9b6c0fa564201c2b442307b5e7069" dependencies = [ "convert_case", "darling 0.24.1", @@ -3374,7 +3374,7 @@ dependencies = [ [[package]] name = "stackable-webhook" version = "0.10.0" -source = "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#ccfff5810f9b5affae4d144b3cf04ca88677e23e" +source = "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#63760c6a00a9b6c0fa564201c2b442307b5e7069" dependencies = [ "arc-swap", "async-trait", diff --git a/Cargo.nix b/Cargo.nix index 691a18e6b..4cc6452f5 100644 --- a/Cargo.nix +++ b/Cargo.nix @@ -5455,7 +5455,7 @@ rec { workspace_member = null; src = pkgs.fetchgit { url = "https://github.com/stackabletech/operator-rs.git"; - rev = "ccfff5810f9b5affae4d144b3cf04ca88677e23e"; + rev = "63760c6a00a9b6c0fa564201c2b442307b5e7069"; sha256 = "052r8sbdgl89fy4jrkl1jjzjp5fj731pips5a95jd5hm51ymwi4s"; }; libName = "k8s_version"; @@ -10388,7 +10388,7 @@ rec { workspace_member = null; src = pkgs.fetchgit { url = "https://github.com/stackabletech/operator-rs.git"; - rev = "ccfff5810f9b5affae4d144b3cf04ca88677e23e"; + rev = "63760c6a00a9b6c0fa564201c2b442307b5e7069"; sha256 = "052r8sbdgl89fy4jrkl1jjzjp5fj731pips5a95jd5hm51ymwi4s"; }; libName = "stackable_agent"; @@ -10431,7 +10431,7 @@ rec { workspace_member = null; src = pkgs.fetchgit { url = "https://github.com/stackabletech/operator-rs.git"; - rev = "ccfff5810f9b5affae4d144b3cf04ca88677e23e"; + rev = "63760c6a00a9b6c0fa564201c2b442307b5e7069"; sha256 = "052r8sbdgl89fy4jrkl1jjzjp5fj731pips5a95jd5hm51ymwi4s"; }; libName = "stackable_certs"; @@ -10703,7 +10703,7 @@ rec { workspace_member = null; src = pkgs.fetchgit { url = "https://github.com/stackabletech/operator-rs.git"; - rev = "ccfff5810f9b5affae4d144b3cf04ca88677e23e"; + rev = "63760c6a00a9b6c0fa564201c2b442307b5e7069"; sha256 = "052r8sbdgl89fy4jrkl1jjzjp5fj731pips5a95jd5hm51ymwi4s"; }; libName = "stackable_operator"; @@ -10908,7 +10908,7 @@ rec { workspace_member = null; src = pkgs.fetchgit { url = "https://github.com/stackabletech/operator-rs.git"; - rev = "ccfff5810f9b5affae4d144b3cf04ca88677e23e"; + rev = "63760c6a00a9b6c0fa564201c2b442307b5e7069"; sha256 = "052r8sbdgl89fy4jrkl1jjzjp5fj731pips5a95jd5hm51ymwi4s"; }; procMacro = true; @@ -10943,7 +10943,7 @@ rec { workspace_member = null; src = pkgs.fetchgit { url = "https://github.com/stackabletech/operator-rs.git"; - rev = "ccfff5810f9b5affae4d144b3cf04ca88677e23e"; + rev = "63760c6a00a9b6c0fa564201c2b442307b5e7069"; sha256 = "052r8sbdgl89fy4jrkl1jjzjp5fj731pips5a95jd5hm51ymwi4s"; }; libName = "stackable_shared"; @@ -11024,7 +11024,7 @@ rec { workspace_member = null; src = pkgs.fetchgit { url = "https://github.com/stackabletech/operator-rs.git"; - rev = "ccfff5810f9b5affae4d144b3cf04ca88677e23e"; + rev = "63760c6a00a9b6c0fa564201c2b442307b5e7069"; sha256 = "052r8sbdgl89fy4jrkl1jjzjp5fj731pips5a95jd5hm51ymwi4s"; }; libName = "stackable_telemetry"; @@ -11134,7 +11134,7 @@ rec { workspace_member = null; src = pkgs.fetchgit { url = "https://github.com/stackabletech/operator-rs.git"; - rev = "ccfff5810f9b5affae4d144b3cf04ca88677e23e"; + rev = "63760c6a00a9b6c0fa564201c2b442307b5e7069"; sha256 = "052r8sbdgl89fy4jrkl1jjzjp5fj731pips5a95jd5hm51ymwi4s"; }; libName = "stackable_versioned"; @@ -11180,7 +11180,7 @@ rec { workspace_member = null; src = pkgs.fetchgit { url = "https://github.com/stackabletech/operator-rs.git"; - rev = "ccfff5810f9b5affae4d144b3cf04ca88677e23e"; + rev = "63760c6a00a9b6c0fa564201c2b442307b5e7069"; sha256 = "052r8sbdgl89fy4jrkl1jjzjp5fj731pips5a95jd5hm51ymwi4s"; }; procMacro = true; @@ -11244,7 +11244,7 @@ rec { workspace_member = null; src = pkgs.fetchgit { url = "https://github.com/stackabletech/operator-rs.git"; - rev = "ccfff5810f9b5affae4d144b3cf04ca88677e23e"; + rev = "63760c6a00a9b6c0fa564201c2b442307b5e7069"; sha256 = "052r8sbdgl89fy4jrkl1jjzjp5fj731pips5a95jd5hm51ymwi4s"; }; libName = "stackable_webhook"; From 3fab70399f4a597ae278ea8985626fb03f38ce6d Mon Sep 17 00:00:00 2001 From: Benedikt Labrenz Date: Wed, 28 Oct 2026 16:27:20 +0100 Subject: [PATCH 55/93] regenerate nix --- Cargo.nix | 20 ++++++++++---------- crate-hashes.json | 20 ++++++++++---------- 2 files changed, 20 insertions(+), 20 deletions(-) diff --git a/Cargo.nix b/Cargo.nix index 8e0e6801c..341aae4b7 100644 --- a/Cargo.nix +++ b/Cargo.nix @@ -5510,7 +5510,7 @@ rec { src = pkgs.fetchgit { url = "https://github.com/stackabletech/operator-rs.git"; rev = "63760c6a00a9b6c0fa564201c2b442307b5e7069"; - sha256 = "052r8sbdgl89fy4jrkl1jjzjp5fj731pips5a95jd5hm51ymwi4s"; + sha256 = "1962ivywrqyzjxa29w6i6gkhblaqd74q3wqwprrggh85z8kfg715"; }; libName = "k8s_version"; authors = [ @@ -10489,7 +10489,7 @@ rec { src = pkgs.fetchgit { url = "https://github.com/stackabletech/operator-rs.git"; rev = "63760c6a00a9b6c0fa564201c2b442307b5e7069"; - sha256 = "052r8sbdgl89fy4jrkl1jjzjp5fj731pips5a95jd5hm51ymwi4s"; + sha256 = "1962ivywrqyzjxa29w6i6gkhblaqd74q3wqwprrggh85z8kfg715"; }; libName = "stackable_agent"; authors = [ @@ -10532,7 +10532,7 @@ rec { src = pkgs.fetchgit { url = "https://github.com/stackabletech/operator-rs.git"; rev = "63760c6a00a9b6c0fa564201c2b442307b5e7069"; - sha256 = "052r8sbdgl89fy4jrkl1jjzjp5fj731pips5a95jd5hm51ymwi4s"; + sha256 = "1962ivywrqyzjxa29w6i6gkhblaqd74q3wqwprrggh85z8kfg715"; }; libName = "stackable_certs"; authors = [ @@ -10934,7 +10934,7 @@ rec { src = pkgs.fetchgit { url = "https://github.com/stackabletech/operator-rs.git"; rev = "63760c6a00a9b6c0fa564201c2b442307b5e7069"; - sha256 = "052r8sbdgl89fy4jrkl1jjzjp5fj731pips5a95jd5hm51ymwi4s"; + sha256 = "1962ivywrqyzjxa29w6i6gkhblaqd74q3wqwprrggh85z8kfg715"; }; libName = "stackable_operator"; authors = [ @@ -11139,7 +11139,7 @@ rec { src = pkgs.fetchgit { url = "https://github.com/stackabletech/operator-rs.git"; rev = "63760c6a00a9b6c0fa564201c2b442307b5e7069"; - sha256 = "052r8sbdgl89fy4jrkl1jjzjp5fj731pips5a95jd5hm51ymwi4s"; + sha256 = "1962ivywrqyzjxa29w6i6gkhblaqd74q3wqwprrggh85z8kfg715"; }; procMacro = true; libName = "stackable_operator_derive"; @@ -11174,7 +11174,7 @@ rec { src = pkgs.fetchgit { url = "https://github.com/stackabletech/operator-rs.git"; rev = "63760c6a00a9b6c0fa564201c2b442307b5e7069"; - sha256 = "052r8sbdgl89fy4jrkl1jjzjp5fj731pips5a95jd5hm51ymwi4s"; + sha256 = "1962ivywrqyzjxa29w6i6gkhblaqd74q3wqwprrggh85z8kfg715"; }; libName = "stackable_shared"; authors = [ @@ -11255,7 +11255,7 @@ rec { src = pkgs.fetchgit { url = "https://github.com/stackabletech/operator-rs.git"; rev = "63760c6a00a9b6c0fa564201c2b442307b5e7069"; - sha256 = "052r8sbdgl89fy4jrkl1jjzjp5fj731pips5a95jd5hm51ymwi4s"; + sha256 = "1962ivywrqyzjxa29w6i6gkhblaqd74q3wqwprrggh85z8kfg715"; }; libName = "stackable_telemetry"; authors = [ @@ -11365,7 +11365,7 @@ rec { src = pkgs.fetchgit { url = "https://github.com/stackabletech/operator-rs.git"; rev = "63760c6a00a9b6c0fa564201c2b442307b5e7069"; - sha256 = "052r8sbdgl89fy4jrkl1jjzjp5fj731pips5a95jd5hm51ymwi4s"; + sha256 = "1962ivywrqyzjxa29w6i6gkhblaqd74q3wqwprrggh85z8kfg715"; }; libName = "stackable_versioned"; authors = [ @@ -11411,7 +11411,7 @@ rec { src = pkgs.fetchgit { url = "https://github.com/stackabletech/operator-rs.git"; rev = "63760c6a00a9b6c0fa564201c2b442307b5e7069"; - sha256 = "052r8sbdgl89fy4jrkl1jjzjp5fj731pips5a95jd5hm51ymwi4s"; + sha256 = "1962ivywrqyzjxa29w6i6gkhblaqd74q3wqwprrggh85z8kfg715"; }; procMacro = true; libName = "stackable_versioned_macros"; @@ -11475,7 +11475,7 @@ rec { src = pkgs.fetchgit { url = "https://github.com/stackabletech/operator-rs.git"; rev = "63760c6a00a9b6c0fa564201c2b442307b5e7069"; - sha256 = "052r8sbdgl89fy4jrkl1jjzjp5fj731pips5a95jd5hm51ymwi4s"; + sha256 = "1962ivywrqyzjxa29w6i6gkhblaqd74q3wqwprrggh85z8kfg715"; }; libName = "stackable_webhook"; authors = [ diff --git a/crate-hashes.json b/crate-hashes.json index 405952ac3..fd83bfb02 100644 --- a/crate-hashes.json +++ b/crate-hashes.json @@ -1,12 +1,12 @@ { - "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#k8s-version@0.1.4": "052r8sbdgl89fy4jrkl1jjzjp5fj731pips5a95jd5hm51ymwi4s", - "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#stackable-agent@0.1.0": "052r8sbdgl89fy4jrkl1jjzjp5fj731pips5a95jd5hm51ymwi4s", - "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#stackable-certs@0.4.1": "052r8sbdgl89fy4jrkl1jjzjp5fj731pips5a95jd5hm51ymwi4s", - "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#stackable-operator-derive@0.3.1": "052r8sbdgl89fy4jrkl1jjzjp5fj731pips5a95jd5hm51ymwi4s", - "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#stackable-operator@0.121.0": "052r8sbdgl89fy4jrkl1jjzjp5fj731pips5a95jd5hm51ymwi4s", - "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#stackable-shared@0.2.0": "052r8sbdgl89fy4jrkl1jjzjp5fj731pips5a95jd5hm51ymwi4s", - "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#stackable-telemetry@0.6.5": "052r8sbdgl89fy4jrkl1jjzjp5fj731pips5a95jd5hm51ymwi4s", - "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#stackable-versioned-macros@0.13.0": "052r8sbdgl89fy4jrkl1jjzjp5fj731pips5a95jd5hm51ymwi4s", - "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#stackable-versioned@0.13.0": "052r8sbdgl89fy4jrkl1jjzjp5fj731pips5a95jd5hm51ymwi4s", - "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#stackable-webhook@0.10.0": "052r8sbdgl89fy4jrkl1jjzjp5fj731pips5a95jd5hm51ymwi4s" + "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#k8s-version@0.1.4": "1962ivywrqyzjxa29w6i6gkhblaqd74q3wqwprrggh85z8kfg715", + "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#stackable-agent@0.1.0": "1962ivywrqyzjxa29w6i6gkhblaqd74q3wqwprrggh85z8kfg715", + "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#stackable-certs@0.4.1": "1962ivywrqyzjxa29w6i6gkhblaqd74q3wqwprrggh85z8kfg715", + "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#stackable-operator-derive@0.3.1": "1962ivywrqyzjxa29w6i6gkhblaqd74q3wqwprrggh85z8kfg715", + "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#stackable-operator@0.121.0": "1962ivywrqyzjxa29w6i6gkhblaqd74q3wqwprrggh85z8kfg715", + "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#stackable-shared@0.2.0": "1962ivywrqyzjxa29w6i6gkhblaqd74q3wqwprrggh85z8kfg715", + "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#stackable-telemetry@0.6.5": "1962ivywrqyzjxa29w6i6gkhblaqd74q3wqwprrggh85z8kfg715", + "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#stackable-versioned-macros@0.13.0": "1962ivywrqyzjxa29w6i6gkhblaqd74q3wqwprrggh85z8kfg715", + "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#stackable-versioned@0.13.0": "1962ivywrqyzjxa29w6i6gkhblaqd74q3wqwprrggh85z8kfg715", + "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#stackable-webhook@0.10.0": "1962ivywrqyzjxa29w6i6gkhblaqd74q3wqwprrggh85z8kfg715" } \ No newline at end of file From 0c7956bc1a1fb260c74eaa14bd929fbe3714e21e Mon Sep 17 00:00:00 2001 From: Benedikt Labrenz Date: Wed, 28 Oct 2026 16:31:18 +0100 Subject: [PATCH 56/93] test correct namespace deletion --- tests/templates/kuttl/management-access/20-assert.yaml | 5 ++++- tests/templates/kuttl/management-access/20-create-topic.yaml | 1 + 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/tests/templates/kuttl/management-access/20-assert.yaml b/tests/templates/kuttl/management-access/20-assert.yaml index 7d87800ab..c062e63ac 100644 --- a/tests/templates/kuttl/management-access/20-assert.yaml +++ b/tests/templates/kuttl/management-access/20-assert.yaml @@ -1,5 +1,6 @@ --- -# The agent authenticates to Kafka and creates the topic. +# The agent authenticates to Kafka and creates the topic. The KafkaTopic keeps its finalizer until +# the end of the test, so the deletion of the namespace only finishes if the operator removes it. apiVersion: kuttl.dev/v1beta1 kind: TestAssert timeout: 300 @@ -8,6 +9,8 @@ apiVersion: kafka.stackable.tech/v1alpha1 kind: KafkaTopic metadata: name: orders + finalizers: + - kafka.stackable.tech/kafkatopic status: conditions: - type: Ready diff --git a/tests/templates/kuttl/management-access/20-create-topic.yaml b/tests/templates/kuttl/management-access/20-create-topic.yaml index 8a57a0e5d..439673153 100644 --- a/tests/templates/kuttl/management-access/20-create-topic.yaml +++ b/tests/templates/kuttl/management-access/20-create-topic.yaml @@ -9,3 +9,4 @@ spec: topicName: orders partitions: 3 replicationFactor: 1 + deletionPolicy: Delete From 0d1d0ac988e5e8790e72567ffecf2f6d284bc416 Mon Sep 17 00:00:00 2001 From: Benedikt Labrenz Date: Wed, 28 Oct 2026 16:32:00 +0100 Subject: [PATCH 57/93] add OPA to management access test --- .../kuttl/management-access/07-assert.yaml | 7 ++++ .../management-access/07-install-opa.yaml.j2 | 34 +++++++++++++++++ .../kuttl/management-access/07_kafka.rego.j2 | 37 +++++++++++++++++++ .../kuttl/management-access/10_kafka.yaml.j2 | 4 ++ .../kuttl/management-access/30-assert.yaml | 25 +++++++++++++ .../30-create-forbidden-topic.yaml | 9 +++++ 6 files changed, 116 insertions(+) create mode 100644 tests/templates/kuttl/management-access/07-assert.yaml create mode 100644 tests/templates/kuttl/management-access/07-install-opa.yaml.j2 create mode 100644 tests/templates/kuttl/management-access/07_kafka.rego.j2 create mode 100644 tests/templates/kuttl/management-access/30-assert.yaml create mode 100644 tests/templates/kuttl/management-access/30-create-forbidden-topic.yaml diff --git a/tests/templates/kuttl/management-access/07-assert.yaml b/tests/templates/kuttl/management-access/07-assert.yaml new file mode 100644 index 000000000..e1112367e --- /dev/null +++ b/tests/templates/kuttl/management-access/07-assert.yaml @@ -0,0 +1,7 @@ +--- +# OPA authorizes the brokers and the agent with the documented permissions. +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +timeout: 300 +commands: + - script: kubectl wait --namespace $NAMESPACE --for=condition=available opaclusters.opa.stackable.tech/test-opa --timeout 301s diff --git a/tests/templates/kuttl/management-access/07-install-opa.yaml.j2 b/tests/templates/kuttl/management-access/07-install-opa.yaml.j2 new file mode 100644 index 000000000..f235db18a --- /dev/null +++ b/tests/templates/kuttl/management-access/07-install-opa.yaml.j2 @@ -0,0 +1,34 @@ +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +commands: + - script: | + envsubst '$NAMESPACE' < 07_kafka.rego | + kubectl create configmap test-rego-kafka --namespace $NAMESPACE --from-file=kafka.rego=/dev/stdin + - script: kubectl label configmap test-rego-kafka --namespace $NAMESPACE opa.stackable.tech/bundle=true +--- +apiVersion: opa.stackable.tech/v1alpha1 +kind: OpaCluster +metadata: + name: test-opa +spec: + image: +{% if test_scenario['values']['opa-latest'].find(",") > 0 %} + custom: "{{ test_scenario['values']['opa-latest'].split(',')[1] }}" + productVersion: "{{ test_scenario['values']['opa-latest'].split(',')[0] }}" +{% else %} + productVersion: "{{ test_scenario['values']['opa-latest'] }}" +{% endif %} + pullPolicy: IfNotPresent + servers: + config: + logging: + containers: + opa: + console: + level: INFO + loggers: + decision: + level: INFO + roleGroups: + default: {} diff --git a/tests/templates/kuttl/management-access/07_kafka.rego.j2 b/tests/templates/kuttl/management-access/07_kafka.rego.j2 new file mode 100644 index 000000000..4121aa2e9 --- /dev/null +++ b/tests/templates/kuttl/management-access/07_kafka.rego.j2 @@ -0,0 +1,37 @@ +{% set authentication = test_scenario['values']['management-access-authentication'] %} +package authz + +broker_principal_pattern := `^DC=local,DC=cluster,DC=svc,DC=$NAMESPACE,DC=test-kafka-broker-default-headless,DC=test-kafka-broker-default-[0-9]+,CN=generated certificate for pod$` + +{% if authentication == 'anonymous' %} +agent_principal_pattern := `^ANONYMOUS$` +{% elif authentication == 'secret-class' %} +# The agent Pod has no subdomain, so its subject DN contains no Service. +agent_principal_pattern := `^DC=local,DC=cluster,DC=svc,DC=$NAMESPACE,DC=test-kafka-agent-[a-z0-9]+-[a-z0-9]+,CN=generated certificate for pod$` +{% else %} +agent_principal_pattern := `^CN=kafka-agent$` +{% endif %} + +default allow := false + +allow if { + input.requestContext.listenerName == "INTERNAL" + regex.match(broker_principal_pattern, input.requestContext.principal.name) +} + +allow if { + regex.match(agent_principal_pattern, input.requestContext.principal.name) + agent_action_is_allowed +} + +# The permissions of the agent as documented, limited to topics starting with "orders" +agent_action_is_allowed if { + input.action.resourcePattern.resourceType == "TOPIC" + startswith(input.action.resourcePattern.name, "orders") + input.action.operation in ["DESCRIBE", "CREATE", "ALTER", "DESCRIBE_CONFIGS", "ALTER_CONFIGS", "DELETE"] +} + +agent_action_is_allowed if { + input.action.resourcePattern.resourceType == "CLUSTER" + input.action.operation == "DESCRIBE" +} diff --git a/tests/templates/kuttl/management-access/10_kafka.yaml.j2 b/tests/templates/kuttl/management-access/10_kafka.yaml.j2 index ca2976982..58608e7be 100644 --- a/tests/templates/kuttl/management-access/10_kafka.yaml.j2 +++ b/tests/templates/kuttl/management-access/10_kafka.yaml.j2 @@ -51,6 +51,10 @@ spec: authentication: - authenticationClass: client-auth-$NAMESPACE {% endif %} + authorization: + opa: + configMapName: test-opa + package: authz managementAccess: enabled: true authentication: diff --git a/tests/templates/kuttl/management-access/30-assert.yaml b/tests/templates/kuttl/management-access/30-assert.yaml new file mode 100644 index 000000000..56bcaf4d1 --- /dev/null +++ b/tests/templates/kuttl/management-access/30-assert.yaml @@ -0,0 +1,25 @@ +--- +# Kafka rejects the requests of the agent, and the KafkaTopic reports why. +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +timeout: 300 +resourceRefs: + - apiVersion: kafka.stackable.tech/v1alpha1 + kind: KafkaTopic + name: forbidden + ref: forbidden +assertAll: + - celExpr: "forbidden.status.conditions.exists(c, c.type == 'Synced' && c.message.contains('TopicAuthorizationFailed'))" +--- +apiVersion: kafka.stackable.tech/v1alpha1 +kind: KafkaTopic +metadata: + name: forbidden +status: + conditions: + - type: Ready + status: Unknown + reason: KafkaRejected + - type: Synced + status: "False" + reason: KafkaRejected diff --git a/tests/templates/kuttl/management-access/30-create-forbidden-topic.yaml b/tests/templates/kuttl/management-access/30-create-forbidden-topic.yaml new file mode 100644 index 000000000..bc5e73322 --- /dev/null +++ b/tests/templates/kuttl/management-access/30-create-forbidden-topic.yaml @@ -0,0 +1,9 @@ +--- +# OPA does not allow the agent to manage this topic. +apiVersion: kafka.stackable.tech/v1alpha1 +kind: KafkaTopic +metadata: + name: forbidden +spec: + clusterRef: + name: test-kafka From a89b5db6d7d678b1ae09c3446c1e268469e0a1f2 Mon Sep 17 00:00:00 2001 From: Benedikt Labrenz Date: Wed, 28 Oct 2026 17:07:16 +0100 Subject: [PATCH 58/93] regenerate nix --- Cargo.nix | 20 ++++++++++---------- crate-hashes.json | 20 ++++++++++---------- 2 files changed, 20 insertions(+), 20 deletions(-) diff --git a/Cargo.nix b/Cargo.nix index 4cc6452f5..b67bd259a 100644 --- a/Cargo.nix +++ b/Cargo.nix @@ -5456,7 +5456,7 @@ rec { src = pkgs.fetchgit { url = "https://github.com/stackabletech/operator-rs.git"; rev = "63760c6a00a9b6c0fa564201c2b442307b5e7069"; - sha256 = "052r8sbdgl89fy4jrkl1jjzjp5fj731pips5a95jd5hm51ymwi4s"; + sha256 = "1962ivywrqyzjxa29w6i6gkhblaqd74q3wqwprrggh85z8kfg715"; }; libName = "k8s_version"; authors = [ @@ -10389,7 +10389,7 @@ rec { src = pkgs.fetchgit { url = "https://github.com/stackabletech/operator-rs.git"; rev = "63760c6a00a9b6c0fa564201c2b442307b5e7069"; - sha256 = "052r8sbdgl89fy4jrkl1jjzjp5fj731pips5a95jd5hm51ymwi4s"; + sha256 = "1962ivywrqyzjxa29w6i6gkhblaqd74q3wqwprrggh85z8kfg715"; }; libName = "stackable_agent"; authors = [ @@ -10432,7 +10432,7 @@ rec { src = pkgs.fetchgit { url = "https://github.com/stackabletech/operator-rs.git"; rev = "63760c6a00a9b6c0fa564201c2b442307b5e7069"; - sha256 = "052r8sbdgl89fy4jrkl1jjzjp5fj731pips5a95jd5hm51ymwi4s"; + sha256 = "1962ivywrqyzjxa29w6i6gkhblaqd74q3wqwprrggh85z8kfg715"; }; libName = "stackable_certs"; authors = [ @@ -10704,7 +10704,7 @@ rec { src = pkgs.fetchgit { url = "https://github.com/stackabletech/operator-rs.git"; rev = "63760c6a00a9b6c0fa564201c2b442307b5e7069"; - sha256 = "052r8sbdgl89fy4jrkl1jjzjp5fj731pips5a95jd5hm51ymwi4s"; + sha256 = "1962ivywrqyzjxa29w6i6gkhblaqd74q3wqwprrggh85z8kfg715"; }; libName = "stackable_operator"; authors = [ @@ -10909,7 +10909,7 @@ rec { src = pkgs.fetchgit { url = "https://github.com/stackabletech/operator-rs.git"; rev = "63760c6a00a9b6c0fa564201c2b442307b5e7069"; - sha256 = "052r8sbdgl89fy4jrkl1jjzjp5fj731pips5a95jd5hm51ymwi4s"; + sha256 = "1962ivywrqyzjxa29w6i6gkhblaqd74q3wqwprrggh85z8kfg715"; }; procMacro = true; libName = "stackable_operator_derive"; @@ -10944,7 +10944,7 @@ rec { src = pkgs.fetchgit { url = "https://github.com/stackabletech/operator-rs.git"; rev = "63760c6a00a9b6c0fa564201c2b442307b5e7069"; - sha256 = "052r8sbdgl89fy4jrkl1jjzjp5fj731pips5a95jd5hm51ymwi4s"; + sha256 = "1962ivywrqyzjxa29w6i6gkhblaqd74q3wqwprrggh85z8kfg715"; }; libName = "stackable_shared"; authors = [ @@ -11025,7 +11025,7 @@ rec { src = pkgs.fetchgit { url = "https://github.com/stackabletech/operator-rs.git"; rev = "63760c6a00a9b6c0fa564201c2b442307b5e7069"; - sha256 = "052r8sbdgl89fy4jrkl1jjzjp5fj731pips5a95jd5hm51ymwi4s"; + sha256 = "1962ivywrqyzjxa29w6i6gkhblaqd74q3wqwprrggh85z8kfg715"; }; libName = "stackable_telemetry"; authors = [ @@ -11135,7 +11135,7 @@ rec { src = pkgs.fetchgit { url = "https://github.com/stackabletech/operator-rs.git"; rev = "63760c6a00a9b6c0fa564201c2b442307b5e7069"; - sha256 = "052r8sbdgl89fy4jrkl1jjzjp5fj731pips5a95jd5hm51ymwi4s"; + sha256 = "1962ivywrqyzjxa29w6i6gkhblaqd74q3wqwprrggh85z8kfg715"; }; libName = "stackable_versioned"; authors = [ @@ -11181,7 +11181,7 @@ rec { src = pkgs.fetchgit { url = "https://github.com/stackabletech/operator-rs.git"; rev = "63760c6a00a9b6c0fa564201c2b442307b5e7069"; - sha256 = "052r8sbdgl89fy4jrkl1jjzjp5fj731pips5a95jd5hm51ymwi4s"; + sha256 = "1962ivywrqyzjxa29w6i6gkhblaqd74q3wqwprrggh85z8kfg715"; }; procMacro = true; libName = "stackable_versioned_macros"; @@ -11245,7 +11245,7 @@ rec { src = pkgs.fetchgit { url = "https://github.com/stackabletech/operator-rs.git"; rev = "63760c6a00a9b6c0fa564201c2b442307b5e7069"; - sha256 = "052r8sbdgl89fy4jrkl1jjzjp5fj731pips5a95jd5hm51ymwi4s"; + sha256 = "1962ivywrqyzjxa29w6i6gkhblaqd74q3wqwprrggh85z8kfg715"; }; libName = "stackable_webhook"; authors = [ diff --git a/crate-hashes.json b/crate-hashes.json index 405952ac3..fd83bfb02 100644 --- a/crate-hashes.json +++ b/crate-hashes.json @@ -1,12 +1,12 @@ { - "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#k8s-version@0.1.4": "052r8sbdgl89fy4jrkl1jjzjp5fj731pips5a95jd5hm51ymwi4s", - "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#stackable-agent@0.1.0": "052r8sbdgl89fy4jrkl1jjzjp5fj731pips5a95jd5hm51ymwi4s", - "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#stackable-certs@0.4.1": "052r8sbdgl89fy4jrkl1jjzjp5fj731pips5a95jd5hm51ymwi4s", - "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#stackable-operator-derive@0.3.1": "052r8sbdgl89fy4jrkl1jjzjp5fj731pips5a95jd5hm51ymwi4s", - "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#stackable-operator@0.121.0": "052r8sbdgl89fy4jrkl1jjzjp5fj731pips5a95jd5hm51ymwi4s", - "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#stackable-shared@0.2.0": "052r8sbdgl89fy4jrkl1jjzjp5fj731pips5a95jd5hm51ymwi4s", - "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#stackable-telemetry@0.6.5": "052r8sbdgl89fy4jrkl1jjzjp5fj731pips5a95jd5hm51ymwi4s", - "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#stackable-versioned-macros@0.13.0": "052r8sbdgl89fy4jrkl1jjzjp5fj731pips5a95jd5hm51ymwi4s", - "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#stackable-versioned@0.13.0": "052r8sbdgl89fy4jrkl1jjzjp5fj731pips5a95jd5hm51ymwi4s", - "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#stackable-webhook@0.10.0": "052r8sbdgl89fy4jrkl1jjzjp5fj731pips5a95jd5hm51ymwi4s" + "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#k8s-version@0.1.4": "1962ivywrqyzjxa29w6i6gkhblaqd74q3wqwprrggh85z8kfg715", + "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#stackable-agent@0.1.0": "1962ivywrqyzjxa29w6i6gkhblaqd74q3wqwprrggh85z8kfg715", + "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#stackable-certs@0.4.1": "1962ivywrqyzjxa29w6i6gkhblaqd74q3wqwprrggh85z8kfg715", + "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#stackable-operator-derive@0.3.1": "1962ivywrqyzjxa29w6i6gkhblaqd74q3wqwprrggh85z8kfg715", + "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#stackable-operator@0.121.0": "1962ivywrqyzjxa29w6i6gkhblaqd74q3wqwprrggh85z8kfg715", + "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#stackable-shared@0.2.0": "1962ivywrqyzjxa29w6i6gkhblaqd74q3wqwprrggh85z8kfg715", + "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#stackable-telemetry@0.6.5": "1962ivywrqyzjxa29w6i6gkhblaqd74q3wqwprrggh85z8kfg715", + "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#stackable-versioned-macros@0.13.0": "1962ivywrqyzjxa29w6i6gkhblaqd74q3wqwprrggh85z8kfg715", + "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#stackable-versioned@0.13.0": "1962ivywrqyzjxa29w6i6gkhblaqd74q3wqwprrggh85z8kfg715", + "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#stackable-webhook@0.10.0": "1962ivywrqyzjxa29w6i6gkhblaqd74q3wqwprrggh85z8kfg715" } \ No newline at end of file From 43657d4c2b69e4679ed4133afd4ad7d48bdc6bf5 Mon Sep 17 00:00:00 2001 From: Benedikt Labrenz Date: Wed, 28 Oct 2026 17:10:54 +0100 Subject: [PATCH 59/93] regenerate nix --- Cargo.lock | 20 ++++++++++---------- Cargo.nix | 40 ++++++++++++++++++++-------------------- crate-hashes.json | 20 ++++++++++---------- 3 files changed, 40 insertions(+), 40 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index e612f604b..ac2068d2e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1712,7 +1712,7 @@ dependencies = [ [[package]] name = "k8s-version" version = "0.1.4" -source = "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#63760c6a00a9b6c0fa564201c2b442307b5e7069" +source = "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#b61f619aa15efa2fa8c88d7eeae6154c3fb3918a" dependencies = [ "darling 0.24.1", "regex", @@ -3168,7 +3168,7 @@ checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596" [[package]] name = "stackable-agent" version = "0.1.0" -source = "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#63760c6a00a9b6c0fa564201c2b442307b5e7069" +source = "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#b61f619aa15efa2fa8c88d7eeae6154c3fb3918a" dependencies = [ "k8s-openapi", "schemars", @@ -3180,7 +3180,7 @@ dependencies = [ [[package]] name = "stackable-certs" version = "0.4.1" -source = "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#63760c6a00a9b6c0fa564201c2b442307b5e7069" +source = "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#b61f619aa15efa2fa8c88d7eeae6154c3fb3918a" dependencies = [ "const-oid", "ecdsa", @@ -3247,7 +3247,7 @@ dependencies = [ [[package]] name = "stackable-operator" version = "0.121.0" -source = "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#63760c6a00a9b6c0fa564201c2b442307b5e7069" +source = "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#b61f619aa15efa2fa8c88d7eeae6154c3fb3918a" dependencies = [ "base64 0.23.1", "clap", @@ -3292,7 +3292,7 @@ dependencies = [ [[package]] name = "stackable-operator-derive" version = "0.3.1" -source = "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#63760c6a00a9b6c0fa564201c2b442307b5e7069" +source = "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#b61f619aa15efa2fa8c88d7eeae6154c3fb3918a" dependencies = [ "darling 0.24.1", "proc-macro2", @@ -3303,7 +3303,7 @@ dependencies = [ [[package]] name = "stackable-shared" version = "0.2.0" -source = "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#63760c6a00a9b6c0fa564201c2b442307b5e7069" +source = "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#b61f619aa15efa2fa8c88d7eeae6154c3fb3918a" dependencies = [ "jiff", "k8s-openapi", @@ -3320,7 +3320,7 @@ dependencies = [ [[package]] name = "stackable-telemetry" version = "0.6.5" -source = "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#63760c6a00a9b6c0fa564201c2b442307b5e7069" +source = "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#b61f619aa15efa2fa8c88d7eeae6154c3fb3918a" dependencies = [ "axum", "clap", @@ -3344,7 +3344,7 @@ dependencies = [ [[package]] name = "stackable-versioned" version = "0.13.0" -source = "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#63760c6a00a9b6c0fa564201c2b442307b5e7069" +source = "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#b61f619aa15efa2fa8c88d7eeae6154c3fb3918a" dependencies = [ "kube", "schemars", @@ -3357,7 +3357,7 @@ dependencies = [ [[package]] name = "stackable-versioned-macros" version = "0.13.0" -source = "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#63760c6a00a9b6c0fa564201c2b442307b5e7069" +source = "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#b61f619aa15efa2fa8c88d7eeae6154c3fb3918a" dependencies = [ "convert_case", "darling 0.24.1", @@ -3374,7 +3374,7 @@ dependencies = [ [[package]] name = "stackable-webhook" version = "0.10.0" -source = "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#63760c6a00a9b6c0fa564201c2b442307b5e7069" +source = "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#b61f619aa15efa2fa8c88d7eeae6154c3fb3918a" dependencies = [ "arc-swap", "async-trait", diff --git a/Cargo.nix b/Cargo.nix index b67bd259a..18130a181 100644 --- a/Cargo.nix +++ b/Cargo.nix @@ -5455,8 +5455,8 @@ rec { workspace_member = null; src = pkgs.fetchgit { url = "https://github.com/stackabletech/operator-rs.git"; - rev = "63760c6a00a9b6c0fa564201c2b442307b5e7069"; - sha256 = "1962ivywrqyzjxa29w6i6gkhblaqd74q3wqwprrggh85z8kfg715"; + rev = "b61f619aa15efa2fa8c88d7eeae6154c3fb3918a"; + sha256 = "0ivrv56a8yzy68d9hr7cvy7hk5w0i2hidzlnns23h0g60ynb3axq"; }; libName = "k8s_version"; authors = [ @@ -10388,8 +10388,8 @@ rec { workspace_member = null; src = pkgs.fetchgit { url = "https://github.com/stackabletech/operator-rs.git"; - rev = "63760c6a00a9b6c0fa564201c2b442307b5e7069"; - sha256 = "1962ivywrqyzjxa29w6i6gkhblaqd74q3wqwprrggh85z8kfg715"; + rev = "b61f619aa15efa2fa8c88d7eeae6154c3fb3918a"; + sha256 = "0ivrv56a8yzy68d9hr7cvy7hk5w0i2hidzlnns23h0g60ynb3axq"; }; libName = "stackable_agent"; authors = [ @@ -10431,8 +10431,8 @@ rec { workspace_member = null; src = pkgs.fetchgit { url = "https://github.com/stackabletech/operator-rs.git"; - rev = "63760c6a00a9b6c0fa564201c2b442307b5e7069"; - sha256 = "1962ivywrqyzjxa29w6i6gkhblaqd74q3wqwprrggh85z8kfg715"; + rev = "b61f619aa15efa2fa8c88d7eeae6154c3fb3918a"; + sha256 = "0ivrv56a8yzy68d9hr7cvy7hk5w0i2hidzlnns23h0g60ynb3axq"; }; libName = "stackable_certs"; authors = [ @@ -10703,8 +10703,8 @@ rec { workspace_member = null; src = pkgs.fetchgit { url = "https://github.com/stackabletech/operator-rs.git"; - rev = "63760c6a00a9b6c0fa564201c2b442307b5e7069"; - sha256 = "1962ivywrqyzjxa29w6i6gkhblaqd74q3wqwprrggh85z8kfg715"; + rev = "b61f619aa15efa2fa8c88d7eeae6154c3fb3918a"; + sha256 = "0ivrv56a8yzy68d9hr7cvy7hk5w0i2hidzlnns23h0g60ynb3axq"; }; libName = "stackable_operator"; authors = [ @@ -10908,8 +10908,8 @@ rec { workspace_member = null; src = pkgs.fetchgit { url = "https://github.com/stackabletech/operator-rs.git"; - rev = "63760c6a00a9b6c0fa564201c2b442307b5e7069"; - sha256 = "1962ivywrqyzjxa29w6i6gkhblaqd74q3wqwprrggh85z8kfg715"; + rev = "b61f619aa15efa2fa8c88d7eeae6154c3fb3918a"; + sha256 = "0ivrv56a8yzy68d9hr7cvy7hk5w0i2hidzlnns23h0g60ynb3axq"; }; procMacro = true; libName = "stackable_operator_derive"; @@ -10943,8 +10943,8 @@ rec { workspace_member = null; src = pkgs.fetchgit { url = "https://github.com/stackabletech/operator-rs.git"; - rev = "63760c6a00a9b6c0fa564201c2b442307b5e7069"; - sha256 = "1962ivywrqyzjxa29w6i6gkhblaqd74q3wqwprrggh85z8kfg715"; + rev = "b61f619aa15efa2fa8c88d7eeae6154c3fb3918a"; + sha256 = "0ivrv56a8yzy68d9hr7cvy7hk5w0i2hidzlnns23h0g60ynb3axq"; }; libName = "stackable_shared"; authors = [ @@ -11024,8 +11024,8 @@ rec { workspace_member = null; src = pkgs.fetchgit { url = "https://github.com/stackabletech/operator-rs.git"; - rev = "63760c6a00a9b6c0fa564201c2b442307b5e7069"; - sha256 = "1962ivywrqyzjxa29w6i6gkhblaqd74q3wqwprrggh85z8kfg715"; + rev = "b61f619aa15efa2fa8c88d7eeae6154c3fb3918a"; + sha256 = "0ivrv56a8yzy68d9hr7cvy7hk5w0i2hidzlnns23h0g60ynb3axq"; }; libName = "stackable_telemetry"; authors = [ @@ -11134,8 +11134,8 @@ rec { workspace_member = null; src = pkgs.fetchgit { url = "https://github.com/stackabletech/operator-rs.git"; - rev = "63760c6a00a9b6c0fa564201c2b442307b5e7069"; - sha256 = "1962ivywrqyzjxa29w6i6gkhblaqd74q3wqwprrggh85z8kfg715"; + rev = "b61f619aa15efa2fa8c88d7eeae6154c3fb3918a"; + sha256 = "0ivrv56a8yzy68d9hr7cvy7hk5w0i2hidzlnns23h0g60ynb3axq"; }; libName = "stackable_versioned"; authors = [ @@ -11180,8 +11180,8 @@ rec { workspace_member = null; src = pkgs.fetchgit { url = "https://github.com/stackabletech/operator-rs.git"; - rev = "63760c6a00a9b6c0fa564201c2b442307b5e7069"; - sha256 = "1962ivywrqyzjxa29w6i6gkhblaqd74q3wqwprrggh85z8kfg715"; + rev = "b61f619aa15efa2fa8c88d7eeae6154c3fb3918a"; + sha256 = "0ivrv56a8yzy68d9hr7cvy7hk5w0i2hidzlnns23h0g60ynb3axq"; }; procMacro = true; libName = "stackable_versioned_macros"; @@ -11244,8 +11244,8 @@ rec { workspace_member = null; src = pkgs.fetchgit { url = "https://github.com/stackabletech/operator-rs.git"; - rev = "63760c6a00a9b6c0fa564201c2b442307b5e7069"; - sha256 = "1962ivywrqyzjxa29w6i6gkhblaqd74q3wqwprrggh85z8kfg715"; + rev = "b61f619aa15efa2fa8c88d7eeae6154c3fb3918a"; + sha256 = "0ivrv56a8yzy68d9hr7cvy7hk5w0i2hidzlnns23h0g60ynb3axq"; }; libName = "stackable_webhook"; authors = [ diff --git a/crate-hashes.json b/crate-hashes.json index fd83bfb02..f2a7eec62 100644 --- a/crate-hashes.json +++ b/crate-hashes.json @@ -1,12 +1,12 @@ { - "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#k8s-version@0.1.4": "1962ivywrqyzjxa29w6i6gkhblaqd74q3wqwprrggh85z8kfg715", - "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#stackable-agent@0.1.0": "1962ivywrqyzjxa29w6i6gkhblaqd74q3wqwprrggh85z8kfg715", - "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#stackable-certs@0.4.1": "1962ivywrqyzjxa29w6i6gkhblaqd74q3wqwprrggh85z8kfg715", - "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#stackable-operator-derive@0.3.1": "1962ivywrqyzjxa29w6i6gkhblaqd74q3wqwprrggh85z8kfg715", - "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#stackable-operator@0.121.0": "1962ivywrqyzjxa29w6i6gkhblaqd74q3wqwprrggh85z8kfg715", - "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#stackable-shared@0.2.0": "1962ivywrqyzjxa29w6i6gkhblaqd74q3wqwprrggh85z8kfg715", - "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#stackable-telemetry@0.6.5": "1962ivywrqyzjxa29w6i6gkhblaqd74q3wqwprrggh85z8kfg715", - "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#stackable-versioned-macros@0.13.0": "1962ivywrqyzjxa29w6i6gkhblaqd74q3wqwprrggh85z8kfg715", - "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#stackable-versioned@0.13.0": "1962ivywrqyzjxa29w6i6gkhblaqd74q3wqwprrggh85z8kfg715", - "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#stackable-webhook@0.10.0": "1962ivywrqyzjxa29w6i6gkhblaqd74q3wqwprrggh85z8kfg715" + "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#k8s-version@0.1.4": "0ivrv56a8yzy68d9hr7cvy7hk5w0i2hidzlnns23h0g60ynb3axq", + "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#stackable-agent@0.1.0": "0ivrv56a8yzy68d9hr7cvy7hk5w0i2hidzlnns23h0g60ynb3axq", + "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#stackable-certs@0.4.1": "0ivrv56a8yzy68d9hr7cvy7hk5w0i2hidzlnns23h0g60ynb3axq", + "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#stackable-operator-derive@0.3.1": "0ivrv56a8yzy68d9hr7cvy7hk5w0i2hidzlnns23h0g60ynb3axq", + "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#stackable-operator@0.121.0": "0ivrv56a8yzy68d9hr7cvy7hk5w0i2hidzlnns23h0g60ynb3axq", + "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#stackable-shared@0.2.0": "0ivrv56a8yzy68d9hr7cvy7hk5w0i2hidzlnns23h0g60ynb3axq", + "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#stackable-telemetry@0.6.5": "0ivrv56a8yzy68d9hr7cvy7hk5w0i2hidzlnns23h0g60ynb3axq", + "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#stackable-versioned-macros@0.13.0": "0ivrv56a8yzy68d9hr7cvy7hk5w0i2hidzlnns23h0g60ynb3axq", + "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#stackable-versioned@0.13.0": "0ivrv56a8yzy68d9hr7cvy7hk5w0i2hidzlnns23h0g60ynb3axq", + "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#stackable-webhook@0.10.0": "0ivrv56a8yzy68d9hr7cvy7hk5w0i2hidzlnns23h0g60ynb3axq" } \ No newline at end of file From f539947a436e4cfe3284916bf552ae7f090cbb0d Mon Sep 17 00:00:00 2001 From: Benedikt Labrenz Date: Wed, 28 Oct 2026 18:10:55 +0100 Subject: [PATCH 60/93] remove OPA from docs and test --- .../pages/usage-guide/management-access.adoc | 2 +- .../kuttl/management-access/07-assert.yaml | 7 ---- .../management-access/07-install-opa.yaml.j2 | 34 ----------------- .../kuttl/management-access/07_kafka.rego.j2 | 37 ------------------- .../kuttl/management-access/10_kafka.yaml.j2 | 4 -- .../kuttl/management-access/30-assert.yaml | 25 ------------- .../30-create-forbidden-topic.yaml | 9 ----- 7 files changed, 1 insertion(+), 117 deletions(-) delete mode 100644 tests/templates/kuttl/management-access/07-assert.yaml delete mode 100644 tests/templates/kuttl/management-access/07-install-opa.yaml.j2 delete mode 100644 tests/templates/kuttl/management-access/07_kafka.rego.j2 delete mode 100644 tests/templates/kuttl/management-access/30-assert.yaml delete mode 100644 tests/templates/kuttl/management-access/30-create-forbidden-topic.yaml diff --git a/docs/modules/kafka/pages/usage-guide/management-access.adoc b/docs/modules/kafka/pages/usage-guide/management-access.adoc index eebeb13e1..650db4b51 100644 --- a/docs/modules/kafka/pages/usage-guide/management-access.adoc +++ b/docs/modules/kafka/pages/usage-guide/management-access.adoc @@ -63,7 +63,7 @@ The agent verifies the brokers with the CA of the SecretClass used by the client == Authorization -The principal of the agent must be allowed to manage topics in Kafka, for example by xref:usage-guide/security.adoc#authorization[OPA authorization]. +The principal of the agent must be allowed to manage topics in Kafka. The principal is derived from the subject of the client certificate. The agent needs the following permissions: diff --git a/tests/templates/kuttl/management-access/07-assert.yaml b/tests/templates/kuttl/management-access/07-assert.yaml deleted file mode 100644 index e1112367e..000000000 --- a/tests/templates/kuttl/management-access/07-assert.yaml +++ /dev/null @@ -1,7 +0,0 @@ ---- -# OPA authorizes the brokers and the agent with the documented permissions. -apiVersion: kuttl.dev/v1beta1 -kind: TestAssert -timeout: 300 -commands: - - script: kubectl wait --namespace $NAMESPACE --for=condition=available opaclusters.opa.stackable.tech/test-opa --timeout 301s diff --git a/tests/templates/kuttl/management-access/07-install-opa.yaml.j2 b/tests/templates/kuttl/management-access/07-install-opa.yaml.j2 deleted file mode 100644 index f235db18a..000000000 --- a/tests/templates/kuttl/management-access/07-install-opa.yaml.j2 +++ /dev/null @@ -1,34 +0,0 @@ ---- -apiVersion: kuttl.dev/v1beta1 -kind: TestStep -commands: - - script: | - envsubst '$NAMESPACE' < 07_kafka.rego | - kubectl create configmap test-rego-kafka --namespace $NAMESPACE --from-file=kafka.rego=/dev/stdin - - script: kubectl label configmap test-rego-kafka --namespace $NAMESPACE opa.stackable.tech/bundle=true ---- -apiVersion: opa.stackable.tech/v1alpha1 -kind: OpaCluster -metadata: - name: test-opa -spec: - image: -{% if test_scenario['values']['opa-latest'].find(",") > 0 %} - custom: "{{ test_scenario['values']['opa-latest'].split(',')[1] }}" - productVersion: "{{ test_scenario['values']['opa-latest'].split(',')[0] }}" -{% else %} - productVersion: "{{ test_scenario['values']['opa-latest'] }}" -{% endif %} - pullPolicy: IfNotPresent - servers: - config: - logging: - containers: - opa: - console: - level: INFO - loggers: - decision: - level: INFO - roleGroups: - default: {} diff --git a/tests/templates/kuttl/management-access/07_kafka.rego.j2 b/tests/templates/kuttl/management-access/07_kafka.rego.j2 deleted file mode 100644 index 4121aa2e9..000000000 --- a/tests/templates/kuttl/management-access/07_kafka.rego.j2 +++ /dev/null @@ -1,37 +0,0 @@ -{% set authentication = test_scenario['values']['management-access-authentication'] %} -package authz - -broker_principal_pattern := `^DC=local,DC=cluster,DC=svc,DC=$NAMESPACE,DC=test-kafka-broker-default-headless,DC=test-kafka-broker-default-[0-9]+,CN=generated certificate for pod$` - -{% if authentication == 'anonymous' %} -agent_principal_pattern := `^ANONYMOUS$` -{% elif authentication == 'secret-class' %} -# The agent Pod has no subdomain, so its subject DN contains no Service. -agent_principal_pattern := `^DC=local,DC=cluster,DC=svc,DC=$NAMESPACE,DC=test-kafka-agent-[a-z0-9]+-[a-z0-9]+,CN=generated certificate for pod$` -{% else %} -agent_principal_pattern := `^CN=kafka-agent$` -{% endif %} - -default allow := false - -allow if { - input.requestContext.listenerName == "INTERNAL" - regex.match(broker_principal_pattern, input.requestContext.principal.name) -} - -allow if { - regex.match(agent_principal_pattern, input.requestContext.principal.name) - agent_action_is_allowed -} - -# The permissions of the agent as documented, limited to topics starting with "orders" -agent_action_is_allowed if { - input.action.resourcePattern.resourceType == "TOPIC" - startswith(input.action.resourcePattern.name, "orders") - input.action.operation in ["DESCRIBE", "CREATE", "ALTER", "DESCRIBE_CONFIGS", "ALTER_CONFIGS", "DELETE"] -} - -agent_action_is_allowed if { - input.action.resourcePattern.resourceType == "CLUSTER" - input.action.operation == "DESCRIBE" -} diff --git a/tests/templates/kuttl/management-access/10_kafka.yaml.j2 b/tests/templates/kuttl/management-access/10_kafka.yaml.j2 index 58608e7be..ca2976982 100644 --- a/tests/templates/kuttl/management-access/10_kafka.yaml.j2 +++ b/tests/templates/kuttl/management-access/10_kafka.yaml.j2 @@ -51,10 +51,6 @@ spec: authentication: - authenticationClass: client-auth-$NAMESPACE {% endif %} - authorization: - opa: - configMapName: test-opa - package: authz managementAccess: enabled: true authentication: diff --git a/tests/templates/kuttl/management-access/30-assert.yaml b/tests/templates/kuttl/management-access/30-assert.yaml deleted file mode 100644 index 56bcaf4d1..000000000 --- a/tests/templates/kuttl/management-access/30-assert.yaml +++ /dev/null @@ -1,25 +0,0 @@ ---- -# Kafka rejects the requests of the agent, and the KafkaTopic reports why. -apiVersion: kuttl.dev/v1beta1 -kind: TestAssert -timeout: 300 -resourceRefs: - - apiVersion: kafka.stackable.tech/v1alpha1 - kind: KafkaTopic - name: forbidden - ref: forbidden -assertAll: - - celExpr: "forbidden.status.conditions.exists(c, c.type == 'Synced' && c.message.contains('TopicAuthorizationFailed'))" ---- -apiVersion: kafka.stackable.tech/v1alpha1 -kind: KafkaTopic -metadata: - name: forbidden -status: - conditions: - - type: Ready - status: Unknown - reason: KafkaRejected - - type: Synced - status: "False" - reason: KafkaRejected diff --git a/tests/templates/kuttl/management-access/30-create-forbidden-topic.yaml b/tests/templates/kuttl/management-access/30-create-forbidden-topic.yaml deleted file mode 100644 index bc5e73322..000000000 --- a/tests/templates/kuttl/management-access/30-create-forbidden-topic.yaml +++ /dev/null @@ -1,9 +0,0 @@ ---- -# OPA does not allow the agent to manage this topic. -apiVersion: kafka.stackable.tech/v1alpha1 -kind: KafkaTopic -metadata: - name: forbidden -spec: - clusterRef: - name: test-kafka From 0f390031254834736c248467f3320cbae7854fd7 Mon Sep 17 00:00:00 2001 From: Benedikt Labrenz Date: Wed, 28 Oct 2026 18:25:07 +0100 Subject: [PATCH 61/93] write status if failing during finalizer reconciliation --- .../kafka/pages/usage-guide/topics.adoc | 3 +- rust/agent-binary/src/topic/controller.rs | 31 ++++++++++++++----- 2 files changed, 25 insertions(+), 9 deletions(-) diff --git a/docs/modules/kafka/pages/usage-guide/topics.adoc b/docs/modules/kafka/pages/usage-guide/topics.adoc index 67d27d876..1578137f3 100644 --- a/docs/modules/kafka/pages/usage-guide/topics.adoc +++ b/docs/modules/kafka/pages/usage-guide/topics.adoc @@ -125,7 +125,8 @@ With the default `deletionPolicy: Retain`, deleting a KafkaTopic leaves the topi With `deletionPolicy: Delete`, the agent deletes the topic in Kafka when the KafkaTopic is deleted, unless another KafkaTopic declares the same topic. The KafkaTopic gets a finalizer once the agent manages the topic, so the deletion waits until the topic is deleted. If the reconciliation of the KafkaTopic is paused, the deletion waits until it is resumed. -If the brokers cannot be reached, for example while the KafkaCluster is xref:usage-guide/operations/cluster-operations.adoc[stopped], switching the KafkaTopic to `deletionPolicy: Retain` releases the deletion. +If the brokers cannot be reached, for example while the KafkaCluster is xref:usage-guide/operations/cluster-operations.adoc[stopped], the KafkaTopic reports `ConnectionFailed` and the agent retries the deletion. +Switching the KafkaTopic to `deletionPolicy: Retain` releases the deletion. When the KafkaCluster or its namespace is deleted, the operator removes the finalizers of its KafkaTopics, so the deletion does not block. The topics are not deleted in Kafka in this case. diff --git a/rust/agent-binary/src/topic/controller.rs b/rust/agent-binary/src/topic/controller.rs index cdc9a4966..1e7203b2e 100644 --- a/rust/agent-binary/src/topic/controller.rs +++ b/rust/agent-binary/src/topic/controller.rs @@ -20,6 +20,7 @@ use tracing::info; use super::{ condition::TopicSyncOutcome, finalizer::{add_finalizer, reconcile_finalizer}, + plan::TopicState, sync::sync, update_status::update_status, }; @@ -103,8 +104,12 @@ pub async fn reconcile_topic( // Left untouched while paused, like operators that only update the status then. A deletion // waits until the reconciliation is resumed. let paused = kafka_topic.spec.topic_operation.reconciliation_paused; - if !paused && let Some(action) = reconcile_finalizer(kafka_topic, &ctx).await? { - return Ok(action); + if !paused { + match reconcile_finalizer(kafka_topic, &ctx).await { + Ok(Some(action)) => return Ok(action), + Ok(None) => {} + Err(error) => return fail_with_status(kafka_topic, &ctx, error, None).await, + } } // sync (Kafka) @@ -123,13 +128,23 @@ pub async fn reconcile_topic( update_status(kafka_topic, &ctx, &sync_outcome, topic_state.as_ref()).await?; Ok(requeue(&sync_outcome)) } - Err(Error::Kafka { source }) => { - let sync_outcome = TopicSyncOutcome::from_client_error(&source); - update_status(kafka_topic, &ctx, &sync_outcome, topic_state.as_ref()).await?; - Err(Error::Kafka { source }) - } - Err(error) => Err(error), + Err(error) => fail_with_status(kafka_topic, &ctx, error, topic_state.as_ref()).await, + } +} + +/// Reports a Kafka error in the status of the `KafkaTopic` before returning it. Other errors are +/// returned as they are. +async fn fail_with_status( + kafka_topic: &KafkaTopic, + ctx: &Ctx, + error: Error, + topic_state: Option<&TopicState>, +) -> Result { + if let Error::Kafka { source } = &error { + let sync_outcome = TopicSyncOutcome::from_client_error(source); + update_status(kafka_topic, ctx, &sync_outcome, topic_state).await?; } + Err(error) } fn requeue(sync_outcome: &TopicSyncOutcome) -> Action { From d4cb53a88f42d583a831a2fdd436ccf634c1df05 Mon Sep 17 00:00:00 2001 From: Benedikt Labrenz Date: Wed, 28 Oct 2026 18:33:30 +0100 Subject: [PATCH 62/93] improve docs on deletion --- docs/modules/kafka/pages/usage-guide/topics.adoc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/modules/kafka/pages/usage-guide/topics.adoc b/docs/modules/kafka/pages/usage-guide/topics.adoc index 1578137f3..c060cdb98 100644 --- a/docs/modules/kafka/pages/usage-guide/topics.adoc +++ b/docs/modules/kafka/pages/usage-guide/topics.adoc @@ -125,12 +125,12 @@ With the default `deletionPolicy: Retain`, deleting a KafkaTopic leaves the topi With `deletionPolicy: Delete`, the agent deletes the topic in Kafka when the KafkaTopic is deleted, unless another KafkaTopic declares the same topic. The KafkaTopic gets a finalizer once the agent manages the topic, so the deletion waits until the topic is deleted. If the reconciliation of the KafkaTopic is paused, the deletion waits until it is resumed. -If the brokers cannot be reached, for example while the KafkaCluster is xref:usage-guide/operations/cluster-operations.adoc[stopped], the KafkaTopic reports `ConnectionFailed` and the agent retries the deletion. +If the brokers cannot be reached, the KafkaTopic reports `ConnectionFailed` and the agent retries the deletion. Switching the KafkaTopic to `deletionPolicy: Retain` releases the deletion. When the KafkaCluster or its namespace is deleted, the operator removes the finalizers of its KafkaTopics, so the deletion does not block. The topics are not deleted in Kafka in this case. If the KafkaCluster is created again with the same PersistentVolumeClaims, its topics still exist, but without KafkaTopics. -If the KafkaCluster exists but has no agent, for example because management access is disabled, the deletion waits for the agent. +If the KafkaCluster exists but has no agent, for example because management access is disabled or the KafkaCluster is xref:usage-guide/operations/cluster-operations.adoc[stopped], the deletion waits for the agent. In this case, the finalizer `kafka.stackable.tech/kafkatopic` can be removed manually, which leaves the topic in Kafka. From c0f9338964b28f5b872a93fd4b552484a90d9ee3 Mon Sep 17 00:00:00 2001 From: Benedikt Labrenz Date: Wed, 28 Oct 2026 18:38:21 +0100 Subject: [PATCH 63/93] report a stopped Kafka cluster as unavailable --- rust/operator-binary/src/topic_controller.rs | 28 +++++++++++++++++--- 1 file changed, 25 insertions(+), 3 deletions(-) diff --git a/rust/operator-binary/src/topic_controller.rs b/rust/operator-binary/src/topic_controller.rs index b7492c94d..11aa540b5 100644 --- a/rust/operator-binary/src/topic_controller.rs +++ b/rust/operator-binary/src/topic_controller.rs @@ -99,7 +99,7 @@ enum ClusterState { #[derive(Debug, Eq, PartialEq)] enum AgentState { Available, - /// `None` if the agent Deployment does not exist + /// `None` if the agent Deployment does not exist or has no replicas UnavailableSince(Option), } @@ -243,11 +243,21 @@ fn operator_action_for( } } -/// A Deployment that has no `Available` condition yet is unavailable since its creation. +/// A Deployment that has no `Available` condition yet is unavailable since its creation. One with +/// zero replicas, e.g. of a stopped KafkaCluster, is unavailable at once, although Kubernetes +/// reports it as `Available`. fn agent_state_from(agent_deployment: Option<&Deployment>) -> AgentState { let Some(agent_deployment) = agent_deployment else { return AgentState::UnavailableSince(None); }; + if agent_deployment + .spec + .as_ref() + .and_then(|spec| spec.replicas) + == Some(0) + { + return AgentState::UnavailableSince(None); + } let available_condition = agent_deployment .status .as_ref() @@ -353,7 +363,7 @@ async fn remove_finalizer(kafka_topic: &KafkaTopic, ctx: &Ctx) -> Result<()> { #[cfg(test)] mod tests { use rstest::rstest; - use stackable_operator::kube::api::ObjectMeta; + use stackable_operator::{k8s_openapi::api::apps::v1::DeploymentSpec, kube::api::ObjectMeta}; use super::*; @@ -496,9 +506,21 @@ mod tests { .expect("the Deployment should be valid") } + fn scaled_to_zero(mut agent_deployment: Deployment) -> Deployment { + agent_deployment.spec = Some(DeploymentSpec { + replicas: Some(0), + ..DeploymentSpec::default() + }); + agent_deployment + } + #[rstest] #[case::missing(None, AgentState::UnavailableSince(None))] #[case::available(Some(agent_deployment(Some("True"))), AgentState::Available)] + #[case::scaled_to_zero( + Some(scaled_to_zero(agent_deployment(Some("True")))), + AgentState::UnavailableSince(None) + )] #[case::unavailable_since_the_transition( Some(agent_deployment(Some("False"))), AgentState::UnavailableSince(Some(Timestamp::from_second(900).expect("valid"))) From 08c6a41682f83ebb80100a9c1faf6d1ce7ecdcac Mon Sep 17 00:00:00 2001 From: Benedikt Labrenz Date: Wed, 28 Oct 2026 18:50:17 +0100 Subject: [PATCH 64/93] derive the topic configs from the applied changes --- rust/agent-binary/src/topic/plan.rs | 38 +++++++++++++++++++++++++++++ rust/agent-binary/src/topic/sync.rs | 18 ++++++-------- 2 files changed, 46 insertions(+), 10 deletions(-) diff --git a/rust/agent-binary/src/topic/plan.rs b/rust/agent-binary/src/topic/plan.rs index ca70fca83..2fea70fd8 100644 --- a/rust/agent-binary/src/topic/plan.rs +++ b/rust/agent-binary/src/topic/plan.rs @@ -49,6 +49,21 @@ impl TopicState { topic_configs: created.configs, } } + + /// The topic-level configs as they are after the changes are applied. + pub fn apply_config_changes( + &mut self, + configs_to_set: &BTreeMap, + configs_to_reset: &BTreeSet, + ) { + self.topic_configs + .retain(|key, _| !configs_to_reset.contains(key)); + self.topic_configs.extend( + configs_to_set + .iter() + .map(|(key, value)| (key.clone(), value.clone())), + ); + } } #[derive(Debug, Eq, PartialEq)] @@ -211,6 +226,8 @@ fn target_replication_factors(observed: &ObservedTopic) -> Vec { #[cfg(test)] mod tests { + use std::str::FromStr; + use stackable_kafka_admin::client::Partition; use super::*; @@ -379,6 +396,27 @@ mod tests { ); } + #[test] + fn apply_config_changes_to_the_topic_state() { + let mut topic_state = TopicState::observed( + TopicName::from_str("orders").expect("the topic name should be valid"), + &observed( + &[1], + &[("retention.ms", "1000"), ("cleanup.policy", "compact")], + ), + ); + + topic_state.apply_config_changes( + &configs(&[("retention.ms", "2000"), ("max.message.bytes", "10")]), + &keys(&["cleanup.policy"]), + ); + + assert_eq!( + configs(&[("max.message.bytes", "10"), ("retention.ms", "2000")]), + topic_state.topic_configs + ); + } + #[test] fn report_a_different_replication_factor_as_unsupported() { let observed = observed(&[3, 3], &[]); diff --git a/rust/agent-binary/src/topic/sync.rs b/rust/agent-binary/src/topic/sync.rs index 97f980288..14bc9b653 100644 --- a/rust/agent-binary/src/topic/sync.rs +++ b/rust/agent-binary/src/topic/sync.rs @@ -136,7 +136,7 @@ async fn execute( // Config if !topic_update.configs_to_set.is_empty() || !topic_update.configs_to_reset.is_empty() { - let topic_configs = replace_configs( + alter_configs( kafka_topic, admin_client, &topic_name, @@ -145,7 +145,10 @@ async fn execute( ) .await?; if let Some(topic_state) = topic_state { - topic_state.topic_configs = topic_configs; + topic_state.apply_config_changes( + &topic_update.configs_to_set, + &topic_update.configs_to_reset, + ); } } } @@ -204,15 +207,13 @@ async fn grow_partitions( Ok(()) } -/// Returns the topic-level configs after the change, described again because -/// IncrementalAlterConfigs does not return them. -async fn replace_configs( +async fn alter_configs( kafka_topic: &KafkaTopic, admin_client: &mut KafkaAdminClient, topic_name: &TopicName, configs_to_set: &BTreeMap, configs_to_reset: &BTreeSet, -) -> Result> { +) -> Result<()> { admin_client .incremental_alter_configs(topic_name.as_ref(), configs_to_set, configs_to_reset) .await @@ -224,8 +225,5 @@ async fn replace_configs( kafka_topic = %kafka_topic.name_any(), "Changed the topic configs" ); - admin_client - .describe_topic_configs(topic_name.as_ref()) - .await - .context(KafkaSnafu) + Ok(()) } From b5be3c18bf6214cc56a855516d9b4c58c4d69500 Mon Sep 17 00:00:00 2001 From: Benedikt Labrenz Date: Wed, 28 Oct 2026 21:04:07 +0100 Subject: [PATCH 65/93] add pull secret to docs --- docs/modules/kafka/pages/usage-guide/management-access.adoc | 4 ++++ docs/modules/kafka/pages/usage-guide/topics.adoc | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/docs/modules/kafka/pages/usage-guide/management-access.adoc b/docs/modules/kafka/pages/usage-guide/management-access.adoc index 650db4b51..5373e7976 100644 --- a/docs/modules/kafka/pages/usage-guide/management-access.adoc +++ b/docs/modules/kafka/pages/usage-guide/management-access.adoc @@ -8,6 +8,9 @@ Management access allows the Stackable Data Platform (SDP) to manage resources i The agent is an optional Deployment next to the Kafka cluster that manages resources inside Kafka, such as topics. The operator deploys it when management access is enabled. +The agent runs the image of the operator, pulled with the pull secrets of the Kafka image in `spec.image.pullSecrets`. +If the operator image is not public, these pull secrets must grant access to its registry as well, or the Deployment of the agent gets its own pull secrets with <>. + == Limitations Management access is not available for all clusters: @@ -88,6 +91,7 @@ The KafkaTopic then reports `KafkaRejected`. The agent connects to the headless Services of the broker role groups, which resolve to the broker Pods. The traffic of the agent therefore stays within the Kubernetes cluster, regardless of the xref:usage-guide/listenerclass.adoc[ListenerClasses] of the cluster. +[[agent-resources]] == Agent resources The operator creates the following resources for the agent, named after the KafkaCluster: diff --git a/docs/modules/kafka/pages/usage-guide/topics.adoc b/docs/modules/kafka/pages/usage-guide/topics.adoc index c060cdb98..c5501fd80 100644 --- a/docs/modules/kafka/pages/usage-guide/topics.adoc +++ b/docs/modules/kafka/pages/usage-guide/topics.adoc @@ -102,7 +102,7 @@ status: The `Ready` condition shows whether the topic exists in Kafka. The `Synced` condition shows whether the topic matches the KafkaTopic. -If it does not, the reason and message of the condition explain why, for example an unsupported change, a request rejected by Kafka or a paused reconciliation. +If it does not, the reason and message of the condition explain why, for example an unsupported change, a request rejected by Kafka, a paused reconciliation or an agent that is not running. == Pausing From 77eb95a86d75d4ffa68b1b9b4169199dad0e8ce5 Mon Sep 17 00:00:00 2001 From: Benedikt Labrenz Date: Wed, 28 Oct 2026 21:05:46 +0100 Subject: [PATCH 66/93] remove the unreachable throttle filter from the planned config changes --- rust/agent-binary/src/topic/plan.rs | 13 +++---------- 1 file changed, 3 insertions(+), 10 deletions(-) diff --git a/rust/agent-binary/src/topic/plan.rs b/rust/agent-binary/src/topic/plan.rs index 2fea70fd8..a204f137e 100644 --- a/rust/agent-binary/src/topic/plan.rs +++ b/rust/agent-binary/src/topic/plan.rs @@ -164,7 +164,7 @@ fn unsupported_changes(spec: &KafkaTopicSpec, observed: &ObservedTopic) -> Vec, current_topic_configs: &BTreeMap, @@ -174,10 +174,7 @@ fn config_changes( // without drift does not write to Kafka. let configs_to_set = desired_topic_configs .iter() - .filter(|(key, value)| { - current_topic_configs.get(*key) != Some(value) - && !THROTTLE_CONFIG_KEYS.contains(&key.as_str()) - }) + .filter(|(key, value)| current_topic_configs.get(*key) != Some(value)) .map(|(key, value)| (key.clone(), value.clone())) .collect(); let configs_to_reset = current_topic_configs @@ -370,11 +367,7 @@ mod tests { &spec( None, None, - &[ - ("retention.ms", "2000"), - ("max.message.bytes", "10"), - ("follower.replication.throttled.replicas", "*"), - ], + &[("retention.ms", "2000"), ("max.message.bytes", "10")], ), Some(&observed), ); From af91710c7efb6965fd619400f2b7587df2d8292f Mon Sep 17 00:00:00 2001 From: Benedikt Labrenz Date: Wed, 28 Oct 2026 21:13:52 +0100 Subject: [PATCH 67/93] clarify the test-support feature of the shared crate --- rust/shared/Cargo.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/rust/shared/Cargo.toml b/rust/shared/Cargo.toml index 9eedd8c66..0e51994c1 100644 --- a/rust/shared/Cargo.toml +++ b/rust/shared/Cargo.toml @@ -22,4 +22,5 @@ indoc.workspace = true rstest.workspace = true [features] +# Referenced by the attributed_string_type! macro of stackable-operator test-support = [] From 45b8a8cb6f6439c2624cbeb76a7cc29feb86722e Mon Sep 17 00:00:00 2001 From: Benedikt Labrenz Date: Wed, 28 Oct 2026 21:21:38 +0100 Subject: [PATCH 68/93] document that the admin client serializes the Kafka requests --- rust/agent-binary/src/topic/controller.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/rust/agent-binary/src/topic/controller.rs b/rust/agent-binary/src/topic/controller.rs index 1e7203b2e..e9b0f7f6a 100644 --- a/rust/agent-binary/src/topic/controller.rs +++ b/rust/agent-binary/src/topic/controller.rs @@ -67,7 +67,8 @@ pub struct Ctx { pub(crate) client: stackable_operator::client::Client, pub(crate) kafka_topics: Store>, /// Reads the TLS files on every new connection and reconnects after a failed request, so - /// rotated certificates are picked up without restarting the agent. + /// rotated certificates are picked up without restarting the agent. The Kafka requests of + /// all reconciles are serialized on its single connection. pub(crate) admin_client: Mutex, } From a326d599ce3950a457cbf32ebc72b3e261079e2b Mon Sep 17 00:00:00 2001 From: Benedikt Labrenz Date: Wed, 28 Oct 2026 21:38:40 +0100 Subject: [PATCH 69/93] retry broken connection requests immediatly with a new connection --- rust/kafka-admin/src/client.rs | 54 +++++++++++++++++++++++++----- rust/kafka-admin/src/connection.rs | 34 ++++++++++++------- 2 files changed, 67 insertions(+), 21 deletions(-) diff --git a/rust/kafka-admin/src/client.rs b/rust/kafka-admin/src/client.rs index 0620aaeff..7766ca2d7 100644 --- a/rust/kafka-admin/src/client.rs +++ b/rust/kafka-admin/src/client.rs @@ -263,17 +263,29 @@ impl KafkaAdminClient { list_partition_reassignments_result(response, topic) } - /// Sends the request on the current connection or a new one. A failed or cancelled request - /// drops the connection, and a connection idle for longer than [`CONNECTION_MAX_IDLE`] is - /// replaced. + /// Sends the request on the current connection or a new one. A request that fails because + /// the broker closed the reused connection is sent once more on a new one. A failed or + /// cancelled request drops the connection, and a connection idle for longer than + /// [`CONNECTION_MAX_IDLE`] is replaced. async fn send(&mut self, request: R, version: i16) -> Result { - let mut connection = match self.connection.take() { - Some((connection, last_used)) if last_used.elapsed() < CONNECTION_MAX_IDLE => { - connection + if let Some((mut connection, last_used)) = self.connection.take() + && last_used.elapsed() < CONNECTION_MAX_IDLE + { + match connection.send(&request, version).await { + Err(error) if error.is_broken_connection() => {} + result => { + let response = result.context(SendSnafu)?; + self.connection = Some((connection, Instant::now())); + return Ok(response); + } } - _ => self.connect().await?, - }; - let response = connection.send(request, version).await.context(SendSnafu)?; + } + // After a broken connections the request is directly send on a new connection again + let mut connection = self.connect().await?; + let response = connection + .send(&request, version) + .await + .context(SendSnafu)?; self.connection = Some((connection, Instant::now())); Ok(response) } @@ -617,6 +629,30 @@ mod tests { result.expect("the client should reconnect"); } + #[tokio::test] + async fn retry_once_if_the_broker_closed_the_reused_connection() { + let (listener, port) = listen().await; + let mut client = kafka_admin_client(&[port]); + + let broker_side = async { + let mut broker = accept(&listener, APIS).await; + answer_delete_topics(&mut broker).await; + drop(broker); + let mut broker = accept(&listener, APIS).await; + answer_delete_topics(&mut broker).await; + }; + let client_side = async { + client + .delete_topic(TOPIC) + .await + .expect("the first request should succeed"); + client.delete_topic(TOPIC).await + }; + let (result, ()) = tokio::join!(client_side, broker_side); + + result.expect("the client should retry on a new connection"); + } + #[test] fn describe_topic_by_name_without_auto_creation() { let request = describe_topic_request(TOPIC); diff --git a/rust/kafka-admin/src/connection.rs b/rust/kafka-admin/src/connection.rs index efb6a606f..c678fd797 100644 --- a/rust/kafka-admin/src/connection.rs +++ b/rust/kafka-admin/src/connection.rs @@ -101,6 +101,16 @@ pub enum Error { }, } +impl Error { + /// Whether the request failed on the socket, so that a new connection may succeed. + pub fn is_broken_connection(&self) -> bool { + matches!( + self, + Error::Write { .. } | Error::Read { .. } | Error::ConnectionClosed + ) + } +} + type Result = std::result::Result; /// How to connect to the broker. Paths point to PEM files. @@ -140,7 +150,7 @@ impl Connection { let mut connection = Self::new(stream, client_id); let response = connection - .send_with_version(api_versions_request(), API_VERSIONS_VERSION) + .send_with_version(&api_versions_request(), API_VERSIONS_VERSION) .await?; connection.broker_api_versions = api_versions_result(response)?; Ok(connection) @@ -162,7 +172,7 @@ impl Connection { /// Sends the request with the given version, if the broker supports it. pub(crate) async fn send( &mut self, - request: R, + request: &R, version: i16, ) -> Result { // Each API that is relevant for us has a version that is supported by all of our Kafka versions. @@ -174,11 +184,11 @@ impl Connection { async fn send_with_version( &mut self, - request: R, + request: &R, version: i16, ) -> Result { self.correlation_id = self.correlation_id.wrapping_add(1); - let frame = encode_request(&request, version, self.correlation_id, &self.client_id) + let frame = encode_request(request, version, self.correlation_id, &self.client_id) .context(CodecSnafu)?; let response = timeout(REQUEST_TIMEOUT, async { @@ -497,8 +507,8 @@ mod tests { ) .await; }; - let (response, ()) = - tokio::join!(connection.send(MetadataRequest::default(), 12), broker_side); + let request = MetadataRequest::default(); + let (response, ()) = tokio::join!(connection.send(&request, 12), broker_side); assert_eq!( 2, @@ -520,8 +530,8 @@ mod tests { ) .await; }; - let (response, ()) = - tokio::join!(connection.send(MetadataRequest::default(), 12), broker_side); + let request = MetadataRequest::default(); + let (response, ()) = tokio::join!(connection.send(&request, 12), broker_side); assert!(matches!( response.expect_err("the correlation id should not match"), @@ -542,8 +552,8 @@ mod tests { .await .expect("the frame size should be writable"); }; - let (response, ()) = - tokio::join!(connection.send(MetadataRequest::default(), 12), broker_side); + let request = MetadataRequest::default(); + let (response, ()) = tokio::join!(connection.send(&request, 12), broker_side); assert!(matches!( response.expect_err("the frame size should be invalid"), @@ -559,8 +569,8 @@ mod tests { let broker_side = async { read_request(&mut broker).await; }; - let (response, ()) = - tokio::join!(connection.send(MetadataRequest::default(), 12), broker_side); + let request = MetadataRequest::default(); + let (response, ()) = tokio::join!(connection.send(&request, 12), broker_side); assert!(matches!( response.expect_err("the request should time out"), From f57945604b6246ba4c27dc6b874f57323ee353cc Mon Sep 17 00:00:00 2001 From: Benedikt Labrenz Date: Thu, 29 Oct 2026 14:41:46 +0100 Subject: [PATCH 70/93] update operator-rs --- Cargo.lock | 374 +++++++------- Cargo.nix | 1212 ++++++++++++++++++++++----------------------- crate-hashes.json | 20 +- extra/crds.yaml | 2 +- 4 files changed, 782 insertions(+), 826 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index ac2068d2e..188851306 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -249,9 +249,9 @@ dependencies = [ [[package]] name = "base16ct" -version = "0.2.0" +version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4c7f02d4ea65f2c1853089ffd8d2787bdbc63de2f0d29dedbcf8ccdfa0ccd4cf" +checksum = "fd307490d624467aa6f74b0eabb77633d1f758a7b25f12bceb0b22e08d9726f6" [[package]] name = "base64" @@ -285,11 +285,11 @@ checksum = "3ded4057c258ba199e2d26386d3af3780957ecaee6c4ef4041c6b4b8b97c0b06" [[package]] name = "block-buffer" -version = "0.10.4" +version = "0.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" +checksum = "d2f6c7dbe95a6ed67ad9f18e57daf93a2f034c524b99fd2b76d18fdfeb6660aa" dependencies = [ - "generic-array", + "hybrid-array", ] [[package]] @@ -339,7 +339,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "65c35e4b699c7e15ccbe7ee35c005e4fc0a278d22238a2857e6ce2dadeda1b06" dependencies = [ "cfg-if", - "cpufeatures 0.3.1", + "cpufeatures", "rand_core 0.10.1", ] @@ -394,6 +394,12 @@ version = "1.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1c133bc6a41be0d194c306b5506d15e6feeea7b1d6604bd3f8310dfb2ca96486" +[[package]] +name = "cmov" +version = "0.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c9ea0ac24bc397ab3c98583a3c9ba74fa56b09a4449bbe172b9b1ddb016027a" + [[package]] name = "colorchoice" version = "1.0.5" @@ -412,15 +418,15 @@ dependencies = [ [[package]] name = "const-oid" -version = "0.9.6" +version = "0.10.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c2459377285ad874054d797f3ccebf984978aa39129f6eafde5cdc8315b612f8" +checksum = "a6ef517f0926dd24a1582492c791b6a4818a4d94e789a334894aa15b0d12f55c" [[package]] name = "const_format" -version = "0.2.36" +version = "0.2.37" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4481a617ad9a412be3b97c5d403fef8ed023103368908b9c50af598ff467cc1e" +checksum = "a3decb4d6965448659923544a634a0571ba44cede4e11ef15a5a317126840454" dependencies = [ "const_format_proc_macros", "konst", @@ -428,9 +434,9 @@ dependencies = [ [[package]] name = "const_format_proc_macros" -version = "0.2.34" +version = "0.2.37" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1d57c2eccfb16dbac1f4e61e206105db5820c9d26c3c472bc17c774259ef7744" +checksum = "941b570269a642506f08cb5b688c04bfa6cfd41a26f8bcdc49db8fd00c0ab659" dependencies = [ "proc-macro2", "quote", @@ -469,13 +475,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7f8f80099a98041a3d1622845c271458a2d73e688351bf3cb999266764b81d48" [[package]] -name = "cpufeatures" -version = "0.2.17" +name = "cpubits" +version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "59ed5838eebb26a2bb2e58f6d5b5316989ae9d08bab10e0e6d103e656d1b0280" -dependencies = [ - "libc", -] +checksum = "15b85f9c39137c3a891689859392b1bd49812121d0d61c9caf00d46ed5ce06ae" [[package]] name = "cpufeatures" @@ -512,24 +515,50 @@ checksum = "a31eee39dddec8330830986fcd7625edb5a24ec90ea038215273bbc3adb08ac6" [[package]] name = "crypto-bigint" -version = "0.5.5" +version = "0.7.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0dc92fb57ca44df6db8059111ab3af99a63d5d0f8375d9972e319a379c6bab76" +checksum = "9c035cb64cba03dad2fb921f3e218785e8cf9a41790da5eb3a1400f62fa2be11" dependencies = [ - "generic-array", - "rand_core 0.6.4", + "cpubits", + "ctutils", + "getrandom 0.4.3", + "hybrid-array", + "num-traits", + "rand_core 0.10.1", + "serdect", "subtle", "zeroize", ] [[package]] name = "crypto-common" -version = "0.1.6" +version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1bfb12502f3fc46cca1bb51ac28df9d618d813cdc3d2f25b9fe775a34af26bb3" +checksum = "ce6e4c961d6cd6c9a86db418387425e8bdeaf05b3c8bc1411e6dca4c252f1453" dependencies = [ - "generic-array", - "typenum", + "getrandom 0.4.3", + "hybrid-array", + "rand_core 0.10.1", +] + +[[package]] +name = "crypto-primes" +version = "0.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3633a51a39c69ebbaa4feaa694bd83d241e4093901c84a0963b19d9bb3f0cf8f" +dependencies = [ + "crypto-bigint", + "rand_core 0.10.1", +] + +[[package]] +name = "ctutils" +version = "0.4.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2376ffd6f96d8a9db59875b4b6b0237d6900b5baf64279714e05493ce64828a" +dependencies = [ + "cmov", + "subtle", ] [[package]] @@ -644,9 +673,9 @@ dependencies = [ [[package]] name = "der" -version = "0.7.10" +version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e7c1832837b905bbfb5101e07cc24c8deddf52f93225eee6ead5f4d63d53ddcb" +checksum = "6a996a108feb8b427a6e12fcf6af4aa83409b79d72c15337d7741368131ee004" dependencies = [ "const-oid", "der_derive", @@ -657,9 +686,9 @@ dependencies = [ [[package]] name = "der_derive" -version = "0.7.3" +version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8034092389675178f570469e6c3b0465d3d30b4505c294a6550db47f3c17ad18" +checksum = "59600e2c2d636fde9b65e99cc6445ac770c63d3628195ff39932b8d6d7409903" dependencies = [ "proc-macro2", "quote", @@ -695,14 +724,14 @@ dependencies = [ [[package]] name = "digest" -version = "0.10.7" +version = "0.11.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" +checksum = "f1dd6dbb5841937940781866fa1281a1ff7bd3bf827091440879f9994983d5c2" dependencies = [ "block-buffer", "const-oid", "crypto-common", - "subtle", + "ctutils", ] [[package]] @@ -744,9 +773,9 @@ checksum = "d0881ea181b1df73ff77ffaaf9c7544ecc11e82fba9b5f27b262a3c73a332555" [[package]] name = "ecdsa" -version = "0.16.9" +version = "0.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ee27f32b5c5292967d2d4a9d7f1e0b0aed2c15daded5a60300e4abb9d8020bca" +checksum = "c0681a4fc24c767085329728d8dfba959af91228aa4610cca4f8ce317ba46ae0" dependencies = [ "der", "digest", @@ -754,6 +783,7 @@ dependencies = [ "rfc6979", "signature", "spki", + "zeroize", ] [[package]] @@ -788,19 +818,20 @@ checksum = "0e9c71c2167ca323c882b99918929403426e2373ea17242ff5653e0d5e1058be" [[package]] name = "elliptic-curve" -version = "0.13.8" +version = "0.14.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b5e6043086bf7973472e0c7dff2142ea0b680d30e18d9cc40f267efbf222bd47" +checksum = "900f37e7e18d74a2347389af6cd69e3a30d5cc9e697f40004865679aa75e9c7c" dependencies = [ "base16ct", "crypto-bigint", + "crypto-common", "digest", "ff", - "generic-array", "group", + "hybrid-array", "pem-rfc7468", "pkcs8", - "rand_core 0.6.4", + "rand_core 0.10.1", "sec1", "subtle", "zeroize", @@ -902,11 +933,11 @@ checksum = "da7c62ceae207dd37ea5b845da6a0696c799f85e97da1ab5b7910be3c1c80223" [[package]] name = "ff" -version = "0.13.1" +version = "0.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c0b50bfb653653f9ca9095b427bed08ab8d75a137839d9ad64eb11810d5b6393" +checksum = "a1f686ab92a9fb0eaf188f6c6c87b89490baa6fdb0db4544ba4dc47f7942489f" dependencies = [ - "rand_core 0.6.4", + "rand_core 0.10.1", "subtle", ] @@ -1048,17 +1079,6 @@ dependencies = [ "slab", ] -[[package]] -name = "generic-array" -version = "0.14.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4bb6743198531e02858aeaea5398fcc883e71851fcbcb5a2f773e2fb6cb1edf2" -dependencies = [ - "typenum", - "version_check", - "zeroize", -] - [[package]] name = "getrandom" version = "0.2.17" @@ -1138,12 +1158,12 @@ dependencies = [ [[package]] name = "group" -version = "0.13.0" +version = "0.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f0f9ef7462f7c099f518d754361858f86d8a07af53ba9af0fe635bbccb151a63" +checksum = "7fd1a1c7a5206c5b7a3f5a0d7ccd3ff85d0c8f5133d62a02680255b0004af5f4" dependencies = [ "ff", - "rand_core 0.6.4", + "rand_core 0.10.1", "subtle", ] @@ -1191,9 +1211,9 @@ checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" [[package]] name = "hmac" -version = "0.12.1" +version = "0.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6c49c37c09c17a53d937dfbb742eb3a961d65a994e6bcdcf37e7399d0cc8ab5e" +checksum = "6303bc9732ae41b04cb554b844a762b4115a61bfaa81e3e83050991eeb56863f" dependencies = [ "digest", ] @@ -1260,6 +1280,17 @@ version = "2.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "15cdd26707701c53297e2fa6afb323d55fbc1d0810c3aec078ae3ef0424c3c15" +[[package]] +name = "hybrid-array" +version = "0.4.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "27f864f10dfb56725ce5ce5472bc52252c8f93a4ab86327122cebf62c5f59a17" +dependencies = [ + "subtle", + "typenum", + "zeroize", +] + [[package]] name = "hyper" version = "1.12.0" @@ -1712,7 +1743,7 @@ dependencies = [ [[package]] name = "k8s-version" version = "0.1.4" -source = "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#b61f619aa15efa2fa8c88d7eeae6154c3fb3918a" +source = "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#46a18a9a4899976b702ca87f73430c6f12d17c56" dependencies = [ "darling 0.24.1", "regex", @@ -1848,9 +1879,6 @@ name = "lazy_static" version = "1.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "20870f649af7073d53e38067b2a84312175d56ea15217e1b15bc83506ec50afb" -dependencies = [ - "spin", -] [[package]] name = "libc" @@ -1870,12 +1898,6 @@ dependencies = [ "pkg-config", ] -[[package]] -name = "libm" -version = "0.2.16" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b6d2cec3eae94f9f509c767b45932f1ada8350c4bdb85af2fcab4a3c14807981" - [[package]] name = "libyaml-rs" version = "0.3.0" @@ -1984,47 +2006,12 @@ dependencies = [ "windows-sys 0.61.2", ] -[[package]] -name = "num-bigint-dig" -version = "0.8.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e661dda6640fad38e827a6d4a310ff4763082116fe217f279885c97f511bb0b7" -dependencies = [ - "lazy_static", - "libm", - "num-integer", - "num-iter", - "num-traits", - "rand 0.8.8", - "smallvec", - "zeroize", -] - [[package]] name = "num-conv" version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "521739c6d2bac4aa25192232afe6841231376b2b26d4d9fae5ecf8ca5772e441" -[[package]] -name = "num-integer" -version = "0.1.47" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7ce2d95d4b3734dc35aa2f45e1aa22cd416814592a4f9d9205e11affd5b8e10b" -dependencies = [ - "num-traits", -] - -[[package]] -name = "num-iter" -version = "0.1.46" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c92800bd69a1eac91786bcfe9da64a897eb72911b8dc3095decbd07429e8048b" -dependencies = [ - "num-integer", - "num-traits", -] - [[package]] name = "num-traits" version = "0.2.19" @@ -2032,7 +2019,6 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" dependencies = [ "autocfg", - "libm", ] [[package]] @@ -2160,12 +2146,13 @@ dependencies = [ [[package]] name = "p256" -version = "0.13.2" +version = "0.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c9863ad85fa8f4460f9c48cb909d38a0d689dba1f6f6988a5e3e0d31071bcd4b" +checksum = "d2c9239b2dbc807adbbe147e8cf72ea7450c3a0aabe62cb8e75ff4ec22e1f72a" dependencies = [ "ecdsa", "elliptic-curve", + "primefield", "primeorder", "sha2", ] @@ -2211,9 +2198,9 @@ dependencies = [ [[package]] name = "pem-rfc7468" -version = "0.7.0" +version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "88b39c9bfcfc231068454382784bb460aae594343fb030d46e9f50a645418412" +checksum = "a6305423e0e7738146434843d1694d621cce767262b2a86910beab705e4493d9" dependencies = [ "base64ct", ] @@ -2296,20 +2283,19 @@ checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" [[package]] name = "pkcs1" -version = "0.7.5" +version = "0.8.0-rc.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c8ffb9f10fa047879315e6625af03c164b16962a5368d724ed16323b68ace47f" +checksum = "f485dbafe293ea6ccaa7d82bc591651a796eb8a9aec3df040b057b706073dbc3" dependencies = [ "der", - "pkcs8", "spki", ] [[package]] name = "pkcs8" -version = "0.10.2" +version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f950b2377845cebe5cf8b5165cb3cc1a5e0fa5cfa3e1f7f55707d8fd82e0a7b7" +checksum = "451913da69c775a56034ea8d9003d27ee8948e12443eae7c038ba100a4f21cb7" dependencies = [ "der", "spki", @@ -2360,13 +2346,31 @@ dependencies = [ "zerocopy", ] +[[package]] +name = "primefield" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c555a6e4eb7d4e158fcb028c835c3b8642206ddc279b5c6b202ef9a8bdb592f4" +dependencies = [ + "crypto-bigint", + "crypto-common", + "ff", + "rand_core 0.10.1", + "subtle", + "zeroize", +] + [[package]] name = "primeorder" -version = "0.13.6" +version = "0.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "353e1ca18966c16d9deb1c69278edbc5f194139612772bd9537af60ac231e1e6" +checksum = "5c9f42978c78a00e3d68f69fc03e57a234debae69da4020a4fb588fcdcd07b06" dependencies = [ "elliptic-curve", + "once_cell", + "primefield", + "serdect", + "wnaf", ] [[package]] @@ -2449,23 +2453,13 @@ version = "6.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" -[[package]] -name = "rand" -version = "0.8.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e058c7de0b26af77780c769414d6257830bb240f3c38477dbc2c16e5f54d6d4c" -dependencies = [ - "rand_chacha 0.3.1", - "rand_core 0.6.4", -] - [[package]] name = "rand" version = "0.9.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b9ef1d0d795eb7d84685bca4f72f3649f064e6641543d3a8c415898726a57b41" dependencies = [ - "rand_chacha 0.9.0", + "rand_chacha", "rand_core 0.9.5", ] @@ -2480,16 +2474,6 @@ dependencies = [ "rand_core 0.10.1", ] -[[package]] -name = "rand_chacha" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" -dependencies = [ - "ppv-lite86", - "rand_core 0.6.4", -] - [[package]] name = "rand_chacha" version = "0.9.0" @@ -2500,15 +2484,6 @@ dependencies = [ "rand_core 0.9.5", ] -[[package]] -name = "rand_core" -version = "0.6.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" -dependencies = [ - "getrandom 0.2.17", -] - [[package]] name = "rand_core" version = "0.9.5" @@ -2621,12 +2596,12 @@ dependencies = [ [[package]] name = "rfc6979" -version = "0.4.0" +version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f8dd2a808d456c4a54e300a23e9f5a67e122c3024119acbfd73e3bf664491cb2" +checksum = "b4a459cddafb3fe76b31fd8f1108007566c40301feb64dc7b54656eb7388172b" dependencies = [ + "crypto-bigint", "hmac", - "subtle", ] [[package]] @@ -2645,22 +2620,20 @@ dependencies = [ [[package]] name = "rsa" -version = "0.9.10" +version = "0.10.0-rc.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b8573f03f5883dcaebdfcf4725caa1ecb9c15b2ef50c43a07b816e06799bb12d" +checksum = "33405e321e16ef941ca66e254f6904c3aafd3c7ff001fb88bf11c73eb75c2b85" dependencies = [ "const-oid", + "crypto-bigint", + "crypto-primes", "digest", - "num-bigint-dig", - "num-integer", - "num-traits", "pkcs1", "pkcs8", - "rand_core 0.6.4", + "rand_core 0.10.1", "sha2", "signature", "spki", - "subtle", "zeroize", ] @@ -2840,14 +2813,14 @@ checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" [[package]] name = "sec1" -version = "0.7.3" +version = "0.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d3e97a565f76233a6003f9f5c54be1d9c5bdfa3eccfb189469f11ec4901c47dc" +checksum = "d56d437c2f19203ce5f7122e507831de96f3d2d4d3be5af44a0b0a09d8a80e4d" dependencies = [ "base16ct", + "ctutils", "der", - "generic-array", - "pkcs8", + "hybrid-array", "subtle", "zeroize", ] @@ -3000,25 +2973,35 @@ dependencies = [ "serde", ] +[[package]] +name = "serdect" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "66cf8fedced2fcf12406bcb34223dffb92eaf34908ede12fed414c82b7f00b3e" +dependencies = [ + "base16ct", + "serde", +] + [[package]] name = "sha1" -version = "0.10.7" +version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a978451301f4db1d02937a4ab3ccce137717b81826e79b7d49ffe3244a13c3b8" +checksum = "aacc4cc499359472b4abe1bf11d0b12e688af9a805fa5e3016f9a386dc2d0214" dependencies = [ "cfg-if", - "cpufeatures 0.2.17", + "cpufeatures", "digest", ] [[package]] name = "sha2" -version = "0.10.9" +version = "0.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" +checksum = "47d7069beb7d6ac7b9acd1039986e73443f24234f41074da099d6f994ac9ad19" dependencies = [ "cfg-if", - "cpufeatures 0.2.17", + "cpufeatures", "digest", ] @@ -3049,12 +3032,12 @@ dependencies = [ [[package]] name = "signature" -version = "2.2.0" +version = "3.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "77549399552de45a898a580c1b41d445bf730df867cc44e6c0233bbc4b8329de" +checksum = "28d567dcbaf0049cb8ac2608a76cd95ff9e4412e1899d389ee400918ca7537f5" dependencies = [ "digest", - "rand_core 0.6.4", + "rand_core 0.10.1", ] [[package]] @@ -3143,17 +3126,11 @@ dependencies = [ "windows-sys 0.61.2", ] -[[package]] -name = "spin" -version = "0.9.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3763264f6b73151db08c50ff20d7d8a0b8796e021cdea7ceedad07b80155fa0e" - [[package]] name = "spki" -version = "0.7.3" +version = "0.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d91ed6c858b01f942cd56b37a94b3e0a1798290327d1236e4d9cf4eaca44d29d" +checksum = "7ef958a98b9d5da290cfc78946e9f3e61e1e62a18db0d92cac0b83cc161491a9" dependencies = [ "base64ct", "der", @@ -3168,7 +3145,7 @@ checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596" [[package]] name = "stackable-agent" version = "0.1.0" -source = "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#b61f619aa15efa2fa8c88d7eeae6154c3fb3918a" +source = "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#46a18a9a4899976b702ca87f73430c6f12d17c56" dependencies = [ "k8s-openapi", "schemars", @@ -3180,7 +3157,7 @@ dependencies = [ [[package]] name = "stackable-certs" version = "0.4.1" -source = "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#b61f619aa15efa2fa8c88d7eeae6154c3fb3918a" +source = "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#46a18a9a4899976b702ca87f73430c6f12d17c56" dependencies = [ "const-oid", "ecdsa", @@ -3188,7 +3165,7 @@ dependencies = [ "kube", "p256", "rand 0.10.3", - "rand_core 0.6.4", + "rand_core 0.10.1", "rsa", "sha2", "signature", @@ -3247,8 +3224,9 @@ dependencies = [ [[package]] name = "stackable-operator" version = "0.121.0" -source = "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#b61f619aa15efa2fa8c88d7eeae6154c3fb3918a" +source = "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#46a18a9a4899976b702ca87f73430c6f12d17c56" dependencies = [ + "base16ct", "base64 0.23.1", "clap", "const_format", @@ -3292,7 +3270,7 @@ dependencies = [ [[package]] name = "stackable-operator-derive" version = "0.3.1" -source = "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#b61f619aa15efa2fa8c88d7eeae6154c3fb3918a" +source = "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#46a18a9a4899976b702ca87f73430c6f12d17c56" dependencies = [ "darling 0.24.1", "proc-macro2", @@ -3303,7 +3281,7 @@ dependencies = [ [[package]] name = "stackable-shared" version = "0.2.0" -source = "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#b61f619aa15efa2fa8c88d7eeae6154c3fb3918a" +source = "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#46a18a9a4899976b702ca87f73430c6f12d17c56" dependencies = [ "jiff", "k8s-openapi", @@ -3320,7 +3298,7 @@ dependencies = [ [[package]] name = "stackable-telemetry" version = "0.6.5" -source = "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#b61f619aa15efa2fa8c88d7eeae6154c3fb3918a" +source = "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#46a18a9a4899976b702ca87f73430c6f12d17c56" dependencies = [ "axum", "clap", @@ -3344,7 +3322,7 @@ dependencies = [ [[package]] name = "stackable-versioned" version = "0.13.0" -source = "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#b61f619aa15efa2fa8c88d7eeae6154c3fb3918a" +source = "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#46a18a9a4899976b702ca87f73430c6f12d17c56" dependencies = [ "kube", "schemars", @@ -3357,7 +3335,7 @@ dependencies = [ [[package]] name = "stackable-versioned-macros" version = "0.13.0" -source = "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#b61f619aa15efa2fa8c88d7eeae6154c3fb3918a" +source = "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#46a18a9a4899976b702ca87f73430c6f12d17c56" dependencies = [ "convert_case", "darling 0.24.1", @@ -3374,7 +3352,7 @@ dependencies = [ [[package]] name = "stackable-webhook" version = "0.10.0" -source = "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#b61f619aa15efa2fa8c88d7eeae6154c3fb3918a" +source = "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#46a18a9a4899976b702ca87f73430c6f12d17c56" dependencies = [ "arc-swap", "async-trait", @@ -3620,9 +3598,9 @@ dependencies = [ [[package]] name = "tokio" -version = "1.54.0" +version = "1.54.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a0c168a474b95bfecfe908a65d4bdffc523144b649a74c09c286afb35d53e0bc" +checksum = "b52fd3255258d26fac7264ab5a76b53c13367136dc011d3d6ad4f8ed070396a3" dependencies = [ "bytes", "libc", @@ -4009,9 +3987,9 @@ checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" [[package]] name = "uuid" -version = "1.28.0" +version = "1.29.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7cc1186384beb7dd8eedea376413fd654937285ea6c9cfbb928dc3043ea4b606" +checksum = "4a0e2d11620c50695fe12bbbfde96b3477fe606e7284c22d505c12ce51214514" dependencies = [ "js-sys", "wasm-bindgen", @@ -4319,6 +4297,18 @@ version = "0.57.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1ebf944e87a7c253233ad6766e082e3cd714b5d03812acc24c318f549614536e" +[[package]] +name = "wnaf" +version = "0.14.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "795ca18b3fdb5e62bf982199278341ddcf7ebf7d32e25e212ad05d496e95f6fa" +dependencies = [ + "ff", + "group", + "hybrid-array", + "primefield", +] + [[package]] name = "writeable" version = "0.6.4" @@ -4327,9 +4317,9 @@ checksum = "3ad82d2a33cdc9674dc7465672f271e096168fcdbe0f799d9e6db8c5892679dc" [[package]] name = "x509-cert" -version = "0.2.5" +version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1301e935010a701ae5f8655edc0ad17c44bad3ac5ce8c39185f75453b720ae94" +checksum = "105ef4642d9cb137ef83d623d0e4bf08b8adf69e9918ca904a174adb6d3d038b" dependencies = [ "const-oid", "der", diff --git a/Cargo.nix b/Cargo.nix index 18130a181..9dd9a010d 100644 --- a/Cargo.nix +++ b/Cargo.nix @@ -833,14 +833,13 @@ rec { }; "base16ct" = rec { crateName = "base16ct"; - version = "0.2.0"; - edition = "2021"; - sha256 = "1kylrjhdzk7qpknrvlphw8ywdnvvg39dizw9622w3wk5xba04zsc"; + version = "1.0.0"; + edition = "2024"; + sha256 = "1xi6jy6y08hbxfy14pxjlxcggl9kfsvsn3jbyyk7lii4ss878c7x"; authors = [ "RustCrypto Developers" ]; features = { - "std" = [ "alloc" ]; }; resolvedDefaultFeatures = [ "alloc" ]; }; @@ -917,20 +916,22 @@ rec { }; "block-buffer" = rec { crateName = "block-buffer"; - version = "0.10.4"; - edition = "2018"; - sha256 = "0w9sa2ypmrsqqvc20nhwr75wbb5cjr4kkyhpjm1z1lv2kdicfy1h"; + version = "0.12.1"; + edition = "2024"; + sha256 = "1ak0cvmxz3yifqmzv6aba9606brsz7d5g3piv5xdcvjsx7dwgxnj"; libName = "block_buffer"; authors = [ "RustCrypto Developers" ]; dependencies = [ { - name = "generic-array"; - packageId = "generic-array"; + name = "hybrid-array"; + packageId = "hybrid-array"; } ]; - + features = { + "zeroize" = [ "dep:zeroize" ]; + }; }; "built" = rec { crateName = "built"; @@ -1058,7 +1059,7 @@ rec { } { name = "cpufeatures"; - packageId = "cpufeatures 0.3.1"; + packageId = "cpufeatures"; target = { target, features }: (("x86_64" == target."arch" or null) || ("x86" == target."arch" or null)); } { @@ -1246,6 +1247,16 @@ rec { edition = "2024"; sha256 = "11k4m4ngn39iz39lnq6nn6kyxzp62mnm1d86qfad3q0vlk33n4qw"; + }; + "cmov" = rec { + crateName = "cmov"; + version = "0.5.4"; + edition = "2024"; + sha256 = "0yh22sqdvcdrfbhvnja4kaq5dyklpb4s70w5r6rplfdw4jna17hc"; + authors = [ + "RustCrypto Developers" + ]; + }; "colorchoice" = rec { crateName = "colorchoice"; @@ -1303,9 +1314,9 @@ rec { }; "const-oid" = rec { crateName = "const-oid"; - version = "0.9.6"; - edition = "2021"; - sha256 = "1y0jnqaq7p2wvspnx7qj76m7hjcqpz73qzvr9l2p9n2s51vr6if2"; + version = "0.10.2"; + edition = "2024"; + sha256 = "0p7m286mp8aai4sa72g7ji6qm0d4ns8wg4i4b2hj9p9615zm3vx6"; libName = "const_oid"; authors = [ "RustCrypto Developers" @@ -1313,13 +1324,13 @@ rec { features = { "arbitrary" = [ "dep:arbitrary" ]; }; - resolvedDefaultFeatures = [ "db" "std" ]; + resolvedDefaultFeatures = [ "db" ]; }; "const_format" = rec { crateName = "const_format"; - version = "0.2.36"; + version = "0.2.37"; edition = "2021"; - sha256 = "07ncczs8yndga2f8p4386c827l4fxwzl0pbwp7ijnhcsmlbsd0a4"; + sha256 = "0m04hhk72casbbqixqg4xm6a86spl0saci1mj9cqci35d56wppm3"; authors = [ "rodrimati1992 " ]; @@ -1356,9 +1367,9 @@ rec { }; "const_format_proc_macros" = rec { crateName = "const_format_proc_macros"; - version = "0.2.34"; + version = "0.2.37"; edition = "2021"; - sha256 = "0i3pxxcl4xvwq4mlfg3csb4j0n6v0mhj07p6yk0vlvdirznc4mqx"; + sha256 = "0ndn186d13yv97fbry163bacz9mz0j66hnyb11pm0hm6d415f6wl"; procMacro = true; authors = [ "rodrimati1992 " @@ -1456,43 +1467,17 @@ rec { }; resolvedDefaultFeatures = [ "default" ]; }; - "cpufeatures 0.2.17" = rec { - crateName = "cpufeatures"; - version = "0.2.17"; - edition = "2018"; - sha256 = "10023dnnaghhdl70xcds12fsx2b966sxbxjq5sxs49mvxqw5ivar"; + "cpubits" = rec { + crateName = "cpubits"; + version = "0.1.1"; + edition = "2024"; + sha256 = "1bh6rvanxm00myf1rmnh44hq2jdxn69971c92s4klz0k76f5zf0m"; authors = [ "RustCrypto Developers" ]; - dependencies = [ - { - name = "libc"; - packageId = "libc"; - usesDefaultFeatures = false; - target = { target, features }: (target.name == "aarch64-linux-android"); - } - { - name = "libc"; - packageId = "libc"; - usesDefaultFeatures = false; - target = { target, features }: (("aarch64" == target."arch" or null) && ("linux" == target."os" or null)); - } - { - name = "libc"; - packageId = "libc"; - usesDefaultFeatures = false; - target = { target, features }: (("aarch64" == target."arch" or null) && ("apple" == target."vendor" or null)); - } - { - name = "libc"; - packageId = "libc"; - usesDefaultFeatures = false; - target = { target, features }: (("loongarch64" == target."arch" or null) && ("linux" == target."os" or null)); - } - ]; }; - "cpufeatures 0.3.1" = rec { + "cpufeatures" = rec { crateName = "cpufeatures"; version = "0.3.1"; edition = "2024"; @@ -1581,27 +1566,54 @@ rec { }; "crypto-bigint" = rec { crateName = "crypto-bigint"; - version = "0.5.5"; - edition = "2021"; - sha256 = "0xmbdff3g6ii5sbxjxc31xfkv9lrmyril4arh3dzckd4gjsjzj8d"; + version = "0.7.10"; + edition = "2024"; + sha256 = "04dyl8pzc00l7bmsa3br86dczs45hwhkw7wjzg9dl0xs9jv5q0ww"; libName = "crypto_bigint"; authors = [ "RustCrypto Developers" ]; dependencies = [ { - name = "generic-array"; - packageId = "generic-array"; + name = "cpubits"; + packageId = "cpubits"; + } + { + name = "ctutils"; + packageId = "ctutils"; + } + { + name = "getrandom"; + packageId = "getrandom 0.4.3"; + optional = true; + features = [ "sys_rng" ]; + } + { + name = "hybrid-array"; + packageId = "hybrid-array"; optional = true; } + { + name = "num-traits"; + packageId = "num-traits"; + usesDefaultFeatures = false; + } { name = "rand_core"; - packageId = "rand_core 0.6.4"; + packageId = "rand_core 0.10.1"; optional = true; + usesDefaultFeatures = false; + } + { + name = "serdect"; + packageId = "serdect"; + optional = true; + usesDefaultFeatures = false; } { name = "subtle"; packageId = "subtle"; + optional = true; usesDefaultFeatures = false; } { @@ -1614,48 +1626,120 @@ rec { devDependencies = [ { name = "rand_core"; - packageId = "rand_core 0.6.4"; - features = [ "std" ]; + packageId = "rand_core 0.10.1"; } ]; features = { "alloc" = [ "serdect?/alloc" ]; - "default" = [ "rand" ]; - "der" = [ "dep:der" ]; - "generic-array" = [ "dep:generic-array" ]; - "rand" = [ "rand_core/std" ]; + "default" = [ "rand_core" ]; + "der" = [ "dep:der" "hybrid-array" ]; + "getrandom" = [ "dep:getrandom" "rand_core" ]; + "hybrid-array" = [ "dep:hybrid-array" ]; "rand_core" = [ "dep:rand_core" ]; "rlp" = [ "dep:rlp" ]; "serde" = [ "dep:serdect" ]; + "subtle" = [ "dep:subtle" "ctutils/subtle" "hybrid-array?/subtle" ]; "zeroize" = [ "dep:zeroize" ]; }; - resolvedDefaultFeatures = [ "generic-array" "rand_core" "zeroize" ]; + resolvedDefaultFeatures = [ "alloc" "getrandom" "hybrid-array" "rand_core" "subtle" "zeroize" ]; }; "crypto-common" = rec { crateName = "crypto-common"; - version = "0.1.6"; - edition = "2018"; - sha256 = "1cvby95a6xg7kxdz5ln3rl9xh66nz66w46mm3g56ri1z5x815yqv"; + version = "0.2.2"; + edition = "2024"; + sha256 = "0lql5wjlrjkd3r0w32rwbgqfmgg84ms3h65ldnlckmkc3nb4qvnf"; libName = "crypto_common"; authors = [ "RustCrypto Developers" ]; dependencies = [ { - name = "generic-array"; - packageId = "generic-array"; - features = [ "more_lengths" ]; + name = "getrandom"; + packageId = "getrandom 0.4.3"; + optional = true; + features = [ "sys_rng" ]; } { - name = "typenum"; - packageId = "typenum"; + name = "hybrid-array"; + packageId = "hybrid-array"; + } + { + name = "rand_core"; + packageId = "rand_core 0.10.1"; + optional = true; } ]; features = { - "getrandom" = [ "rand_core/getrandom" ]; + "getrandom" = [ "rand_core" "dep:getrandom" ]; "rand_core" = [ "dep:rand_core" ]; + "zeroize" = [ "hybrid-array/zeroize" ]; }; - resolvedDefaultFeatures = [ "std" ]; + resolvedDefaultFeatures = [ "getrandom" "rand_core" ]; + }; + "crypto-primes" = rec { + crateName = "crypto-primes"; + version = "0.7.2"; + edition = "2024"; + sha256 = "13ygy2rrp7dicc4lmj01744y8hfjhfyr99pa9ymbp7n674daacrn"; + libName = "crypto_primes"; + dependencies = [ + { + name = "crypto-bigint"; + packageId = "crypto-bigint"; + usesDefaultFeatures = false; + features = [ "rand_core" ]; + } + { + name = "rand_core"; + packageId = "rand_core 0.10.1"; + usesDefaultFeatures = false; + } + ]; + devDependencies = [ + { + name = "crypto-bigint"; + packageId = "crypto-bigint"; + usesDefaultFeatures = false; + features = [ "alloc" ]; + } + ]; + features = { + "default" = [ "alloc" ]; + "glass_pumpkin" = [ "dep:glass_pumpkin" ]; + "multicore" = [ "rayon" ]; + "openssl" = [ "dep:openssl" ]; + "rayon" = [ "dep:rayon" ]; + "rug" = [ "dep:rug" ]; + "tests-all" = [ "tests-openssl" "tests-gmp" "tests-exhaustive" "tests-glass-pumpkin" ]; + "tests-glass-pumpkin" = [ "glass_pumpkin" ]; + "tests-gmp" = [ "rug/std" ]; + "tests-openssl" = [ "openssl" ]; + }; + }; + "ctutils" = rec { + crateName = "ctutils"; + version = "0.4.7"; + edition = "2024"; + sha256 = "12l2ck796m702jbjfr5gbc5r1mip09mlnnw7b7dskn4ndzynydx2"; + authors = [ + "RustCrypto Developers" + ]; + dependencies = [ + { + name = "cmov"; + packageId = "cmov"; + } + { + name = "subtle"; + packageId = "subtle"; + optional = true; + usesDefaultFeatures = false; + } + ]; + features = { + "subtle" = [ "dep:subtle" ]; + }; + resolvedDefaultFeatures = [ "subtle" ]; }; "darling 0.23.0" = rec { crateName = "darling"; @@ -1940,9 +2024,9 @@ rec { }; "der" = rec { crateName = "der"; - version = "0.7.10"; - edition = "2021"; - sha256 = "1jyxacyxdx6mxbkfw99jz59dzvcd9k17rq01a7xvn1dr6wl87hg7"; + version = "0.8.3"; + edition = "2024"; + sha256 = "01703q9nh4vlswvm7hbjknvhjd589apzdz0jdrx452zbiw86m6ba"; authors = [ "RustCrypto Developers" ]; @@ -1977,10 +2061,11 @@ rec { ]; features = { "alloc" = [ "zeroize?/alloc" ]; - "arbitrary" = [ "dep:arbitrary" "const-oid?/arbitrary" "std" ]; + "arbitrary" = [ "dep:arbitrary" "dep:derive_arbitrary" "const-oid?/arbitrary" "std" ]; "bytes" = [ "dep:bytes" "alloc" ]; "derive" = [ "dep:der_derive" ]; "flagset" = [ "dep:flagset" ]; + "heapless" = [ "dep:heapless" ]; "oid" = [ "dep:const-oid" ]; "pem" = [ "dep:pem-rfc7468" "alloc" "zeroize" ]; "std" = [ "alloc" ]; @@ -1991,9 +2076,9 @@ rec { }; "der_derive" = rec { crateName = "der_derive"; - version = "0.7.3"; - edition = "2021"; - sha256 = "065d2wy7zd0dank99hh58l5x7lv50hxnr7j6f3sphlb7i4ihjd40"; + version = "0.8.0"; + edition = "2024"; + sha256 = "00wr83bxdf1jk7rmy6986qywcw67b92cd779cndxwvv35ln0wq2r"; procMacro = true; authors = [ "RustCrypto Developers" @@ -2137,9 +2222,9 @@ rec { }; "digest" = rec { crateName = "digest"; - version = "0.10.7"; - edition = "2018"; - sha256 = "14p2n6ih29x81akj097lvz7wi9b6b9hvls0lwrv7b6xwyy0s5ncy"; + version = "0.11.3"; + edition = "2024"; + sha256 = "1hnmhd4rkybr11292w42pz9ppzx1h49glrhqg107k4s1b2xnvpgi"; authors = [ "RustCrypto Developers" ]; @@ -2157,28 +2242,26 @@ rec { { name = "crypto-common"; packageId = "crypto-common"; + rename = "common"; } { - name = "subtle"; - packageId = "subtle"; + name = "ctutils"; + packageId = "ctutils"; optional = true; - usesDefaultFeatures = false; } ]; features = { "blobby" = [ "dep:blobby" ]; - "block-buffer" = [ "dep:block-buffer" ]; - "const-oid" = [ "dep:const-oid" ]; - "core-api" = [ "block-buffer" ]; - "default" = [ "core-api" ]; + "block-api" = [ "dep:block-buffer" ]; + "default" = [ "block-api" ]; "dev" = [ "blobby" ]; - "mac" = [ "subtle" ]; - "oid" = [ "const-oid" ]; - "rand_core" = [ "crypto-common/rand_core" ]; - "std" = [ "alloc" "crypto-common/std" ]; - "subtle" = [ "dep:subtle" ]; + "getrandom" = [ "common/getrandom" "rand_core" ]; + "mac" = [ "dep:ctutils" ]; + "oid" = [ "dep:const-oid" ]; + "rand_core" = [ "common/rand_core" ]; + "zeroize" = [ "dep:zeroize" "block-buffer?/zeroize" ]; }; - resolvedDefaultFeatures = [ "alloc" "block-buffer" "const-oid" "core-api" "default" "mac" "oid" "std" "subtle" ]; + resolvedDefaultFeatures = [ "alloc" "block-api" "default" "mac" "oid" ]; }; "displaydoc" = rec { crateName = "displaydoc"; @@ -2269,9 +2352,9 @@ rec { }; "ecdsa" = rec { crateName = "ecdsa"; - version = "0.16.9"; - edition = "2021"; - sha256 = "1jhb0bcbkaz4001sdmfyv8ajrv8a1cg7z7aa5myrd4jjbhmz69zf"; + version = "0.17.0"; + edition = "2024"; + sha256 = "1q3alixk3kpqlk610ima509gk6lmpbgxha4p6a2p0xjcq97ils60"; authors = [ "RustCrypto Developers" ]; @@ -2292,7 +2375,7 @@ rec { name = "elliptic-curve"; packageId = "elliptic-curve"; usesDefaultFeatures = false; - features = [ "digest" "sec1" ]; + features = [ "sec1" ]; } { name = "rfc6979"; @@ -2311,6 +2394,11 @@ rec { optional = true; usesDefaultFeatures = false; } + { + name = "zeroize"; + packageId = "zeroize"; + usesDefaultFeatures = false; + } ]; devDependencies = [ { @@ -2321,24 +2409,21 @@ rec { } ]; features = { + "algorithm" = [ "dep:rfc6979" "digest" "elliptic-curve/arithmetic" ]; "alloc" = [ "elliptic-curve/alloc" "signature/alloc" "spki/alloc" ]; - "arithmetic" = [ "elliptic-curve/arithmetic" ]; "default" = [ "digest" ]; "der" = [ "dep:der" ]; - "dev" = [ "arithmetic" "digest" "elliptic-curve/dev" "hazmat" ]; - "digest" = [ "dep:digest" "signature/digest" ]; + "dev" = [ "algorithm" "digest/dev" "elliptic-curve/dev" ]; + "digest" = [ "dep:digest" "elliptic-curve/digest" "signature/digest" ]; + "getrandom" = [ "elliptic-curve/getrandom" ]; "pem" = [ "elliptic-curve/pem" "pkcs8" ]; - "pkcs8" = [ "digest" "elliptic-curve/pkcs8" "der" ]; - "rfc6979" = [ "dep:rfc6979" ]; - "serde" = [ "elliptic-curve/serde" "serdect" ]; - "serdect" = [ "dep:serdect" ]; + "pkcs8" = [ "der" "digest" "elliptic-curve/pkcs8" ]; + "serde" = [ "dep:serdect" "elliptic-curve/serde" "pkcs8" ]; "sha2" = [ "dep:sha2" ]; - "signing" = [ "arithmetic" "digest" "hazmat" "rfc6979" ]; "spki" = [ "dep:spki" ]; - "std" = [ "alloc" "elliptic-curve/std" "signature/std" ]; - "verifying" = [ "arithmetic" "digest" "hazmat" ]; + "std" = [ "alloc" "elliptic-curve/std" ]; }; - resolvedDefaultFeatures = [ "alloc" "arithmetic" "default" "der" "digest" "hazmat" "pem" "pkcs8" "rfc6979" "signing" "spki" "std" "verifying" ]; + resolvedDefaultFeatures = [ "algorithm" "alloc" "default" "der" "digest" "pem" "pkcs8" "spki" "std" ]; }; "educe 0.6.0" = rec { crateName = "educe"; @@ -2432,9 +2517,9 @@ rec { }; "elliptic-curve" = rec { crateName = "elliptic-curve"; - version = "0.13.8"; - edition = "2021"; - sha256 = "0ixx4brgnzi61z29r3g1606nh2za88hzyz8c5r3p6ydzhqq09rmm"; + version = "0.14.3"; + edition = "2024"; + sha256 = "0z4wbskrlrv590040zv9kv6dac1skvb6rbw9fcsa4x4dw7kkf3wh"; libName = "elliptic_curve"; authors = [ "RustCrypto Developers" @@ -2447,8 +2532,15 @@ rec { { name = "crypto-bigint"; packageId = "crypto-bigint"; + rename = "bigint"; usesDefaultFeatures = false; - features = [ "rand_core" "generic-array" "zeroize" ]; + features = [ "hybrid-array" "rand_core" "subtle" "zeroize" ]; + } + { + name = "crypto-common"; + packageId = "crypto-common"; + rename = "common"; + features = [ "rand_core" ]; } { name = "digest"; @@ -2461,18 +2553,19 @@ rec { optional = true; usesDefaultFeatures = false; } - { - name = "generic-array"; - packageId = "generic-array"; - usesDefaultFeatures = false; - features = [ "zeroize" ]; - } { name = "group"; packageId = "group"; optional = true; usesDefaultFeatures = false; } + { + name = "hybrid-array"; + packageId = "hybrid-array"; + rename = "array"; + usesDefaultFeatures = false; + features = [ "zeroize" ]; + } { name = "pem-rfc7468"; packageId = "pem-rfc7468"; @@ -2487,14 +2580,14 @@ rec { } { name = "rand_core"; - packageId = "rand_core 0.6.4"; + packageId = "rand_core 0.10.1"; usesDefaultFeatures = false; } { name = "sec1"; packageId = "sec1"; optional = true; - features = [ "subtle" "zeroize" ]; + features = [ "ctutils" "subtle" "zeroize" ]; } { name = "subtle"; @@ -2508,25 +2601,22 @@ rec { } ]; features = { - "alloc" = [ "base16ct/alloc" "ff?/alloc" "group?/alloc" "pkcs8?/alloc" "sec1?/alloc" "zeroize/alloc" ]; + "alloc" = [ "base16ct/alloc" "ff?/alloc" "group?/alloc" "array/alloc" "pkcs8?/alloc" "sec1?/alloc" "zeroize/alloc" ]; "arithmetic" = [ "group" ]; - "bits" = [ "arithmetic" "ff/bits" "dep:tap" ]; "default" = [ "arithmetic" ]; "dev" = [ "arithmetic" "dep:hex-literal" "pem" "pkcs8" ]; "digest" = [ "dep:digest" ]; "ecdh" = [ "arithmetic" "digest" "dep:hkdf" ]; "ff" = [ "dep:ff" ]; + "getrandom" = [ "arithmetic" "bigint/getrandom" "common/getrandom" ]; "group" = [ "dep:group" "ff" ]; - "hash2curve" = [ "arithmetic" "digest" ]; - "jwk" = [ "dep:base64ct" "dep:serde_json" "alloc" "serde" "zeroize/alloc" ]; - "pem" = [ "dep:pem-rfc7468" "alloc" "arithmetic" "pkcs8" "sec1/pem" ]; + "pem" = [ "dep:pem-rfc7468" "alloc" "arithmetic" "pkcs8/pem" "sec1/pem" ]; "pkcs8" = [ "dep:pkcs8" "sec1" ]; "sec1" = [ "dep:sec1" ]; "serde" = [ "dep:serdect" "alloc" "pkcs8" "sec1/serde" ]; - "std" = [ "alloc" "rand_core/std" "pkcs8?/std" "sec1?/std" ]; - "voprf" = [ "digest" ]; + "std" = [ "alloc" "pkcs8?/std" "sec1?/std" ]; }; - resolvedDefaultFeatures = [ "alloc" "arithmetic" "digest" "ff" "group" "hazmat" "pem" "pkcs8" "sec1" "std" ]; + resolvedDefaultFeatures = [ "alloc" "arithmetic" "digest" "ff" "getrandom" "group" "pem" "pkcs8" "sec1" "std" ]; }; "encoding_rs" = rec { crateName = "encoding_rs"; @@ -2790,9 +2880,9 @@ rec { }; "ff" = rec { crateName = "ff"; - version = "0.13.1"; + version = "0.14.0"; edition = "2021"; - sha256 = "14v3bc6q24gbcjnxjfbq2dddgf4as2z2gd4mj35gjlrncpxhpdf0"; + sha256 = "17s889wpzi2dp924bnxhznkbm44lp23nqv4g32phxyx9jamqdxm1"; authors = [ "Sean Bowe " "Jack Grigg " @@ -2800,7 +2890,7 @@ rec { dependencies = [ { name = "rand_core"; - packageId = "rand_core 0.6.4"; + packageId = "rand_core 0.10.1"; usesDefaultFeatures = false; } { @@ -2811,12 +2901,11 @@ rec { } ]; features = { - "bits" = [ "bitvec" ]; + "bits" = [ "bitvec" "ff_derive?/bits" ]; "bitvec" = [ "dep:bitvec" ]; "byteorder" = [ "dep:byteorder" ]; "default" = [ "bits" "std" ]; "derive" = [ "byteorder" "ff_derive" ]; - "derive_bits" = [ "bits" "ff_derive/bits" ]; "ff_derive" = [ "dep:ff_derive" ]; "std" = [ "alloc" ]; }; @@ -3235,40 +3324,6 @@ rec { }; resolvedDefaultFeatures = [ "alloc" "async-await" "async-await-macro" "channel" "default" "futures-channel" "futures-io" "futures-macro" "futures-sink" "io" "memchr" "sink" "slab" "std" ]; }; - "generic-array" = rec { - crateName = "generic-array"; - version = "0.14.9"; - edition = "2015"; - sha256 = "1wpdn5ngpqkkyyibbg7wa4cfg0y8zjc57spaia2h47jkk0qp9djb"; - libName = "generic_array"; - authors = [ - "Bartłomiej Kamiński " - "Aaron Trent " - ]; - dependencies = [ - { - name = "typenum"; - packageId = "typenum"; - } - { - name = "zeroize"; - packageId = "zeroize"; - optional = true; - usesDefaultFeatures = false; - } - ]; - buildDependencies = [ - { - name = "version_check"; - packageId = "version_check"; - } - ]; - features = { - "serde" = [ "dep:serde" ]; - "zeroize" = [ "dep:zeroize" ]; - }; - resolvedDefaultFeatures = [ "more_lengths" "zeroize" ]; - }; "getrandom 0.2.17" = rec { crateName = "getrandom"; version = "0.2.17"; @@ -3303,7 +3358,6 @@ rec { "rustc-dep-of-std" = [ "compiler_builtins" "core" "libc/rustc-dep-of-std" "wasi/rustc-dep-of-std" ]; "wasm-bindgen" = [ "dep:wasm-bindgen" ]; }; - resolvedDefaultFeatures = [ "std" ]; }; "getrandom 0.3.4" = rec { crateName = "getrandom"; @@ -3591,9 +3645,9 @@ rec { }; "group" = rec { crateName = "group"; - version = "0.13.0"; + version = "0.14.0"; edition = "2021"; - sha256 = "0qqs2p5vqnv3zvq9mfjkmw3qlvgqb0c3cm6p33srkh7pc9sfzygh"; + sha256 = "1x7m980b0m82d012mmika67hqpgq7z6pq3as7xx5nv10lp3s3lbz"; authors = [ "Sean Bowe " "Jack Grigg " @@ -3606,7 +3660,7 @@ rec { } { name = "rand_core"; - packageId = "rand_core 0.6.4"; + packageId = "rand_core 0.10.1"; usesDefaultFeatures = false; } { @@ -3767,9 +3821,9 @@ rec { }; "hmac" = rec { crateName = "hmac"; - version = "0.12.1"; - edition = "2018"; - sha256 = "0pmbr069sfg76z7wsssfk5ddcqd9ncp79fyz6zcm6yn115yc6jbc"; + version = "0.13.0"; + edition = "2024"; + sha256 = "0gw6avmix6ah63lf70dapxhml4dlcakl9f2lnm6b0hdf6abvq0v3"; authors = [ "RustCrypto Developers" ]; @@ -3788,9 +3842,8 @@ rec { } ]; features = { - "std" = [ "digest/std" ]; + "zeroize" = [ "digest/zeroize" ]; }; - resolvedDefaultFeatures = [ "reset" ]; }; "hostname" = rec { crateName = "hostname"; @@ -3936,6 +3989,46 @@ rec { features = { }; }; + "hybrid-array" = rec { + crateName = "hybrid-array"; + version = "0.4.15"; + edition = "2024"; + sha256 = "05wsyp2n5gyf49qk51mblj9qyb15aay74m6fwmf74mpv1pqn9y17"; + libName = "hybrid_array"; + authors = [ + "RustCrypto Developers" + ]; + dependencies = [ + { + name = "subtle"; + packageId = "subtle"; + optional = true; + usesDefaultFeatures = false; + features = [ "const-generics" ]; + } + { + name = "typenum"; + packageId = "typenum"; + features = [ "const-generics" ]; + } + { + name = "zeroize"; + packageId = "zeroize"; + optional = true; + usesDefaultFeatures = false; + } + ]; + features = { + "arbitrary" = [ "dep:arbitrary" ]; + "bytemuck" = [ "dep:bytemuck" ]; + "ctutils" = [ "dep:ctutils" ]; + "serde" = [ "dep:serde" ]; + "subtle" = [ "dep:subtle" ]; + "zerocopy" = [ "dep:zerocopy" ]; + "zeroize" = [ "dep:zeroize" ]; + }; + resolvedDefaultFeatures = [ "alloc" "subtle" "zeroize" ]; + }; "hyper" = rec { crateName = "hyper"; version = "1.12.0"; @@ -5455,8 +5548,8 @@ rec { workspace_member = null; src = pkgs.fetchgit { url = "https://github.com/stackabletech/operator-rs.git"; - rev = "b61f619aa15efa2fa8c88d7eeae6154c3fb3918a"; - sha256 = "0ivrv56a8yzy68d9hr7cvy7hk5w0i2hidzlnns23h0g60ynb3axq"; + rev = "46a18a9a4899976b702ca87f73430c6f12d17c56"; + sha256 = "10va8k2dwshcb6c7pp01q6m6bx2ad58fzqxf19mp90hhcp00bxm7"; }; libName = "k8s_version"; authors = [ @@ -6098,20 +6191,10 @@ rec { authors = [ "Marvin Löbel " ]; - dependencies = [ - { - name = "spin"; - packageId = "spin"; - optional = true; - usesDefaultFeatures = false; - features = [ "once" ]; - } - ]; features = { "spin" = [ "dep:spin" ]; "spin_no_std" = [ "spin" ]; }; - resolvedDefaultFeatures = [ "spin" "spin_no_std" ]; }; "libc" = rec { crateName = "libc"; @@ -6170,23 +6253,6 @@ rec { "zlib-ng-compat" = [ "libz-sys/zlib-ng" "libssh2-sys?/zlib-ng-compat" ]; }; }; - "libm" = rec { - crateName = "libm"; - version = "0.2.16"; - edition = "2021"; - sha256 = "10brh0a3qjmbzkr5mf5xqi887nhs5y9layvnki89ykz9xb1wxlmn"; - authors = [ - "Alex Crichton " - "Amanieu d'Antras " - "Jorge Aparicio " - "Trevor Gross " - ]; - features = { - "default" = [ "arch" ]; - "unstable" = [ "unstable-intrinsics" "unstable-float" ]; - }; - resolvedDefaultFeatures = [ "arch" "default" ]; - }; "libyaml-rs" = rec { crateName = "libyaml-rs"; version = "0.3.0"; @@ -6489,81 +6555,6 @@ rec { }; resolvedDefaultFeatures = [ "default" "std" ]; }; - "num-bigint-dig" = rec { - crateName = "num-bigint-dig"; - version = "0.8.6"; - edition = "2021"; - sha256 = "1dxh3d8pzjc5k0kpy8gy2qhhhqs7zw8a7m564zl3ib8gcjkdsqg6"; - libName = "num_bigint_dig"; - authors = [ - "dignifiedquire " - "The Rust Project Developers" - ]; - dependencies = [ - { - name = "lazy_static"; - packageId = "lazy_static"; - usesDefaultFeatures = false; - features = [ "spin_no_std" ]; - } - { - name = "libm"; - packageId = "libm"; - } - { - name = "num-integer"; - packageId = "num-integer"; - usesDefaultFeatures = false; - features = [ "i128" ]; - } - { - name = "num-iter"; - packageId = "num-iter"; - usesDefaultFeatures = false; - } - { - name = "num-traits"; - packageId = "num-traits"; - usesDefaultFeatures = false; - features = [ "i128" ]; - } - { - name = "rand"; - packageId = "rand 0.8.8"; - optional = true; - usesDefaultFeatures = false; - } - { - name = "smallvec"; - packageId = "smallvec"; - usesDefaultFeatures = false; - } - { - name = "zeroize"; - packageId = "zeroize"; - optional = true; - usesDefaultFeatures = false; - } - ]; - devDependencies = [ - { - name = "rand"; - packageId = "rand 0.8.8"; - features = [ "small_rng" ]; - } - ]; - features = { - "arbitrary" = [ "dep:arbitrary" ]; - "default" = [ "std" "u64_digit" ]; - "fuzz" = [ "arbitrary" "smallvec/arbitrary" ]; - "prime" = [ "rand/std_rng" ]; - "rand" = [ "dep:rand" ]; - "serde" = [ "dep:serde" ]; - "std" = [ "num-integer/std" "num-traits/std" "smallvec/write" "rand/std" "serde/std" ]; - "zeroize" = [ "dep:zeroize" ]; - }; - resolvedDefaultFeatures = [ "i128" "prime" "rand" "u64_digit" "zeroize" ]; - }; "num-conv" = rec { crateName = "num-conv"; version = "0.2.2"; @@ -6575,57 +6566,6 @@ rec { ]; }; - "num-integer" = rec { - crateName = "num-integer"; - version = "0.1.47"; - edition = "2018"; - sha256 = "02z1p3azy6p10n99skrab4a6hhfd4amf2i9gm8sxqd1p9dfxkqkw"; - libName = "num_integer"; - authors = [ - "The Rust Project Developers" - ]; - dependencies = [ - { - name = "num-traits"; - packageId = "num-traits"; - usesDefaultFeatures = false; - features = [ "i128" ]; - } - ]; - features = { - "default" = [ "std" ]; - "std" = [ "num-traits/std" ]; - }; - resolvedDefaultFeatures = [ "i128" ]; - }; - "num-iter" = rec { - crateName = "num-iter"; - version = "0.1.46"; - edition = "2018"; - sha256 = "12q4x0lp9l6bvsak1p5q24lvfzl99ak9vzmwhqbwksm1d6yh0a69"; - libName = "num_iter"; - authors = [ - "The Rust Project Developers" - ]; - dependencies = [ - { - name = "num-integer"; - packageId = "num-integer"; - usesDefaultFeatures = false; - features = [ "i128" ]; - } - { - name = "num-traits"; - packageId = "num-traits"; - usesDefaultFeatures = false; - features = [ "i128" ]; - } - ]; - features = { - "default" = [ "std" ]; - "std" = [ "num-integer/std" "num-traits/std" ]; - }; - }; "num-traits" = rec { crateName = "num-traits"; version = "0.2.19"; @@ -6635,13 +6575,6 @@ rec { authors = [ "The Rust Project Developers" ]; - dependencies = [ - { - name = "libm"; - packageId = "libm"; - optional = true; - } - ]; buildDependencies = [ { name = "autocfg"; @@ -6652,7 +6585,7 @@ rec { "default" = [ "std" ]; "libm" = [ "dep:libm" ]; }; - resolvedDefaultFeatures = [ "default" "i128" "libm" "std" ]; + resolvedDefaultFeatures = [ "default" "std" ]; }; "once_cell" = rec { crateName = "once_cell"; @@ -7185,9 +7118,9 @@ rec { }; "p256" = rec { crateName = "p256"; - version = "0.13.2"; - edition = "2021"; - sha256 = "0jyd3c3k239ybs59ixpnl7dqkmm072fr1js8kh7ldx58bzc3m1n9"; + version = "0.14.0"; + edition = "2024"; + sha256 = "0appw4ifrx2zwyw2rrmb18x0qid75vvqqzhlpvdpm05w5ndj7jfj"; authors = [ "RustCrypto Developers" ]; @@ -7204,7 +7137,12 @@ rec { name = "elliptic-curve"; packageId = "elliptic-curve"; usesDefaultFeatures = false; - features = [ "hazmat" "sec1" ]; + features = [ "sec1" ]; + } + { + name = "primefield"; + packageId = "primefield"; + optional = true; } { name = "primeorder"; @@ -7226,6 +7164,10 @@ rec { usesDefaultFeatures = false; features = [ "dev" ]; } + { + name = "primefield"; + packageId = "primefield"; + } { name = "primeorder"; packageId = "primeorder"; @@ -7233,28 +7175,28 @@ rec { } ]; features = { - "alloc" = [ "ecdsa-core?/alloc" "elliptic-curve/alloc" ]; - "arithmetic" = [ "dep:primeorder" "elliptic-curve/arithmetic" ]; - "bits" = [ "arithmetic" "elliptic-curve/bits" ]; - "default" = [ "arithmetic" "ecdsa" "pem" "std" ]; - "digest" = [ "ecdsa-core/digest" "ecdsa-core/hazmat" ]; + "alloc" = [ "ecdsa-core?/alloc" "elliptic-curve/alloc" "primeorder?/alloc" ]; + "arithmetic" = [ "dep:primefield" "dep:primeorder" "elliptic-curve/arithmetic" ]; + "default" = [ "arithmetic" "ecdsa" "pem" "precomputed-tables" "std" ]; + "digest" = [ "ecdsa-core/digest" ]; "ecdh" = [ "arithmetic" "elliptic-curve/ecdh" ]; - "ecdsa" = [ "arithmetic" "ecdsa-core/signing" "ecdsa-core/verifying" "sha256" ]; + "ecdsa" = [ "arithmetic" "ecdsa-core/algorithm" "sha256" ]; "ecdsa-core" = [ "dep:ecdsa-core" ]; - "expose-field" = [ "arithmetic" ]; - "hash2curve" = [ "arithmetic" "elliptic-curve/hash2curve" ]; - "jwk" = [ "elliptic-curve/jwk" ]; + "getrandom" = [ "elliptic-curve/getrandom" ]; + "group-digest" = [ "hash2curve" "sha2" ]; + "hash2curve" = [ "arithmetic" "dep:hash2curve" "primeorder/hash2curve" ]; + "oprf" = [ "group-digest" ]; "pem" = [ "elliptic-curve/pem" "ecdsa-core/pem" "pkcs8" ]; "pkcs8" = [ "ecdsa-core?/pkcs8" "elliptic-curve/pkcs8" ]; + "precomputed-tables" = [ "arithmetic" "primeorder/basepoint-table" ]; "serde" = [ "ecdsa-core?/serde" "elliptic-curve/serde" "primeorder?/serde" "serdect" ]; "serdect" = [ "dep:serdect" ]; "sha2" = [ "dep:sha2" ]; "sha256" = [ "digest" "sha2" ]; - "std" = [ "alloc" "ecdsa-core?/std" "elliptic-curve/std" ]; + "std" = [ "alloc" "ecdsa-core?/std" "elliptic-curve/std" "getrandom" "primeorder?/std" ]; "test-vectors" = [ "dep:hex-literal" ]; - "voprf" = [ "elliptic-curve/voprf" "sha2" ]; }; - resolvedDefaultFeatures = [ "alloc" "arithmetic" "default" "digest" "ecdsa" "ecdsa-core" "pem" "pkcs8" "sha2" "sha256" "std" ]; + resolvedDefaultFeatures = [ "alloc" "arithmetic" "default" "digest" "ecdsa" "ecdsa-core" "getrandom" "pem" "pkcs8" "precomputed-tables" "sha2" "sha256" "std" ]; }; "parking" = rec { crateName = "parking"; @@ -7366,9 +7308,9 @@ rec { }; "pem-rfc7468" = rec { crateName = "pem-rfc7468"; - version = "0.7.0"; - edition = "2021"; - sha256 = "04l4852scl4zdva31c1z6jafbak0ni5pi0j38ml108zwzjdrrcw8"; + version = "1.0.0"; + edition = "2024"; + sha256 = "1nck8ig71axy21lsick2f9vcw7329mlx2hs88d382wz7w0im8c56"; libName = "pem_rfc7468"; authors = [ "RustCrypto Developers" @@ -7572,9 +7514,9 @@ rec { }; "pkcs1" = rec { crateName = "pkcs1"; - version = "0.7.5"; - edition = "2021"; - sha256 = "0zz4mil3nchnxljdfs2k5ab1cjqn7kq5lqp62n9qfix01zqvkzy8"; + version = "0.8.0-rc.5"; + edition = "2024"; + sha256 = "1hyvfdh70yq51c2dzhxfm6w6wy8scn8waayqlz56rslkwapxp1gl"; authors = [ "RustCrypto Developers" ]; @@ -7584,31 +7526,24 @@ rec { packageId = "der"; features = [ "oid" ]; } - { - name = "pkcs8"; - packageId = "pkcs8"; - optional = true; - usesDefaultFeatures = false; - } { name = "spki"; packageId = "spki"; } ]; features = { - "alloc" = [ "der/alloc" "zeroize" "pkcs8?/alloc" ]; - "pem" = [ "alloc" "der/pem" "pkcs8?/pem" ]; - "pkcs8" = [ "dep:pkcs8" ]; + "alloc" = [ "der/alloc" "spki/alloc" "zeroize" ]; + "pem" = [ "alloc" "der/pem" ]; "std" = [ "der/std" "alloc" ]; "zeroize" = [ "der/zeroize" ]; }; - resolvedDefaultFeatures = [ "alloc" "pem" "pkcs8" "std" "zeroize" ]; + resolvedDefaultFeatures = [ "alloc" "pem" "std" "zeroize" ]; }; "pkcs8" = rec { crateName = "pkcs8"; - version = "0.10.2"; - edition = "2021"; - sha256 = "1dx7w21gvn07azszgqd3ryjhyphsrjrmq5mmz1fbxkj5g0vv4l7r"; + version = "0.11.0"; + edition = "2024"; + sha256 = "1dqwyaj018cb0dyawgj42a799s3ys81r13ga6ihaaxf7d7d166a5"; authors = [ "RustCrypto Developers" ]; @@ -7626,15 +7561,14 @@ rec { features = { "3des" = [ "encryption" "pkcs5/3des" ]; "alloc" = [ "der/alloc" "der/zeroize" "spki/alloc" ]; + "ctutils" = [ "dep:ctutils" ]; "des-insecure" = [ "encryption" "pkcs5/des-insecure" ]; - "encryption" = [ "alloc" "pkcs5/alloc" "pkcs5/pbes2" "rand_core" ]; - "getrandom" = [ "rand_core/getrandom" ]; + "encryption" = [ "alloc" "pkcs5/alloc" "pkcs5/pbes2" "dep:rand_core" ]; + "getrandom" = [ "encryption" "pkcs5/getrandom" "dep:getrandom" ]; "pem" = [ "alloc" "der/pem" "spki/pem" ]; "pkcs5" = [ "dep:pkcs5" ]; - "rand_core" = [ "dep:rand_core" ]; "sha1-insecure" = [ "encryption" "pkcs5/sha1-insecure" ]; "std" = [ "alloc" "der/std" "spki/std" ]; - "subtle" = [ "dep:subtle" ]; }; resolvedDefaultFeatures = [ "alloc" "pem" "std" ]; }; @@ -7733,21 +7667,67 @@ rec { ]; dependencies = [ { - name = "zerocopy"; - packageId = "zerocopy"; - features = [ "simd" ]; + name = "zerocopy"; + packageId = "zerocopy"; + features = [ "simd" ]; + } + ]; + features = { + "default" = [ "std" ]; + }; + resolvedDefaultFeatures = [ "simd" "std" ]; + }; + "primefield" = rec { + crateName = "primefield"; + version = "0.14.0"; + edition = "2024"; + sha256 = "1x4jnnysiy9f41mmr6r7vinj0hl67df87302rf7iakkxxgjacmf5"; + authors = [ + "RustCrypto Developers" + ]; + dependencies = [ + { + name = "crypto-bigint"; + packageId = "crypto-bigint"; + rename = "bigint"; + usesDefaultFeatures = false; + features = [ "rand_core" "hybrid-array" "subtle" ]; + } + { + name = "crypto-common"; + packageId = "crypto-common"; + rename = "common"; + features = [ "rand_core" ]; + } + { + name = "ff"; + packageId = "ff"; + usesDefaultFeatures = false; + } + { + name = "rand_core"; + packageId = "rand_core 0.10.1"; + usesDefaultFeatures = false; + } + { + name = "subtle"; + packageId = "subtle"; + usesDefaultFeatures = false; + features = [ "const-generics" ]; + } + { + name = "zeroize"; + packageId = "zeroize"; + usesDefaultFeatures = false; } ]; - features = { - "default" = [ "std" ]; - }; - resolvedDefaultFeatures = [ "simd" "std" ]; + }; "primeorder" = rec { crateName = "primeorder"; - version = "0.13.6"; - edition = "2021"; - sha256 = "1rp16710mxksagcjnxqjjq9r9wf5vf72fs8wxffnvhb6i6hiqgim"; + version = "0.14.0"; + edition = "2024"; + sha256 = "01kvs3fgr25m9w50594xwsxdwd52awzc17znd0yhx83qijbl57sw"; authors = [ "RustCrypto Developers" ]; @@ -7758,13 +7738,37 @@ rec { usesDefaultFeatures = false; features = [ "arithmetic" "sec1" ]; } + { + name = "once_cell"; + packageId = "once_cell"; + optional = true; + usesDefaultFeatures = false; + } + { + name = "primefield"; + packageId = "primefield"; + } + { + name = "serdect"; + packageId = "serdect"; + optional = true; + usesDefaultFeatures = false; + } + { + name = "wnaf"; + packageId = "wnaf"; + usesDefaultFeatures = false; + } ]; features = { - "alloc" = [ "elliptic-curve/alloc" ]; + "alloc" = [ "elliptic-curve/alloc" "serdect?/alloc" ]; + "critical-section" = [ "basepoint-table" "once_cell/critical-section" ]; + "once_cell" = [ "dep:once_cell" ]; "serde" = [ "elliptic-curve/serde" "serdect" ]; "serdect" = [ "dep:serdect" ]; - "std" = [ "alloc" "elliptic-curve/std" ]; + "std" = [ "alloc" "elliptic-curve/std" "once_cell?/std" ]; }; + resolvedDefaultFeatures = [ "alloc" "basepoint-table" "std" ]; }; "proc-macro-crate" = rec { crateName = "proc-macro-crate"; @@ -8000,40 +8004,6 @@ rec { }; resolvedDefaultFeatures = [ "alloc" "default" "std" "std_rng" "sys_rng" "thread_rng" ]; }; - "rand 0.8.8" = rec { - crateName = "rand"; - version = "0.8.8"; - edition = "2018"; - sha256 = "0k3d9psya5icpiylff1w1wjbnc3q4pb1953n1iw7gbr61ggcfn70"; - authors = [ - "The Rand Project Developers" - "The Rust Project Developers" - ]; - dependencies = [ - { - name = "rand_chacha"; - packageId = "rand_chacha 0.3.1"; - optional = true; - usesDefaultFeatures = false; - } - { - name = "rand_core"; - packageId = "rand_core 0.6.4"; - } - ]; - features = { - "alloc" = [ "rand_core/alloc" ]; - "default" = [ "std" "std_rng" ]; - "getrandom" = [ "rand_core/getrandom" ]; - "libc" = [ "dep:libc" ]; - "rand_chacha" = [ "dep:rand_chacha" ]; - "serde" = [ "dep:serde" ]; - "serde1" = [ "serde" "rand_core/serde1" ]; - "std" = [ "rand_core/std" "rand_chacha/std" "alloc" "getrandom" "libc" ]; - "std_rng" = [ "rand_chacha" ]; - }; - resolvedDefaultFeatures = [ "rand_chacha" "std_rng" ]; - }; "rand 0.9.5" = rec { crateName = "rand"; version = "0.9.5"; @@ -8046,7 +8016,7 @@ rec { dependencies = [ { name = "rand_chacha"; - packageId = "rand_chacha 0.9.0"; + packageId = "rand_chacha"; optional = true; usesDefaultFeatures = false; } @@ -8066,36 +8036,7 @@ rec { }; resolvedDefaultFeatures = [ "alloc" "os_rng" "small_rng" "std" "std_rng" "thread_rng" ]; }; - "rand_chacha 0.3.1" = rec { - crateName = "rand_chacha"; - version = "0.3.1"; - edition = "2018"; - sha256 = "123x2adin558xbhvqb8w4f6syjsdkmqff8cxwhmjacpsl1ihmhg6"; - authors = [ - "The Rand Project Developers" - "The Rust Project Developers" - "The CryptoCorrosion Contributors" - ]; - dependencies = [ - { - name = "ppv-lite86"; - packageId = "ppv-lite86"; - usesDefaultFeatures = false; - features = [ "simd" ]; - } - { - name = "rand_core"; - packageId = "rand_core 0.6.4"; - } - ]; - features = { - "default" = [ "std" ]; - "serde" = [ "dep:serde" ]; - "serde1" = [ "serde" ]; - "std" = [ "ppv-lite86/std" ]; - }; - }; - "rand_chacha 0.9.0" = rec { + "rand_chacha" = rec { crateName = "rand_chacha"; version = "0.9.0"; edition = "2021"; @@ -8142,30 +8083,6 @@ rec { ]; }; - "rand_core 0.6.4" = rec { - crateName = "rand_core"; - version = "0.6.4"; - edition = "2018"; - sha256 = "0b4j2v4cb5krak1pv6kakv4sz6xcwbrmy2zckc32hsigbrwy82zc"; - authors = [ - "The Rand Project Developers" - "The Rust Project Developers" - ]; - dependencies = [ - { - name = "getrandom"; - packageId = "getrandom 0.2.17"; - optional = true; - } - ]; - features = { - "getrandom" = [ "dep:getrandom" ]; - "serde" = [ "dep:serde" ]; - "serde1" = [ "serde" ]; - "std" = [ "alloc" "getrandom" "getrandom/std" ]; - }; - resolvedDefaultFeatures = [ "alloc" "getrandom" "std" ]; - }; "rand_core 0.9.5" = rec { crateName = "rand_core"; version = "0.9.5"; @@ -8612,23 +8529,32 @@ rec { }; "rfc6979" = rec { crateName = "rfc6979"; - version = "0.4.0"; - edition = "2021"; - sha256 = "1chw95jgcfrysyzsq6a10b1j5qb7bagkx8h0wda4lv25in02mpgq"; + version = "0.6.0"; + edition = "2024"; + sha256 = "0aqpi1rynmj6np3lvdpy041w8rkm004133zx65myfgzvvb6mk95l"; authors = [ "RustCrypto Developers" ]; dependencies = [ + { + name = "crypto-bigint"; + packageId = "crypto-bigint"; + rename = "bigint"; + usesDefaultFeatures = false; + } { name = "hmac"; packageId = "hmac"; usesDefaultFeatures = false; - features = [ "reset" ]; } + ]; + devDependencies = [ { - name = "subtle"; - packageId = "subtle"; + name = "crypto-bigint"; + packageId = "crypto-bigint"; + rename = "bigint"; usesDefaultFeatures = false; + features = [ "alloc" ]; } ]; @@ -8696,9 +8622,9 @@ rec { }; "rsa" = rec { crateName = "rsa"; - version = "0.9.10"; + version = "0.10.0-rc.19"; edition = "2021"; - sha256 = "0bdikdwhcvl1gfh4637m5rdw3fgcl752aiygvzmwlgc8yl1kymxq"; + sha256 = "119bbjvkxiqipy4gn0ghgwygvan30illy9bflqf99vqn3qr5wh1k"; authors = [ "RustCrypto Developers" "dignifiedquire " @@ -8710,44 +8636,39 @@ rec { usesDefaultFeatures = false; } { - name = "digest"; - packageId = "digest"; - usesDefaultFeatures = false; - features = [ "alloc" "oid" ]; - } - { - name = "num-bigint-dig"; - packageId = "num-bigint-dig"; - rename = "num-bigint"; + name = "crypto-bigint"; + packageId = "crypto-bigint"; usesDefaultFeatures = false; - features = [ "i128" "prime" "zeroize" ]; + features = [ "zeroize" "alloc" ]; } { - name = "num-integer"; - packageId = "num-integer"; + name = "crypto-primes"; + packageId = "crypto-primes"; usesDefaultFeatures = false; } { - name = "num-traits"; - packageId = "num-traits"; + name = "digest"; + packageId = "digest"; usesDefaultFeatures = false; - features = [ "libm" ]; + features = [ "alloc" "oid" ]; } { name = "pkcs1"; packageId = "pkcs1"; + optional = true; usesDefaultFeatures = false; - features = [ "alloc" "pkcs8" ]; + features = [ "alloc" "pem" ]; } { name = "pkcs8"; packageId = "pkcs8"; + optional = true; usesDefaultFeatures = false; - features = [ "alloc" ]; + features = [ "alloc" "pem" ]; } { name = "rand_core"; - packageId = "rand_core 0.6.4"; + packageId = "rand_core 0.10.1"; usesDefaultFeatures = false; } { @@ -8766,14 +8687,10 @@ rec { { name = "spki"; packageId = "spki"; + optional = true; usesDefaultFeatures = false; features = [ "alloc" ]; } - { - name = "subtle"; - packageId = "subtle"; - usesDefaultFeatures = false; - } { name = "zeroize"; packageId = "zeroize"; @@ -8783,7 +8700,7 @@ rec { devDependencies = [ { name = "rand_core"; - packageId = "rand_core 0.6.4"; + packageId = "rand_core 0.10.1"; usesDefaultFeatures = false; } { @@ -8794,18 +8711,17 @@ rec { } ]; features = { - "default" = [ "std" "pem" "u64_digit" ]; - "getrandom" = [ "rand_core/getrandom" ]; - "nightly" = [ "num-bigint/nightly" ]; - "pem" = [ "pkcs1/pem" "pkcs8/pem" ]; + "crypto-common" = [ "dep:crypto-common" ]; + "default" = [ "std" "encoding" ]; + "encoding" = [ "dep:pkcs1" "dep:pkcs8" "dep:spki" ]; + "getrandom" = [ "crypto-bigint/getrandom" "crypto-common" ]; "pkcs5" = [ "pkcs8/encryption" ]; - "serde" = [ "dep:serde" "num-bigint/serde" ]; + "serde" = [ "encoding" "dep:serde" "dep:serdect" "crypto-bigint/serde" ]; "sha1" = [ "dep:sha1" ]; "sha2" = [ "dep:sha2" ]; - "std" = [ "digest/std" "pkcs1/std" "pkcs8/std" "rand_core/std" "signature/std" ]; - "u64_digit" = [ "num-bigint/u64_digit" ]; + "std" = [ "pkcs1?/std" "pkcs8?/std" ]; }; - resolvedDefaultFeatures = [ "default" "pem" "sha2" "std" "u64_digit" ]; + resolvedDefaultFeatures = [ "default" "encoding" "sha2" "std" ]; }; "rstest" = rec { crateName = "rstest"; @@ -9399,9 +9315,9 @@ rec { }; "sec1" = rec { crateName = "sec1"; - version = "0.7.3"; - edition = "2021"; - sha256 = "1p273j8c87pid6a1iyyc7vxbvifrw55wbxgr0dh3l8vnbxb7msfk"; + version = "0.8.1"; + edition = "2024"; + sha256 = "0k8fm3c0j2hb9bs5mgnksk9g75ny65w50bhjyzjkq80r5xy46vfm"; authors = [ "RustCrypto Developers" ]; @@ -9413,20 +9329,19 @@ rec { usesDefaultFeatures = false; } { - name = "der"; - packageId = "der"; + name = "ctutils"; + packageId = "ctutils"; optional = true; - features = [ "oid" ]; } { - name = "generic-array"; - packageId = "generic-array"; + name = "der"; + packageId = "der"; optional = true; - usesDefaultFeatures = false; + features = [ "oid" ]; } { - name = "pkcs8"; - packageId = "pkcs8"; + name = "hybrid-array"; + packageId = "hybrid-array"; optional = true; usesDefaultFeatures = false; } @@ -9444,18 +9359,18 @@ rec { } ]; features = { - "alloc" = [ "der?/alloc" "pkcs8?/alloc" "zeroize?/alloc" ]; + "alloc" = [ "der?/alloc" "zeroize?/alloc" ]; + "ctutils" = [ "dep:ctutils" ]; "default" = [ "der" "point" ]; "der" = [ "dep:der" "zeroize" ]; - "pem" = [ "alloc" "der/pem" "pkcs8/pem" ]; - "pkcs8" = [ "dep:pkcs8" ]; - "point" = [ "dep:base16ct" "dep:generic-array" ]; + "pem" = [ "alloc" "der/pem" ]; + "point" = [ "dep:base16ct" "dep:hybrid-array" ]; "serde" = [ "dep:serdect" ]; "std" = [ "alloc" "der?/std" ]; "subtle" = [ "dep:subtle" ]; "zeroize" = [ "dep:zeroize" "der?/zeroize" ]; }; - resolvedDefaultFeatures = [ "alloc" "default" "der" "pem" "pkcs8" "point" "std" "subtle" "zeroize" ]; + resolvedDefaultFeatures = [ "alloc" "ctutils" "default" "der" "pem" "point" "std" "subtle" "zeroize" ]; }; "secrecy" = rec { crateName = "secrecy"; @@ -9909,11 +9824,47 @@ rec { ]; }; + "serdect" = rec { + crateName = "serdect"; + version = "0.4.3"; + edition = "2024"; + sha256 = "0ghby2vq4k21xlpy3v8897rym4pvvwil5cxw0qjg3z6jrvnqzkv6"; + authors = [ + "RustCrypto Developers" + ]; + dependencies = [ + { + name = "base16ct"; + packageId = "base16ct"; + usesDefaultFeatures = false; + } + { + name = "serde"; + packageId = "serde"; + usesDefaultFeatures = false; + } + ]; + devDependencies = [ + { + name = "serde"; + packageId = "serde"; + usesDefaultFeatures = false; + features = [ "derive" ]; + } + ]; + features = { + "alloc" = [ "base16ct/alloc" "serde/alloc" ]; + "default" = [ "alloc" ]; + "derive" = [ "serde/derive" ]; + "zeroize" = [ "dep:zeroize" ]; + }; + resolvedDefaultFeatures = [ "alloc" ]; + }; "sha1" = rec { crateName = "sha1"; - version = "0.10.7"; - edition = "2018"; - sha256 = "1f632d529qzz95yrprr632w1fxqkrv6b6jksjc11vnzl049lay59"; + version = "0.11.0"; + edition = "2024"; + sha256 = "05025pf8d8zr2qq5xyh5m3wqls1fn7813gz1mfs7551mk724rk5a"; authors = [ "RustCrypto Developers" ]; @@ -9924,7 +9875,7 @@ rec { } { name = "cpufeatures"; - packageId = "cpufeatures 0.2.17"; + packageId = "cpufeatures"; target = { target, features }: (("aarch64" == target."arch" or null) || ("x86" == target."arch" or null) || ("x86_64" == target."arch" or null)); } { @@ -9940,17 +9891,17 @@ rec { } ]; features = { - "default" = [ "std" ]; + "alloc" = [ "digest/alloc" ]; + "default" = [ "alloc" "oid" ]; "oid" = [ "digest/oid" ]; - "std" = [ "digest/std" ]; + "zeroize" = [ "digest/zeroize" ]; }; - resolvedDefaultFeatures = [ "default" "std" ]; }; "sha2" = rec { crateName = "sha2"; - version = "0.10.9"; - edition = "2018"; - sha256 = "10xjj843v31ghsksd9sl9y12qfc48157j1xpb8v1ml39jy0psl57"; + version = "0.11.1"; + edition = "2024"; + sha256 = "06ddr559jvwx17d7847l6i1g4hrlwy39j0yimjwwfskxxfdhdms7"; authors = [ "RustCrypto Developers" ]; @@ -9961,7 +9912,7 @@ rec { } { name = "cpufeatures"; - packageId = "cpufeatures 0.2.17"; + packageId = "cpufeatures"; target = { target, features }: (("aarch64" == target."arch" or null) || ("x86_64" == target."arch" or null) || ("x86" == target."arch" or null)); } { @@ -9977,14 +9928,12 @@ rec { } ]; features = { - "asm" = [ "sha2-asm" ]; - "asm-aarch64" = [ "asm" ]; - "default" = [ "std" ]; + "alloc" = [ "digest/alloc" ]; + "default" = [ "alloc" "oid" ]; "oid" = [ "digest/oid" ]; - "sha2-asm" = [ "dep:sha2-asm" ]; - "std" = [ "digest/std" ]; + "zeroize" = [ "digest/zeroize" ]; }; - resolvedDefaultFeatures = [ "default" "oid" "std" ]; + resolvedDefaultFeatures = [ "alloc" "default" "oid" ]; }; "sharded-slab" = rec { crateName = "sharded-slab"; @@ -10047,9 +9996,9 @@ rec { }; "signature" = rec { crateName = "signature"; - version = "2.2.0"; - edition = "2021"; - sha256 = "1pi9hd5vqfr3q3k49k37z06p7gs5si0in32qia4mmr1dancr6m3p"; + version = "3.0.0"; + edition = "2024"; + sha256 = "1x9pfp51h2a0xs4x768q5r0y9yazv5naf216mjw9q17hpbf6gm98"; authors = [ "RustCrypto Developers" ]; @@ -10062,18 +10011,16 @@ rec { } { name = "rand_core"; - packageId = "rand_core 0.6.4"; + packageId = "rand_core 0.10.1"; optional = true; usesDefaultFeatures = false; } ]; features = { - "derive" = [ "dep:derive" ]; "digest" = [ "dep:digest" ]; "rand_core" = [ "dep:rand_core" ]; - "std" = [ "alloc" "rand_core?/std" ]; }; - resolvedDefaultFeatures = [ "alloc" "digest" "rand_core" "std" ]; + resolvedDefaultFeatures = [ "alloc" "digest" "rand_core" ]; }; "simd-adler32" = rec { crateName = "simd-adler32"; @@ -10311,36 +10258,11 @@ rec { }; resolvedDefaultFeatures = [ "all" ]; }; - "spin" = rec { - crateName = "spin"; - version = "0.9.9"; - edition = "2015"; - sha256 = "03psal0vh1xdxp7agphw09p7kf50v3bj1zshijq1s5bkdd7jcqrp"; - authors = [ - "Mathijs van de Nes " - "John Ericson " - "Joshua Barretto " - ]; - features = { - "barrier" = [ "mutex" ]; - "default" = [ "lock_api" "mutex" "spin_mutex" "rwlock" "once" "lazy" "barrier" ]; - "fair_mutex" = [ "mutex" ]; - "lazy" = [ "once" ]; - "lock_api" = [ "lock_api_crate" ]; - "lock_api_crate" = [ "dep:lock_api_crate" ]; - "portable-atomic" = [ "dep:portable-atomic" ]; - "portable_atomic" = [ "portable-atomic" ]; - "spin_mutex" = [ "mutex" ]; - "ticket_mutex" = [ "mutex" ]; - "use_ticket_mutex" = [ "mutex" "ticket_mutex" ]; - }; - resolvedDefaultFeatures = [ "once" ]; - }; "spki" = rec { crateName = "spki"; - version = "0.7.3"; - edition = "2021"; - sha256 = "17fj8k5fmx4w9mp27l970clrh5qa7r5sjdvbsln987xhb34dc7nr"; + version = "0.8.1"; + edition = "2024"; + sha256 = "1aci2hbcr0qbmhndkc4dl5i1w7p6yglld2f7ry8a4pcxiflmiyby"; authors = [ "RustCrypto Developers" ]; @@ -10361,7 +10283,8 @@ rec { "alloc" = [ "base64ct?/alloc" "der/alloc" ]; "arbitrary" = [ "std" "dep:arbitrary" "der/arbitrary" ]; "base64" = [ "dep:base64ct" ]; - "fingerprint" = [ "sha2" ]; + "digest" = [ "dep:digest" ]; + "fingerprint" = [ "digest" "sha2" ]; "pem" = [ "alloc" "der/pem" ]; "sha2" = [ "dep:sha2" ]; "std" = [ "der/std" "alloc" ]; @@ -10388,8 +10311,8 @@ rec { workspace_member = null; src = pkgs.fetchgit { url = "https://github.com/stackabletech/operator-rs.git"; - rev = "b61f619aa15efa2fa8c88d7eeae6154c3fb3918a"; - sha256 = "0ivrv56a8yzy68d9hr7cvy7hk5w0i2hidzlnns23h0g60ynb3axq"; + rev = "46a18a9a4899976b702ca87f73430c6f12d17c56"; + sha256 = "10va8k2dwshcb6c7pp01q6m6bx2ad58fzqxf19mp90hhcp00bxm7"; }; libName = "stackable_agent"; authors = [ @@ -10431,8 +10354,8 @@ rec { workspace_member = null; src = pkgs.fetchgit { url = "https://github.com/stackabletech/operator-rs.git"; - rev = "b61f619aa15efa2fa8c88d7eeae6154c3fb3918a"; - sha256 = "0ivrv56a8yzy68d9hr7cvy7hk5w0i2hidzlnns23h0g60ynb3axq"; + rev = "46a18a9a4899976b702ca87f73430c6f12d17c56"; + sha256 = "10va8k2dwshcb6c7pp01q6m6bx2ad58fzqxf19mp90hhcp00bxm7"; }; libName = "stackable_certs"; authors = [ @@ -10472,7 +10395,7 @@ rec { } { name = "rand_core"; - packageId = "rand_core 0.6.4"; + packageId = "rand_core 0.10.1"; } { name = "rsa"; @@ -10703,14 +10626,18 @@ rec { workspace_member = null; src = pkgs.fetchgit { url = "https://github.com/stackabletech/operator-rs.git"; - rev = "b61f619aa15efa2fa8c88d7eeae6154c3fb3918a"; - sha256 = "0ivrv56a8yzy68d9hr7cvy7hk5w0i2hidzlnns23h0g60ynb3axq"; + rev = "46a18a9a4899976b702ca87f73430c6f12d17c56"; + sha256 = "10va8k2dwshcb6c7pp01q6m6bx2ad58fzqxf19mp90hhcp00bxm7"; }; libName = "stackable_operator"; authors = [ "Stackable GmbH " ]; dependencies = [ + { + name = "base16ct"; + packageId = "base16ct"; + } { name = "base64"; packageId = "base64 0.23.1"; @@ -10908,8 +10835,8 @@ rec { workspace_member = null; src = pkgs.fetchgit { url = "https://github.com/stackabletech/operator-rs.git"; - rev = "b61f619aa15efa2fa8c88d7eeae6154c3fb3918a"; - sha256 = "0ivrv56a8yzy68d9hr7cvy7hk5w0i2hidzlnns23h0g60ynb3axq"; + rev = "46a18a9a4899976b702ca87f73430c6f12d17c56"; + sha256 = "10va8k2dwshcb6c7pp01q6m6bx2ad58fzqxf19mp90hhcp00bxm7"; }; procMacro = true; libName = "stackable_operator_derive"; @@ -10943,8 +10870,8 @@ rec { workspace_member = null; src = pkgs.fetchgit { url = "https://github.com/stackabletech/operator-rs.git"; - rev = "b61f619aa15efa2fa8c88d7eeae6154c3fb3918a"; - sha256 = "0ivrv56a8yzy68d9hr7cvy7hk5w0i2hidzlnns23h0g60ynb3axq"; + rev = "46a18a9a4899976b702ca87f73430c6f12d17c56"; + sha256 = "10va8k2dwshcb6c7pp01q6m6bx2ad58fzqxf19mp90hhcp00bxm7"; }; libName = "stackable_shared"; authors = [ @@ -11024,8 +10951,8 @@ rec { workspace_member = null; src = pkgs.fetchgit { url = "https://github.com/stackabletech/operator-rs.git"; - rev = "b61f619aa15efa2fa8c88d7eeae6154c3fb3918a"; - sha256 = "0ivrv56a8yzy68d9hr7cvy7hk5w0i2hidzlnns23h0g60ynb3axq"; + rev = "46a18a9a4899976b702ca87f73430c6f12d17c56"; + sha256 = "10va8k2dwshcb6c7pp01q6m6bx2ad58fzqxf19mp90hhcp00bxm7"; }; libName = "stackable_telemetry"; authors = [ @@ -11134,8 +11061,8 @@ rec { workspace_member = null; src = pkgs.fetchgit { url = "https://github.com/stackabletech/operator-rs.git"; - rev = "b61f619aa15efa2fa8c88d7eeae6154c3fb3918a"; - sha256 = "0ivrv56a8yzy68d9hr7cvy7hk5w0i2hidzlnns23h0g60ynb3axq"; + rev = "46a18a9a4899976b702ca87f73430c6f12d17c56"; + sha256 = "10va8k2dwshcb6c7pp01q6m6bx2ad58fzqxf19mp90hhcp00bxm7"; }; libName = "stackable_versioned"; authors = [ @@ -11180,8 +11107,8 @@ rec { workspace_member = null; src = pkgs.fetchgit { url = "https://github.com/stackabletech/operator-rs.git"; - rev = "b61f619aa15efa2fa8c88d7eeae6154c3fb3918a"; - sha256 = "0ivrv56a8yzy68d9hr7cvy7hk5w0i2hidzlnns23h0g60ynb3axq"; + rev = "46a18a9a4899976b702ca87f73430c6f12d17c56"; + sha256 = "10va8k2dwshcb6c7pp01q6m6bx2ad58fzqxf19mp90hhcp00bxm7"; }; procMacro = true; libName = "stackable_versioned_macros"; @@ -11244,8 +11171,8 @@ rec { workspace_member = null; src = pkgs.fetchgit { url = "https://github.com/stackabletech/operator-rs.git"; - rev = "b61f619aa15efa2fa8c88d7eeae6154c3fb3918a"; - sha256 = "0ivrv56a8yzy68d9hr7cvy7hk5w0i2hidzlnns23h0g60ynb3axq"; + rev = "46a18a9a4899976b702ca87f73430c6f12d17c56"; + sha256 = "10va8k2dwshcb6c7pp01q6m6bx2ad58fzqxf19mp90hhcp00bxm7"; }; libName = "stackable_webhook"; authors = [ @@ -11488,7 +11415,7 @@ rec { features = { "default" = [ "std" "i128" ]; }; - resolvedDefaultFeatures = [ "i128" ]; + resolvedDefaultFeatures = [ "const-generics" "i128" ]; }; "symlink" = rec { crateName = "symlink"; @@ -11960,9 +11887,9 @@ rec { }; "tokio" = rec { crateName = "tokio"; - version = "1.54.0"; + version = "1.54.1"; edition = "2021"; - sha256 = "1g70adfv7bw6q84lr9s9nr232lpwvx5mv9h8x77zwnxrfjj6ihd0"; + sha256 = "18wn0c3yvy6ld8yis0fw6rqkc4rwnmv5mav4fan6zljqa8jx6bxm"; authors = [ "Tokio Contributors " ]; @@ -13310,6 +13237,7 @@ rec { "scale-info" = [ "dep:scale-info" ]; "scale_info" = [ "scale-info/derive" ]; }; + resolvedDefaultFeatures = [ "const-generics" ]; }; "ucd-trie" = rec { crateName = "ucd-trie"; @@ -13474,9 +13402,9 @@ rec { }; "uuid" = rec { crateName = "uuid"; - version = "1.28.0"; + version = "1.29.0"; edition = "2021"; - sha256 = "01mnlhz09hwdjaxwzjd6bql3fjb5zl9n8dzaxn7dvdxyhiiiihbw"; + sha256 = "0525458ww4jwa0nw513jdrhgwxrldglzvfrbw5gnjl0cc88js3ja"; authors = [ "Ashley Mannix" "Dylan DPC" @@ -15180,6 +15108,41 @@ rec { "rustc-dep-of-std" = [ "dep:core" "dep:alloc" ]; }; }; + "wnaf" = rec { + crateName = "wnaf"; + version = "0.14.1"; + edition = "2024"; + sha256 = "1ypnjmp4jpfh58hmxqijgnzpxkyx861jg691k2zn4pnv7y5s2p3r"; + authors = [ + "RustCrypto Developers" + ]; + dependencies = [ + { + name = "ff"; + packageId = "ff"; + usesDefaultFeatures = false; + } + { + name = "group"; + packageId = "group"; + usesDefaultFeatures = false; + } + { + name = "hybrid-array"; + packageId = "hybrid-array"; + rename = "array"; + } + { + name = "primefield"; + packageId = "primefield"; + usesDefaultFeatures = false; + } + ]; + features = { + "alloc" = [ "array/alloc" "ff/alloc" "group/alloc" ]; + "default" = [ "alloc" ]; + }; + }; "writeable" = rec { crateName = "writeable"; version = "0.6.4"; @@ -15195,9 +15158,9 @@ rec { }; "x509-cert" = rec { crateName = "x509-cert"; - version = "0.2.5"; - edition = "2021"; - sha256 = "155f42vm6m7phn8w7s2wmk9vli3ws45dqpk5z3jilw0a04syj08k"; + version = "0.3.0"; + edition = "2024"; + sha256 = "12q37mnxnjhp9a8cl64rkvvavf08pzjd08ynhgpkgccw5mjg8phh"; libName = "x509_cert"; authors = [ "RustCrypto Developers" @@ -15217,6 +15180,7 @@ rec { name = "sha1"; packageId = "sha1"; optional = true; + usesDefaultFeatures = false; } { name = "signature"; @@ -15239,15 +15203,17 @@ rec { ]; features = { "arbitrary" = [ "dep:arbitrary" "std" "der/arbitrary" "spki/arbitrary" ]; - "builder" = [ "std" "sha1/default" "signature" ]; + "base64" = [ "spki/base64" ]; + "builder" = [ "dep:sha1" "signature" ]; "default" = [ "pem" "std" ]; + "digest" = [ "dep:digest" "spki/digest" ]; + "fingerprint" = [ "spki/fingerprint" ]; "pem" = [ "der/pem" "spki/pem" ]; "sct" = [ "dep:tls_codec" ]; - "sha1" = [ "dep:sha1" ]; "signature" = [ "dep:signature" ]; - "std" = [ "const-oid/std" "der/std" "spki/std" "tls_codec?/std" ]; + "std" = [ "der/std" "spki/std" "tls_codec?/std" ]; }; - resolvedDefaultFeatures = [ "builder" "default" "pem" "sha1" "signature" "std" ]; + resolvedDefaultFeatures = [ "builder" "default" "pem" "signature" "std" ]; }; "xml" = rec { crateName = "xml"; diff --git a/crate-hashes.json b/crate-hashes.json index f2a7eec62..e08caca83 100644 --- a/crate-hashes.json +++ b/crate-hashes.json @@ -1,12 +1,12 @@ { - "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#k8s-version@0.1.4": "0ivrv56a8yzy68d9hr7cvy7hk5w0i2hidzlnns23h0g60ynb3axq", - "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#stackable-agent@0.1.0": "0ivrv56a8yzy68d9hr7cvy7hk5w0i2hidzlnns23h0g60ynb3axq", - "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#stackable-certs@0.4.1": "0ivrv56a8yzy68d9hr7cvy7hk5w0i2hidzlnns23h0g60ynb3axq", - "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#stackable-operator-derive@0.3.1": "0ivrv56a8yzy68d9hr7cvy7hk5w0i2hidzlnns23h0g60ynb3axq", - "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#stackable-operator@0.121.0": "0ivrv56a8yzy68d9hr7cvy7hk5w0i2hidzlnns23h0g60ynb3axq", - "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#stackable-shared@0.2.0": "0ivrv56a8yzy68d9hr7cvy7hk5w0i2hidzlnns23h0g60ynb3axq", - "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#stackable-telemetry@0.6.5": "0ivrv56a8yzy68d9hr7cvy7hk5w0i2hidzlnns23h0g60ynb3axq", - "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#stackable-versioned-macros@0.13.0": "0ivrv56a8yzy68d9hr7cvy7hk5w0i2hidzlnns23h0g60ynb3axq", - "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#stackable-versioned@0.13.0": "0ivrv56a8yzy68d9hr7cvy7hk5w0i2hidzlnns23h0g60ynb3axq", - "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#stackable-webhook@0.10.0": "0ivrv56a8yzy68d9hr7cvy7hk5w0i2hidzlnns23h0g60ynb3axq" + "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#k8s-version@0.1.4": "10va8k2dwshcb6c7pp01q6m6bx2ad58fzqxf19mp90hhcp00bxm7", + "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#stackable-agent@0.1.0": "10va8k2dwshcb6c7pp01q6m6bx2ad58fzqxf19mp90hhcp00bxm7", + "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#stackable-certs@0.4.1": "10va8k2dwshcb6c7pp01q6m6bx2ad58fzqxf19mp90hhcp00bxm7", + "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#stackable-operator-derive@0.3.1": "10va8k2dwshcb6c7pp01q6m6bx2ad58fzqxf19mp90hhcp00bxm7", + "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#stackable-operator@0.121.0": "10va8k2dwshcb6c7pp01q6m6bx2ad58fzqxf19mp90hhcp00bxm7", + "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#stackable-shared@0.2.0": "10va8k2dwshcb6c7pp01q6m6bx2ad58fzqxf19mp90hhcp00bxm7", + "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#stackable-telemetry@0.6.5": "10va8k2dwshcb6c7pp01q6m6bx2ad58fzqxf19mp90hhcp00bxm7", + "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#stackable-versioned-macros@0.13.0": "10va8k2dwshcb6c7pp01q6m6bx2ad58fzqxf19mp90hhcp00bxm7", + "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#stackable-versioned@0.13.0": "10va8k2dwshcb6c7pp01q6m6bx2ad58fzqxf19mp90hhcp00bxm7", + "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#stackable-webhook@0.10.0": "10va8k2dwshcb6c7pp01q6m6bx2ad58fzqxf19mp90hhcp00bxm7" } \ No newline at end of file diff --git a/extra/crds.yaml b/extra/crds.yaml index d814d144f..48821787b 100644 --- a/extra/crds.yaml +++ b/extra/crds.yaml @@ -1053,7 +1053,7 @@ spec: - secretName properties: secretClassName: - description: An AutoTLS SecretClass used to provision the certificate. + description: A SecretClass providing the certificate. maxLength: 253 minLength: 1 pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ From 28efc851cbf0444ceb55150bdb6718307706966f Mon Sep 17 00:00:00 2001 From: Benedikt Labrenz Date: Thu, 29 Oct 2026 16:29:26 +0100 Subject: [PATCH 71/93] improve broker address Display impl --- rust/kafka-admin/src/bootstrap.rs | 22 ++++++++++++++++++---- 1 file changed, 18 insertions(+), 4 deletions(-) diff --git a/rust/kafka-admin/src/bootstrap.rs b/rust/kafka-admin/src/bootstrap.rs index 1ab97bb93..27f14d751 100644 --- a/rust/kafka-admin/src/bootstrap.rs +++ b/rust/kafka-admin/src/bootstrap.rs @@ -1,6 +1,6 @@ //! The bootstrap servers of a Kafka cluster. -use std::fmt; +use std::{fmt, net::Ipv6Addr}; use snafu::{OptionExt, Snafu, ensure}; @@ -23,10 +23,11 @@ pub struct BrokerAddress { impl fmt::Display for BrokerAddress { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - if self.host.contains(':') { - write!(f, "[{}]:{}", self.host, self.port) + let Self { host, port } = self; + if host.parse::().is_ok() { + write!(f, "[{host}]:{port}") } else { - write!(f, "{}:{}", self.host, self.port) + write!(f, "{host}:{port}") } } } @@ -91,6 +92,19 @@ mod tests { ); } + #[rstest] + #[case("kafka", "kafka:9092")] + #[case("10.0.0.1", "10.0.0.1:9092")] + #[case("fd00::1", "[fd00::1]:9092")] + fn format_address_with_brackets_around_ipv6(#[case] host: &str, #[case] expected: &str) { + let address = BrokerAddress { + host: host.to_owned(), + port: 9092, + }; + + assert_eq!(expected, address.to_string()); + } + #[rstest] #[case("")] #[case(" , ")] From 82f895032ee6944bee05fa8a35056597ae2fa74d Mon Sep 17 00:00:00 2001 From: Benedikt Labrenz Date: Thu, 29 Oct 2026 16:45:31 +0100 Subject: [PATCH 72/93] build the tls config once instead of per-server --- rust/kafka-admin/src/client.rs | 10 +++++++++- rust/kafka-admin/src/connection.rs | 17 +++++++---------- 2 files changed, 16 insertions(+), 11 deletions(-) diff --git a/rust/kafka-admin/src/client.rs b/rust/kafka-admin/src/client.rs index 7766ca2d7..1b81917b6 100644 --- a/rust/kafka-admin/src/client.rs +++ b/rust/kafka-admin/src/client.rs @@ -2,6 +2,7 @@ use std::{ collections::{BTreeMap, BTreeSet}, + sync::Arc, time::Duration, }; @@ -30,6 +31,7 @@ use tokio::time::Instant; use crate::{ bootstrap::BrokerAddress, connection::{self, Connection, Security}, + tls, }; // The highest versions that Kafka 3.9.2 supports and 4.3.1 still supports, see `validVersions` in @@ -70,6 +72,9 @@ pub enum Error { #[snafu(display("no bootstrap servers given"))] NoBootstrapServers, + #[snafu(display("failed to configure TLS"))] + Tls { source: tls::Error }, + #[snafu(display("failed to connect to the bootstrap server {server}"))] Connect { source: connection::Error, @@ -293,9 +298,12 @@ impl KafkaAdminClient { /// Connects to the first bootstrap server that answers. Any broker works, because brokers in /// KRaft mode forward admin requests to the active controller. async fn connect(&self) -> Result { + let tls_config = tls::client_config(&self.security) + .context(TlsSnafu)? + .map(Arc::new); let mut last_error = Error::NoBootstrapServers; for server in &self.bootstrap_servers { - match Connection::connect(server, &self.security, &self.client_id).await { + match Connection::connect(server, tls_config.clone(), &self.client_id).await { Ok(connection) => return Ok(connection), Err(source) => { last_error = Error::Connect { diff --git a/rust/kafka-admin/src/connection.rs b/rust/kafka-admin/src/connection.rs index c678fd797..f34134dbd 100644 --- a/rust/kafka-admin/src/connection.rs +++ b/rust/kafka-admin/src/connection.rs @@ -24,7 +24,6 @@ use tokio_util::{ use crate::{ bootstrap::BrokerAddress, codec::{self, api_key, decode_response, encode_request}, - tls, }; // The defaults of Kafka's Java admin client, in `clients/src/main/java/org/apache/kafka/clients/`: @@ -55,9 +54,6 @@ pub enum Error { address: BrokerAddress, }, - #[snafu(display("failed to configure TLS"))] - Tls { source: tls::Error }, - #[snafu(display("{host:?} is not a valid TLS server name"))] InvalidServerName { host: String }, @@ -136,14 +132,15 @@ pub(crate) struct Connection { } impl Connection { - /// Connects to the broker and fetches the API versions it supports. + /// Connects to the broker and fetches the API versions it supports. Without a `tls_config`, + /// the connection is plaintext. pub(crate) async fn connect( address: &BrokerAddress, - security: &Security, + tls_config: Option>, client_id: &str, ) -> Result { let tcp_stream = connect_tcp(address).await?; - let stream = match tls::client_config(security).context(TlsSnafu)? { + let stream = match tls_config { None => Either::Left(tcp_stream), Some(config) => Either::Right(tls_handshake(tcp_stream, address, config).await?), }; @@ -274,7 +271,7 @@ fn check_version( async fn tls_handshake( tcp_stream: TcpStream, address: &BrokerAddress, - config: ClientConfig, + config: Arc, ) -> Result> { let host = &address.host; let server_name = ServerName::try_from(host.clone()) @@ -282,7 +279,7 @@ async fn tls_handshake( .context(InvalidServerNameSnafu { host })?; timeout( CONNECT_TIMEOUT, - TlsConnector::from(Arc::new(config)).connect(server_name, tcp_stream), + TlsConnector::from(config).connect(server_name, tcp_stream), ) .await .context(ConnectTimeoutSnafu { @@ -423,7 +420,7 @@ mod tests { port, }; let (connection, broker) = tokio::join!( - Connection::connect(&address, &Security::Plaintext, "test"), + Connection::connect(&address, None, "test"), accept(&listener, &[(MetadataRequest::KEY, 0, 13)]) ); (connection.expect("the connection should succeed"), broker) From 1937b5317bed6b1a51f4734b67dcada24d39aca7 Mon Sep 17 00:00:00 2001 From: Benedikt Labrenz Date: Thu, 29 Oct 2026 16:53:31 +0100 Subject: [PATCH 73/93] keep the connection if only the version check fails --- rust/kafka-admin/src/client.rs | 26 ++++++++++++++++++++++---- rust/kafka-admin/src/connection.rs | 17 +++++++---------- 2 files changed, 29 insertions(+), 14 deletions(-) diff --git a/rust/kafka-admin/src/client.rs b/rust/kafka-admin/src/client.rs index 1b81917b6..0287ee02e 100644 --- a/rust/kafka-admin/src/client.rs +++ b/rust/kafka-admin/src/client.rs @@ -270,12 +270,13 @@ impl KafkaAdminClient { /// Sends the request on the current connection or a new one. A request that fails because /// the broker closed the reused connection is sent once more on a new one. A failed or - /// cancelled request drops the connection, and a connection idle for longer than - /// [`CONNECTION_MAX_IDLE`] is replaced. + /// cancelled request drops the connection, unless only the version check failed, and a + /// connection idle for longer than [`CONNECTION_MAX_IDLE`] is replaced. async fn send(&mut self, request: R, version: i16) -> Result { - if let Some((mut connection, last_used)) = self.connection.take() + if let Some((connection, last_used)) = self.connection.take() && last_used.elapsed() < CONNECTION_MAX_IDLE { + let mut connection = self.with_version_checked::(connection, last_used, version)?; match connection.send(&request, version).await { Err(error) if error.is_broken_connection() => {} result => { @@ -286,7 +287,8 @@ impl KafkaAdminClient { } } // After a broken connections the request is directly send on a new connection again - let mut connection = self.connect().await?; + let connection = self.connect().await?; + let mut connection = self.with_version_checked::(connection, Instant::now(), version)?; let response = connection .send(&request, version) .await @@ -295,6 +297,22 @@ impl KafkaAdminClient { Ok(response) } + /// Keeps the connection if the broker does not support the version, as the check does no I/O. + fn with_version_checked( + &mut self, + connection: Connection, + last_used: Instant, + version: i16, + ) -> Result { + match connection.check_version::(version) { + Ok(()) => Ok(connection), + Err(source) => { + self.connection = Some((connection, last_used)); + Err(Error::Send { source }) + } + } + } + /// Connects to the first bootstrap server that answers. Any broker works, because brokers in /// KRaft mode forward admin requests to the active controller. async fn connect(&self) -> Result { diff --git a/rust/kafka-admin/src/connection.rs b/rust/kafka-admin/src/connection.rs index f34134dbd..ce57663af 100644 --- a/rust/kafka-admin/src/connection.rs +++ b/rust/kafka-admin/src/connection.rs @@ -147,7 +147,7 @@ impl Connection { let mut connection = Self::new(stream, client_id); let response = connection - .send_with_version(&api_versions_request(), API_VERSIONS_VERSION) + .send(&api_versions_request(), API_VERSIONS_VERSION) .await?; connection.broker_api_versions = api_versions_result(response)?; Ok(connection) @@ -166,20 +166,17 @@ impl Connection { } } - /// Sends the request with the given version, if the broker supports it. - pub(crate) async fn send( - &mut self, - request: &R, - version: i16, - ) -> Result { + /// Whether the broker supports the version of the API of `R`. No request is sent, so the + /// connection stays usable after a failed check. + pub(crate) fn check_version(&self, version: i16) -> Result<()> { // Each API that is relevant for us has a version that is supported by all of our Kafka versions. // We use these versions and pin them in client.rs. // We will have to add API version negotiation at some point in the future. - check_version(&self.broker_api_versions, api_key::(), version)?; - self.send_with_version(request, version).await + check_version(&self.broker_api_versions, api_key::(), version) } - async fn send_with_version( + /// Sends the request with the given version, see [`Self::check_version`]. + pub(crate) async fn send( &mut self, request: &R, version: i16, From b00fe6590a97dafd0de83ad5fe5c0160fec32ef7 Mon Sep 17 00:00:00 2001 From: Benedikt Labrenz Date: Thu, 29 Oct 2026 17:01:11 +0100 Subject: [PATCH 74/93] rename helpers and doc comments to be more descriptive --- rust/kafka-admin/src/client.rs | 44 +++++++++++++++++------------- rust/kafka-admin/src/connection.rs | 18 ++++++------ 2 files changed, 34 insertions(+), 28 deletions(-) diff --git a/rust/kafka-admin/src/client.rs b/rust/kafka-admin/src/client.rs index 0287ee02e..c896e9bad 100644 --- a/rust/kafka-admin/src/client.rs +++ b/rust/kafka-admin/src/client.rs @@ -186,7 +186,8 @@ impl KafkaAdminClient { describe_topic_configs_result(response, topic) } - /// `None` uses the broker default for the number of partitions or the replication factor. + /// Creates the topic. Without `partitions` or `replication_factor`, the broker applies its + /// `num.partitions` or `default.replication.factor`. pub async fn create_topic( &mut self, topic: &str, @@ -212,8 +213,8 @@ impl KafkaAdminClient { ) .await?; - let result = first(response.results, ApiKey::CreatePartitions, topic)?; - check( + let result = first_result(response.results, ApiKey::CreatePartitions, topic)?; + check_error_code( ApiKey::CreatePartitions, topic, result.error_code, @@ -236,8 +237,8 @@ impl KafkaAdminClient { ) .await?; - let result = first(response.responses, ApiKey::IncrementalAlterConfigs, topic)?; - check( + let result = first_result(response.responses, ApiKey::IncrementalAlterConfigs, topic)?; + check_error_code( ApiKey::IncrementalAlterConfigs, topic, result.error_code, @@ -344,11 +345,11 @@ fn describe_topic_request(topic: &str) -> MetadataRequest { } fn describe_topic_result(response: MetadataResponse, topic: &str) -> Result> { - let result = first(response.topics, ApiKey::Metadata, topic)?; + let result = first_result(response.topics, ApiKey::Metadata, topic)?; if result.error_code.err() == Some(ResponseError::UnknownTopicOrPartition) { return Ok(None); } - check(ApiKey::Metadata, topic, result.error_code, None)?; + check_error_code(ApiKey::Metadata, topic, result.error_code, None)?; let partitions = result .partitions @@ -381,8 +382,8 @@ fn describe_topic_configs_result( response: DescribeConfigsResponse, topic: &str, ) -> Result> { - let result = first(response.results, ApiKey::DescribeConfigs, topic)?; - check( + let result = first_result(response.results, ApiKey::DescribeConfigs, topic)?; + check_error_code( ApiKey::DescribeConfigs, topic, result.error_code, @@ -431,15 +432,15 @@ fn create_topic_request( } fn create_topic_result(response: CreateTopicsResponse, topic: &str) -> Result { - let result = first(response.topics, ApiKey::CreateTopics, topic)?; - check( + let result = first_result(response.topics, ApiKey::CreateTopics, topic)?; + check_error_code( ApiKey::CreateTopics, topic, result.error_code, result.error_message, )?; // Without the details, e.g. if `DESCRIBE_CONFIGS` is not allowed, the topic was still created. - check( + check_error_code( ApiKey::CreateTopics, topic, result.topic_config_error_code, @@ -505,7 +506,7 @@ fn delete_topic_request(topic: &str) -> DeleteTopicsRequest { } fn delete_topic_result(response: DeleteTopicsResponse, topic: &str) -> Result<()> { - let result = first(response.responses, ApiKey::DeleteTopics, topic)?; + let result = first_result(response.responses, ApiKey::DeleteTopics, topic)?; // It's ok if topic or partition are already being deleted if matches!( result.error_code.err(), @@ -513,7 +514,7 @@ fn delete_topic_result(response: DeleteTopicsResponse, topic: &str) -> Result<() ) { return Ok(()); } - check( + check_error_code( ApiKey::DeleteTopics, topic, result.error_code, @@ -540,7 +541,7 @@ fn list_partition_reassignments_result( response: ListPartitionReassignmentsResponse, topic: &str, ) -> Result> { - check( + check_error_code( ApiKey::ListPartitionReassignments, topic, response.error_code, @@ -564,14 +565,19 @@ fn topic_name(topic: &str) -> TopicName { } // Used instead of [0] to correctly handle empty results & take ownership -fn first(results: Vec, api_key: ApiKey, topic: &str) -> Result { +fn first_result(results: Vec, api_key: ApiKey, topic: &str) -> Result { results .into_iter() .next() .context(MissingResultSnafu { api_key, topic }) } -fn check(api_key: ApiKey, topic: &str, error_code: i16, message: Option) -> Result<()> { +fn check_error_code( + api_key: ApiKey, + topic: &str, + error_code: i16, + message: Option, +) -> Result<()> { match error_code.err() { None => Ok(()), Some(error) => KafkaSnafu { @@ -985,7 +991,7 @@ mod tests { #[test] fn report_rejected_request() { - let error = check( + let error = check_error_code( ApiKey::CreateTopics, TOPIC, ResponseError::TopicAlreadyExists.code(), @@ -1006,7 +1012,7 @@ mod tests { #[test] fn reject_response_without_result() { - let error = first( + let error = first_result( Vec::::new(), ApiKey::DeleteTopics, TOPIC, diff --git a/rust/kafka-admin/src/connection.rs b/rust/kafka-admin/src/connection.rs index ce57663af..abd0eb482 100644 --- a/rust/kafka-admin/src/connection.rs +++ b/rust/kafka-admin/src/connection.rs @@ -172,7 +172,7 @@ impl Connection { // Each API that is relevant for us has a version that is supported by all of our Kafka versions. // We use these versions and pin them in client.rs. // We will have to add API version negotiation at some point in the future. - check_version(&self.broker_api_versions, api_key::(), version) + check_version_in_range(&self.broker_api_versions, api_key::(), version) } /// Sends the request with the given version, see [`Self::check_version`]. @@ -246,7 +246,7 @@ fn api_versions_result(response: ApiVersionsResponse) -> Result, api_key: ApiKey, version: i16, @@ -330,7 +330,7 @@ pub(crate) mod test_support { &mut broker, &request, request.correlation_id, - api_versions(apis), + api_versions_response(apis), ) .await; broker @@ -385,7 +385,7 @@ pub(crate) mod test_support { .expect("the response should be writable"); } - pub(crate) fn api_versions(apis: &[(i16, i16, i16)]) -> ApiVersionsResponse { + pub(crate) fn api_versions_response(apis: &[(i16, i16, i16)]) -> ApiVersionsResponse { ApiVersionsResponse::default().with_api_keys( apis.iter() .map(|(api_key, min_version, max_version)| { @@ -405,7 +405,7 @@ mod tests { use tokio::io::AsyncWriteExt; use super::{ - test_support::{accept, api_versions, listen, read_request, write_response}, + test_support::{accept, api_versions_response, listen, read_request, write_response}, *, }; @@ -425,7 +425,7 @@ mod tests { #[test] fn read_api_versions() { - let response = api_versions(&[(MetadataRequest::KEY, 0, 13)]); + let response = api_versions_response(&[(MetadataRequest::KEY, 0, 13)]); assert_eq!( BTreeMap::from([(MetadataRequest::KEY, VersionRange { min: 0, max: 13 })]), @@ -454,13 +454,13 @@ mod tests { let broker_api_versions = BTreeMap::from([(MetadataRequest::KEY, VersionRange { min: 0, max: 13 })]); - check_version(&broker_api_versions, ApiKey::Metadata, 12) + check_version_in_range(&broker_api_versions, ApiKey::Metadata, 12) .expect("the version should be supported"); } #[test] fn reject_api_unknown_to_the_broker() { - let error = check_version(&BTreeMap::new(), ApiKey::Metadata, 12) + let error = check_version_in_range(&BTreeMap::new(), ApiKey::Metadata, 12) .expect_err("the API should be unsupported"); assert!(matches!( @@ -476,7 +476,7 @@ mod tests { let broker_api_versions = BTreeMap::from([(MetadataRequest::KEY, VersionRange { min: 0, max: 9 })]); - let error = check_version(&broker_api_versions, ApiKey::Metadata, 12) + let error = check_version_in_range(&broker_api_versions, ApiKey::Metadata, 12) .expect_err("the version should be unsupported"); assert!(matches!( From c475bdb854585e5ae3171909aed20b837ba1c45c Mon Sep 17 00:00:00 2001 From: Benedikt Labrenz Date: Fri, 30 Oct 2026 10:30:17 +0100 Subject: [PATCH 75/93] trim kuttl tests --- .../management-access/20-create-topic.yaml | 2 -- .../kuttl/topic/20-create-topic.yaml | 1 + .../kuttl/topic/30-change-topic.yaml | 5 --- .../topic/41-remove-config-override.yaml | 15 ++++---- .../kuttl/topic/50-unsupported-change.yaml | 5 --- tests/templates/kuttl/topic/51-assert.yaml | 20 +++++++++++ .../topic/51-revert-unsupported-change.yaml | 8 +++++ .../kuttl/topic/60-invalid-spec.yaml | 3 -- .../kuttl/topic/71-adopt-foreign-topic.yaml | 3 -- .../templates/kuttl/topic/75-pause-topic.yaml | 5 --- .../kuttl/topic/76-resume-topic.yaml | 7 ---- .../kuttl/topic/77-invalid-config-value.yaml | 2 -- .../topic/80-create-conflicting-topic.yaml | 2 -- .../kuttl/topic/92-create-cleanup-topic.yaml | 2 -- .../topic/95-disable-management-access.yaml | 9 +++++ .../95-disable-management-access.yaml.j2 | 35 ------------------- .../topic/96-delete-topic-without-agent.yaml | 2 +- 17 files changed, 47 insertions(+), 79 deletions(-) create mode 100644 tests/templates/kuttl/topic/51-assert.yaml create mode 100644 tests/templates/kuttl/topic/51-revert-unsupported-change.yaml create mode 100644 tests/templates/kuttl/topic/95-disable-management-access.yaml delete mode 100644 tests/templates/kuttl/topic/95-disable-management-access.yaml.j2 diff --git a/tests/templates/kuttl/management-access/20-create-topic.yaml b/tests/templates/kuttl/management-access/20-create-topic.yaml index 439673153..9da42728f 100644 --- a/tests/templates/kuttl/management-access/20-create-topic.yaml +++ b/tests/templates/kuttl/management-access/20-create-topic.yaml @@ -6,7 +6,5 @@ metadata: spec: clusterRef: name: test-kafka - topicName: orders partitions: 3 - replicationFactor: 1 deletionPolicy: Delete diff --git a/tests/templates/kuttl/topic/20-create-topic.yaml b/tests/templates/kuttl/topic/20-create-topic.yaml index d64165b49..2cba7e028 100644 --- a/tests/templates/kuttl/topic/20-create-topic.yaml +++ b/tests/templates/kuttl/topic/20-create-topic.yaml @@ -6,6 +6,7 @@ metadata: spec: clusterRef: name: test-kafka + # Equal to the name on purpose, so that step 31 can test that topicName is immutable. topicName: orders partitions: 3 replicationFactor: 1 diff --git a/tests/templates/kuttl/topic/30-change-topic.yaml b/tests/templates/kuttl/topic/30-change-topic.yaml index ef186c7b8..911cc3843 100644 --- a/tests/templates/kuttl/topic/30-change-topic.yaml +++ b/tests/templates/kuttl/topic/30-change-topic.yaml @@ -4,11 +4,6 @@ kind: KafkaTopic metadata: name: orders spec: - clusterRef: - name: test-kafka - topicName: orders partitions: 5 - replicationFactor: 1 configOverrides: retention.ms: "7200000" - max.message.bytes: "2097152" diff --git a/tests/templates/kuttl/topic/41-remove-config-override.yaml b/tests/templates/kuttl/topic/41-remove-config-override.yaml index ca1786493..31e7866f3 100644 --- a/tests/templates/kuttl/topic/41-remove-config-override.yaml +++ b/tests/templates/kuttl/topic/41-remove-config-override.yaml @@ -1,8 +1,9 @@ --- -# kuttl merges applied objects into the existing ones, so a removed key has to be removed explicitly. -apiVersion: kuttl.dev/v1beta1 -kind: TestStep -commands: - - script: >- - kubectl patch kafkatopic orders --namespace $NAMESPACE --type json - --patch '[{"op": "remove", "path": "/spec/configOverrides/max.message.bytes"}]' +# kuttl merges applied objects into the existing ones, so a removed key has to be set to null. +apiVersion: kafka.stackable.tech/v1alpha1 +kind: KafkaTopic +metadata: + name: orders +spec: + configOverrides: + max.message.bytes: null diff --git a/tests/templates/kuttl/topic/50-unsupported-change.yaml b/tests/templates/kuttl/topic/50-unsupported-change.yaml index 9424a96d4..9831ad248 100644 --- a/tests/templates/kuttl/topic/50-unsupported-change.yaml +++ b/tests/templates/kuttl/topic/50-unsupported-change.yaml @@ -5,10 +5,5 @@ kind: KafkaTopic metadata: name: orders spec: - clusterRef: - name: test-kafka - topicName: orders partitions: 2 replicationFactor: 3 - configOverrides: - retention.ms: "7200000" diff --git a/tests/templates/kuttl/topic/51-assert.yaml b/tests/templates/kuttl/topic/51-assert.yaml new file mode 100644 index 000000000..2c43c1564 --- /dev/null +++ b/tests/templates/kuttl/topic/51-assert.yaml @@ -0,0 +1,20 @@ +--- +# The topic is in sync again once the spec matches it. +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +timeout: 300 +--- +apiVersion: kafka.stackable.tech/v1alpha1 +kind: KafkaTopic +metadata: + name: orders +status: + conditions: + - type: Ready + status: "True" + reason: Available + - type: Synced + status: "True" + reason: Synced + partitions: 5 + replicationFactor: 1 diff --git a/tests/templates/kuttl/topic/51-revert-unsupported-change.yaml b/tests/templates/kuttl/topic/51-revert-unsupported-change.yaml new file mode 100644 index 000000000..cfaeab1f0 --- /dev/null +++ b/tests/templates/kuttl/topic/51-revert-unsupported-change.yaml @@ -0,0 +1,8 @@ +--- +apiVersion: kafka.stackable.tech/v1alpha1 +kind: KafkaTopic +metadata: + name: orders +spec: + partitions: 5 + replicationFactor: 1 diff --git a/tests/templates/kuttl/topic/60-invalid-spec.yaml b/tests/templates/kuttl/topic/60-invalid-spec.yaml index e331d3661..85c919ced 100644 --- a/tests/templates/kuttl/topic/60-invalid-spec.yaml +++ b/tests/templates/kuttl/topic/60-invalid-spec.yaml @@ -7,8 +7,5 @@ metadata: spec: clusterRef: name: test-kafka - topicName: throttled - partitions: 1 - replicationFactor: 1 configOverrides: leader.replication.throttled.replicas: "*" diff --git a/tests/templates/kuttl/topic/71-adopt-foreign-topic.yaml b/tests/templates/kuttl/topic/71-adopt-foreign-topic.yaml index 71b6bedf9..10395380c 100644 --- a/tests/templates/kuttl/topic/71-adopt-foreign-topic.yaml +++ b/tests/templates/kuttl/topic/71-adopt-foreign-topic.yaml @@ -6,7 +6,4 @@ metadata: spec: clusterRef: name: test-kafka - topicName: legacy - partitions: 1 - replicationFactor: 1 deletionPolicy: Delete diff --git a/tests/templates/kuttl/topic/75-pause-topic.yaml b/tests/templates/kuttl/topic/75-pause-topic.yaml index ea35cd524..f66a88641 100644 --- a/tests/templates/kuttl/topic/75-pause-topic.yaml +++ b/tests/templates/kuttl/topic/75-pause-topic.yaml @@ -5,11 +5,6 @@ kind: KafkaTopic metadata: name: orders spec: - clusterRef: - name: test-kafka - topicName: orders - partitions: 5 - replicationFactor: 1 configOverrides: retention.ms: "10800000" topicOperation: diff --git a/tests/templates/kuttl/topic/76-resume-topic.yaml b/tests/templates/kuttl/topic/76-resume-topic.yaml index 8cc896dc1..f20505613 100644 --- a/tests/templates/kuttl/topic/76-resume-topic.yaml +++ b/tests/templates/kuttl/topic/76-resume-topic.yaml @@ -4,12 +4,5 @@ kind: KafkaTopic metadata: name: orders spec: - clusterRef: - name: test-kafka - topicName: orders - partitions: 5 - replicationFactor: 1 - configOverrides: - retention.ms: "10800000" topicOperation: reconciliationPaused: false diff --git a/tests/templates/kuttl/topic/77-invalid-config-value.yaml b/tests/templates/kuttl/topic/77-invalid-config-value.yaml index a230cce9c..f23d4445f 100644 --- a/tests/templates/kuttl/topic/77-invalid-config-value.yaml +++ b/tests/templates/kuttl/topic/77-invalid-config-value.yaml @@ -5,7 +5,5 @@ kind: KafkaTopic metadata: name: defaults spec: - clusterRef: - name: test-kafka configOverrides: retention.ms: "abc" diff --git a/tests/templates/kuttl/topic/80-create-conflicting-topic.yaml b/tests/templates/kuttl/topic/80-create-conflicting-topic.yaml index 7314dcc8e..b213bb82c 100644 --- a/tests/templates/kuttl/topic/80-create-conflicting-topic.yaml +++ b/tests/templates/kuttl/topic/80-create-conflicting-topic.yaml @@ -8,6 +8,4 @@ spec: clusterRef: name: test-kafka topicName: orders - partitions: 5 - replicationFactor: 1 deletionPolicy: Delete diff --git a/tests/templates/kuttl/topic/92-create-cleanup-topic.yaml b/tests/templates/kuttl/topic/92-create-cleanup-topic.yaml index f14412ee9..6be86881a 100644 --- a/tests/templates/kuttl/topic/92-create-cleanup-topic.yaml +++ b/tests/templates/kuttl/topic/92-create-cleanup-topic.yaml @@ -8,6 +8,4 @@ metadata: spec: clusterRef: name: test-kafka - partitions: 1 - replicationFactor: 1 deletionPolicy: Delete diff --git a/tests/templates/kuttl/topic/95-disable-management-access.yaml b/tests/templates/kuttl/topic/95-disable-management-access.yaml new file mode 100644 index 000000000..a48f7063a --- /dev/null +++ b/tests/templates/kuttl/topic/95-disable-management-access.yaml @@ -0,0 +1,9 @@ +--- +apiVersion: kafka.stackable.tech/v1alpha1 +kind: KafkaCluster +metadata: + name: test-kafka +spec: + clusterConfig: + managementAccess: + enabled: false diff --git a/tests/templates/kuttl/topic/95-disable-management-access.yaml.j2 b/tests/templates/kuttl/topic/95-disable-management-access.yaml.j2 deleted file mode 100644 index 10563bfa0..000000000 --- a/tests/templates/kuttl/topic/95-disable-management-access.yaml.j2 +++ /dev/null @@ -1,35 +0,0 @@ ---- -apiVersion: kafka.stackable.tech/v1alpha1 -kind: KafkaCluster -metadata: - name: test-kafka -spec: - image: -{% if test_scenario['values']['kafka-kraft'].find(",") > 0 %} - custom: "{{ test_scenario['values']['kafka-kraft'].split(',')[1] }}" - productVersion: "{{ test_scenario['values']['kafka-kraft'].split(',')[0] }}" -{% else %} - productVersion: "{{ test_scenario['values']['kafka-kraft'] }}" -{% endif %} - pullPolicy: IfNotPresent - clusterConfig: - metadataManager: KRaft - tls: - serverSecretClass: tls - managementAccess: - enabled: false - authentication: - userProvided: - anonymous: {} - controllers: - config: - gracefulShutdownTimeout: 30s # speed up tests - roleGroups: - default: - replicas: 1 - brokers: - config: - gracefulShutdownTimeout: 30s # speed up tests - roleGroups: - default: - replicas: 1 diff --git a/tests/templates/kuttl/topic/96-delete-topic-without-agent.yaml b/tests/templates/kuttl/topic/96-delete-topic-without-agent.yaml index 57614fa46..4962acaaf 100644 --- a/tests/templates/kuttl/topic/96-delete-topic-without-agent.yaml +++ b/tests/templates/kuttl/topic/96-delete-topic-without-agent.yaml @@ -1,5 +1,5 @@ --- -# kuttl would wait for the deletion, which only finishes with the KafkaCluster. +# Not a kuttl delete, as that waits for the deletion, which only finishes with the KafkaCluster. apiVersion: kuttl.dev/v1beta1 kind: TestStep commands: From 45d37f3190b60c925e14db713e9fe92b5f9a2ddd Mon Sep 17 00:00:00 2001 From: Benedikt Labrenz Date: Fri, 30 Oct 2026 11:07:39 +0100 Subject: [PATCH 76/93] add keep environment util to topic test --- .../kuttl/topic/98-keep-environment.yaml | 7 ++++++ tests/templates/kuttl/topic/98_topic.yaml | 9 ++++++++ .../templates/kuttl/topic/keep-environment.sh | 23 +++++++++++++++++++ 3 files changed, 39 insertions(+) create mode 100644 tests/templates/kuttl/topic/98-keep-environment.yaml create mode 100644 tests/templates/kuttl/topic/98_topic.yaml create mode 100644 tests/templates/kuttl/topic/keep-environment.sh diff --git a/tests/templates/kuttl/topic/98-keep-environment.yaml b/tests/templates/kuttl/topic/98-keep-environment.yaml new file mode 100644 index 000000000..910bec3e8 --- /dev/null +++ b/tests/templates/kuttl/topic/98-keep-environment.yaml @@ -0,0 +1,7 @@ +--- +# With --skip-delete, the namespace is kept, so the deleted cluster is created again to experiment +# with. See the script for the details. +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +commands: + - script: bash keep-environment.sh diff --git a/tests/templates/kuttl/topic/98_topic.yaml b/tests/templates/kuttl/topic/98_topic.yaml new file mode 100644 index 000000000..d8aa89eaf --- /dev/null +++ b/tests/templates/kuttl/topic/98_topic.yaml @@ -0,0 +1,9 @@ +--- +# Adopts the topic that the KafkaTopic orders retained in Kafka. +apiVersion: kafka.stackable.tech/v1alpha1 +kind: KafkaTopic +metadata: + name: orders +spec: + clusterRef: + name: test-kafka diff --git a/tests/templates/kuttl/topic/keep-environment.sh b/tests/templates/kuttl/topic/keep-environment.sh new file mode 100644 index 000000000..fcd274f9e --- /dev/null +++ b/tests/templates/kuttl/topic/keep-environment.sh @@ -0,0 +1,23 @@ +#!/usr/bin/env bash +# Creates the KafkaCluster and a KafkaTopic again after the deletion steps, if kuttl was started +# with --skip-delete, so that the kept namespace has a running cluster to experiment with. kuttl +# does not expose the flag to test steps, hence looking for it in the argv of the kuttl ancestor. +set -euo pipefail + +pid=$PPID +while [ "${pid:-0}" -gt 1 ]; do + args=$(ps -o args= -p "$pid" 2>/dev/null || true) + case "$args" in + *kuttl*--skip-delete*) break ;; + esac + pid=$(ps -o ppid= -p "$pid" 2>/dev/null | tr -d '[:space:]' || true) +done +if [ "${pid:-0}" -le 1 ]; then + echo "kuttl deletes the namespace, nothing to keep" + exit 0 +fi + +# The cluster id is the name of the KafkaCluster, so the topics in the kept PersistentVolumeClaims +# are back once the cluster runs. +kubectl apply --namespace "$NAMESPACE" --filename 10-install-kafka.yaml +kubectl apply --namespace "$NAMESPACE" --filename 98_topic.yaml From cc89f97ecef492927fb3bdd55ed09cad8d42e130 Mon Sep 17 00:00:00 2001 From: Benedikt Labrenz Date: Tue, 3 Nov 2026 15:01:13 +0100 Subject: [PATCH 77/93] use stackable-operator 0.122.0 --- Cargo.lock | 157 ++++++++------------- Cargo.nix | 345 ++++++++++++++-------------------------------- Cargo.toml | 2 +- crate-hashes.json | 20 +-- 4 files changed, 167 insertions(+), 357 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 188851306..606cd0f20 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -561,37 +561,14 @@ dependencies = [ "subtle", ] -[[package]] -name = "darling" -version = "0.23.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "25ae13da2f202d56bd7f91c25fba009e7717a1e4a1cc98a76d844b65ae912e9d" -dependencies = [ - "darling_core 0.23.0", - "darling_macro 0.23.0", -] - [[package]] name = "darling" version = "0.24.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ed17f5901b6630b993ca003def43f2f8ef4014fc13b047b57aad617ff32bc2ec" dependencies = [ - "darling_core 0.24.1", - "darling_macro 0.24.1", -] - -[[package]] -name = "darling_core" -version = "0.23.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9865a50f7c335f53564bb694ef660825eb8610e0a53d3e11bf1b0d3df31e03b0" -dependencies = [ - "ident_case", - "proc-macro2", - "quote", - "strsim", - "syn 2.0.119", + "darling_core", + "darling_macro", ] [[package]] @@ -607,24 +584,13 @@ dependencies = [ "syn 3.0.9", ] -[[package]] -name = "darling_macro" -version = "0.23.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ac3984ec7bd6cfa798e62b4a642426a5be0e68f9401cfc2a01e3fa9ea2fcdb8d" -dependencies = [ - "darling_core 0.23.0", - "quote", - "syn 2.0.119", -] - [[package]] name = "darling_macro" version = "0.24.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2ac7135c3ef02b2f7833bbeb1be5ba7f966dcde8a87c6b87f65a778d71a02785" dependencies = [ - "darling_core 0.24.1", + "darling_core", "quote", "syn 3.0.9", ] @@ -786,18 +752,6 @@ dependencies = [ "zeroize", ] -[[package]] -name = "educe" -version = "0.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1d7bc049e1bd8cdeb31b68bbd586a9464ecf9f3944af3958a7a9d0f8b9799417" -dependencies = [ - "enum-ordinalize", - "proc-macro2", - "quote", - "syn 2.0.119", -] - [[package]] name = "educe" version = "0.8.1" @@ -1097,11 +1051,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "899def5c37c4fd7b2664648c28120ecec138e4d395b459e5ca34f9cce2dd77fd" dependencies = [ "cfg-if", - "js-sys", "libc", "r-efi 5.3.0", "wasip2", - "wasm-bindgen", ] [[package]] @@ -1148,9 +1100,9 @@ dependencies = [ [[package]] name = "granit-parser" -version = "0.0.7" +version = "2.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d03f81ad4732830d85cfd417a9f62cde6dadda4354d37d078a6084a19560aa2d" +checksum = "05f05586d4d4df3d47123207826c40b252728b438a1a07bad3c01ee6bea401df" dependencies = [ "arraydeque", "smallvec", @@ -1729,9 +1681,9 @@ dependencies = [ [[package]] name = "k8s-openapi" -version = "0.28.0" +version = "0.29.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d9c6922f6afe80418dd6019818af5d0d34584c371780ff09b9752370c25b4abb" +checksum = "ab0d13801242174b1f3966907c3f32039f2d6b36c7ce82c61225d66ed5d1eb70" dependencies = [ "base64 0.22.1", "jiff", @@ -1742,10 +1694,10 @@ dependencies = [ [[package]] name = "k8s-version" -version = "0.1.4" -source = "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#46a18a9a4899976b702ca87f73430c6f12d17c56" +version = "0.1.5" +source = "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.122.0#b0b8ca31a79da03b3fa20507b709390d2647efe4" dependencies = [ - "darling 0.24.1", + "darling", "regex", "snafu 0.9.2", ] @@ -1767,9 +1719,9 @@ checksum = "a4933f3f57a8e9d9da04db23fb153356ecaf00cbd14aee46279c33dc80925c37" [[package]] name = "kube" -version = "4.2.0" +version = "5.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "208d7fe1380066abb194812a8a8e303cb896a33bb3d7b3177b71bd03ff39bf18" +checksum = "c73b082f549411623a4f8949bd24dcaedacb1511379c461c76d579fde301d021" dependencies = [ "k8s-openapi", "kube-client", @@ -1780,11 +1732,11 @@ dependencies = [ [[package]] name = "kube-client" -version = "4.2.0" +version = "5.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "31e940a73033a7c5c7918b5ece7851d84571b58be25e937198da37fa13f116d3" +checksum = "ae56b223bca490db1f1b115bfc606d411177511d7748d8cc019a7e3b2935430b" dependencies = [ - "base64 0.22.1", + "base64 0.23.1", "bytes", "either", "futures", @@ -1800,6 +1752,7 @@ dependencies = [ "k8s-openapi", "kube-core", "pem", + "percent-encoding", "rustls", "rustls-platform-verifier", "secrecy", @@ -1810,15 +1763,15 @@ dependencies = [ "tokio", "tokio-util", "tower", - "tower-http 0.6.11", + "tower-http 0.7.1", "tracing", ] [[package]] name = "kube-core" -version = "4.2.0" +version = "5.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a9d2353c118cf3462c352ee0b5bd5b0cf17990af456dfd8662d136ff9812eeb4" +checksum = "6de84b47fc1aa35ba0ef9059ef54b47c8b47f0979afd15487e525282f2d73fb9" dependencies = [ "derive_more", "form_urlencoded", @@ -1835,29 +1788,29 @@ dependencies = [ [[package]] name = "kube-derive" -version = "4.2.0" +version = "5.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "92141c19e1fa83bf91633c1234c66b03375c29aa8ca5486331ddb47e3a3da9c0" +checksum = "de1fb46ac34371491ac0277c13e0e1bb9288000e64c86ca3d91343754818e64e" dependencies = [ - "darling 0.23.0", + "darling", "proc-macro2", "quote", "serde", "serde_json", - "syn 2.0.119", + "syn 3.0.9", ] [[package]] name = "kube-runtime" -version = "4.2.0" +version = "5.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3eb064ef71c7bea55e934a443960da9e9ec31fbb2ba63e47e4aeca32603d5cc7" +checksum = "9dea60005024b6ec1819c426209aa29b9eaca64f6b3988ab73c55f43fa484392" dependencies = [ "ahash", "async-broadcast", "async-stream", "backon", - "educe 0.6.0", + "educe", "futures", "hashbrown 0.16.1", "hostname", @@ -2188,11 +2141,11 @@ dependencies = [ [[package]] name = "pem" -version = "3.0.6" +version = "4.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1d30c53c26bc5b31a98cd02d20f25a7c8567146caf63ed593a9d87b2775291be" +checksum = "d354a98a3d1251555de99e8fdd8afda05573c31b82f59063a7b0a29b5527f120" dependencies = [ - "base64 0.22.1", + "base64 0.23.1", "serde_core", ] @@ -2879,15 +2832,13 @@ dependencies = [ [[package]] name = "serde-saphyr" -version = "0.0.29" +version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7bd22781911de0ca6debda95f073c8f18bec65d1a94f1fa9573f3102e514cea4" +checksum = "a253edfdea34032da21af84e16fcc94f4c7a6ba237f9f3bc2dbf43cea8535c44" dependencies = [ - "ahash", "annotate-snippets", "base64 0.22.1", "encoding_rs_io", - "getrandom 0.3.4", "granit-parser", "nohash-hasher", "num-traits", @@ -3145,7 +3096,7 @@ checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596" [[package]] name = "stackable-agent" version = "0.1.0" -source = "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#46a18a9a4899976b702ca87f73430c6f12d17c56" +source = "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.122.0#b0b8ca31a79da03b3fa20507b709390d2647efe4" dependencies = [ "k8s-openapi", "schemars", @@ -3156,8 +3107,8 @@ dependencies = [ [[package]] name = "stackable-certs" -version = "0.4.1" -source = "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#46a18a9a4899976b702ca87f73430c6f12d17c56" +version = "0.5.0" +source = "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.122.0#b0b8ca31a79da03b3fa20507b709390d2647efe4" dependencies = [ "const-oid", "ecdsa", @@ -3223,8 +3174,8 @@ dependencies = [ [[package]] name = "stackable-operator" -version = "0.121.0" -source = "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#46a18a9a4899976b702ca87f73430c6f12d17c56" +version = "0.122.0" +source = "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.122.0#b0b8ca31a79da03b3fa20507b709390d2647efe4" dependencies = [ "base16ct", "base64 0.23.1", @@ -3232,7 +3183,7 @@ dependencies = [ "const_format", "delegate", "dockerfile-parser", - "educe 0.8.1", + "educe", "either", "futures", "http", @@ -3269,10 +3220,10 @@ dependencies = [ [[package]] name = "stackable-operator-derive" -version = "0.3.1" -source = "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#46a18a9a4899976b702ca87f73430c6f12d17c56" +version = "0.3.2" +source = "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.122.0#b0b8ca31a79da03b3fa20507b709390d2647efe4" dependencies = [ - "darling 0.24.1", + "darling", "proc-macro2", "quote", "syn 3.0.9", @@ -3280,8 +3231,8 @@ dependencies = [ [[package]] name = "stackable-shared" -version = "0.2.0" -source = "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#46a18a9a4899976b702ca87f73430c6f12d17c56" +version = "0.2.1" +source = "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.122.0#b0b8ca31a79da03b3fa20507b709390d2647efe4" dependencies = [ "jiff", "k8s-openapi", @@ -3297,8 +3248,8 @@ dependencies = [ [[package]] name = "stackable-telemetry" -version = "0.6.5" -source = "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#46a18a9a4899976b702ca87f73430c6f12d17c56" +version = "0.6.6" +source = "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.122.0#b0b8ca31a79da03b3fa20507b709390d2647efe4" dependencies = [ "axum", "clap", @@ -3321,8 +3272,8 @@ dependencies = [ [[package]] name = "stackable-versioned" -version = "0.13.0" -source = "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#46a18a9a4899976b702ca87f73430c6f12d17c56" +version = "0.14.0" +source = "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.122.0#b0b8ca31a79da03b3fa20507b709390d2647efe4" dependencies = [ "kube", "schemars", @@ -3334,11 +3285,11 @@ dependencies = [ [[package]] name = "stackable-versioned-macros" -version = "0.13.0" -source = "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#46a18a9a4899976b702ca87f73430c6f12d17c56" +version = "0.14.0" +source = "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.122.0#b0b8ca31a79da03b3fa20507b709390d2647efe4" dependencies = [ "convert_case", - "darling 0.24.1", + "darling", "indoc", "itertools 0.15.0", "k8s-openapi", @@ -3351,8 +3302,8 @@ dependencies = [ [[package]] name = "stackable-webhook" -version = "0.10.0" -source = "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#46a18a9a4899976b702ca87f73430c6f12d17c56" +version = "0.11.0" +source = "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.122.0#b0b8ca31a79da03b3fa20507b709390d2647efe4" dependencies = [ "arc-swap", "async-trait", @@ -3764,18 +3715,15 @@ version = "0.6.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4cfcf7e2740e6fc6d4d688b4ef00650406bb94adf4731e43c096c3a19fe40840" dependencies = [ - "base64 0.22.1", "bitflags 2.13.2", "bytes", "futures-util", "http", "http-body", - "mime", "pin-project-lite", "tower", "tower-layer", "tower-service", - "tracing", "url", ] @@ -3785,12 +3733,15 @@ version = "0.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "08a05a66a4fdd61cbbe0a1d755ffe0ca6aba159dd4820936a0ff8a8278245b9c" dependencies = [ + "base64 0.22.1", "bitflags 2.13.2", "bytes", "http", "http-body", + "mime", "percent-encoding", "pin-project-lite", + "tower", "tower-layer", "tower-service", "tracing", diff --git a/Cargo.nix b/Cargo.nix index 9dd9a010d..4133e4d1d 100644 --- a/Cargo.nix +++ b/Cargo.nix @@ -1741,33 +1741,7 @@ rec { }; resolvedDefaultFeatures = [ "subtle" ]; }; - "darling 0.23.0" = rec { - crateName = "darling"; - version = "0.23.0"; - edition = "2021"; - sha256 = "179fj6p6ajw4dnkrik51wjhifxwy02x5zhligyymcb905zd17bi5"; - authors = [ - "Ted Driggs " - ]; - dependencies = [ - { - name = "darling_core"; - packageId = "darling_core 0.23.0"; - } - { - name = "darling_macro"; - packageId = "darling_macro 0.23.0"; - } - ]; - features = { - "default" = [ "suggestions" ]; - "diagnostics" = [ "darling_core/diagnostics" ]; - "serde" = [ "darling_core/serde" ]; - "suggestions" = [ "darling_core/suggestions" ]; - }; - resolvedDefaultFeatures = [ "default" "suggestions" ]; - }; - "darling 0.24.1" = rec { + "darling" = rec { crateName = "darling"; version = "0.24.1"; edition = "2021"; @@ -1778,11 +1752,11 @@ rec { dependencies = [ { name = "darling_core"; - packageId = "darling_core 0.24.1"; + packageId = "darling_core"; } { name = "darling_macro"; - packageId = "darling_macro 0.24.1"; + packageId = "darling_macro"; } ]; features = { @@ -1793,46 +1767,7 @@ rec { }; resolvedDefaultFeatures = [ "default" "suggestions" ]; }; - "darling_core 0.23.0" = rec { - crateName = "darling_core"; - version = "0.23.0"; - edition = "2021"; - sha256 = "1c033vrks38vpw8kwgd5w088dsr511kfz55n9db56prkgh7sarcq"; - authors = [ - "Ted Driggs " - ]; - dependencies = [ - { - name = "ident_case"; - packageId = "ident_case"; - } - { - name = "proc-macro2"; - packageId = "proc-macro2"; - } - { - name = "quote"; - packageId = "quote"; - } - { - name = "strsim"; - packageId = "strsim"; - optional = true; - } - { - name = "syn"; - packageId = "syn 2.0.119"; - features = [ "full" "extra-traits" ]; - } - ]; - features = { - "serde" = [ "dep:serde" ]; - "strsim" = [ "dep:strsim" ]; - "suggestions" = [ "strsim" ]; - }; - resolvedDefaultFeatures = [ "strsim" "suggestions" ]; - }; - "darling_core 0.24.1" = rec { + "darling_core" = rec { crateName = "darling_core"; version = "0.24.1"; edition = "2021"; @@ -1871,32 +1806,7 @@ rec { }; resolvedDefaultFeatures = [ "strsim" "suggestions" ]; }; - "darling_macro 0.23.0" = rec { - crateName = "darling_macro"; - version = "0.23.0"; - edition = "2021"; - sha256 = "13fvzji9xyp304mgq720z5l0xgm54qj68jibwscagkynggn88fdc"; - procMacro = true; - authors = [ - "Ted Driggs " - ]; - dependencies = [ - { - name = "darling_core"; - packageId = "darling_core 0.23.0"; - } - { - name = "quote"; - packageId = "quote"; - } - { - name = "syn"; - packageId = "syn 2.0.119"; - } - ]; - - }; - "darling_macro 0.24.1" = rec { + "darling_macro" = rec { crateName = "darling_macro"; version = "0.24.1"; edition = "2021"; @@ -1908,7 +1818,7 @@ rec { dependencies = [ { name = "darling_core"; - packageId = "darling_core 0.24.1"; + packageId = "darling_core"; } { name = "quote"; @@ -2425,49 +2335,7 @@ rec { }; resolvedDefaultFeatures = [ "algorithm" "alloc" "default" "der" "digest" "pem" "pkcs8" "spki" "std" ]; }; - "educe 0.6.0" = rec { - crateName = "educe"; - version = "0.6.0"; - edition = "2021"; - sha256 = "05wlg6wzil59lxc3kbs476gwykj6m63dbfv83frxx35xw54w0yqx"; - procMacro = true; - authors = [ - "Magic Len " - ]; - dependencies = [ - { - name = "enum-ordinalize"; - packageId = "enum-ordinalize"; - usesDefaultFeatures = false; - features = [ "derive" ]; - } - { - name = "proc-macro2"; - packageId = "proc-macro2"; - } - { - name = "quote"; - packageId = "quote"; - } - { - name = "syn"; - packageId = "syn 2.0.119"; - } - ]; - devDependencies = [ - { - name = "syn"; - packageId = "syn 2.0.119"; - features = [ "full" ]; - } - ]; - features = { - "default" = [ "Debug" "Clone" "Copy" "PartialEq" "Eq" "PartialOrd" "Ord" "Hash" "Default" "Deref" "DerefMut" "Into" ]; - "full" = [ "syn/full" ]; - }; - resolvedDefaultFeatures = [ "Clone" "Debug" "Hash" "PartialEq" ]; - }; - "educe 0.8.1" = rec { + "educe" = rec { crateName = "educe"; version = "0.8.1"; edition = "2024"; @@ -2501,7 +2369,7 @@ rec { "default" = [ "Debug" "Clone" "Copy" "PartialEq" "Eq" "PartialOrd" "Ord" "Hash" "Default" "Deref" "DerefMut" "Into" ]; "full" = [ "syn/full" ]; }; - resolvedDefaultFeatures = [ "Clone" "Debug" "Default" "Eq" "PartialEq" ]; + resolvedDefaultFeatures = [ "Clone" "Debug" "Default" "Eq" "Hash" "PartialEq" ]; }; "either" = rec { crateName = "either"; @@ -3372,13 +3240,6 @@ rec { name = "cfg-if"; packageId = "cfg-if"; } - { - name = "js-sys"; - packageId = "js-sys"; - optional = true; - usesDefaultFeatures = false; - target = { target, features }: (("wasm32" == target."arch" or null) && (("unknown" == target."os" or null) || ("none" == target."os" or null)) && (builtins.elem "atomics" targetFeatures)); - } { name = "libc"; packageId = "libc"; @@ -3439,18 +3300,11 @@ rec { usesDefaultFeatures = false; target = { target, features }: (("wasm32" == target."arch" or null) && ("wasi" == target."os" or null) && ("p2" == target."env" or null)); } - { - name = "wasm-bindgen"; - packageId = "wasm-bindgen"; - optional = true; - usesDefaultFeatures = false; - target = { target, features }: (("wasm32" == target."arch" or null) && (("unknown" == target."os" or null) || ("none" == target."os" or null))); - } ]; features = { "wasm_js" = [ "dep:wasm-bindgen" "dep:js-sys" ]; }; - resolvedDefaultFeatures = [ "std" "wasm_js" ]; + resolvedDefaultFeatures = [ "std" ]; }; "getrandom 0.4.3" = rec { crateName = "getrandom"; @@ -3618,10 +3472,10 @@ rec { }; "granit-parser" = rec { crateName = "granit-parser"; - version = "0.0.7"; + version = "2.1.0"; edition = "2021"; crateBin = []; - sha256 = "0bdac2as3130i83pvlsl8gdasvfy5kvaj5ylry2hv0rj8ynq2gyh"; + sha256 = "1pq1ljzfc7n0sfx0f6la8f5p4lmj81n841rj293kvpylsj35bw05"; libName = "granit_parser"; authors = [ "Ethiraric " @@ -3641,7 +3495,10 @@ rec { } ]; features = { + "debug_prints" = [ "std" ]; + "default" = [ "error_messages" "std" "comments" ]; }; + resolvedDefaultFeatures = [ "comments" "error_messages" "std" ]; }; "group" = rec { crateName = "group"; @@ -5493,10 +5350,10 @@ rec { }; "k8s-openapi" = rec { crateName = "k8s-openapi"; - version = "0.28.0"; - edition = "2021"; - links = "k8s-openapi-0.28.0"; - sha256 = "1fsabg1708vmp44zz00p6x65hd0dbnpii601ss6l307yd8pr5inr"; + version = "0.29.0"; + edition = "2024"; + links = "k8s-openapi-0.29.0"; + sha256 = "0w7bs7anxmi52b385kn76rmjv7q368zpr43674gln5s22a0163db"; libName = "k8s_openapi"; authors = [ "Arnav Singh " @@ -5534,21 +5391,20 @@ rec { ]; features = { "default" = [ "std" ]; - "earliest" = [ "v1_32" ]; - "latest" = [ "v1_36" ]; + "earliest" = [ "v1_33" ]; + "latest" = [ "v1_37" ]; "schemars" = [ "dep:schemars" ]; - "schemars08" = [ "dep:schemars08" ]; }; resolvedDefaultFeatures = [ "schemars" "v1_36" ]; }; "k8s-version" = rec { crateName = "k8s-version"; - version = "0.1.4"; + version = "0.1.5"; edition = "2024"; workspace_member = null; src = pkgs.fetchgit { url = "https://github.com/stackabletech/operator-rs.git"; - rev = "46a18a9a4899976b702ca87f73430c6f12d17c56"; + rev = "b0b8ca31a79da03b3fa20507b709390d2647efe4"; sha256 = "10va8k2dwshcb6c7pp01q6m6bx2ad58fzqxf19mp90hhcp00bxm7"; }; libName = "k8s_version"; @@ -5558,7 +5414,7 @@ rec { dependencies = [ { name = "darling"; - packageId = "darling 0.24.1"; + packageId = "darling"; optional = true; } { @@ -5625,9 +5481,9 @@ rec { }; "kube" = rec { crateName = "kube"; - version = "4.2.0"; + version = "5.0.0"; edition = "2024"; - sha256 = "065z77zh7gbigcbv7mxk7firdf1w6278lal1jjqsnrh073hpz390"; + sha256 = "08fh07izsyfmfqf4d71p24awpnmfvhjbsjc99wx644clahphhfy7"; authors = [ "clux " "Natalie Klestrup Röijezon " @@ -5687,6 +5543,7 @@ rec { "oauth" = [ "kube-client/oauth" "client" ]; "oidc" = [ "kube-client/oidc" "client" ]; "openssl-tls" = [ "kube-client/openssl-tls" "client" ]; + "proxy" = [ "kube-client/proxy" "client" ]; "ring" = [ "kube-client?/ring" ]; "runtime" = [ "kube-runtime" ]; "rustls-tls" = [ "kube-client/rustls-tls" "client" ]; @@ -5700,9 +5557,9 @@ rec { }; "kube-client" = rec { crateName = "kube-client"; - version = "4.2.0"; + version = "5.0.0"; edition = "2024"; - sha256 = "1lqny49zldysk1qr6pp2ifsp2ifqa5wcwplbj73wb9rk62kl1s9i"; + sha256 = "02s36llknzls076dhj3p3m8pf4a1dmhgqnqi3cgxp454phiv4mmf"; libName = "kube_client"; authors = [ "clux " @@ -5712,7 +5569,7 @@ rec { dependencies = [ { name = "base64"; - packageId = "base64 0.22.1"; + packageId = "base64 0.23.1"; optional = true; } { @@ -5796,6 +5653,10 @@ rec { packageId = "pem"; optional = true; } + { + name = "percent-encoding"; + packageId = "percent-encoding"; + } { name = "rustls"; packageId = "rustls"; @@ -5849,7 +5710,7 @@ rec { } { name = "tower-http"; - packageId = "tower-http 0.6.11"; + packageId = "tower-http 0.7.1"; optional = true; features = [ "auth" "map-response-body" "trace" "util" ]; } @@ -5899,7 +5760,7 @@ rec { "gzip" = [ "client" "tower-http/decompression-gzip" ]; "http-body" = [ "dep:http-body" ]; "http-body-util" = [ "dep:http-body-util" ]; - "http-proxy" = [ "hyper-util/client-proxy" ]; + "http-proxy" = [ "hyper-util/client-proxy" "dep:ipnet" ]; "hyper" = [ "dep:hyper" ]; "hyper-openssl" = [ "dep:hyper-openssl" ]; "hyper-rustls" = [ "dep:hyper-rustls" ]; @@ -5915,11 +5776,12 @@ rec { "openssl" = [ "dep:openssl" ]; "openssl-tls" = [ "openssl" "hyper-openssl" ]; "pem" = [ "dep:pem" ]; + "proxy" = [ "socks5" "http-proxy" ]; "ring" = [ "hyper-rustls?/ring" ]; "rustls" = [ "dep:rustls" ]; "rustls-tls" = [ "rustls" "hyper-rustls" "dep:rustls-platform-verifier" ]; "serde-saphyr" = [ "dep:serde-saphyr" ]; - "socks5" = [ "hyper-util/client-proxy" ]; + "socks5" = [ "hyper-util/client-proxy" "dep:ipnet" ]; "tame-oauth" = [ "dep:tame-oauth" ]; "tokio" = [ "dep:tokio" ]; "tokio-tungstenite" = [ "dep:tokio-tungstenite" ]; @@ -5934,9 +5796,9 @@ rec { }; "kube-core" = rec { crateName = "kube-core"; - version = "4.2.0"; + version = "5.0.0"; edition = "2024"; - sha256 = "1d7f2acgydnica3gsva5my87kw8cbfyvbq1f6ln4dwwc24y3blm9"; + sha256 = "1f9zszr84ljjgr41bzcsjzq4g2vwniafynchxyh5p8qszi3lps3d"; libName = "kube_core"; authors = [ "clux " @@ -6017,9 +5879,9 @@ rec { }; "kube-derive" = rec { crateName = "kube-derive"; - version = "4.2.0"; + version = "5.0.0"; edition = "2024"; - sha256 = "1h597lx7xd6x65ili9ccm8lmqdq3dg3384iwcf8vz0zsw4ciq54j"; + sha256 = "0kp63147ahqkv6inrj341q08i4mvw7h16z17q0d4jwa3qdmb87yy"; procMacro = true; libName = "kube_derive"; authors = [ @@ -6030,7 +5892,7 @@ rec { dependencies = [ { name = "darling"; - packageId = "darling 0.23.0"; + packageId = "darling"; } { name = "proc-macro2"; @@ -6051,7 +5913,7 @@ rec { } { name = "syn"; - packageId = "syn 2.0.119"; + packageId = "syn 3.0.9"; features = [ "extra-traits" ]; } ]; @@ -6066,9 +5928,9 @@ rec { }; "kube-runtime" = rec { crateName = "kube-runtime"; - version = "4.2.0"; + version = "5.0.0"; edition = "2024"; - sha256 = "1isw7mh35jmfwi3kx9ibpcgw77lyv9h3ji2ajdgabgn7f7pn9c1y"; + sha256 = "14j393x46py5ffmqhfbb9ykar7lvlad209n434cfrdi4a0061slx"; libName = "kube_runtime"; authors = [ "clux " @@ -6094,7 +5956,7 @@ rec { } { name = "educe"; - packageId = "educe 0.6.0"; + packageId = "educe"; usesDefaultFeatures = false; features = [ "Clone" "Debug" "Hash" "PartialEq" ]; } @@ -7279,16 +7141,16 @@ rec { }; "pem" = rec { crateName = "pem"; - version = "3.0.6"; + version = "4.0.0"; edition = "2021"; - sha256 = "1glia9vv51wx79cysqxgdha6g1bwbbr20bfhijlk2nxw4qycac0x"; + sha256 = "087i4xarp8mhlxir1xc23g1p6md0zn5dv3wyx5fmal8j7n5ajm6k"; authors = [ "Jonathan Creekmore " ]; dependencies = [ { name = "base64"; - packageId = "base64 0.22.1"; + packageId = "base64 0.23.1"; usesDefaultFeatures = false; features = [ "alloc" ]; } @@ -9524,17 +9386,12 @@ rec { }; "serde-saphyr" = rec { crateName = "serde-saphyr"; - version = "0.0.29"; + version = "2.0.0"; edition = "2024"; crateBin = []; - sha256 = "196f2kjh4c9zayliykx9s5jyr2zir1rz15fsxdnwmq0xj60jglkv"; + sha256 = "0i2waflcwhxz5nyg7y9pl9mplk2gr7y1ckpq3ai2s0rlxbyyslx2"; libName = "serde_saphyr"; dependencies = [ - { - name = "ahash"; - packageId = "ahash"; - optional = true; - } { name = "annotate-snippets"; packageId = "annotate-snippets"; @@ -9550,16 +9407,12 @@ rec { packageId = "encoding_rs_io"; optional = true; } - { - name = "getrandom"; - packageId = "getrandom 0.3.4"; - target = { target, features }: ("wasm32" == target."arch" or null); - features = [ "wasm_js" ]; - } { name = "granit-parser"; packageId = "granit-parser"; optional = true; + usesDefaultFeatures = false; + features = [ "std" "error_messages" ]; } { name = "nohash-hasher"; @@ -9587,21 +9440,23 @@ rec { } ]; features = { - "default" = [ "serialize" "deserialize" ]; - "deserialize" = [ "dep:num-traits" "dep:annotate-snippets" "dep:granit-parser" "dep:smallvec" "dep:encoding_rs_io" "dep:ahash" ]; + "comments" = [ "granit-parser?/comments" ]; + "default" = [ "serialize" "deserialize" "comments" ]; + "deserialize" = [ "dep:num-traits" "dep:annotate-snippets" "dep:granit-parser" "dep:smallvec" "dep:encoding_rs_io" ]; "figment" = [ "dep:figment" "deserialize" ]; "figment2" = [ "dep:figment2" "deserialize" ]; "garde" = [ "dep:garde" "deserialize" ]; + "huge_documents" = [ "deserialize" ]; "include" = [ "deserialize" ]; "include_fs" = [ "include" ]; "miette" = [ "dep:miette" "deserialize" ]; "properties" = [ "deserialize" ]; "robotics" = [ "deserialize" ]; - "serde_derived_types" = [ "dep:serde" ]; + "serde_derived_types" = [ "dep:serde" "deserialize" ]; "serialize" = [ "dep:base64" "dep:num-traits" "dep:zmij" "dep:nohash-hasher" ]; "validator" = [ "dep:validator" "deserialize" ]; }; - resolvedDefaultFeatures = [ "default" "deserialize" "serialize" ]; + resolvedDefaultFeatures = [ "comments" "default" "deserialize" "serialize" ]; }; "serde-value" = rec { crateName = "serde-value"; @@ -10311,7 +10166,7 @@ rec { workspace_member = null; src = pkgs.fetchgit { url = "https://github.com/stackabletech/operator-rs.git"; - rev = "46a18a9a4899976b702ca87f73430c6f12d17c56"; + rev = "b0b8ca31a79da03b3fa20507b709390d2647efe4"; sha256 = "10va8k2dwshcb6c7pp01q6m6bx2ad58fzqxf19mp90hhcp00bxm7"; }; libName = "stackable_agent"; @@ -10349,12 +10204,12 @@ rec { }; "stackable-certs" = rec { crateName = "stackable-certs"; - version = "0.4.1"; + version = "0.5.0"; edition = "2024"; workspace_member = null; src = pkgs.fetchgit { url = "https://github.com/stackabletech/operator-rs.git"; - rev = "46a18a9a4899976b702ca87f73430c6f12d17c56"; + rev = "b0b8ca31a79da03b3fa20507b709390d2647efe4"; sha256 = "10va8k2dwshcb6c7pp01q6m6bx2ad58fzqxf19mp90hhcp00bxm7"; }; libName = "stackable_certs"; @@ -10621,12 +10476,12 @@ rec { }; "stackable-operator" = rec { crateName = "stackable-operator"; - version = "0.121.0"; + version = "0.122.0"; edition = "2024"; workspace_member = null; src = pkgs.fetchgit { url = "https://github.com/stackabletech/operator-rs.git"; - rev = "46a18a9a4899976b702ca87f73430c6f12d17c56"; + rev = "b0b8ca31a79da03b3fa20507b709390d2647efe4"; sha256 = "10va8k2dwshcb6c7pp01q6m6bx2ad58fzqxf19mp90hhcp00bxm7"; }; libName = "stackable_operator"; @@ -10661,7 +10516,7 @@ rec { } { name = "educe"; - packageId = "educe 0.8.1"; + packageId = "educe"; usesDefaultFeatures = false; features = [ "Clone" "Debug" "Default" "PartialEq" "Eq" ]; } @@ -10830,12 +10685,12 @@ rec { }; "stackable-operator-derive" = rec { crateName = "stackable-operator-derive"; - version = "0.3.1"; + version = "0.3.2"; edition = "2024"; workspace_member = null; src = pkgs.fetchgit { url = "https://github.com/stackabletech/operator-rs.git"; - rev = "46a18a9a4899976b702ca87f73430c6f12d17c56"; + rev = "b0b8ca31a79da03b3fa20507b709390d2647efe4"; sha256 = "10va8k2dwshcb6c7pp01q6m6bx2ad58fzqxf19mp90hhcp00bxm7"; }; procMacro = true; @@ -10846,7 +10701,7 @@ rec { dependencies = [ { name = "darling"; - packageId = "darling 0.24.1"; + packageId = "darling"; } { name = "proc-macro2"; @@ -10865,12 +10720,12 @@ rec { }; "stackable-shared" = rec { crateName = "stackable-shared"; - version = "0.2.0"; + version = "0.2.1"; edition = "2024"; workspace_member = null; src = pkgs.fetchgit { url = "https://github.com/stackabletech/operator-rs.git"; - rev = "46a18a9a4899976b702ca87f73430c6f12d17c56"; + rev = "b0b8ca31a79da03b3fa20507b709390d2647efe4"; sha256 = "10va8k2dwshcb6c7pp01q6m6bx2ad58fzqxf19mp90hhcp00bxm7"; }; libName = "stackable_shared"; @@ -10946,12 +10801,12 @@ rec { }; "stackable-telemetry" = rec { crateName = "stackable-telemetry"; - version = "0.6.5"; + version = "0.6.6"; edition = "2024"; workspace_member = null; src = pkgs.fetchgit { url = "https://github.com/stackabletech/operator-rs.git"; - rev = "46a18a9a4899976b702ca87f73430c6f12d17c56"; + rev = "b0b8ca31a79da03b3fa20507b709390d2647efe4"; sha256 = "10va8k2dwshcb6c7pp01q6m6bx2ad58fzqxf19mp90hhcp00bxm7"; }; libName = "stackable_telemetry"; @@ -11056,12 +10911,12 @@ rec { }; "stackable-versioned" = rec { crateName = "stackable-versioned"; - version = "0.13.0"; + version = "0.14.0"; edition = "2024"; workspace_member = null; src = pkgs.fetchgit { url = "https://github.com/stackabletech/operator-rs.git"; - rev = "46a18a9a4899976b702ca87f73430c6f12d17c56"; + rev = "b0b8ca31a79da03b3fa20507b709390d2647efe4"; sha256 = "10va8k2dwshcb6c7pp01q6m6bx2ad58fzqxf19mp90hhcp00bxm7"; }; libName = "stackable_versioned"; @@ -11102,12 +10957,12 @@ rec { }; "stackable-versioned-macros" = rec { crateName = "stackable-versioned-macros"; - version = "0.13.0"; + version = "0.14.0"; edition = "2024"; workspace_member = null; src = pkgs.fetchgit { url = "https://github.com/stackabletech/operator-rs.git"; - rev = "46a18a9a4899976b702ca87f73430c6f12d17c56"; + rev = "b0b8ca31a79da03b3fa20507b709390d2647efe4"; sha256 = "10va8k2dwshcb6c7pp01q6m6bx2ad58fzqxf19mp90hhcp00bxm7"; }; procMacro = true; @@ -11122,7 +10977,7 @@ rec { } { name = "darling"; - packageId = "darling 0.24.1"; + packageId = "darling"; } { name = "indoc"; @@ -11166,12 +11021,12 @@ rec { }; "stackable-webhook" = rec { crateName = "stackable-webhook"; - version = "0.10.0"; + version = "0.11.0"; edition = "2024"; workspace_member = null; src = pkgs.fetchgit { url = "https://github.com/stackabletech/operator-rs.git"; - rev = "46a18a9a4899976b702ca87f73430c6f12d17c56"; + rev = "b0b8ca31a79da03b3fa20507b709390d2647efe4"; sha256 = "10va8k2dwshcb6c7pp01q6m6bx2ad58fzqxf19mp90hhcp00bxm7"; }; libName = "stackable_webhook"; @@ -12571,11 +12426,6 @@ rec { "Tower Maintainers " ]; dependencies = [ - { - name = "base64"; - packageId = "base64 0.22.1"; - optional = true; - } { name = "bitflags"; packageId = "bitflags 2.13.2"; @@ -12599,12 +12449,6 @@ rec { packageId = "http-body"; optional = true; } - { - name = "mime"; - packageId = "mime"; - optional = true; - usesDefaultFeatures = false; - } { name = "pin-project-lite"; packageId = "pin-project-lite"; @@ -12622,12 +12466,6 @@ rec { name = "tower-service"; packageId = "tower-service"; } - { - name = "tracing"; - packageId = "tracing"; - optional = true; - usesDefaultFeatures = false; - } { name = "url"; packageId = "url"; @@ -12689,7 +12527,7 @@ rec { "uuid" = [ "dep:uuid" ]; "validate-request" = [ "mime" ]; }; - resolvedDefaultFeatures = [ "auth" "base64" "default" "follow-redirect" "futures-util" "map-response-body" "mime" "tower" "trace" "tracing" "util" "validate-request" ]; + resolvedDefaultFeatures = [ "follow-redirect" "futures-util" "tower" ]; }; "tower-http 0.7.1" = rec { crateName = "tower-http"; @@ -12701,6 +12539,11 @@ rec { "Tower Maintainers " ]; dependencies = [ + { + name = "base64"; + packageId = "base64 0.22.1"; + optional = true; + } { name = "bitflags"; packageId = "bitflags 2.13.2"; @@ -12718,6 +12561,12 @@ rec { packageId = "http-body"; optional = true; } + { + name = "mime"; + packageId = "mime"; + optional = true; + usesDefaultFeatures = false; + } { name = "percent-encoding"; packageId = "percent-encoding"; @@ -12726,6 +12575,11 @@ rec { name = "pin-project-lite"; packageId = "pin-project-lite"; } + { + name = "tower"; + packageId = "tower"; + optional = true; + } { name = "tower-layer"; packageId = "tower-layer"; @@ -12750,6 +12604,11 @@ rec { name = "http-body"; packageId = "http-body"; } + { + name = "tower"; + packageId = "tower"; + features = [ "buffer" "util" "retry" "make" "timeout" ]; + } ]; features = { "auth" = [ "base64" "validate-request" ]; @@ -12786,7 +12645,7 @@ rec { "uuid" = [ "dep:uuid" ]; "validate-request" = [ "mime" ]; }; - resolvedDefaultFeatures = [ "default" "trace" "tracing" ]; + resolvedDefaultFeatures = [ "auth" "base64" "default" "map-response-body" "mime" "tower" "trace" "tracing" "util" "validate-request" ]; }; "tower-layer" = rec { crateName = "tower-layer"; diff --git a/Cargo.toml b/Cargo.toml index ae973805a..bb57cebb7 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -10,7 +10,7 @@ edition = "2024" repository = "https://github.com/stackabletech/kafka-operator" [workspace.dependencies] -stackable-operator = { git = "https://github.com/stackabletech/operator-rs.git", branch = "feat/platform-access", features = ["agent", "crds", "webhook"] } +stackable-operator = { git = "https://github.com/stackabletech/operator-rs.git", tag = "stackable-operator-0.122.0", features = ["agent", "crds", "webhook"] } stackable-kafka-shared = { path = "rust/shared" } anyhow = "1.0" diff --git a/crate-hashes.json b/crate-hashes.json index e08caca83..e49d4b765 100644 --- a/crate-hashes.json +++ b/crate-hashes.json @@ -1,12 +1,12 @@ { - "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#k8s-version@0.1.4": "10va8k2dwshcb6c7pp01q6m6bx2ad58fzqxf19mp90hhcp00bxm7", - "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#stackable-agent@0.1.0": "10va8k2dwshcb6c7pp01q6m6bx2ad58fzqxf19mp90hhcp00bxm7", - "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#stackable-certs@0.4.1": "10va8k2dwshcb6c7pp01q6m6bx2ad58fzqxf19mp90hhcp00bxm7", - "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#stackable-operator-derive@0.3.1": "10va8k2dwshcb6c7pp01q6m6bx2ad58fzqxf19mp90hhcp00bxm7", - "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#stackable-operator@0.121.0": "10va8k2dwshcb6c7pp01q6m6bx2ad58fzqxf19mp90hhcp00bxm7", - "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#stackable-shared@0.2.0": "10va8k2dwshcb6c7pp01q6m6bx2ad58fzqxf19mp90hhcp00bxm7", - "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#stackable-telemetry@0.6.5": "10va8k2dwshcb6c7pp01q6m6bx2ad58fzqxf19mp90hhcp00bxm7", - "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#stackable-versioned-macros@0.13.0": "10va8k2dwshcb6c7pp01q6m6bx2ad58fzqxf19mp90hhcp00bxm7", - "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#stackable-versioned@0.13.0": "10va8k2dwshcb6c7pp01q6m6bx2ad58fzqxf19mp90hhcp00bxm7", - "git+https://github.com/stackabletech/operator-rs.git?branch=feat%2Fplatform-access#stackable-webhook@0.10.0": "10va8k2dwshcb6c7pp01q6m6bx2ad58fzqxf19mp90hhcp00bxm7" + "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.122.0#k8s-version@0.1.5": "10va8k2dwshcb6c7pp01q6m6bx2ad58fzqxf19mp90hhcp00bxm7", + "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.122.0#stackable-agent@0.1.0": "10va8k2dwshcb6c7pp01q6m6bx2ad58fzqxf19mp90hhcp00bxm7", + "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.122.0#stackable-certs@0.5.0": "10va8k2dwshcb6c7pp01q6m6bx2ad58fzqxf19mp90hhcp00bxm7", + "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.122.0#stackable-operator-derive@0.3.2": "10va8k2dwshcb6c7pp01q6m6bx2ad58fzqxf19mp90hhcp00bxm7", + "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.122.0#stackable-operator@0.122.0": "10va8k2dwshcb6c7pp01q6m6bx2ad58fzqxf19mp90hhcp00bxm7", + "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.122.0#stackable-shared@0.2.1": "10va8k2dwshcb6c7pp01q6m6bx2ad58fzqxf19mp90hhcp00bxm7", + "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.122.0#stackable-telemetry@0.6.6": "10va8k2dwshcb6c7pp01q6m6bx2ad58fzqxf19mp90hhcp00bxm7", + "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.122.0#stackable-versioned-macros@0.14.0": "10va8k2dwshcb6c7pp01q6m6bx2ad58fzqxf19mp90hhcp00bxm7", + "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.122.0#stackable-versioned@0.14.0": "10va8k2dwshcb6c7pp01q6m6bx2ad58fzqxf19mp90hhcp00bxm7", + "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.122.0#stackable-webhook@0.11.0": "10va8k2dwshcb6c7pp01q6m6bx2ad58fzqxf19mp90hhcp00bxm7" } \ No newline at end of file From efb02c4a017b52df2a317cdf406c5ae70a999c6c Mon Sep 17 00:00:00 2001 From: Benedikt Labrenz Date: Tue, 3 Nov 2026 16:36:22 +0100 Subject: [PATCH 78/93] add readiness and liveness probes --- Cargo.lock | 3 + Cargo.nix | 12 ++++ Cargo.toml | 1 + rust/agent-binary/Cargo.toml | 3 + rust/agent-binary/src/health.rs | 58 +++++++++++++++++++ rust/agent-binary/src/main.rs | 9 ++- .../src/controller/build/resource/agent.rs | 36 +++++++++++- rust/shared/src/agent.rs | 5 ++ 8 files changed, 123 insertions(+), 4 deletions(-) create mode 100644 rust/agent-binary/src/health.rs diff --git a/Cargo.lock b/Cargo.lock index 606cd0f20..9ce432d82 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3134,8 +3134,11 @@ name = "stackable-kafka-agent" version = "0.0.0-dev" dependencies = [ "anyhow", + "axum", "built", "clap", + "snafu 0.9.2", + "stackable-kafka-shared", "stackable-operator", "tokio", "tracing", diff --git a/Cargo.nix b/Cargo.nix index 4133e4d1d..172de3666 100644 --- a/Cargo.nix +++ b/Cargo.nix @@ -10325,10 +10325,22 @@ rec { name = "anyhow"; packageId = "anyhow"; } + { + name = "axum"; + packageId = "axum"; + } { name = "clap"; packageId = "clap"; } + { + name = "snafu"; + packageId = "snafu 0.9.2"; + } + { + name = "stackable-kafka-shared"; + packageId = "stackable-kafka-shared"; + } { name = "stackable-operator"; packageId = "stackable-operator"; diff --git a/Cargo.toml b/Cargo.toml index bb57cebb7..877f7c176 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -14,6 +14,7 @@ stackable-operator = { git = "https://github.com/stackabletech/operator-rs.git", stackable-kafka-shared = { path = "rust/shared" } anyhow = "1.0" +axum = "0.8" built = { version = "0.8", features = ["chrono", "git2"] } clap = "4.6" const_format = "0.2" diff --git a/rust/agent-binary/Cargo.toml b/rust/agent-binary/Cargo.toml index c138e675b..700eece71 100644 --- a/rust/agent-binary/Cargo.toml +++ b/rust/agent-binary/Cargo.toml @@ -9,10 +9,13 @@ repository.workspace = true publish = false [dependencies] +stackable-kafka-shared.workspace = true stackable-operator.workspace = true anyhow.workspace = true +axum.workspace = true clap.workspace = true +snafu.workspace = true tokio.workspace = true tracing.workspace = true diff --git a/rust/agent-binary/src/health.rs b/rust/agent-binary/src/health.rs new file mode 100644 index 000000000..13a79ef6f --- /dev/null +++ b/rust/agent-binary/src/health.rs @@ -0,0 +1,58 @@ +//! The health endpoints of the agent, which the probes of its Deployment call. + +use std::{ + future::Future, + net::{IpAddr, Ipv4Addr, SocketAddr}, + sync::Arc, +}; + +use axum::{ + Router, + extract::State, + http::StatusCode, + response::{IntoResponse, Response}, + routing::get, +}; +use snafu::{ResultExt, Snafu}; +use stackable_kafka_shared::agent::{HEALTH_PORT, LIVENESS_PATH, READINESS_PATH}; +use stackable_operator::webhook::health::HealthCheckRegistry; +use tokio::net::TcpListener; +use tracing::info; + +const SOCKET_ADDRESS: SocketAddr = SocketAddr::new(IpAddr::V4(Ipv4Addr::UNSPECIFIED), HEALTH_PORT); + +#[derive(Debug, Snafu)] +pub enum Error { + #[snafu(display("failed to listen on {address} for the health endpoints"))] + Bind { + source: std::io::Error, + address: SocketAddr, + }, + + #[snafu(display("failed to serve the health endpoints"))] + Serve { source: std::io::Error }, +} + +/// Serves the endpoints until `shutdown` completes. Liveness only shows that the agent answers, +/// readiness is decided by the checks of the registry. +pub async fn serve( + readiness: Arc, + shutdown: impl Future + Send + 'static, +) -> Result<(), Error> { + let router = Router::new() + .route(LIVENESS_PATH, get(StatusCode::OK)) + .route(READINESS_PATH, get(ready)) + .with_state(readiness); + let listener = TcpListener::bind(SOCKET_ADDRESS).await.context(BindSnafu { + address: SOCKET_ADDRESS, + })?; + info!(address = %SOCKET_ADDRESS, "Serving the health endpoints"); + axum::serve(listener, router) + .with_graceful_shutdown(shutdown) + .await + .context(ServeSnafu) +} + +async fn ready(State(readiness): State>) -> Response { + (&*readiness).into_response() +} diff --git a/rust/agent-binary/src/main.rs b/rust/agent-binary/src/main.rs index 0934f803f..2341559f1 100644 --- a/rust/agent-binary/src/main.rs +++ b/rust/agent-binary/src/main.rs @@ -1,12 +1,17 @@ //! The Stackable Kafka agent: a per-cluster controller that reconciles in-cluster resources like `KafkaTopic`s. +use std::sync::Arc; + use clap::Parser; use stackable_operator::{ cli::{Command, CommonOptions}, telemetry::Tracing, utils::signal::SignalWatcher, + webhook::health::HealthCheckRegistry, }; +mod health; + mod built_info { include!(concat!(env!("OUT_DIR"), "/built.rs")); } @@ -35,8 +40,10 @@ async fn main() -> anyhow::Result<()> { description = built_info::PKG_DESCRIPTION ); + let readiness = Arc::new(HealthCheckRegistry::new()); + let sigterm_watcher = SignalWatcher::sigterm()?; - sigterm_watcher.handle().await; + health::serve(readiness, sigterm_watcher.handle()).await?; } } diff --git a/rust/operator-binary/src/controller/build/resource/agent.rs b/rust/operator-binary/src/controller/build/resource/agent.rs index a97886a17..f1fd3a84a 100644 --- a/rust/operator-binary/src/controller/build/resource/agent.rs +++ b/rust/operator-binary/src/controller/build/resource/agent.rs @@ -4,7 +4,7 @@ use std::str::FromStr; use stackable_kafka_shared::agent::{ ClientSecurity, ENV_BOOTSTRAP_SERVERS, ENV_CLIENT_SECURITY, ENV_CLUSTER_NAME, - ENV_CLUSTER_NAMESPACE, KAFKA_TLS_DIR, + ENV_CLUSTER_NAMESPACE, HEALTH_PORT, KAFKA_TLS_DIR, LIVENESS_PATH, READINESS_PATH, }; use stackable_operator::{ builder::{ @@ -12,6 +12,7 @@ use stackable_operator::{ pod::{ PodBuilder, container::ContainerBuilder, + probe::ProbeBuilder, resources::ResourceRequirementsBuilder, security::PodSecurityContextBuilder, volume::{SecretFormat, SecretOperatorVolumeSourceBuilder, VolumeBuilder}, @@ -22,11 +23,14 @@ use stackable_operator::{ k8s_openapi::{ api::{ apps::v1::{Deployment, DeploymentSpec, DeploymentStrategy}, - core::v1::{EnvVar, EnvVarSource, ObjectFieldSelector, ServiceAccount}, + core::v1::{ + EnvVar, EnvVarSource, HTTPGetAction, ObjectFieldSelector, Probe, ServiceAccount, + }, rbac::v1::RoleBinding, }, - apimachinery::pkg::apis::meta::v1::LabelSelector, + apimachinery::pkg::{apis::meta::v1::LabelSelector, util::intstr::IntOrString}, }, + shared::time::Duration, v2::{ builder::{ meta::ownerreference_from_resource, @@ -49,6 +53,7 @@ use crate::{ }; constant!(AGENT_CONTAINER_NAME: ContainerName = "agent"); +const HEALTH_PORT_NAME: &str = "health"; constant!(KAFKA_TLS_VOLUME_NAME: VolumeName = "tls-kafka"); constant!(KUBERNETES_NODE_NAME: EnvVarName = "KUBERNETES_NODE_NAME"); constant!(KUBERNETES_CLUSTER_DOMAIN: EnvVarName = "KUBERNETES_CLUSTER_DOMAIN"); @@ -92,6 +97,9 @@ pub fn build_agent_deployment( .command(vec!["stackable-kafka-agent".to_owned()]) .args(vec!["run".to_owned()]) .add_env_vars(agent_env_vars(cluster, agent_config)) + .add_container_port(HEALTH_PORT_NAME, HEALTH_PORT.into()) + .readiness_probe(http_probe(READINESS_PATH, Duration::from_secs(10))) + .liveness_probe(http_probe(LIVENESS_PATH, Duration::from_secs(60))) .resources( ResourceRequirementsBuilder::new() .with_cpu_request("100m") @@ -141,6 +149,18 @@ pub fn build_agent_deployment( } } +fn http_probe(path: &str, period: Duration) -> Probe { + ProbeBuilder::http_get(HTTPGetAction { + path: Some(path.to_owned()), + port: IntOrString::String(HEALTH_PORT_NAME.to_owned()), + ..HTTPGetAction::default() + }) + .with_period(period) + .with_failure_threshold(3) + .build() + .expect("the probe period is a constant that fits into an i32") +} + pub fn build_agent_service_account(cluster: &ValidatedCluster) -> ServiceAccount { rbac::build_service_account( cluster, @@ -357,6 +377,16 @@ mod tests { json!(["stackable-kafka-agent"]) ); assert_eq!(pod_spec["containers"][0]["args"], json!(["run"])); + assert_eq!(pod_spec["containers"][0]["ports"][0]["name"], "health"); + assert_eq!(pod_spec["containers"][0]["ports"][0]["containerPort"], 8080); + assert_eq!( + pod_spec["containers"][0]["readinessProbe"]["httpGet"], + json!({"path": "/readyz", "port": "health"}) + ); + assert_eq!( + pod_spec["containers"][0]["livenessProbe"]["httpGet"], + json!({"path": "/healthz", "port": "health"}) + ); assert_eq!( pod_spec["containers"][0]["volumeMounts"], json!([ diff --git a/rust/shared/src/agent.rs b/rust/shared/src/agent.rs index c481cbc47..802ebc951 100644 --- a/rust/shared/src/agent.rs +++ b/rust/shared/src/agent.rs @@ -8,6 +8,11 @@ use strum::{Display, EnumString}; /// Directory of the CA certificate the Kafka brokers are verified with. pub const KAFKA_TLS_DIR: &str = "/stackable/secrets/tls-kafka"; +/// Port of the HTTP health endpoints, which the probes of the agent Deployment call. +pub const HEALTH_PORT: u16 = 8080; +pub const LIVENESS_PATH: &str = "/healthz"; +pub const READINESS_PATH: &str = "/readyz"; + constant!(pub ENV_CLUSTER_NAME: EnvVarName = "KAFKA_CLUSTER_NAME"); constant!(pub ENV_CLUSTER_NAMESPACE: EnvVarName = "KAFKA_CLUSTER_NAMESPACE"); constant!(pub ENV_CLIENT_SECURITY: EnvVarName = "KAFKA_CLIENT_SECURITY"); From b4111e0dc17adfea34313ed47ad23d80ebecc2c5 Mon Sep 17 00:00:00 2001 From: Benedikt Labrenz Date: Tue, 3 Nov 2026 16:39:51 +0100 Subject: [PATCH 79/93] fix Cargo.toml --- Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index 034694c0c..98d10beda 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -14,8 +14,8 @@ stackable-operator = { git = "https://github.com/stackabletech/operator-rs.git", stackable-kafka-shared = { path = "rust/shared" } anyhow = "1.0" -base64 = "0.22" axum = "0.8" +base64 = "0.22" built = { version = "0.8", features = ["chrono", "git2"] } bytes = "1.12" clap = "4.6" From aab52d9637fd727f2012c2bad565dd5692a33515 Mon Sep 17 00:00:00 2001 From: Benedikt Labrenz Date: Wed, 4 Nov 2026 16:30:17 +0100 Subject: [PATCH 80/93] regenerate nix --- Cargo.nix | 20 ++++++++++---------- crate-hashes.json | 20 ++++++++++---------- 2 files changed, 20 insertions(+), 20 deletions(-) diff --git a/Cargo.nix b/Cargo.nix index 985ab90d4..6f1ea933e 100644 --- a/Cargo.nix +++ b/Cargo.nix @@ -5459,7 +5459,7 @@ rec { src = pkgs.fetchgit { url = "https://github.com/stackabletech/operator-rs.git"; rev = "b0b8ca31a79da03b3fa20507b709390d2647efe4"; - sha256 = "10va8k2dwshcb6c7pp01q6m6bx2ad58fzqxf19mp90hhcp00bxm7"; + sha256 = "1yfmjslv9s111zc4zfqzi2aankchiw8yjxx2zlc17wgayqrbmxrj"; }; libName = "k8s_version"; authors = [ @@ -10267,7 +10267,7 @@ rec { src = pkgs.fetchgit { url = "https://github.com/stackabletech/operator-rs.git"; rev = "b0b8ca31a79da03b3fa20507b709390d2647efe4"; - sha256 = "10va8k2dwshcb6c7pp01q6m6bx2ad58fzqxf19mp90hhcp00bxm7"; + sha256 = "1yfmjslv9s111zc4zfqzi2aankchiw8yjxx2zlc17wgayqrbmxrj"; }; libName = "stackable_agent"; authors = [ @@ -10310,7 +10310,7 @@ rec { src = pkgs.fetchgit { url = "https://github.com/stackabletech/operator-rs.git"; rev = "b0b8ca31a79da03b3fa20507b709390d2647efe4"; - sha256 = "10va8k2dwshcb6c7pp01q6m6bx2ad58fzqxf19mp90hhcp00bxm7"; + sha256 = "1yfmjslv9s111zc4zfqzi2aankchiw8yjxx2zlc17wgayqrbmxrj"; }; libName = "stackable_certs"; authors = [ @@ -10716,7 +10716,7 @@ rec { src = pkgs.fetchgit { url = "https://github.com/stackabletech/operator-rs.git"; rev = "b0b8ca31a79da03b3fa20507b709390d2647efe4"; - sha256 = "10va8k2dwshcb6c7pp01q6m6bx2ad58fzqxf19mp90hhcp00bxm7"; + sha256 = "1yfmjslv9s111zc4zfqzi2aankchiw8yjxx2zlc17wgayqrbmxrj"; }; libName = "stackable_operator"; authors = [ @@ -10925,7 +10925,7 @@ rec { src = pkgs.fetchgit { url = "https://github.com/stackabletech/operator-rs.git"; rev = "b0b8ca31a79da03b3fa20507b709390d2647efe4"; - sha256 = "10va8k2dwshcb6c7pp01q6m6bx2ad58fzqxf19mp90hhcp00bxm7"; + sha256 = "1yfmjslv9s111zc4zfqzi2aankchiw8yjxx2zlc17wgayqrbmxrj"; }; procMacro = true; libName = "stackable_operator_derive"; @@ -10960,7 +10960,7 @@ rec { src = pkgs.fetchgit { url = "https://github.com/stackabletech/operator-rs.git"; rev = "b0b8ca31a79da03b3fa20507b709390d2647efe4"; - sha256 = "10va8k2dwshcb6c7pp01q6m6bx2ad58fzqxf19mp90hhcp00bxm7"; + sha256 = "1yfmjslv9s111zc4zfqzi2aankchiw8yjxx2zlc17wgayqrbmxrj"; }; libName = "stackable_shared"; authors = [ @@ -11041,7 +11041,7 @@ rec { src = pkgs.fetchgit { url = "https://github.com/stackabletech/operator-rs.git"; rev = "b0b8ca31a79da03b3fa20507b709390d2647efe4"; - sha256 = "10va8k2dwshcb6c7pp01q6m6bx2ad58fzqxf19mp90hhcp00bxm7"; + sha256 = "1yfmjslv9s111zc4zfqzi2aankchiw8yjxx2zlc17wgayqrbmxrj"; }; libName = "stackable_telemetry"; authors = [ @@ -11151,7 +11151,7 @@ rec { src = pkgs.fetchgit { url = "https://github.com/stackabletech/operator-rs.git"; rev = "b0b8ca31a79da03b3fa20507b709390d2647efe4"; - sha256 = "10va8k2dwshcb6c7pp01q6m6bx2ad58fzqxf19mp90hhcp00bxm7"; + sha256 = "1yfmjslv9s111zc4zfqzi2aankchiw8yjxx2zlc17wgayqrbmxrj"; }; libName = "stackable_versioned"; authors = [ @@ -11197,7 +11197,7 @@ rec { src = pkgs.fetchgit { url = "https://github.com/stackabletech/operator-rs.git"; rev = "b0b8ca31a79da03b3fa20507b709390d2647efe4"; - sha256 = "10va8k2dwshcb6c7pp01q6m6bx2ad58fzqxf19mp90hhcp00bxm7"; + sha256 = "1yfmjslv9s111zc4zfqzi2aankchiw8yjxx2zlc17wgayqrbmxrj"; }; procMacro = true; libName = "stackable_versioned_macros"; @@ -11261,7 +11261,7 @@ rec { src = pkgs.fetchgit { url = "https://github.com/stackabletech/operator-rs.git"; rev = "b0b8ca31a79da03b3fa20507b709390d2647efe4"; - sha256 = "10va8k2dwshcb6c7pp01q6m6bx2ad58fzqxf19mp90hhcp00bxm7"; + sha256 = "1yfmjslv9s111zc4zfqzi2aankchiw8yjxx2zlc17wgayqrbmxrj"; }; libName = "stackable_webhook"; authors = [ diff --git a/crate-hashes.json b/crate-hashes.json index e49d4b765..aa6c876cf 100644 --- a/crate-hashes.json +++ b/crate-hashes.json @@ -1,12 +1,12 @@ { - "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.122.0#k8s-version@0.1.5": "10va8k2dwshcb6c7pp01q6m6bx2ad58fzqxf19mp90hhcp00bxm7", - "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.122.0#stackable-agent@0.1.0": "10va8k2dwshcb6c7pp01q6m6bx2ad58fzqxf19mp90hhcp00bxm7", - "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.122.0#stackable-certs@0.5.0": "10va8k2dwshcb6c7pp01q6m6bx2ad58fzqxf19mp90hhcp00bxm7", - "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.122.0#stackable-operator-derive@0.3.2": "10va8k2dwshcb6c7pp01q6m6bx2ad58fzqxf19mp90hhcp00bxm7", - "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.122.0#stackable-operator@0.122.0": "10va8k2dwshcb6c7pp01q6m6bx2ad58fzqxf19mp90hhcp00bxm7", - "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.122.0#stackable-shared@0.2.1": "10va8k2dwshcb6c7pp01q6m6bx2ad58fzqxf19mp90hhcp00bxm7", - "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.122.0#stackable-telemetry@0.6.6": "10va8k2dwshcb6c7pp01q6m6bx2ad58fzqxf19mp90hhcp00bxm7", - "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.122.0#stackable-versioned-macros@0.14.0": "10va8k2dwshcb6c7pp01q6m6bx2ad58fzqxf19mp90hhcp00bxm7", - "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.122.0#stackable-versioned@0.14.0": "10va8k2dwshcb6c7pp01q6m6bx2ad58fzqxf19mp90hhcp00bxm7", - "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.122.0#stackable-webhook@0.11.0": "10va8k2dwshcb6c7pp01q6m6bx2ad58fzqxf19mp90hhcp00bxm7" + "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.122.0#k8s-version@0.1.5": "1yfmjslv9s111zc4zfqzi2aankchiw8yjxx2zlc17wgayqrbmxrj", + "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.122.0#stackable-agent@0.1.0": "1yfmjslv9s111zc4zfqzi2aankchiw8yjxx2zlc17wgayqrbmxrj", + "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.122.0#stackable-certs@0.5.0": "1yfmjslv9s111zc4zfqzi2aankchiw8yjxx2zlc17wgayqrbmxrj", + "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.122.0#stackable-operator-derive@0.3.2": "1yfmjslv9s111zc4zfqzi2aankchiw8yjxx2zlc17wgayqrbmxrj", + "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.122.0#stackable-operator@0.122.0": "1yfmjslv9s111zc4zfqzi2aankchiw8yjxx2zlc17wgayqrbmxrj", + "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.122.0#stackable-shared@0.2.1": "1yfmjslv9s111zc4zfqzi2aankchiw8yjxx2zlc17wgayqrbmxrj", + "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.122.0#stackable-telemetry@0.6.6": "1yfmjslv9s111zc4zfqzi2aankchiw8yjxx2zlc17wgayqrbmxrj", + "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.122.0#stackable-versioned-macros@0.14.0": "1yfmjslv9s111zc4zfqzi2aankchiw8yjxx2zlc17wgayqrbmxrj", + "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.122.0#stackable-versioned@0.14.0": "1yfmjslv9s111zc4zfqzi2aankchiw8yjxx2zlc17wgayqrbmxrj", + "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.122.0#stackable-webhook@0.11.0": "1yfmjslv9s111zc4zfqzi2aankchiw8yjxx2zlc17wgayqrbmxrj" } \ No newline at end of file From d5eeca98599a919d4a314b88782c59c8b2832381 Mon Sep 17 00:00:00 2001 From: Benedikt Labrenz Date: Wed, 4 Nov 2026 16:57:10 +0100 Subject: [PATCH 81/93] Try every address of a bootstrap server when connecting --- rust/kafka-admin/src/connection.rs | 148 +++++++++++++++++++++-------- 1 file changed, 107 insertions(+), 41 deletions(-) diff --git a/rust/kafka-admin/src/connection.rs b/rust/kafka-admin/src/connection.rs index abd0eb482..ed785ad28 100644 --- a/rust/kafka-admin/src/connection.rs +++ b/rust/kafka-admin/src/connection.rs @@ -1,6 +1,6 @@ //! A connection to a single Kafka broker. -use std::{collections::BTreeMap, io, path::PathBuf, sync::Arc, time::Duration}; +use std::{collections::BTreeMap, io, net::SocketAddr, path::PathBuf, sync::Arc, time::Duration}; use futures::{SinkExt, StreamExt}; use kafka_protocol::{ @@ -12,7 +12,7 @@ use kafka_protocol::{ use rustls::{ClientConfig, pki_types::ServerName}; use snafu::{OptionExt, ResultExt, Snafu, ensure}; use tokio::{ - net::TcpStream, + net::{TcpStream, lookup_host}, time::{error::Elapsed, timeout}, }; use tokio_rustls::{TlsConnector, client::TlsStream}; @@ -42,6 +42,15 @@ const API_VERSIONS_VERSION: i16 = 3; #[derive(Debug, Snafu)] pub enum Error { + #[snafu(display("failed to resolve {address}"))] + Resolve { + source: io::Error, + address: BrokerAddress, + }, + + #[snafu(display("{address} does not resolve to any address"))] + NoAddresses { address: BrokerAddress }, + #[snafu(display("failed to connect to {address}"))] Connect { source: io::Error, @@ -139,10 +148,74 @@ impl Connection { tls_config: Option>, client_id: &str, ) -> Result { - let tcp_stream = connect_tcp(address).await?; - let stream = match tls_config { + let socket_addresses = lookup_host((address.host.as_str(), address.port)) + .await + .context(ResolveSnafu { + address: address.clone(), + })?; + Self::connect_any(socket_addresses, address, tls_config, client_id).await + } + + /// One [`CONNECT_TIMEOUT`] per address, so a broker that accepts but never answers does not + /// hide the others. + async fn connect_any( + socket_addresses: impl IntoIterator, + address: &BrokerAddress, + tls_config: Option>, + client_id: &str, + ) -> Result { + let tls = match tls_config { + None => None, + Some(config) => { + let host = &address.host; + let server_name = ServerName::try_from(host.clone()) + .ok() + .context(InvalidServerNameSnafu { host })?; + Some((server_name, config)) + } + }; + + let mut last_error = NoAddressesSnafu { + address: address.clone(), + } + .build(); + for socket_address in socket_addresses { + let attempt = Self::connect_to(socket_address, address, tls.as_ref(), client_id); + match timeout(CONNECT_TIMEOUT, attempt).await { + Ok(Ok(connection)) => return Ok(connection), + Ok(Err(error)) => last_error = error, + Err(source) => { + last_error = Error::ConnectTimeout { + source, + address: address.clone(), + } + } + } + } + Err(last_error) + } + + async fn connect_to( + socket_address: SocketAddr, + address: &BrokerAddress, + tls: Option<&(ServerName<'static>, Arc)>, + client_id: &str, + ) -> Result { + let tcp_stream = TcpStream::connect(socket_address) + .await + .context(ConnectSnafu { + address: address.clone(), + })?; + let stream = match tls { None => Either::Left(tcp_stream), - Some(config) => Either::Right(tls_handshake(tcp_stream, address, config).await?), + Some((server_name, config)) => Either::Right( + TlsConnector::from(config.clone()) + .connect(server_name.clone(), tcp_stream) + .await + .context(TlsHandshakeSnafu { + address: address.clone(), + })?, + ), }; let mut connection = Self::new(stream, client_id); @@ -202,20 +275,6 @@ impl Connection { } } -async fn connect_tcp(address: &BrokerAddress) -> Result { - timeout( - CONNECT_TIMEOUT, - TcpStream::connect((address.host.as_str(), address.port)), - ) - .await - .context(ConnectTimeoutSnafu { - address: address.clone(), - })? - .context(ConnectSnafu { - address: address.clone(), - }) -} - fn api_versions_request() -> ApiVersionsRequest { ApiVersionsRequest::default() .with_client_software_name(StrBytes::from_static_str("stackable-kafka-admin")) @@ -265,28 +324,6 @@ fn check_version_in_range( Ok(()) } -async fn tls_handshake( - tcp_stream: TcpStream, - address: &BrokerAddress, - config: Arc, -) -> Result> { - let host = &address.host; - let server_name = ServerName::try_from(host.clone()) - .ok() - .context(InvalidServerNameSnafu { host })?; - timeout( - CONNECT_TIMEOUT, - TlsConnector::from(config).connect(server_name, tcp_stream), - ) - .await - .context(ConnectTimeoutSnafu { - address: address.clone(), - })? - .context(TlsHandshakeSnafu { - address: address.clone(), - }) -} - /// A mock Kafka broker for tests, speaking the wire protocol over a local TCP socket. #[cfg(test)] pub(crate) mod test_support { @@ -485,6 +522,35 @@ mod tests { )); } + #[tokio::test] + async fn try_the_next_address_if_the_broker_does_not_answer() { + let (silent_listener, silent_port) = listen().await; + let (listener, port) = listen().await; + let address = BrokerAddress { + host: "127.0.0.1".to_owned(), + port, + }; + let socket_addresses = + [silent_port, port].map(|port| SocketAddr::from(([127, 0, 0, 1], port))); + tokio::time::pause(); + + let silent_side = async { + let (mut broker, _) = silent_listener + .accept() + .await + .expect("the silent broker should accept the connection"); + read_request(&mut broker).await; + broker + }; + let (connection, _silent_broker, _broker) = tokio::join!( + Connection::connect_any(socket_addresses, &address, None, "test"), + silent_side, + accept(&listener, &[(MetadataRequest::KEY, 0, 13)]) + ); + + connection.expect("the second address should be used"); + } + #[tokio::test] async fn send_request_and_receive_response() { let (mut connection, mut broker) = connect().await; From 187fe586da6aa4eca073cd0d00e0aa25b7398687 Mon Sep 17 00:00:00 2001 From: Benedikt Labrenz Date: Wed, 4 Nov 2026 17:14:52 +0100 Subject: [PATCH 82/93] Skip connecting for 30 s after all bootstrap servers failed --- rust/kafka-admin/src/client.rs | 91 +++++++++++++++++++++++++++++----- 1 file changed, 79 insertions(+), 12 deletions(-) diff --git a/rust/kafka-admin/src/client.rs b/rust/kafka-admin/src/client.rs index c896e9bad..f511a12e4 100644 --- a/rust/kafka-admin/src/client.rs +++ b/rust/kafka-admin/src/client.rs @@ -53,6 +53,9 @@ const _: () = assert!((BROKER_TIMEOUT_MS as u128) < connection::REQUEST_TIMEOUT. /// `server/src/main/java/org/apache/kafka/network/SocketServerConfigs.java`. const CONNECTION_MAX_IDLE: Duration = Duration::from_secs(5 * 60); +/// Connecting is skipped for this long after all bootstrap servers failed, so an outage fails fast. +const CONNECT_COOLDOWN: Duration = Duration::from_secs(30); + /// Lets the broker apply `num.partitions` or `default.replication.factor` when creating a topic, /// see `NumPartitions` and `ReplicationFactor` in /// `clients/src/main/resources/common/message/CreateTopicsRequest.json`. @@ -81,6 +84,12 @@ pub enum Error { server: BrokerAddress, }, + #[snafu(display( + "not connecting for {}s after all bootstrap servers failed", + CONNECT_COOLDOWN.as_secs() + ))] + ConnectCooldown { source: Arc }, + #[snafu(display("failed to send the request to the broker"))] Send { source: connection::Error }, @@ -151,6 +160,8 @@ pub struct KafkaAdminClient { client_id: String, /// The connection and the time of its last successful request connection: Option<(Connection, Instant)>, + /// When and why all bootstrap servers last failed, see [`CONNECT_COOLDOWN`] + connect_failure: Option<(Instant, Arc)>, } impl KafkaAdminClient { @@ -161,6 +172,7 @@ impl KafkaAdminClient { security, client_id: client_id.to_owned(), connection: None, + connect_failure: None, } } @@ -316,14 +328,25 @@ impl KafkaAdminClient { /// Connects to the first bootstrap server that answers. Any broker works, because brokers in /// KRaft mode forward admin requests to the active controller. - async fn connect(&self) -> Result { + async fn connect(&mut self) -> Result { + if let Some((failed_at, failure)) = &self.connect_failure + && failed_at.elapsed() < CONNECT_COOLDOWN + { + return Err(Error::ConnectCooldown { + source: Arc::clone(failure), + }); + } + let tls_config = tls::client_config(&self.security) .context(TlsSnafu)? .map(Arc::new); let mut last_error = Error::NoBootstrapServers; for server in &self.bootstrap_servers { match Connection::connect(server, tls_config.clone(), &self.client_id).await { - Ok(connection) => return Ok(connection), + Ok(connection) => { + self.connect_failure = None; + return Ok(connection); + } Err(source) => { last_error = Error::Connect { source, @@ -332,7 +355,9 @@ impl KafkaAdminClient { } } } - Err(last_error) + let failure = Arc::new(last_error); + self.connect_failure = Some((Instant::now(), Arc::clone(&failure))); + Err(Error::ConnectCooldown { source: failure }) } } @@ -615,15 +640,7 @@ mod tests { #[tokio::test] async fn try_the_next_bootstrap_server() { - // Use a port that is always rejected - let unreachable = TcpSocket::new_v4().expect("the socket should be creatable"); - unreachable - .bind("127.0.0.1:0".parse().expect("the address should be valid")) - .expect("the socket should be bindable"); - let unreachable_port = unreachable - .local_addr() - .expect("the socket should have an address") - .port(); + let (_unreachable, unreachable_port) = unreachable_socket(); let (listener, port) = listen().await; let mut client = kafka_admin_client(&[unreachable_port, port]); @@ -636,6 +653,43 @@ mod tests { result.expect("the second bootstrap server should be used"); } + #[tokio::test] + async fn skip_connecting_during_the_cooldown() { + let (unreachable, unreachable_port) = unreachable_socket(); + let mut client = kafka_admin_client(&[unreachable_port]); + tokio::time::pause(); + + let failure = client + .delete_topic(TOPIC) + .await + .expect_err("the connection should fail"); + assert!(matches!( + failure, + Error::ConnectCooldown { ref source } if matches!(**source, Error::Connect { .. }) + )); + + let skipped = client + .delete_topic(TOPIC) + .await + .expect_err("connecting should be skipped"); + assert!(matches!( + skipped, + Error::ConnectCooldown { ref source } if matches!(**source, Error::Connect { .. }) + )); + assert_eq!(failure.to_string(), skipped.to_string()); + + tokio::time::advance(CONNECT_COOLDOWN).await; + tokio::time::resume(); + let listener = unreachable.listen(1).expect("the socket should listen"); + let broker_side = async { + let mut broker = accept(&listener, APIS).await; + answer_delete_topics(&mut broker).await; + }; + let (result, ()) = tokio::join!(client.delete_topic(TOPIC), broker_side); + + result.expect("the client should connect again after the cooldown"); + } + #[tokio::test] async fn reconnect_after_a_failed_request() { let (listener, port) = listen().await; @@ -1028,6 +1082,19 @@ mod tests { )); } + /// A bound but not listening socket, so connections to its port are rejected. + fn unreachable_socket() -> (TcpSocket, u16) { + let socket = TcpSocket::new_v4().expect("the socket should be creatable"); + socket + .bind("127.0.0.1:0".parse().expect("the address should be valid")) + .expect("the socket should be bindable"); + let port = socket + .local_addr() + .expect("the socket should have an address") + .port(); + (socket, port) + } + fn kafka_admin_client(ports: &[u16]) -> KafkaAdminClient { let bootstrap_servers = ports .iter() From b9b15829f4bfbd76642470244e4effa05cfac08d Mon Sep 17 00:00:00 2001 From: Benedikt Labrenz Date: Thu, 5 Nov 2026 09:17:55 +0100 Subject: [PATCH 83/93] Compare declared topic names without parsing them in the store scans --- rust/agent-binary/src/topic/validate.rs | 4 +--- rust/shared/src/crd/topic.rs | 8 ++++++++ 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/rust/agent-binary/src/topic/validate.rs b/rust/agent-binary/src/topic/validate.rs index d5055d4dc..807a297f9 100644 --- a/rust/agent-binary/src/topic/validate.rs +++ b/rust/agent-binary/src/topic/validate.rs @@ -90,9 +90,7 @@ fn is_same_topic( topic_name: &TopicName, ) -> bool { other_kafka_topic.spec.cluster_ref.name == kafka_topic.spec.cluster_ref.name - && other_kafka_topic - .topic_name() - .is_ok_and(|other_topic_name| &other_topic_name == topic_name) + && other_kafka_topic.declared_topic_name() == topic_name.as_ref() } #[cfg(test)] diff --git a/rust/shared/src/crd/topic.rs b/rust/shared/src/crd/topic.rs index d2816c979..72c3b124a 100644 --- a/rust/shared/src/crd/topic.rs +++ b/rust/shared/src/crd/topic.rs @@ -142,6 +142,14 @@ impl v1alpha1::KafkaTopic { None => TopicName::from_str(&self.name_any()), } } + + /// [`Self::topic_name`] as a plain string, for comparisons, where validating it is redundant. + pub fn declared_topic_name(&self) -> &str { + match &self.spec.topic_name { + Some(topic_name) => topic_name.as_ref(), + None => self.metadata.name.as_deref().unwrap_or_default(), + } + } } #[cfg(test)] From de63aa7049dc0320ed8d9be4fc6fb3592913bf96 Mon Sep 17 00:00:00 2001 From: Benedikt Labrenz Date: Thu, 5 Nov 2026 09:38:09 +0100 Subject: [PATCH 84/93] Reconcile one KafkaTopic at a time in the agent --- rust/agent-binary/src/main.rs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/rust/agent-binary/src/main.rs b/rust/agent-binary/src/main.rs index 87360abe6..da5a7c7db 100644 --- a/rust/agent-binary/src/main.rs +++ b/rust/agent-binary/src/main.rs @@ -21,6 +21,7 @@ use stackable_operator::{ core::DeserializeGuard, runtime::{ Controller, + controller::Config as ControllerConfig, events::{Recorder, Reporter}, watcher, }, @@ -135,7 +136,9 @@ async fn main() -> anyhow::Result<()> { client.get_api::>(cluster_namespace.as_ref()), // Works because the CRD declares the field as a selectableField. watcher::Config::default().fields(&format!("spec.clusterRef.name={cluster_name}")), - ); + ) + // Each reconcile then acts on the current object; Kafka requests are serialized anyway. + .with_config(ControllerConfig::default().concurrency(1)); let kafka_topic_store = topic_controller.store(); let topic_controller = topic_controller .graceful_shutdown_on(sigterm_watcher.handle()) From 4dd7893fe92d5560022a92c7cdd4e6423b7cf661 Mon Sep 17 00:00:00 2001 From: Benedikt Labrenz Date: Thu, 5 Nov 2026 09:59:45 +0100 Subject: [PATCH 85/93] Leave KafkaTopics to a running agent whatever the cluster state --- rust/operator-binary/src/topic_controller.rs | 79 ++++++++++++++------ 1 file changed, 56 insertions(+), 23 deletions(-) diff --git a/rust/operator-binary/src/topic_controller.rs b/rust/operator-binary/src/topic_controller.rs index 11aa540b5..52c8d5937 100644 --- a/rust/operator-binary/src/topic_controller.rs +++ b/rust/operator-binary/src/topic_controller.rs @@ -207,8 +207,12 @@ fn operator_action_for( agent_state: &AgentState, now: Timestamp, ) -> OperatorAction { - match cluster_state { - ClusterState::MissingOrDeleting if kafka_topic.metadata.deletion_timestamp.is_some() => { + match (cluster_state, agent_state) { + // A running agent manages the KafkaTopics, whatever the KafkaCluster says. + (_, AgentState::Available) => OperatorAction::LeaveToAgent, + (ClusterState::MissingOrDeleting, _) + if kafka_topic.metadata.deletion_timestamp.is_some() => + { if kafka_topic .finalizers() .iter() @@ -219,27 +223,28 @@ fn operator_action_for( OperatorAction::LeaveToAgent } } - ClusterState::MissingOrDeleting => OperatorAction::SetAgentUnavailableConditions { + (ClusterState::MissingOrDeleting, _) => OperatorAction::SetAgentUnavailableConditions { message: "the KafkaCluster does not exist", }, - ClusterState::ManagementAccessDisabled => OperatorAction::SetAgentUnavailableConditions { - message: "the KafkaCluster has no agent, because clusterConfig.managementAccess is \ - not enabled", - }, - ClusterState::ManagementAccessEnabled => match agent_state { - AgentState::Available => OperatorAction::LeaveToAgent, - AgentState::UnavailableSince(Some(since)) - if now.duration_since(*since) < AGENT_UNAVAILABLE_GRACE_PERIOD => - { - OperatorAction::WaitForAgent { - remaining: (AGENT_UNAVAILABLE_GRACE_PERIOD - now.duration_since(*since)) - .unsigned_abs(), - } + (ClusterState::ManagementAccessDisabled, _) => { + OperatorAction::SetAgentUnavailableConditions { + message: "the KafkaCluster has no agent, because clusterConfig.managementAccess \ + is not enabled", } - AgentState::UnavailableSince(_) => OperatorAction::SetAgentUnavailableConditions { + } + (ClusterState::ManagementAccessEnabled, AgentState::UnavailableSince(Some(since))) + if now.duration_since(*since) < AGENT_UNAVAILABLE_GRACE_PERIOD => + { + OperatorAction::WaitForAgent { + remaining: (AGENT_UNAVAILABLE_GRACE_PERIOD - now.duration_since(*since)) + .unsigned_abs(), + } + } + (ClusterState::ManagementAccessEnabled, AgentState::UnavailableSince(_)) => { + OperatorAction::SetAgentUnavailableConditions { message: "the agent of the KafkaCluster is not available", - }, - }, + } + } } } @@ -394,12 +399,21 @@ mod tests { } #[rstest] - #[case::cluster_missing(ClusterState::MissingOrDeleting, AgentState::Available, true)] + #[case::cluster_missing_with_agent( + ClusterState::MissingOrDeleting, + AgentState::Available, + false + )] #[case::cluster_missing_without_agent( ClusterState::MissingOrDeleting, AgentState::UnavailableSince(None), true )] + #[case::cluster_missing_within_the_grace_period( + ClusterState::MissingOrDeleting, + AgentState::UnavailableSince(Some(seconds_ago(10))), + true + )] #[case::management_access_disabled( ClusterState::ManagementAccessDisabled, AgentState::UnavailableSince(None), @@ -411,7 +425,7 @@ mod tests { AgentState::UnavailableSince(None), false )] - fn remove_the_finalizer_only_if_the_cluster_is_missing_or_deleting( + fn remove_the_finalizer_only_if_the_cluster_is_missing_or_deleting_without_agent( #[case] cluster_state: ClusterState, #[case] agent_state: AgentState, #[case] removes_finalizer: bool, @@ -428,10 +442,29 @@ mod tests { } #[rstest] - #[case::cluster_missing(ClusterState::MissingOrDeleting, AgentState::Available, true)] - #[case::management_access_disabled( + #[case::cluster_missing_with_agent( + ClusterState::MissingOrDeleting, + AgentState::Available, + false + )] + #[case::cluster_missing_without_agent( + ClusterState::MissingOrDeleting, + AgentState::UnavailableSince(None), + true + )] + #[case::cluster_missing_within_the_grace_period( + ClusterState::MissingOrDeleting, + AgentState::UnavailableSince(Some(seconds_ago(10))), + true + )] + #[case::management_access_disabled_with_agent( ClusterState::ManagementAccessDisabled, AgentState::Available, + false + )] + #[case::management_access_disabled_without_agent( + ClusterState::ManagementAccessDisabled, + AgentState::UnavailableSince(None), true )] #[case::agent_available(ClusterState::ManagementAccessEnabled, AgentState::Available, false)] From 4ae14d968f7382fbf5e057d941962120986b93d0 Mon Sep 17 00:00:00 2001 From: Benedikt Labrenz Date: Thu, 5 Nov 2026 13:05:52 +0100 Subject: [PATCH 86/93] Report the agent not ready while it cannot list its KafkaTopics --- .../pages/usage-guide/management-access.adoc | 2 + rust/agent-binary/src/health.rs | 45 +++++++++++++++++-- rust/agent-binary/src/main.rs | 18 +++++++- 3 files changed, 59 insertions(+), 6 deletions(-) diff --git a/docs/modules/kafka/pages/usage-guide/management-access.adoc b/docs/modules/kafka/pages/usage-guide/management-access.adoc index 5373e7976..c8578abb2 100644 --- a/docs/modules/kafka/pages/usage-guide/management-access.adoc +++ b/docs/modules/kafka/pages/usage-guide/management-access.adoc @@ -100,6 +100,8 @@ The operator creates the following resources for the agent, named after the Kafk * A ServiceAccount `-agent`. * A RoleBinding `-agent`, which binds the ClusterRole `kafka-agent-clusterrole` deployed by the Helm chart. +The agent reports not ready while it cannot list the KafkaTopics of its cluster, and the operator then reports `AgentUnavailable` after a minute. + The Deployment can be adjusted with xref:concepts:overrides.adoc#object-overrides[object overrides], for example to change the resources of the agent: [source,yaml] diff --git a/rust/agent-binary/src/health.rs b/rust/agent-binary/src/health.rs index 13a79ef6f..baf862881 100644 --- a/rust/agent-binary/src/health.rs +++ b/rust/agent-binary/src/health.rs @@ -1,9 +1,11 @@ -//! The health endpoints of the agent, which the probes of its Deployment call. +//! The health endpoints of the agent, which the probes of its Deployment call, and the checks +//! behind them. use std::{ future::Future, net::{IpAddr, Ipv4Addr, SocketAddr}, sync::Arc, + time::Duration, }; use axum::{ @@ -14,13 +16,21 @@ use axum::{ routing::get, }; use snafu::{ResultExt, Snafu}; -use stackable_kafka_shared::agent::{HEALTH_PORT, LIVENESS_PATH, READINESS_PATH}; -use stackable_operator::webhook::health::HealthCheckRegistry; +use stackable_kafka_shared::{ + agent::{HEALTH_PORT, LIVENESS_PATH, READINESS_PATH}, + crd::topic::v1alpha1::KafkaTopic, +}; +use stackable_operator::{ + kube::{Api, api::ListParams}, + webhook::health::{HealthCheck, HealthCheckRegistry}, +}; use tokio::net::TcpListener; -use tracing::info; +use tracing::{info, warn}; const SOCKET_ADDRESS: SocketAddr = SocketAddr::new(IpAddr::V4(Ipv4Addr::UNSPECIFIED), HEALTH_PORT); +const API_ACCESS_INTERVAL: Duration = Duration::from_secs(30); + #[derive(Debug, Snafu)] pub enum Error { #[snafu(display("failed to listen on {address} for the health endpoints"))] @@ -56,3 +66,30 @@ pub async fn serve( async fn ready(State(readiness): State>) -> Response { (&*readiness).into_response() } + +/// Marks `check` by listing the KafkaTopics selected by `params` every [`API_ACCESS_INTERVAL`], +/// because a failing watch of the controller is not visible otherwise. +pub async fn check_k8s_api_access( + check: HealthCheck, + api: Api, + params: ListParams, + shutdown: impl Future, +) { + let mut interval = tokio::time::interval(API_ACCESS_INTERVAL); + tokio::pin!(shutdown); + loop { + tokio::select! { + () = &mut shutdown => break, + _ = interval.tick() => match api.list_metadata(¶ms).await { + Ok(_) => check.mark_passed(), + Err(error) => { + warn!( + error = &error as &dyn std::error::Error, + "Cannot list the KafkaTopics, reporting not ready" + ); + check.mark_not_passed(); + } + }, + } + } +} diff --git a/rust/agent-binary/src/main.rs b/rust/agent-binary/src/main.rs index da5a7c7db..691558639 100644 --- a/rust/agent-binary/src/main.rs +++ b/rust/agent-binary/src/main.rs @@ -18,6 +18,7 @@ use stackable_operator::{ cli::{Command, CommonOptions}, client, kube::{ + api::ListParams, core::DeserializeGuard, runtime::{ Controller, @@ -116,6 +117,7 @@ async fn main() -> anyhow::Result<()> { let mut readiness = HealthCheckRegistry::new(); let kafka_topics_synced = readiness.register("kafka-topics-synced"); + let kafka_topics_access = readiness.register("kafka-topics-access"); let readiness = Arc::new(readiness); // Watches for the SIGTERM signal and gracefully shuts down the controller. @@ -132,10 +134,18 @@ async fn main() -> anyhow::Result<()> { }, )); + // Works because the CRD declares the field as a selectableField. + let cluster_selector = format!("spec.clusterRef.name={cluster_name}"); + let check_api_access = health::check_k8s_api_access( + kafka_topics_access, + client.get_api::(cluster_namespace.as_ref()), + ListParams::default().fields(&cluster_selector).limit(1), + sigterm_watcher.handle(), + ); + let topic_controller = Controller::new( client.get_api::>(cluster_namespace.as_ref()), - // Works because the CRD declares the field as a selectableField. - watcher::Config::default().fields(&format!("spec.clusterRef.name={cluster_name}")), + watcher::Config::default().fields(&cluster_selector), ) // Each reconcile then acts on the current object; Kafka requests are serialized anyway. .with_config(ControllerConfig::default().concurrency(1)); @@ -180,6 +190,10 @@ async fn main() -> anyhow::Result<()> { anyhow::Ok(()) }, mark_ready, + async { + check_api_access.await; + anyhow::Ok(()) + }, async { Ok(health::serve(readiness, sigterm_watcher.handle()).await?) }, )?; } From d6513afbd46d8a75d36a80f7b1936da350adba0d Mon Sep 17 00:00:00 2001 From: Benedikt Labrenz Date: Thu, 5 Nov 2026 13:51:47 +0100 Subject: [PATCH 87/93] Reconcile at most four KafkaTopics at a time in the operator --- rust/operator-binary/src/main.rs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/rust/operator-binary/src/main.rs b/rust/operator-binary/src/main.rs index 7df6ab26e..2dfe0b24a 100644 --- a/rust/operator-binary/src/main.rs +++ b/rust/operator-binary/src/main.rs @@ -27,6 +27,7 @@ use stackable_operator::{ core::DeserializeGuard, runtime::{ Controller, + controller::Config as ControllerConfig, events::{Recorder, Reporter}, reflector::ObjectRef, watcher, @@ -254,7 +255,9 @@ async fn main() -> anyhow::Result<()> { let topic_controller = Controller::new( watch_namespace.get_api::>(&client), watcher::Config::default(), - ); + ) + // Bounds the memory of the startup burst, which reconciles every KafkaTopic at once. + .with_config(ControllerConfig::default().concurrency(4)); let kafka_topic_store_for_clusters = topic_controller.store(); let kafka_topic_store_for_agents = topic_controller.store(); let topic_controller = topic_controller From 08facc70767efab0f8513bd1db1528186b2ae938 Mon Sep 17 00:00:00 2001 From: Benedikt Labrenz Date: Thu, 5 Nov 2026 14:09:35 +0100 Subject: [PATCH 88/93] Limit configOverrides to 64 entries with values of up to 1024 characters --- extra/crds.yaml | 5 ++++- rust/shared/src/crd/topic.rs | 12 +++++++++++- 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/extra/crds.yaml b/extra/crds.yaml index b598f1a02..6df9d422a 100644 --- a/extra/crds.yaml +++ b/extra/crds.yaml @@ -2288,11 +2288,14 @@ spec: rule: self == oldSelf configOverrides: additionalProperties: + maxLength: 1024 type: string default: {} description: |- Topic configs like `retention.ms`. All other topic-level configs are reset to the broker - defaults, except the replication throttles set by partition reassignments. + defaults, except the replication throttles set by partition reassignments. At most 64 + configs with values of up to 1024 characters. + maxProperties: 64 type: object deletionPolicy: default: Retain diff --git a/rust/shared/src/crd/topic.rs b/rust/shared/src/crd/topic.rs index 72c3b124a..020adb3b4 100644 --- a/rust/shared/src/crd/topic.rs +++ b/rust/shared/src/crd/topic.rs @@ -82,8 +82,13 @@ pub mod versioned { pub replication_factor: Option, /// Topic configs like `retention.ms`. All other topic-level configs are reset to the broker - /// defaults, except the replication throttles set by partition reassignments. + /// defaults, except the replication throttles set by partition reassignments. At most 64 + /// configs with values of up to 1024 characters. #[serde(default)] + #[schemars(extend( + "maxProperties" = 64, + "additionalProperties" = { "type": "string", "maxLength": 1024 } + ))] pub config_overrides: BTreeMap, // no doc - docs in DeletionPolicy enum. @@ -286,5 +291,10 @@ mod tests { Some(1.0), properties["replicationFactor"]["minimum"].as_f64() ); + assert_eq!(64, properties["configOverrides"]["maxProperties"]); + assert_eq!( + serde_json::json!({ "type": "string", "maxLength": 1024 }), + properties["configOverrides"]["additionalProperties"] + ); } } From 616e4e6d12b898b349b967a39cebc587844ae80d Mon Sep 17 00:00:00 2001 From: Benedikt Labrenz Date: Thu, 5 Nov 2026 14:19:39 +0100 Subject: [PATCH 89/93] Document that a topic is retained when its owner and successor are both deleted --- docs/modules/kafka/pages/usage-guide/topics.adoc | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/modules/kafka/pages/usage-guide/topics.adoc b/docs/modules/kafka/pages/usage-guide/topics.adoc index c5501fd80..9e01504f8 100644 --- a/docs/modules/kafka/pages/usage-guide/topics.adoc +++ b/docs/modules/kafka/pages/usage-guide/topics.adoc @@ -123,6 +123,7 @@ While paused, the agent does not change the topic in Kafka, but still updates th With the default `deletionPolicy: Retain`, deleting a KafkaTopic leaves the topic in Kafka. With `deletionPolicy: Delete`, the agent deletes the topic in Kafka when the KafkaTopic is deleted, unless another KafkaTopic declares the same topic. +If that KafkaTopic is deleted as well before it takes over the topic, the topic stays in Kafka. The KafkaTopic gets a finalizer once the agent manages the topic, so the deletion waits until the topic is deleted. If the reconciliation of the KafkaTopic is paused, the deletion waits until it is resumed. If the brokers cannot be reached, the KafkaTopic reports `ConnectionFailed` and the agent retries the deletion. From 153a4f1240b9f1b66074cfef02bcdd8c16915fa8 Mon Sep 17 00:00:00 2001 From: Benedikt Labrenz Date: Thu, 5 Nov 2026 14:21:12 +0100 Subject: [PATCH 90/93] Name the length limit when metadata.name is not a valid topic name --- rust/agent-binary/src/topic/validate.rs | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/rust/agent-binary/src/topic/validate.rs b/rust/agent-binary/src/topic/validate.rs index 807a297f9..fd01e4b0f 100644 --- a/rust/agent-binary/src/topic/validate.rs +++ b/rust/agent-binary/src/topic/validate.rs @@ -14,8 +14,9 @@ pub(crate) fn validate( .topic_name() .map_err(|error| TopicSyncOutcome::InvalidSpec { message: format!( - "metadata.name is not a valid topic name, recreate the KafkaTopic with topicName \ - set: {error}" + "metadata.name is not a valid topic name of at most {} characters, recreate the \ + KafkaTopic with topicName set: {error}", + TopicName::MAX_LENGTH ), })?; From d4046de17e05ce1e89d5fc9335baa3771d3a0db7 Mon Sep 17 00:00:00 2001 From: Benedikt Labrenz Date: Thu, 5 Nov 2026 16:09:29 +0100 Subject: [PATCH 91/93] Document the topic count the default resources are sized for --- docs/modules/kafka/pages/usage-guide/management-access.adoc | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docs/modules/kafka/pages/usage-guide/management-access.adoc b/docs/modules/kafka/pages/usage-guide/management-access.adoc index c8578abb2..f94945a1a 100644 --- a/docs/modules/kafka/pages/usage-guide/management-access.adoc +++ b/docs/modules/kafka/pages/usage-guide/management-access.adoc @@ -122,6 +122,9 @@ spec: memory: 256Mi ---- +The default resources of the agent are sufficient for a few thousand topics. +The operator keeps all KafkaTopics of all clusters it watches in memory and reconciles them whenever it starts, so with many thousands of topics, increase its CPU and memory in the Helm values of the operator. + == Disabling management access Setting `enabled` to `false` or removing `managementAccess` removes the agent. From f8fe3c897456c48b5e1af185a080ceb4d696a514 Mon Sep 17 00:00:00 2001 From: Benedikt Labrenz Date: Thu, 5 Nov 2026 16:19:28 +0100 Subject: [PATCH 92/93] Start the connect cooldown only after a slow failure --- rust/kafka-admin/src/client.rs | 34 ++++++++++++++++++++++++++++++---- 1 file changed, 30 insertions(+), 4 deletions(-) diff --git a/rust/kafka-admin/src/client.rs b/rust/kafka-admin/src/client.rs index f511a12e4..8f44a19d7 100644 --- a/rust/kafka-admin/src/client.rs +++ b/rust/kafka-admin/src/client.rs @@ -56,6 +56,9 @@ const CONNECTION_MAX_IDLE: Duration = Duration::from_secs(5 * 60); /// Connecting is skipped for this long after all bootstrap servers failed, so an outage fails fast. const CONNECT_COOLDOWN: Duration = Duration::from_secs(30); +/// Failures faster than this, like a refused connection, do not start the cooldown. +const CONNECT_COOLDOWN_THRESHOLD: Duration = Duration::from_secs(1); + /// Lets the broker apply `num.partitions` or `default.replication.factor` when creating a topic, /// see `NumPartitions` and `ReplicationFactor` in /// `clients/src/main/resources/common/message/CreateTopicsRequest.json`. @@ -340,6 +343,7 @@ impl KafkaAdminClient { let tls_config = tls::client_config(&self.security) .context(TlsSnafu)? .map(Arc::new); + let started = Instant::now(); let mut last_error = Error::NoBootstrapServers; for server in &self.bootstrap_servers { match Connection::connect(server, tls_config.clone(), &self.client_id).await { @@ -355,6 +359,9 @@ impl KafkaAdminClient { } } } + if started.elapsed() < CONNECT_COOLDOWN_THRESHOLD { + return Err(last_error); + } let failure = Arc::new(last_error); self.connect_failure = Some((Instant::now(), Arc::clone(&failure))); Err(Error::ConnectCooldown { source: failure }) @@ -654,15 +661,30 @@ mod tests { } #[tokio::test] - async fn skip_connecting_during_the_cooldown() { - let (unreachable, unreachable_port) = unreachable_socket(); + async fn retry_at_once_after_a_fast_failure() { + let (_unreachable, unreachable_port) = unreachable_socket(); let mut client = kafka_admin_client(&[unreachable_port]); + + for _ in 0..2 { + let error = client + .delete_topic(TOPIC) + .await + .expect_err("the connection should be refused"); + assert!(matches!(error, Error::Connect { .. })); + } + } + + #[tokio::test] + async fn skip_connecting_during_the_cooldown() { + // Accepts connections but never answers, until the test takes them over + let (listener, port) = listen().await; + let mut client = kafka_admin_client(&[port]); tokio::time::pause(); let failure = client .delete_topic(TOPIC) .await - .expect_err("the connection should fail"); + .expect_err("the connection should time out"); assert!(matches!( failure, Error::ConnectCooldown { ref source } if matches!(**source, Error::Connect { .. }) @@ -680,8 +702,12 @@ mod tests { tokio::time::advance(CONNECT_COOLDOWN).await; tokio::time::resume(); - let listener = unreachable.listen(1).expect("the socket should listen"); let broker_side = async { + let (timed_out, _) = listener + .accept() + .await + .expect("the timed out connection should be accepted"); + drop(timed_out); let mut broker = accept(&listener, APIS).await; answer_delete_topics(&mut broker).await; }; From e60b478b733df8e068326faee27d7a7622547e18 Mon Sep 17 00:00:00 2001 From: Benedikt Labrenz Date: Fri, 6 Nov 2026 11:12:59 +0100 Subject: [PATCH 93/93] update serde-saphyr to 2.1.1. 2.1.0 was yanked --- Cargo.lock | 4 ++-- Cargo.nix | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index dcbd173f2..016542354 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2860,9 +2860,9 @@ dependencies = [ [[package]] name = "serde-saphyr" -version = "2.1.0" +version = "2.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3f4dc2714fc5632417a6e979602d7044bad8f2880c63fc7217087180acec31b6" +checksum = "d43fb82d550b07e6f929a58d7d0935182602f48ddead875f5ee313c41c8f07fe" dependencies = [ "annotate-snippets", "base64 0.23.1", diff --git a/Cargo.nix b/Cargo.nix index 87fad5d3f..862d10642 100644 --- a/Cargo.nix +++ b/Cargo.nix @@ -9807,10 +9807,10 @@ rec { }; "serde-saphyr" = rec { crateName = "serde-saphyr"; - version = "2.1.0"; + version = "2.1.1"; edition = "2024"; crateBin = []; - sha256 = "1diixjn80w882xrgqqqci3rdifj4f0nn0yg9lqbj8qy59xqw4k9z"; + sha256 = "1zh7iwfc84z3brgqgbfyips049hq6l4pv3d557wyc1qbalnvhgyl"; libName = "serde_saphyr"; license = "MIT OR Apache-2.0"; dependencies = [