diff --git a/rust/operator-binary/src/controller.rs b/rust/operator-binary/src/controller.rs index 9e7b64fb..f9db48a2 100644 --- a/rust/operator-binary/src/controller.rs +++ b/rust/operator-binary/src/controller.rs @@ -18,7 +18,12 @@ use stackable_operator::{ rbac::build_rbac_resources, resources::{NoRuntimeLimits, Resources}, }, - k8s_openapi::api::core::v1::Secret, + crd::listener, + k8s_openapi::api::{ + apps::v1::{Deployment, StatefulSet}, + core::v1::{ConfigMap, Secret, Service}, + policy::v1::PodDisruptionBudget, + }, kube::{ Resource, ResourceExt, api::ObjectMeta, @@ -54,14 +59,6 @@ use tracing::instrument; use crate::{ OPERATOR_NAME, - controller::build::resource::{ - config_map::build_rolegroup_config_map, - deployment::build_rolegroup_deployment, - listener::build_group_listener, - pdb::build_pdb, - service::{build_rolegroup_headless_service, build_rolegroup_metrics_service}, - statefulset::build_node_rolegroup_statefulset, - }, crd::{ APP_NAME, INTERNAL_SECRET_SECRET_KEY, SupersetRole, authentication::SupersetClientAuthenticationDetailsResolved, @@ -86,6 +83,19 @@ pub struct Ctx { pub operator_environment: OperatorEnvironmentOptions, } +/// Every Kubernetes resource produced by the build step. +/// +/// The `Node` role is provisioned via a `StatefulSet` (it serves the Superset web UI), while the +/// `Worker`/`Beat` Celery roles are provisioned via `Deployment`s; the build step collects both. +pub struct KubernetesResources { + pub stateful_sets: Vec, + pub deployments: Vec, + pub services: Vec, + pub listeners: Vec, + pub config_maps: Vec, + pub pod_disruption_budgets: Vec, +} + /// Per-role configuration extracted during validation. #[derive(Clone, Debug)] pub struct ValidatedRoleConfig { @@ -386,28 +396,12 @@ pub enum Error { source: stackable_operator::cluster_resources::Error, }, - #[snafu(display("failed to apply Service for role group {role_group_name}"))] - ApplyRoleGroupService { - source: stackable_operator::cluster_resources::Error, - role_group_name: RoleGroupName, - }, - - #[snafu(display("failed to apply ConfigMap for role group {role_group_name}"))] - ApplyRoleGroupConfig { - source: stackable_operator::cluster_resources::Error, - role_group_name: RoleGroupName, - }, + #[snafu(display("failed to build the Kubernetes resources"))] + BuildResources { source: build::Error }, - #[snafu(display("failed to apply StatefulSet for role group {role_group_name}"))] - ApplyRoleGroupStatefulSet { + #[snafu(display("failed to apply Kubernetes resource"))] + ApplyResource { source: stackable_operator::cluster_resources::Error, - role_group_name: RoleGroupName, - }, - - #[snafu(display("failed to apply Deployment for role group {role_group_name}"))] - ApplyRoleGroupDeployment { - source: stackable_operator::cluster_resources::Error, - role_group_name: RoleGroupName, }, #[snafu(display("failed to update status"))] @@ -430,11 +424,6 @@ pub enum Error { source: stackable_operator::commons::rbac::Error, }, - #[snafu(display("failed to apply PodDisruptionBudget"))] - ApplyPdb { - source: stackable_operator::cluster_resources::Error, - }, - #[snafu(display("failed to get required Labels"))] GetRequiredLabels { source: @@ -446,26 +435,6 @@ pub enum Error { source: error_boundary::InvalidObject, }, - #[snafu(display("failed to apply group listener"))] - ApplyGroupListener { - source: stackable_operator::cluster_resources::Error, - }, - - #[snafu(display("failed to build statefulset"))] - BuildStatefulSet { - source: crate::controller::build::resource::statefulset::Error, - }, - - #[snafu(display("failed to build deployment"))] - BuildDeployment { - source: crate::controller::build::resource::deployment::Error, - }, - - #[snafu(display("failed to build configmap"))] - BuildConfigMap { - source: crate::controller::build::resource::config_map::Error, - }, - #[snafu(display("failed to create SECRET_KEY secret"))] CreateSecretKeySecret { source: random_secret_creation::Error, @@ -567,129 +536,53 @@ pub async fn reconcile_superset( .await .context(CreateSecretKeySecretSnafu)?; + let resources = build::build(&validated, &rbac_sa.name_any()).context(BuildResourcesSnafu)?; + let mut statefulset_cond_builder = StatefulSetConditionBuilder::default(); let mut deployment_cond_builder = DeploymentConditionBuilder::default(); - for (superset_role, rolegroup_configs) in &validated.role_groups { - for (rolegroup_name, validated_rolegroup) in rolegroup_configs { - let config = &validated_rolegroup.config; - - let rg_configmap = build_rolegroup_config_map( - &validated, - superset_role, - rolegroup_name, - config, - &validated_rolegroup.config_overrides, - ) - .context(BuildConfigMapSnafu)?; - - // Every role exposes metrics via the statsd-exporter sidecar, so each rolegroup gets a - // metrics Service. The headless Service is built per role below: only the `Node` role's - // StatefulSet references one (as its `serviceName`); the `Worker`/`Beat` Deployments have - // no `serviceName` and do not serve the HTTP port, so they get no headless Service. - let rg_metrics_service = - build_rolegroup_metrics_service(&validated, superset_role, rolegroup_name); - + // The StatefulSets/Deployments are applied last, so every ConfigMap and Secret they mount + // already exists — otherwise a changed mount would restart the Pods. + // See https://github.com/stackabletech/commons-operator/issues/111 for details. + for service in resources.services { + cluster_resources + .add(client, service) + .await + .context(ApplyResourceSnafu)?; + } + for config_map in resources.config_maps { + cluster_resources + .add(client, config_map) + .await + .context(ApplyResourceSnafu)?; + } + for listener in resources.listeners { + cluster_resources + .add(client, listener) + .await + .context(ApplyResourceSnafu)?; + } + for pdb in resources.pod_disruption_budgets { + cluster_resources + .add(client, pdb) + .await + .context(ApplyResourceSnafu)?; + } + for statefulset in resources.stateful_sets { + statefulset_cond_builder.add( cluster_resources - .add(client, rg_metrics_service) + .add(client, statefulset) .await - .with_context(|_| ApplyRoleGroupServiceSnafu { - role_group_name: rolegroup_name.clone(), - })?; - + .context(ApplyResourceSnafu)?, + ); + } + for deployment in resources.deployments { + deployment_cond_builder.add( cluster_resources - .add(client, rg_configmap) + .add(client, deployment) .await - .with_context(|_| ApplyRoleGroupConfigSnafu { - role_group_name: rolegroup_name.clone(), - })?; - - match superset_role { - SupersetRole::Node => { - let rg_headless_service = - build_rolegroup_headless_service(&validated, superset_role, rolegroup_name); - - cluster_resources - .add(client, rg_headless_service) - .await - .with_context(|_| ApplyRoleGroupServiceSnafu { - role_group_name: rolegroup_name.clone(), - })?; - - let rg_statefulset = build_node_rolegroup_statefulset( - &validated, - superset_role, - rolegroup_name, - validated_rolegroup, - &rbac_sa.name_any(), - ) - .context(BuildStatefulSetSnafu)?; - - // Note: The StatefulSet needs to be applied after all ConfigMaps and Secrets it mounts - // to prevent unnecessary Pod restarts. - // See https://github.com/stackabletech/commons-operator/issues/111 for details. - statefulset_cond_builder.add( - cluster_resources - .add(client, rg_statefulset) - .await - .with_context(|_| ApplyRoleGroupStatefulSetSnafu { - role_group_name: rolegroup_name.clone(), - })?, - ); - } - SupersetRole::Worker | SupersetRole::Beat => { - let rg_deployment = build_rolegroup_deployment( - &validated, - superset_role, - rolegroup_name, - validated_rolegroup, - &rbac_sa.name_any(), - ) - .context(BuildDeploymentSnafu)?; - - // Note: The Deployment needs to be applied after all ConfigMaps and Secrets it mounts - // to prevent unnecessary Pod restarts. - // See https://github.com/stackabletech/commons-operator/issues/111 for details. - deployment_cond_builder.add( - cluster_resources - .add(client, rg_deployment) - .await - .with_context(|_| ApplyRoleGroupDeploymentSnafu { - role_group_name: rolegroup_name.clone(), - })?, - ); - } - } - } - - // Role-level resources (group listener, PDB) are built once per role, after - // its role groups — not once per role group. - if let Some(role_config) = validated.role_configs.get(superset_role) { - if let (Some(listener_class), Some(listener_group_name)) = ( - &role_config.listener_class, - &role_config.group_listener_name, - ) { - let group_listener = build_group_listener( - &validated, - superset_role, - listener_class, - listener_group_name.to_string(), - ); - cluster_resources - .add(client, group_listener) - .await - .context(ApplyGroupListenerSnafu)?; - } - - if let Some(pdb) = &role_config.pdb - && let Some(pdb) = build_pdb(pdb, &validated, superset_role) - { - cluster_resources - .add(client, pdb) - .await - .context(ApplyPdbSnafu)?; - } - } + .context(ApplyResourceSnafu)?, + ); } cluster_resources diff --git a/rust/operator-binary/src/controller/build/mod.rs b/rust/operator-binary/src/controller/build/mod.rs index a40d49e3..58840c79 100644 --- a/rust/operator-binary/src/controller/build/mod.rs +++ b/rust/operator-binary/src/controller/build/mod.rs @@ -2,8 +2,24 @@ use std::str::FromStr; +use snafu::{ResultExt, Snafu}; use stackable_operator::v2::types::operator::{ProductVersion, RoleGroupName}; +use crate::{ + controller::{ + KubernetesResources, ValidatedCluster, + build::resource::{ + config_map::build_rolegroup_config_map, + deployment::build_rolegroup_deployment, + listener::build_group_listener, + pdb::build_pdb, + service::{build_rolegroup_headless_service, build_rolegroup_metrics_service}, + statefulset::build_node_rolegroup_statefulset, + }, + }, + crd::SupersetRole, +}; + pub mod command; pub mod properties; pub mod resource; @@ -15,3 +31,235 @@ stackable_operator::constant!(pub(crate) PLACEHOLDER_LISTENER_ROLE_GROUP: RoleGr // Product version used for the recommended labels of PVC templates, which cannot be modified after // deployment. A constant `none` keeps those labels stable across version upgrades. stackable_operator::constant!(pub(crate) UNVERSIONED_PRODUCT_VERSION: ProductVersion = "none"); + +#[derive(Snafu, Debug)] +pub enum Error { + #[snafu(display("failed to build ConfigMap for role group {role_group}"))] + ConfigMap { + source: resource::config_map::Error, + role_group: RoleGroupName, + }, + + #[snafu(display("failed to build StatefulSet for role group {role_group}"))] + StatefulSet { + source: resource::statefulset::Error, + role_group: RoleGroupName, + }, + + #[snafu(display("failed to build Deployment for role group {role_group}"))] + Deployment { + source: resource::deployment::Error, + role_group: RoleGroupName, + }, +} + +/// Builds every Kubernetes resource for the given validated cluster. +/// +/// `service_account_name` is the name of the RBAC `ServiceAccount` the role-group Pods run under. +/// The RBAC resources themselves are built and applied separately in the reconcile step. +pub fn build( + cluster: &ValidatedCluster, + service_account_name: &str, +) -> Result { + let mut stateful_sets = vec![]; + let mut deployments = vec![]; + let mut services = vec![]; + let mut listeners = vec![]; + let mut config_maps = vec![]; + let mut pod_disruption_budgets = vec![]; + + for (superset_role, role_group_configs) in &cluster.role_groups { + for (role_group_name, rolegroup_config) in role_group_configs { + let config = &rolegroup_config.config; + + config_maps.push( + build_rolegroup_config_map( + cluster, + superset_role, + role_group_name, + config, + &rolegroup_config.config_overrides, + ) + .context(ConfigMapSnafu { + role_group: role_group_name.clone(), + })?, + ); + + // Every role exposes metrics via the statsd-exporter sidecar, so each rolegroup gets a + // metrics Service. + services.push(build_rolegroup_metrics_service( + cluster, + superset_role, + role_group_name, + )); + + match superset_role { + SupersetRole::Node => { + // Only the `Node` role's StatefulSet references a headless Service (as its + // `serviceName`); the `Worker`/`Beat` Deployments have no `serviceName` and do + // not serve the HTTP port, so they get no headless Service. + services.push(build_rolegroup_headless_service( + cluster, + superset_role, + role_group_name, + )); + + stateful_sets.push( + build_node_rolegroup_statefulset( + cluster, + superset_role, + role_group_name, + rolegroup_config, + service_account_name, + ) + .context(StatefulSetSnafu { + role_group: role_group_name.clone(), + })?, + ); + } + SupersetRole::Worker | SupersetRole::Beat => { + deployments.push( + build_rolegroup_deployment( + cluster, + superset_role, + role_group_name, + rolegroup_config, + service_account_name, + ) + .context(DeploymentSnafu { + role_group: role_group_name.clone(), + })?, + ); + } + } + } + + // Role-level resources (group listener, PDB) are built once per role, after its role + // groups — not once per role group. + if let Some(role_config) = cluster.role_configs.get(superset_role) { + if let (Some(listener_class), Some(listener_group_name)) = ( + &role_config.listener_class, + &role_config.group_listener_name, + ) { + listeners.push(build_group_listener( + cluster, + superset_role, + listener_class, + listener_group_name.to_string(), + )); + } + + if let Some(pdb_config) = &role_config.pdb { + pod_disruption_budgets.extend(build_pdb(pdb_config, cluster, superset_role)); + } + } + } + + Ok(KubernetesResources { + stateful_sets, + deployments, + services, + listeners, + config_maps, + pod_disruption_budgets, + }) +} + +#[cfg(test)] +mod tests { + use stackable_operator::{kube::Resource, utils::yaml_from_str_singleton_map}; + + use super::build; + use crate::{ + controller::{ + ValidatedCluster, test_support::default_dereferenced, validate::validate_cluster, + }, + crd::v1alpha1, + }; + + /// A validated cluster with a `node`, `worker` and `beat` role (one `default` role group each). + fn validated_cluster() -> ValidatedCluster { + let input = r#" + apiVersion: superset.stackable.tech/v1alpha1 + kind: SupersetCluster + metadata: + name: simple-superset + namespace: default + spec: + image: + productVersion: 4.1.4 + clusterConfig: + credentialsSecret: superset-admin-credentials + metadataDatabase: + postgresql: + host: superset-postgresql + database: superset + credentialsSecretName: superset-postgresql-credentials + nodes: + roleGroups: + default: + replicas: 1 + workers: + roleGroups: + default: + replicas: 1 + beat: + roleGroups: + default: + replicas: 1 + "#; + let superset: v1alpha1::SupersetCluster = + yaml_from_str_singleton_map(input).expect("illegal test input"); + validate_cluster(&superset, default_dereferenced(), "test-repo").expect("validated") + } + + fn sorted_names(resources: &[impl Resource]) -> Vec<&str> { + let mut names: Vec<&str> = resources + .iter() + .filter_map(|resource| resource.meta().name.as_deref()) + .collect(); + names.sort(); + names + } + + /// The build step turns a validated cluster into the full set of Kubernetes resources: the + /// `node` role becomes a StatefulSet, the `worker`/`beat` Celery roles become Deployments, and + /// each role group additionally gets a ConfigMap and a metrics Service (plus a headless Service + /// for the `node` role). Role-level Listeners and PDBs are emitted once per role. + #[test] + fn build_produces_expected_resource_names() { + let cluster = validated_cluster(); + let resources = build(&cluster, "simple-superset-serviceaccount").expect("build succeeds"); + + assert_eq!( + sorted_names(&resources.stateful_sets), + ["simple-superset-node-default"] + ); + assert_eq!( + sorted_names(&resources.deployments), + [ + "simple-superset-beat-default", + "simple-superset-worker-default" + ] + ); + assert_eq!( + sorted_names(&resources.config_maps), + [ + "simple-superset-beat-default", + "simple-superset-node-default", + "simple-superset-worker-default", + ] + ); + // Only the `node` role serves the web UI and gets a group Listener. + assert_eq!(sorted_names(&resources.listeners), ["simple-superset-node"]); + // A default PDB per role. + assert_eq!( + sorted_names(&resources.pod_disruption_budgets), + [ + "simple-superset-beat", + "simple-superset-node", + "simple-superset-worker" + ] + ); + } +}