From ca25b787373f988bbeb2a7c40655f548df1cfb46 Mon Sep 17 00:00:00 2001 From: Linying Assad Date: Wed, 23 Sep 2026 21:06:34 +0800 Subject: [PATCH] feat: reconcile TDE in Doris Operator --- .../v1/disaggregatedcluster_webhook.go | 22 +- api/disaggregated/v1/types.go | 7 + api/disaggregated/v1/zz_generated.deepcopy.go | 8 + api/doris/v1/doriscluster_webhook.go | 17 + api/doris/v1/types.go | 7 + api/doris/v1/zz_generated.deepcopy.go | 8 + api/tde/types.go | 324 +++++++ api/tde/validation.go | 303 ++++++ api/tde/validation_test.go | 205 ++++ config/crd/bases/crds.yaml | 860 +++++++++++++++++ ....doris.com_dorisdisaggregatedclusters.yaml | 430 +++++++++ .../bases/doris.apache.com_dorisclusters.yaml | 430 +++++++++ .../doris.selectdb.com_dorisclusters.yaml | 430 +++++++++ config/operator/disaggregated-operator.yaml | 4 +- config/operator/operator.yaml | 4 +- config/rbac/role.yaml | 3 + go.mod | 32 + go.sum | 167 ++++ ....doris.com_dorisdisaggregatedclusters.yaml | 430 +++++++++ .../crds/doris.apache.com_dorisclusters.yaml | 430 +++++++++ .../doris-operator/templates/clusterrole.yaml | 3 + pkg/controller/controllers_utils.go | 3 +- pkg/controller/controllers_utils_test.go | 36 + .../disaggregated_cluster_controller.go | 39 +- pkg/controller/doriscluster_controller.go | 37 +- .../disaggregated_fe/controller.go | 13 +- .../disaggregated_fe/statefulset.go | 2 + .../sub_controller/fe/controller.go | 13 +- pkg/controller/sub_controller/fe/pod.go | 2 + pkg/controller/tde_watch.go | 48 + pkg/tde/config.go | 248 +++++ pkg/tde/kms.go | 194 ++++ pkg/tde/lifecycle.go | 63 ++ pkg/tde/pod.go | 212 +++++ pkg/tde/reconciler.go | 874 ++++++++++++++++++ pkg/tde/tde_test.go | 624 +++++++++++++ 36 files changed, 6513 insertions(+), 19 deletions(-) create mode 100644 api/tde/types.go create mode 100644 api/tde/validation.go create mode 100644 api/tde/validation_test.go create mode 100644 pkg/controller/controllers_utils_test.go create mode 100644 pkg/controller/tde_watch.go create mode 100644 pkg/tde/config.go create mode 100644 pkg/tde/kms.go create mode 100644 pkg/tde/lifecycle.go create mode 100644 pkg/tde/pod.go create mode 100644 pkg/tde/reconciler.go create mode 100644 pkg/tde/tde_test.go diff --git a/api/disaggregated/v1/disaggregatedcluster_webhook.go b/api/disaggregated/v1/disaggregatedcluster_webhook.go index bf2098d8..8110d54c 100644 --- a/api/disaggregated/v1/disaggregatedcluster_webhook.go +++ b/api/disaggregated/v1/disaggregatedcluster_webhook.go @@ -20,6 +20,8 @@ package v1 import ( "context" "fmt" + tdev1 "github.com/apache/doris-operator/api/tde" + "reflect" "k8s.io/apimachinery/pkg/runtime" kerrors "k8s.io/apimachinery/pkg/util/errors" @@ -50,7 +52,7 @@ func (ddc *DorisDisaggregatedCluster) Default(ctx context.Context, obj runtime.O } // TODO(user): change verbs to "verbs=create;update;delete" if you want to enable deletion validation. -// +kubebuilder:unnamedwatches:path=/validate-disaggregated-doris-com-v1-dorisdisaggregatedcluster,mutating=false,failurePolicy=ignore,sideEffects=None,groups=disaggregated.cluster.doris.com,resources=dorisdisaggregatedclusters,verbs=create;update,versions=v1,name=vdorisdisaggregatedcluster.kb.io,admissionReviewVersions=v1 +// +kubebuilder:unnamedwatches:path=/validate-disaggregated-cluster-doris-com-v1-dorisdisaggregatedcluster,mutating=false,failurePolicy=fail,sideEffects=None,groups=disaggregated.cluster.doris.com,resources=dorisdisaggregatedclusters,verbs=create;update,versions=v1,name=vdorisdisaggregatedcluster.kb.io,admissionReviewVersions=v1 var _ webhook.CustomValidator = &DorisDisaggregatedCluster{} // ValidateCreate implements webhook.Validator so a unnamedwatches will be registered for the type @@ -64,6 +66,9 @@ func (ddc *DorisDisaggregatedCluster) ValidateCreate(ctx context.Context, obj ru if errs := cluster.validate(); len(errs) != 0 { return nil, kerrors.NewAggregate(errs) } + if errs := tdev1.ValidateCreate(cluster.Spec.TDE); len(errs) != 0 { + return nil, kerrors.NewAggregate(errs) + } return nil, nil } @@ -76,7 +81,20 @@ func (ddc *DorisDisaggregatedCluster) ValidateUpdate(ctx context.Context, oldObj } klog.Info("validate update", "name", cluster.Name) - if errs := cluster.validate(); len(errs) != 0 { + oldCluster, ok := oldObj.(*DorisDisaggregatedCluster) + if !ok { + return nil, fmt.Errorf("expected an old DorisDisaggregatedCluster but got %T", oldObj) + } + errs := cluster.validate() + errs = append(errs, tdev1.ValidateUpdate(oldCluster.Spec.TDE, cluster.Spec.TDE, oldCluster.Status.TDE)...) + if tdev1.BlocksFELifecycle(oldCluster.Status.TDE) && + !reflect.DeepEqual(oldCluster.Spec.FeSpec, cluster.Spec.FeSpec) { + errs = append(errs, fmt.Errorf("spec.feSpec cannot change while a TDE operation or configuration sync is pending")) + } + if !reflect.DeepEqual(oldCluster.Spec.TDE, cluster.Spec.TDE) && !reflect.DeepEqual(oldCluster.Spec.FeSpec, cluster.Spec.FeSpec) { + errs = append(errs, fmt.Errorf("spec.tde and spec.feSpec cannot change in the same update")) + } + if len(errs) != 0 { return nil, kerrors.NewAggregate(errs) } diff --git a/api/disaggregated/v1/types.go b/api/disaggregated/v1/types.go index 002968ff..52415d9b 100644 --- a/api/disaggregated/v1/types.go +++ b/api/disaggregated/v1/types.go @@ -18,11 +18,15 @@ package v1 import ( + tdev1 "github.com/apache/doris-operator/api/tde" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) type DorisDisaggregatedClusterSpec struct { + // TDE declares cluster-wide transparent data encryption management. + TDE *tdev1.TDEConfig `json:"tde,omitempty"` + //VaultConfigmap specify the configmap that have configuration of file object information. example S3. //configmap have to config, please reference the doc. //InstanceConfigMap string `json:"instanceConfigMap,omitempty"` @@ -344,6 +348,9 @@ type PortMap struct { } type DorisDisaggregatedClusterStatus struct { + // TDE reports the observed encryption configuration and operation state. + TDE *tdev1.TDEStatus `json:"tde,omitempty"` + //describe the metaservice status now. MetaServiceStatus MetaServiceStatus `json:"metaServiceStatus,omitempty"` diff --git a/api/disaggregated/v1/zz_generated.deepcopy.go b/api/disaggregated/v1/zz_generated.deepcopy.go index 02162b73..2da015c7 100644 --- a/api/disaggregated/v1/zz_generated.deepcopy.go +++ b/api/disaggregated/v1/zz_generated.deepcopy.go @@ -289,6 +289,10 @@ func (in *DorisDisaggregatedClusterList) DeepCopyObject() runtime.Object { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *DorisDisaggregatedClusterSpec) DeepCopyInto(out *DorisDisaggregatedClusterSpec) { *out = *in + if in.TDE != nil { + in, out := &in.TDE, &out.TDE + *out = (*in).DeepCopy() + } in.MetaService.DeepCopyInto(&out.MetaService) in.FeSpec.DeepCopyInto(&out.FeSpec) if in.ComputeGroups != nil { @@ -323,6 +327,10 @@ func (in *DorisDisaggregatedClusterSpec) DeepCopy() *DorisDisaggregatedClusterSp // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *DorisDisaggregatedClusterStatus) DeepCopyInto(out *DorisDisaggregatedClusterStatus) { *out = *in + if in.TDE != nil { + in, out := &in.TDE, &out.TDE + *out = (*in).DeepCopy() + } out.MetaServiceStatus = in.MetaServiceStatus out.FEStatus = in.FEStatus out.ClusterHealth = in.ClusterHealth diff --git a/api/doris/v1/doriscluster_webhook.go b/api/doris/v1/doriscluster_webhook.go index a2bf40f6..34d8e67d 100644 --- a/api/doris/v1/doriscluster_webhook.go +++ b/api/doris/v1/doriscluster_webhook.go @@ -36,9 +36,11 @@ package v1 import ( "context" "fmt" + tdev1 "github.com/apache/doris-operator/api/tde" "k8s.io/apimachinery/pkg/runtime" kerrors "k8s.io/apimachinery/pkg/util/errors" "k8s.io/klog/v2" + "reflect" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/webhook" "sigs.k8s.io/controller-runtime/pkg/webhook/admission" @@ -81,6 +83,9 @@ func (r *DorisCluster) ValidateCreate(ctx context.Context, obj runtime.Object) ( if errs := cluster.validateManagementUser(); len(errs) != 0 { return nil, kerrors.NewAggregate(errs) } + if errs := tdev1.ValidateCreate(cluster.Spec.TDE); len(errs) != 0 { + return nil, kerrors.NewAggregate(errs) + } return nil, nil } @@ -94,6 +99,18 @@ func (r *DorisCluster) ValidateUpdate(ctx context.Context, oldObj, newObj runtim klog.Info("validate update", "name", cluster.Name) var errors []error errors = append(errors, cluster.validateManagementUser()...) + oldCluster, ok := oldObj.(*DorisCluster) + if !ok { + return nil, fmt.Errorf("expected an old DorisCluster but got %T", oldObj) + } + errors = append(errors, tdev1.ValidateUpdate(oldCluster.Spec.TDE, cluster.Spec.TDE, oldCluster.Status.TDE)...) + if tdev1.BlocksFELifecycle(oldCluster.Status.TDE) && + !reflect.DeepEqual(oldCluster.Spec.FeSpec, cluster.Spec.FeSpec) { + errors = append(errors, fmt.Errorf("spec.feSpec cannot change while a TDE operation or configuration sync is pending")) + } + if !reflect.DeepEqual(oldCluster.Spec.TDE, cluster.Spec.TDE) && !reflect.DeepEqual(oldCluster.Spec.FeSpec, cluster.Spec.FeSpec) { + errors = append(errors, fmt.Errorf("spec.tde and spec.feSpec cannot change in the same update")) + } // fe FeSpec.Replicas must greater than or equal to FeSpec.ElectionNumber if cluster.Spec.FeSpec.Replicas != nil && *cluster.Spec.FeSpec.Replicas < cluster.GetElectionNumber() { errors = append(errors, fmt.Errorf("'FeSpec.Replicas' error: the number of FeSpec.Replicas should greater than or equal to FeSpec.ElectionNumber")) diff --git a/api/doris/v1/types.go b/api/doris/v1/types.go index f533e3fd..a316ea28 100644 --- a/api/doris/v1/types.go +++ b/api/doris/v1/types.go @@ -18,6 +18,7 @@ package v1 import ( + tdev1 "github.com/apache/doris-operator/api/tde" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) @@ -30,6 +31,9 @@ var ( // DorisClusterSpec defines the desired state of DorisCluster type DorisClusterSpec struct { + // TDE declares cluster-wide transparent data encryption management. + TDE *tdev1.TDEConfig `json:"tde,omitempty"` + //defines the fe cluster state that will be created by operator. FeSpec *FeSpec `json:"feSpec,omitempty"` @@ -444,6 +448,9 @@ type DorisServicePort struct { // DorisClusterStatus defines the observed state of DorisCluster type DorisClusterStatus struct { + // TDE reports the observed encryption configuration and operation state. + TDE *tdev1.TDEStatus `json:"tde,omitempty"` + //describe fe cluster status, record running, creating and failed pods. FEStatus *ComponentStatus `json:"feStatus,omitempty"` diff --git a/api/doris/v1/zz_generated.deepcopy.go b/api/doris/v1/zz_generated.deepcopy.go index fdcc5be8..b21da0da 100644 --- a/api/doris/v1/zz_generated.deepcopy.go +++ b/api/doris/v1/zz_generated.deepcopy.go @@ -417,6 +417,10 @@ func (in *DorisClusterList) DeepCopyObject() runtime.Object { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *DorisClusterSpec) DeepCopyInto(out *DorisClusterSpec) { *out = *in + if in.TDE != nil { + in, out := &in.TDE, &out.TDE + *out = (*in).DeepCopy() + } if in.FeSpec != nil { in, out := &in.FeSpec, &out.FeSpec *out = new(FeSpec) @@ -469,6 +473,10 @@ func (in *DorisClusterSpec) DeepCopy() *DorisClusterSpec { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *DorisClusterStatus) DeepCopyInto(out *DorisClusterStatus) { *out = *in + if in.TDE != nil { + in, out := &in.TDE, &out.TDE + *out = (*in).DeepCopy() + } if in.FEStatus != nil { in, out := &in.FEStatus, &out.FEStatus *out = new(ComponentStatus) diff --git a/api/tde/types.go b/api/tde/types.go new file mode 100644 index 00000000..22150d53 --- /dev/null +++ b/api/tde/types.go @@ -0,0 +1,324 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package tde + +import metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + +type ManagementPolicy string + +const ( + ManagementPolicyManaged ManagementPolicy = "Managed" + ManagementPolicyObserveOnly ManagementPolicy = "ObserveOnly" +) + +type ProviderType string + +const ( + ProviderLocal ProviderType = "Local" + ProviderAwsKms ProviderType = "AwsKms" + ProviderAliyunKms ProviderType = "AliyunKms" +) + +type KmsAuthType string + +const ( + KmsAuthEnvironmentSecret KmsAuthType = "EnvironmentSecret" + KmsAuthInstanceRole KmsAuthType = "InstanceRole" +) + +type TDEConfig struct { + // +kubebuilder:validation:Enum=Managed;ObserveOnly + ManagementPolicy ManagementPolicy `json:"managementPolicy"` + Provider ProviderSpec `json:"provider"` + // +kubebuilder:validation:Enum=PLAINTEXT;AES256;SM4 + DefaultAlgorithm string `json:"defaultAlgorithm"` + MasterKeyRotation *MasterKeyRotationSpec `json:"masterKeyRotation,omitempty"` + RootKeyRotation *RotationRequest `json:"rootKeyRotation,omitempty"` + CredentialRotation *RotationRequest `json:"credentialRotation,omitempty"` + Recovery *RecoveryRequest `json:"recovery,omitempty"` +} + +type ProviderSpec struct { + // +kubebuilder:validation:Enum=Local;AwsKms;AliyunKms + Type ProviderType `json:"type"` + Local *LocalProviderSpec `json:"local,omitempty"` + Kms *KmsProviderSpec `json:"kms,omitempty"` +} + +type LocalProviderSpec struct { + SecretKeyRef SecretKeyReference `json:"secretKeyRef"` +} + +type SecretKeyReference struct { + Name string `json:"name"` + Key string `json:"key"` +} + +type KmsProviderSpec struct { + KeyID string `json:"keyId"` + Endpoint string `json:"endpoint"` + Region string `json:"region"` + Auth KmsAuthSpec `json:"auth"` +} + +type KmsAuthSpec struct { + // +kubebuilder:validation:Enum=EnvironmentSecret;InstanceRole + Type KmsAuthType `json:"type"` + CredentialSecretRef *KmsCredentialSecretReference `json:"credentialSecretRef,omitempty"` +} + +type KmsCredentialSecretReference struct { + Name string `json:"name"` + AccessKeyKey string `json:"accessKeyKey"` + SecretKeyKey string `json:"secretKeyKey"` +} + +type MasterKeyRotationSpec struct { + // +kubebuilder:validation:Minimum=1 + RotateIntervalMs int64 `json:"rotateIntervalMs"` + // +kubebuilder:validation:Minimum=1 + CheckIntervalMs int64 `json:"checkIntervalMs"` +} + +type RotationRequest struct { + // +kubebuilder:validation:MinLength=1 + RequestID string `json:"requestId"` +} + +type RecoveryDecision string + +const ( + RecoveryConfirmApplied RecoveryDecision = "ConfirmApplied" + RecoveryConfirmNotApplied RecoveryDecision = "ConfirmNotApplied" +) + +type RecoveryRequest struct { + // +kubebuilder:validation:MinLength=1 + RequestID string `json:"requestId"` + // +kubebuilder:validation:MinLength=1 + RotationRequestID string `json:"rotationRequestId"` + // +kubebuilder:validation:Enum=ConfirmApplied;ConfirmNotApplied + Decision RecoveryDecision `json:"decision"` +} + +type TDEStatus struct { + ObservedGeneration int64 `json:"observedGeneration,omitempty"` + State TDEState `json:"state,omitempty"` + MetadataInitialized bool `json:"metadataInitialized,omitempty"` + Current *ProviderStatus `json:"current,omitempty"` + Conditions []metav1.Condition `json:"conditions,omitempty"` + Operation *OperationStatus `json:"operation,omitempty"` + UsedRequestIDs []string `json:"usedRequestIds,omitempty"` + ActiveConfigHash string `json:"activeConfigHash,omitempty"` + ActiveFESpecHash string `json:"activeFeSpecHash,omitempty"` + FEChecks []FECheck `json:"feChecks,omitempty"` +} + +type TDEState string + +const ( + StateUnconfigured TDEState = "Unconfigured" + StateConfiguredUninitialized TDEState = "ConfiguredUninitialized" + StateActive TDEState = "Active" + StateReconciling TDEState = "Reconciling" + StateConfigInconsistent TDEState = "ConfigInconsistent" + StateUnsupportedProvider TDEState = "UnsupportedProvider" + StateLocalKeyNotReady TDEState = "LocalKeyNotReady" + StateKmsNotReady TDEState = "KmsNotReady" + StateUnknown TDEState = "Unknown" +) + +type ProviderStatus struct { + Provider ProviderType `json:"provider"` + DefaultAlgorithm string `json:"defaultAlgorithm,omitempty"` + RootKeyRef *RootKeyRefStatus `json:"rootKeyRef,omitempty"` + Kms *KmsProviderStatus `json:"kms,omitempty"` +} + +type RootKeyRefStatus struct { + SecretName string `json:"secretName"` + Key string `json:"key"` + SecretUID string `json:"secretUid"` + ResolvedPath string `json:"resolvedPath"` +} + +type KmsProviderStatus struct { + KeyID string `json:"keyId"` + Endpoint string `json:"endpoint"` + Region string `json:"region"` + Auth KmsAuthStatus `json:"auth"` +} + +type KmsAuthStatus struct { + Type KmsAuthType `json:"type"` + CredentialSecretName string `json:"credentialSecretName,omitempty"` + CredentialSecretUID string `json:"credentialSecretUid,omitempty"` + AccessKeyKey string `json:"accessKeyKey,omitempty"` + SecretKeyKey string `json:"secretKeyKey,omitempty"` +} + +type OperationType string + +const ( + OperationEnableTDE OperationType = "EnableTDE" + OperationUpdateAlgorithm OperationType = "UpdateAlgorithm" + OperationRotateKmsCredential OperationType = "RotateKmsCredential" + OperationRotateRootKey OperationType = "RotateRootKey" +) + +type OperationStage string + +const ( + StagePreparingMaterial OperationStage = "PreparingMaterial" + StageRollingOutMaterial OperationStage = "RollingOutMaterial" + StageRotatingRootKey OperationStage = "RotatingRootKey" + StageWaitingForFEReplay OperationStage = "WaitingForFEReplay" + StageSyncingConfiguration OperationStage = "SyncingConfiguration" + StageCompleted OperationStage = "Completed" + StageFailed OperationStage = "Failed" +) + +type SQLState string + +const ( + SQLNotStarted SQLState = "NotStarted" + SQLSubmitting SQLState = "Submitting" + SQLOutcomeUnknown SQLState = "OutcomeUnknown" + SQLRejected SQLState = "Rejected" + SQLApplied SQLState = "Applied" + SQLNotApplied SQLState = "NotApplied" +) + +type RecoveryStatus struct { + RequestID string `json:"requestId"` + Decision RecoveryDecision `json:"decision"` + ResolvedAt metav1.Time `json:"resolvedAt"` +} + +type OperationStatus struct { + Type OperationType `json:"type"` + RequestID string `json:"requestId,omitempty"` + SpecGeneration int64 `json:"specGeneration"` + Stage OperationStage `json:"stage"` + SQLState SQLState `json:"sqlState,omitempty"` + Source *ProviderStatus `json:"source,omitempty"` + Target *ProviderStatus `json:"target,omitempty"` + Recovery *RecoveryStatus `json:"recovery,omitempty"` + CommitJournalID string `json:"commitJournalId,omitempty"` + StartedAt metav1.Time `json:"startedAt"` + LastTransitionTime metav1.Time `json:"lastTransitionTime"` +} + +type FECheck struct { + PodName string `json:"podName"` + PodReady bool `json:"podReady"` + ConfigConsistent bool `json:"configConsistent"` + MaterialReady bool `json:"materialReady"` +} + +func (in *TDEConfig) DeepCopyInto(out *TDEConfig) { + *out = *in + if in.Provider.Local != nil { + out.Provider.Local = new(LocalProviderSpec) + *out.Provider.Local = *in.Provider.Local + } + if in.Provider.Kms != nil { + out.Provider.Kms = new(KmsProviderSpec) + *out.Provider.Kms = *in.Provider.Kms + if in.Provider.Kms.Auth.CredentialSecretRef != nil { + out.Provider.Kms.Auth.CredentialSecretRef = new(KmsCredentialSecretReference) + *out.Provider.Kms.Auth.CredentialSecretRef = *in.Provider.Kms.Auth.CredentialSecretRef + } + } + if in.MasterKeyRotation != nil { + out.MasterKeyRotation = new(MasterKeyRotationSpec) + *out.MasterKeyRotation = *in.MasterKeyRotation + } + if in.RootKeyRotation != nil { + out.RootKeyRotation = new(RotationRequest) + *out.RootKeyRotation = *in.RootKeyRotation + } + if in.CredentialRotation != nil { + out.CredentialRotation = new(RotationRequest) + *out.CredentialRotation = *in.CredentialRotation + } + if in.Recovery != nil { + out.Recovery = new(RecoveryRequest) + *out.Recovery = *in.Recovery + } +} + +func (in *TDEConfig) DeepCopy() *TDEConfig { + if in == nil { + return nil + } + out := new(TDEConfig) + in.DeepCopyInto(out) + return out +} + +func (in *TDEStatus) DeepCopyInto(out *TDEStatus) { + *out = *in + if in.Current != nil { + out.Current = in.Current.DeepCopy() + } + if in.Conditions != nil { + out.Conditions = append([]metav1.Condition(nil), in.Conditions...) + } + if in.Operation != nil { + out.Operation = new(OperationStatus) + *out.Operation = *in.Operation + out.Operation.Source = in.Operation.Source.DeepCopy() + out.Operation.Target = in.Operation.Target.DeepCopy() + if in.Operation.Recovery != nil { + out.Operation.Recovery = new(RecoveryStatus) + *out.Operation.Recovery = *in.Operation.Recovery + } + } + if in.UsedRequestIDs != nil { + out.UsedRequestIDs = append([]string(nil), in.UsedRequestIDs...) + } + if in.FEChecks != nil { + out.FEChecks = append([]FECheck(nil), in.FEChecks...) + } +} + +func (in *TDEStatus) DeepCopy() *TDEStatus { + if in == nil { + return nil + } + out := new(TDEStatus) + in.DeepCopyInto(out) + return out +} + +func (in *ProviderStatus) DeepCopy() *ProviderStatus { + if in == nil { + return nil + } + out := new(ProviderStatus) + *out = *in + if in.RootKeyRef != nil { + out.RootKeyRef = new(RootKeyRefStatus) + *out.RootKeyRef = *in.RootKeyRef + } + if in.Kms != nil { + out.Kms = new(KmsProviderStatus) + *out.Kms = *in.Kms + } + return out +} diff --git a/api/tde/validation.go b/api/tde/validation.go new file mode 100644 index 00000000..71469f67 --- /dev/null +++ b/api/tde/validation.go @@ -0,0 +1,303 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 + +package tde + +import ( + "fmt" + "net/url" + "reflect" +) + +func Validate(config *TDEConfig) []error { + if config == nil { + return nil + } + var errs []error + if config.ManagementPolicy != ManagementPolicyManaged && config.ManagementPolicy != ManagementPolicyObserveOnly { + errs = append(errs, fmt.Errorf("spec.tde.managementPolicy must be Managed or ObserveOnly")) + } + if config.DefaultAlgorithm != "PLAINTEXT" && config.DefaultAlgorithm != "AES256" && config.DefaultAlgorithm != "SM4" { + errs = append(errs, fmt.Errorf("spec.tde.defaultAlgorithm must be PLAINTEXT, AES256, or SM4")) + } + if config.Recovery != nil { + if config.Recovery.RequestID == "" || config.Recovery.RotationRequestID == "" { + errs = append(errs, fmt.Errorf("spec.tde.recovery.requestId and rotationRequestId are required")) + } + if config.Recovery.Decision != RecoveryConfirmApplied && config.Recovery.Decision != RecoveryConfirmNotApplied { + errs = append(errs, fmt.Errorf("spec.tde.recovery.decision must be ConfirmApplied or ConfirmNotApplied")) + } + } + switch config.Provider.Type { + case ProviderLocal: + if config.Provider.Local == nil || config.Provider.Local.SecretKeyRef.Name == "" || config.Provider.Local.SecretKeyRef.Key == "" { + errs = append(errs, fmt.Errorf("spec.tde.provider.local.secretKeyRef.name and key are required for Local")) + } + if config.Provider.Kms != nil { + errs = append(errs, fmt.Errorf("spec.tde.provider.kms must be empty for Local")) + } + case ProviderAwsKms, ProviderAliyunKms: + kms := config.Provider.Kms + if kms == nil || kms.KeyID == "" || kms.Endpoint == "" || kms.Region == "" { + errs = append(errs, fmt.Errorf("spec.tde.provider.kms.keyId, endpoint, and region are required for %s", config.Provider.Type)) + } else if endpoint, err := url.Parse(kms.Endpoint); err != nil || endpoint.Scheme != "https" || endpoint.Host == "" { + errs = append(errs, fmt.Errorf("spec.tde.provider.kms.endpoint must be a valid HTTPS URL")) + } + if config.Provider.Local != nil { + errs = append(errs, fmt.Errorf("spec.tde.provider.local must be empty for %s", config.Provider.Type)) + } + if kms != nil { + switch kms.Auth.Type { + case KmsAuthEnvironmentSecret: + ref := kms.Auth.CredentialSecretRef + if ref == nil || ref.Name == "" || ref.AccessKeyKey == "" || ref.SecretKeyKey == "" { + errs = append(errs, fmt.Errorf("spec.tde.provider.kms.auth.credentialSecretRef fields are required for EnvironmentSecret")) + } + case KmsAuthInstanceRole: + if kms.Auth.CredentialSecretRef != nil { + errs = append(errs, fmt.Errorf("credentialSecretRef must be empty for InstanceRole")) + } + default: + errs = append(errs, fmt.Errorf("spec.tde.provider.kms.auth.type must be EnvironmentSecret or InstanceRole")) + } + } + default: + errs = append(errs, fmt.Errorf("spec.tde.provider.type must be Local, AwsKms, or AliyunKms")) + } + return errs +} + +func ValidateCreate(config *TDEConfig) []error { + errs := Validate(config) + if config != nil && config.Recovery != nil { + errs = append(errs, fmt.Errorf("spec.tde.recovery is only valid for an existing RotationOutcomeUnknown operation")) + } + return errs +} + +func ValidateUpdate(oldConfig, newConfig *TDEConfig, status *TDEStatus) []error { + errs := Validate(newConfig) + if oldConfig == nil { + return errs + } + if newConfig == nil { + if OperationInProgress(status) || RotationOutcomeUnknown(status) || status != nil && status.Current != nil { + errs = append(errs, fmt.Errorf("spec.tde cannot be removed after TDE reconciliation has started")) + } + return errs + } + if RotationOutcomeUnknown(status) { + return append(errs, validateRecoveryUpdate(oldConfig, newConfig, status)...) + } + if rootRotationCancellation(newConfig, status) { + if oldConfig.RootKeyRotation == nil || oldConfig.RootKeyRotation.RequestID != status.Operation.RequestID || + oldConfig.ManagementPolicy != newConfig.ManagementPolicy || + !reflect.DeepEqual(oldConfig.MasterKeyRotation, newConfig.MasterKeyRotation) || + !reflect.DeepEqual(oldConfig.CredentialRotation, newConfig.CredentialRotation) || + !reflect.DeepEqual(oldConfig.Recovery, newConfig.Recovery) { + errs = append(errs, fmt.Errorf("canceling a root key rotation may only restore the captured source and clear rootKeyRotation")) + } + return errs + } + if status != nil && status.Current != nil && newConfig.ManagementPolicy != ManagementPolicyManaged { + errs = append(errs, fmt.Errorf("spec.tde cannot leave Managed after TDE initialization")) + } + if !reflect.DeepEqual(oldConfig.Recovery, newConfig.Recovery) { + clearingCompletedRecovery := oldConfig.Recovery != nil && newConfig.Recovery == nil && status != nil && + status.Operation != nil && status.Operation.Stage == StageCompleted && status.Operation.Recovery != nil && + status.Operation.Recovery.RequestID == oldConfig.Recovery.RequestID + if !clearingCompletedRecovery { + errs = append(errs, fmt.Errorf("spec.tde.recovery may only change while RotationOutcomeUnknown is true or be cleared after completion")) + } + } + rootChanged := !sameRootIdentity(oldConfig.Provider, newConfig.Provider) + authChanged := !sameAuth(oldConfig.Provider, newConfig.Provider) + if rootChanged { + if newConfig.RootKeyRotation == nil || newConfig.RootKeyRotation.RequestID == "" || + (oldConfig.RootKeyRotation != nil && oldConfig.RootKeyRotation.RequestID == newConfig.RootKeyRotation.RequestID) { + errs = append(errs, fmt.Errorf("changing the TDE root key requires a new spec.tde.rootKeyRotation.requestId")) + } + if (oldConfig.Provider.Type == ProviderAwsKms && newConfig.Provider.Type == ProviderAliyunKms) || + (oldConfig.Provider.Type == ProviderAliyunKms && newConfig.Provider.Type == ProviderAwsKms) { + errs = append(errs, fmt.Errorf("direct AwsKms/AliyunKms rotation is unsupported; rotate through Local using two requests")) + } + if newConfig.RootKeyRotation != nil && RequestIDUsed(status, newConfig.RootKeyRotation.RequestID) && + (status == nil || status.Operation == nil || status.Operation.Type != OperationRotateRootKey || + status.Operation.RequestID != newConfig.RootKeyRotation.RequestID) { + errs = append(errs, fmt.Errorf("spec.tde.rootKeyRotation.requestId %q was already used", newConfig.RootKeyRotation.RequestID)) + } + } + if authChanged { + if rootChanged { + errs = append(errs, fmt.Errorf("KMS credentials and root key cannot change in the same update")) + } + if newConfig.CredentialRotation == nil || newConfig.CredentialRotation.RequestID == "" || + (oldConfig.CredentialRotation != nil && oldConfig.CredentialRotation.RequestID == newConfig.CredentialRotation.RequestID) { + errs = append(errs, fmt.Errorf("changing KMS credentials requires a new spec.tde.credentialRotation.requestId")) + } + if newConfig.CredentialRotation != nil && RequestIDUsed(status, newConfig.CredentialRotation.RequestID) && + (status == nil || status.Operation == nil || status.Operation.Type != OperationRotateKmsCredential || + status.Operation.RequestID != newConfig.CredentialRotation.RequestID) { + errs = append(errs, fmt.Errorf("spec.tde.credentialRotation.requestId %q was already used", newConfig.CredentialRotation.RequestID)) + } + } + if status != nil && status.Operation != nil && status.Operation.Stage != StageCompleted && status.Operation.Stage != StageFailed && !reflect.DeepEqual(oldConfig, newConfig) { + errs = append(errs, fmt.Errorf("spec.tde cannot be changed while operation %q is in stage %q", status.Operation.RequestID, status.Operation.Stage)) + } + return errs +} + +func validateRecoveryUpdate(oldConfig, newConfig *TDEConfig, status *TDEStatus) []error { + operation := status.Operation + if operation == nil || operation.Type != OperationRotateRootKey || operation.SQLState != SQLOutcomeUnknown { + return []error{fmt.Errorf("spec.tde cannot be changed while RotationOutcomeUnknown is true")} + } + recovery := newConfig.Recovery + if recovery == nil { + return []error{fmt.Errorf("spec.tde.recovery is required while RotationOutcomeUnknown is true")} + } + var errs []error + if recovery.RotationRequestID != operation.RequestID { + errs = append(errs, fmt.Errorf("spec.tde.recovery.rotationRequestId must match the unknown rotation request %q", operation.RequestID)) + } + if operation.Recovery != nil && operation.Recovery.RequestID == recovery.RequestID { + errs = append(errs, fmt.Errorf("spec.tde.recovery.requestId %q was already used", recovery.RequestID)) + } + if RequestIDUsed(status, recovery.RequestID) && (operation.Recovery == nil || operation.Recovery.RequestID != recovery.RequestID) { + errs = append(errs, fmt.Errorf("spec.tde.recovery.requestId %q was already used", recovery.RequestID)) + } + switch recovery.Decision { + case RecoveryConfirmApplied: + oldWithoutRecovery := oldConfig.DeepCopy() + oldWithoutRecovery.Recovery = nil + newWithoutRecovery := newConfig.DeepCopy() + newWithoutRecovery.Recovery = nil + if !reflect.DeepEqual(oldWithoutRecovery, newWithoutRecovery) { + errs = append(errs, fmt.Errorf("ConfirmApplied may only add spec.tde.recovery; the target TDE configuration must not change")) + } + case RecoveryConfirmNotApplied: + if operation.Source == nil || !configMatchesProviderStatus(newConfig, operation.Source) { + errs = append(errs, fmt.Errorf("ConfirmNotApplied must restore provider and algorithm to the captured source")) + } + if newConfig.RootKeyRotation != nil { + errs = append(errs, fmt.Errorf("ConfirmNotApplied must clear spec.tde.rootKeyRotation")) + } + if !reflect.DeepEqual(oldConfig.MasterKeyRotation, newConfig.MasterKeyRotation) || + !reflect.DeepEqual(oldConfig.CredentialRotation, newConfig.CredentialRotation) || + oldConfig.ManagementPolicy != newConfig.ManagementPolicy { + errs = append(errs, fmt.Errorf("ConfirmNotApplied cannot change unrelated TDE fields")) + } + } + return errs +} + +func configMatchesProviderStatus(config *TDEConfig, provider *ProviderStatus) bool { + if config == nil || provider == nil || config.Provider.Type != provider.Provider || config.DefaultAlgorithm != provider.DefaultAlgorithm { + return false + } + switch provider.Provider { + case ProviderLocal: + return config.Provider.Local != nil && provider.RootKeyRef != nil && + config.Provider.Local.SecretKeyRef.Name == provider.RootKeyRef.SecretName && + config.Provider.Local.SecretKeyRef.Key == provider.RootKeyRef.Key + case ProviderAwsKms, ProviderAliyunKms: + if config.Provider.Kms == nil || provider.Kms == nil { + return false + } + kms, current := config.Provider.Kms, provider.Kms + if kms.KeyID != current.KeyID || kms.Endpoint != current.Endpoint || kms.Region != current.Region || kms.Auth.Type != current.Auth.Type { + return false + } + if kms.Auth.Type == KmsAuthInstanceRole { + return kms.Auth.CredentialSecretRef == nil + } + return kms.Auth.CredentialSecretRef != nil && + kms.Auth.CredentialSecretRef.Name == current.Auth.CredentialSecretName && + kms.Auth.CredentialSecretRef.AccessKeyKey == current.Auth.AccessKeyKey && + kms.Auth.CredentialSecretRef.SecretKeyKey == current.Auth.SecretKeyKey + default: + return false + } +} + +func rootRotationCancellation(config *TDEConfig, status *TDEStatus) bool { + return config != nil && config.RootKeyRotation == nil && status != nil && status.Operation != nil && + status.Operation.Type == OperationRotateRootKey && status.Operation.Stage != StageCompleted && + (status.Operation.SQLState == SQLNotStarted || status.Operation.SQLState == SQLRejected) && status.Operation.Source != nil && + configMatchesProviderStatus(config, status.Operation.Source) +} + +func RotationOutcomeUnknown(status *TDEStatus) bool { + if status == nil { + return false + } + for i := range status.Conditions { + if status.Conditions[i].Type == "RotationOutcomeUnknown" && status.Conditions[i].Status == "True" { + return true + } + } + return false +} + +func OperationInProgress(status *TDEStatus) bool { + return status != nil && status.Operation != nil && status.Operation.Stage != StageCompleted && status.Operation.Stage != StageFailed +} + +func BlocksFELifecycle(status *TDEStatus) bool { + return OperationInProgress(status) || RotationOutcomeUnknown(status) || conditionTrue(status, "ConfigSyncPending") +} + +func conditionTrue(status *TDEStatus, conditionType string) bool { + if status == nil { + return false + } + for i := range status.Conditions { + if status.Conditions[i].Type == conditionType && status.Conditions[i].Status == "True" { + return true + } + } + return false +} + +func RequestIDUsed(status *TDEStatus, requestID string) bool { + if status == nil || requestID == "" { + return false + } + for _, used := range status.UsedRequestIDs { + if used == requestID { + return true + } + } + return false +} + +func sameRootIdentity(a, b ProviderSpec) bool { + if a.Type != b.Type { + return false + } + switch a.Type { + case ProviderLocal: + return a.Local != nil && b.Local != nil && a.Local.SecretKeyRef == b.Local.SecretKeyRef + case ProviderAwsKms, ProviderAliyunKms: + return a.Kms != nil && b.Kms != nil && a.Kms.KeyID == b.Kms.KeyID && a.Kms.Endpoint == b.Kms.Endpoint && a.Kms.Region == b.Kms.Region + default: + return false + } +} + +func sameAuth(a, b ProviderSpec) bool { + if a.Type == ProviderLocal || b.Type == ProviderLocal { + return true + } + if a.Kms == nil || b.Kms == nil { + return a.Kms == b.Kms + } + return reflect.DeepEqual(a.Kms.Auth, b.Kms.Auth) +} diff --git a/api/tde/validation_test.go b/api/tde/validation_test.go new file mode 100644 index 00000000..0fbf1059 --- /dev/null +++ b/api/tde/validation_test.go @@ -0,0 +1,205 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 + +package tde + +import ( + "strings" + "testing" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +func TestValidateLocal(t *testing.T) { + config := localConfig("key-v1") + if errs := Validate(config); len(errs) != 0 { + t.Fatalf("valid Local config rejected: %v", errs) + } + config.Provider.Local.SecretKeyRef.Key = "" + if errs := Validate(config); len(errs) == 0 { + t.Fatal("missing Secret key was accepted") + } +} + +func TestValidatePlaintextAlgorithm(t *testing.T) { + config := localConfig("key-v1") + config.DefaultAlgorithm = "PLAINTEXT" + if errs := Validate(config); len(errs) != 0 { + t.Fatalf("PLAINTEXT algorithm rejected: %v", errs) + } +} + +func TestValidateRejectsUnsupportedAES128(t *testing.T) { + config := localConfig("key-v1") + config.DefaultAlgorithm = "AES128" + if errs := Validate(config); len(errs) == 0 { + t.Fatal("AES128 was accepted even though the current FE does not support it") + } +} + +func TestValidateCreateRejectsRecovery(t *testing.T) { + config := localConfig("key-v1") + config.Recovery = &RecoveryRequest{RequestID: "recover-1", RotationRequestID: "rotate-1", Decision: RecoveryConfirmApplied} + if errs := ValidateCreate(config); len(errs) == 0 { + t.Fatal("recovery was accepted on create") + } +} + +func TestValidateUpdateRequiresRotationRequest(t *testing.T) { + oldConfig := localConfig("key-v1") + newConfig := localConfig("key-v2") + if errs := ValidateUpdate(oldConfig, newConfig, nil); len(errs) == 0 { + t.Fatal("root key change without requestId was accepted") + } + newConfig.RootKeyRotation = &RotationRequest{RequestID: "rotate-1"} + if errs := ValidateUpdate(oldConfig, newConfig, nil); len(errs) != 0 { + t.Fatalf("root key change with requestId rejected: %v", errs) + } +} + +func TestValidateUpdateRejectsRequestIDFromHistory(t *testing.T) { + oldConfig := localConfig("key-v2") + newConfig := localConfig("key-v3") + newConfig.RootKeyRotation = &RotationRequest{RequestID: "rotate-1"} + status := &TDEStatus{ + Current: &ProviderStatus{Provider: ProviderLocal, DefaultAlgorithm: "AES256", RootKeyRef: &RootKeyRefStatus{SecretName: "key-v2", Key: "root.key"}}, + UsedRequestIDs: []string{"rotate-1"}, + } + + errs := ValidateUpdate(oldConfig, newConfig, status) + if len(errs) == 0 || !strings.Contains(errs[len(errs)-1].Error(), "already used") { + t.Fatalf("expected reused requestId error, got %v", errs) + } +} + +func TestValidateUpdateRejectsDirectCrossKMSRotation(t *testing.T) { + oldConfig := kmsConfig(ProviderAwsKms, "aws-key") + newConfig := kmsConfig(ProviderAliyunKms, "aliyun-key") + newConfig.RootKeyRotation = &RotationRequest{RequestID: "rotate-1"} + if errs := ValidateUpdate(oldConfig, newConfig, nil); len(errs) == 0 { + t.Fatal("direct AwsKms to AliyunKms rotation was accepted") + } +} + +func TestValidateUpdateRejectsRemovalAfterReconciliationStarts(t *testing.T) { + oldConfig := localConfig("key-v1") + status := &TDEStatus{Operation: &OperationStatus{Type: OperationEnableTDE, Stage: StageRollingOutMaterial}} + if errs := ValidateUpdate(oldConfig, nil, status); len(errs) == 0 { + t.Fatal("removing spec.tde during enable was accepted") + } +} + +func TestValidateUpdateAcceptsConfirmAppliedRecovery(t *testing.T) { + oldConfig := localConfig("key-v2") + oldConfig.RootKeyRotation = &RotationRequest{RequestID: "rotate-1"} + newConfig := oldConfig.DeepCopy() + newConfig.Recovery = &RecoveryRequest{RequestID: "recover-1", RotationRequestID: "rotate-1", Decision: RecoveryConfirmApplied} + status := unknownRotationStatus() + if errs := ValidateUpdate(oldConfig, newConfig, status); len(errs) != 0 { + t.Fatalf("valid ConfirmApplied recovery rejected: %v", errs) + } +} + +func TestValidateUpdateAcceptsConfirmNotAppliedRecovery(t *testing.T) { + oldConfig := localConfig("key-v2") + oldConfig.RootKeyRotation = &RotationRequest{RequestID: "rotate-1"} + newConfig := localConfig("key-v1") + newConfig.Recovery = &RecoveryRequest{RequestID: "recover-1", RotationRequestID: "rotate-1", Decision: RecoveryConfirmNotApplied} + status := unknownRotationStatus() + if errs := ValidateUpdate(oldConfig, newConfig, status); len(errs) != 0 { + t.Fatalf("valid ConfirmNotApplied recovery rejected: %v", errs) + } +} + +func TestValidateUpdateRejectsUnknownOutcomeWithoutRecovery(t *testing.T) { + config := localConfig("key-v2") + config.RootKeyRotation = &RotationRequest{RequestID: "rotate-1"} + newConfig := config.DeepCopy() + newConfig.RootKeyRotation.RequestID = "rotate-2" + if errs := ValidateUpdate(config, newConfig, unknownRotationStatus()); len(errs) == 0 { + t.Fatal("unknown rotation outcome accepted a new request without recovery") + } +} + +func TestValidateUpdateAllowsClearingCompletedRecovery(t *testing.T) { + oldConfig := localConfig("key-v2") + oldConfig.RootKeyRotation = &RotationRequest{RequestID: "rotate-1"} + oldConfig.Recovery = &RecoveryRequest{RequestID: "recover-1", RotationRequestID: "rotate-1", Decision: RecoveryConfirmApplied} + newConfig := oldConfig.DeepCopy() + newConfig.Recovery = nil + status := &TDEStatus{Current: &ProviderStatus{Provider: ProviderLocal, DefaultAlgorithm: "AES256", RootKeyRef: &RootKeyRefStatus{SecretName: "key-v2", Key: "root.key"}}, + Operation: &OperationStatus{Type: OperationRotateRootKey, RequestID: "rotate-1", Stage: StageCompleted, + Recovery: &RecoveryStatus{RequestID: "recover-1", Decision: RecoveryConfirmApplied}}} + + if errs := ValidateUpdate(oldConfig, newConfig, status); len(errs) != 0 { + t.Fatalf("clearing a completed recovery was rejected: %v", errs) + } +} + +func TestValidateUpdateAllowsRootRotationCancellationBeforeSQL(t *testing.T) { + oldConfig := localConfig("key-v2") + oldConfig.RootKeyRotation = &RotationRequest{RequestID: "rotate-1"} + newConfig := localConfig("key-v1") + status := &TDEStatus{Current: &ProviderStatus{Provider: ProviderLocal, DefaultAlgorithm: "AES256", RootKeyRef: &RootKeyRefStatus{SecretName: "key-v1", Key: "root.key"}}, + Operation: &OperationStatus{Type: OperationRotateRootKey, RequestID: "rotate-1", Stage: StageRollingOutMaterial, SQLState: SQLNotStarted, + Source: &ProviderStatus{Provider: ProviderLocal, DefaultAlgorithm: "AES256", RootKeyRef: &RootKeyRefStatus{SecretName: "key-v1", Key: "root.key"}}, + Target: &ProviderStatus{Provider: ProviderLocal, DefaultAlgorithm: "AES256", RootKeyRef: &RootKeyRefStatus{SecretName: "key-v2", Key: "root.key"}}}} + + if errs := ValidateUpdate(oldConfig, newConfig, status); len(errs) != 0 { + t.Fatalf("safe pre-SQL cancellation was rejected: %v", errs) + } + status.Operation.SQLState = SQLSubmitting + if errs := ValidateUpdate(oldConfig, newConfig, status); len(errs) == 0 { + t.Fatal("cancellation was accepted after SQL submission started") + } +} + +func TestBlocksFELifecycleOnlyForUnsafeTDETransitions(t *testing.T) { + status := &TDEStatus{Current: &ProviderStatus{Provider: ProviderLocal}, State: StateLocalKeyNotReady} + if BlocksFELifecycle(status) { + t.Fatal("material validation failure without an active operation blocked FE lifecycle") + } + status.Operation = &OperationStatus{Type: OperationRotateRootKey, Stage: StageRollingOutMaterial} + if !BlocksFELifecycle(status) { + t.Fatal("active root key rotation did not block FE lifecycle") + } + status.Operation.Stage = StageFailed + status.Conditions = []metav1.Condition{{Type: "RotationOutcomeUnknown", Status: metav1.ConditionTrue}} + if !BlocksFELifecycle(status) { + t.Fatal("unknown rotation outcome did not block FE lifecycle") + } + status.Conditions = []metav1.Condition{{Type: "ConfigSyncPending", Status: metav1.ConditionTrue}} + if !BlocksFELifecycle(status) { + t.Fatal("pending TDE configuration sync did not block FE lifecycle") + } +} + +func unknownRotationStatus() *TDEStatus { + return &TDEStatus{ + Current: &ProviderStatus{Provider: ProviderLocal, DefaultAlgorithm: "AES256", RootKeyRef: &RootKeyRefStatus{SecretName: "key-v1", Key: "root.key"}}, + Operation: &OperationStatus{ + Type: OperationRotateRootKey, RequestID: "rotate-1", Stage: StageFailed, SQLState: SQLOutcomeUnknown, + Source: &ProviderStatus{Provider: ProviderLocal, DefaultAlgorithm: "AES256", RootKeyRef: &RootKeyRefStatus{SecretName: "key-v1", Key: "root.key"}}, + Target: &ProviderStatus{Provider: ProviderLocal, DefaultAlgorithm: "AES256", RootKeyRef: &RootKeyRefStatus{SecretName: "key-v2", Key: "root.key"}}, + }, + Conditions: []metav1.Condition{{Type: "RotationOutcomeUnknown", Status: metav1.ConditionTrue}}, + } +} + +func localConfig(secret string) *TDEConfig { + return &TDEConfig{ManagementPolicy: ManagementPolicyManaged, DefaultAlgorithm: "AES256", + Provider: ProviderSpec{Type: ProviderLocal, Local: &LocalProviderSpec{SecretKeyRef: SecretKeyReference{Name: secret, Key: "root.key"}}}} +} + +func kmsConfig(provider ProviderType, keyID string) *TDEConfig { + return &TDEConfig{ManagementPolicy: ManagementPolicyManaged, DefaultAlgorithm: "AES256", Provider: ProviderSpec{Type: provider, + Kms: &KmsProviderSpec{KeyID: keyID, Endpoint: "https://kms.example.com", Region: "region-1", Auth: KmsAuthSpec{Type: KmsAuthEnvironmentSecret, + CredentialSecretRef: &KmsCredentialSecretReference{Name: "credentials", AccessKeyKey: "accessKey", SecretKeyKey: "secretKey"}}}}} +} diff --git a/config/crd/bases/crds.yaml b/config/crd/bases/crds.yaml index 86cc968f..66cd35b8 100644 --- a/config/crd/bases/crds.yaml +++ b/config/crd/bases/crds.yaml @@ -9263,6 +9263,137 @@ spec: type: array type: object type: array + tde: + description: TDE declares cluster-wide transparent data encryption + management. + properties: + credentialRotation: + properties: + requestId: + minLength: 1 + type: string + required: + - requestId + type: object + defaultAlgorithm: + enum: + - PLAINTEXT + - AES256 + - SM4 + type: string + managementPolicy: + enum: + - Managed + - ObserveOnly + type: string + masterKeyRotation: + properties: + checkIntervalMs: + format: int64 + minimum: 1 + type: integer + rotateIntervalMs: + format: int64 + minimum: 1 + type: integer + required: + - checkIntervalMs + - rotateIntervalMs + type: object + provider: + properties: + kms: + properties: + auth: + properties: + credentialSecretRef: + properties: + accessKeyKey: + type: string + name: + type: string + secretKeyKey: + type: string + required: + - accessKeyKey + - name + - secretKeyKey + type: object + type: + enum: + - EnvironmentSecret + - InstanceRole + type: string + required: + - type + type: object + endpoint: + type: string + keyId: + type: string + region: + type: string + required: + - auth + - endpoint + - keyId + - region + type: object + local: + properties: + secretKeyRef: + properties: + key: + type: string + name: + type: string + required: + - key + - name + type: object + required: + - secretKeyRef + type: object + type: + enum: + - Local + - AwsKms + - AliyunKms + type: string + required: + - type + type: object + recovery: + properties: + decision: + enum: + - ConfirmApplied + - ConfirmNotApplied + type: string + requestId: + minLength: 1 + type: string + rotationRequestId: + minLength: 1 + type: string + required: + - decision + - requestId + - rotationRequestId + type: object + rootKeyRotation: + properties: + requestId: + minLength: 1 + type: string + required: + - requestId + type: object + required: + - defaultAlgorithm + - managementPolicy + - provider + type: object type: object status: description: DorisClusterStatus defines the observed state of DorisCluster @@ -9489,6 +9620,305 @@ spec: required: - componentCondition type: object + tde: + description: TDE reports the observed encryption configuration and + operation state. + properties: + activeConfigHash: + type: string + activeFeSpecHash: + type: string + conditions: + items: + description: Condition contains details for one aspect of the + current state of this API Resource. + properties: + lastTransitionTime: + description: |- + lastTransitionTime is the last time the condition transitioned from one status to another. + This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. + format: date-time + type: string + message: + description: |- + message is a human readable message indicating details about the transition. + This may be an empty string. + maxLength: 32768 + type: string + observedGeneration: + description: |- + observedGeneration represents the .metadata.generation that the condition was set based upon. + For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date + with respect to the current state of the instance. + format: int64 + minimum: 0 + type: integer + reason: + description: |- + reason contains a programmatic identifier indicating the reason for the condition's last transition. + Producers of specific condition types may define expected values and meanings for this field, + and whether the values are considered a guaranteed API. + The value should be a CamelCase string. + This field may not be empty. + maxLength: 1024 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ + type: string + status: + description: status of the condition, one of True, False, + Unknown. + enum: + - "True" + - "False" + - Unknown + type: string + type: + description: type of condition in CamelCase or in foo.example.com/CamelCase. + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + required: + - lastTransitionTime + - message + - reason + - status + - type + type: object + type: array + current: + properties: + defaultAlgorithm: + type: string + kms: + properties: + auth: + properties: + accessKeyKey: + type: string + credentialSecretName: + type: string + credentialSecretUid: + type: string + secretKeyKey: + type: string + type: + type: string + required: + - type + type: object + endpoint: + type: string + keyId: + type: string + region: + type: string + required: + - auth + - endpoint + - keyId + - region + type: object + provider: + type: string + rootKeyRef: + properties: + key: + type: string + resolvedPath: + type: string + secretName: + type: string + secretUid: + type: string + required: + - key + - resolvedPath + - secretName + - secretUid + type: object + required: + - provider + type: object + feChecks: + items: + properties: + configConsistent: + type: boolean + materialReady: + type: boolean + podName: + type: string + podReady: + type: boolean + required: + - configConsistent + - materialReady + - podName + - podReady + type: object + type: array + metadataInitialized: + type: boolean + observedGeneration: + format: int64 + type: integer + operation: + properties: + commitJournalId: + type: string + lastTransitionTime: + format: date-time + type: string + recovery: + properties: + decision: + type: string + requestId: + type: string + resolvedAt: + format: date-time + type: string + required: + - decision + - requestId + - resolvedAt + type: object + requestId: + type: string + source: + properties: + defaultAlgorithm: + type: string + kms: + properties: + auth: + properties: + accessKeyKey: + type: string + credentialSecretName: + type: string + credentialSecretUid: + type: string + secretKeyKey: + type: string + type: + type: string + required: + - type + type: object + endpoint: + type: string + keyId: + type: string + region: + type: string + required: + - auth + - endpoint + - keyId + - region + type: object + provider: + type: string + rootKeyRef: + properties: + key: + type: string + resolvedPath: + type: string + secretName: + type: string + secretUid: + type: string + required: + - key + - resolvedPath + - secretName + - secretUid + type: object + required: + - provider + type: object + specGeneration: + format: int64 + type: integer + sqlState: + type: string + stage: + type: string + startedAt: + format: date-time + type: string + target: + properties: + defaultAlgorithm: + type: string + kms: + properties: + auth: + properties: + accessKeyKey: + type: string + credentialSecretName: + type: string + credentialSecretUid: + type: string + secretKeyKey: + type: string + type: + type: string + required: + - type + type: object + endpoint: + type: string + keyId: + type: string + region: + type: string + required: + - auth + - endpoint + - keyId + - region + type: object + provider: + type: string + rootKeyRef: + properties: + key: + type: string + resolvedPath: + type: string + secretName: + type: string + secretUid: + type: string + required: + - key + - resolvedPath + - secretName + - secretUid + type: object + required: + - provider + type: object + type: + type: string + required: + - lastTransitionTime + - specGeneration + - stage + - startedAt + - type + type: object + state: + type: string + usedRequestIds: + items: + type: string + type: array + type: object type: object type: object served: true @@ -16544,6 +16974,137 @@ spec: type: object type: array type: object + tde: + description: TDE declares cluster-wide transparent data encryption + management. + properties: + credentialRotation: + properties: + requestId: + minLength: 1 + type: string + required: + - requestId + type: object + defaultAlgorithm: + enum: + - PLAINTEXT + - AES256 + - SM4 + type: string + managementPolicy: + enum: + - Managed + - ObserveOnly + type: string + masterKeyRotation: + properties: + checkIntervalMs: + format: int64 + minimum: 1 + type: integer + rotateIntervalMs: + format: int64 + minimum: 1 + type: integer + required: + - checkIntervalMs + - rotateIntervalMs + type: object + provider: + properties: + kms: + properties: + auth: + properties: + credentialSecretRef: + properties: + accessKeyKey: + type: string + name: + type: string + secretKeyKey: + type: string + required: + - accessKeyKey + - name + - secretKeyKey + type: object + type: + enum: + - EnvironmentSecret + - InstanceRole + type: string + required: + - type + type: object + endpoint: + type: string + keyId: + type: string + region: + type: string + required: + - auth + - endpoint + - keyId + - region + type: object + local: + properties: + secretKeyRef: + properties: + key: + type: string + name: + type: string + required: + - key + - name + type: object + required: + - secretKeyRef + type: object + type: + enum: + - Local + - AwsKms + - AliyunKms + type: string + required: + - type + type: object + recovery: + properties: + decision: + enum: + - ConfirmApplied + - ConfirmNotApplied + type: string + requestId: + minLength: 1 + type: string + rotationRequestId: + minLength: 1 + type: string + required: + - decision + - requestId + - rotationRequestId + type: object + rootKeyRotation: + properties: + requestId: + minLength: 1 + type: string + required: + - requestId + type: object + required: + - defaultAlgorithm + - managementPolicy + - provider + type: object type: object status: properties: @@ -16751,6 +17312,305 @@ spec: description: is the most recent generation observed for DorisDisaggregatedCluster format: int64 type: integer + tde: + description: TDE reports the observed encryption configuration and + operation state. + properties: + activeConfigHash: + type: string + activeFeSpecHash: + type: string + conditions: + items: + description: Condition contains details for one aspect of the + current state of this API Resource. + properties: + lastTransitionTime: + description: |- + lastTransitionTime is the last time the condition transitioned from one status to another. + This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. + format: date-time + type: string + message: + description: |- + message is a human readable message indicating details about the transition. + This may be an empty string. + maxLength: 32768 + type: string + observedGeneration: + description: |- + observedGeneration represents the .metadata.generation that the condition was set based upon. + For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date + with respect to the current state of the instance. + format: int64 + minimum: 0 + type: integer + reason: + description: |- + reason contains a programmatic identifier indicating the reason for the condition's last transition. + Producers of specific condition types may define expected values and meanings for this field, + and whether the values are considered a guaranteed API. + The value should be a CamelCase string. + This field may not be empty. + maxLength: 1024 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ + type: string + status: + description: status of the condition, one of True, False, + Unknown. + enum: + - "True" + - "False" + - Unknown + type: string + type: + description: type of condition in CamelCase or in foo.example.com/CamelCase. + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + required: + - lastTransitionTime + - message + - reason + - status + - type + type: object + type: array + current: + properties: + defaultAlgorithm: + type: string + kms: + properties: + auth: + properties: + accessKeyKey: + type: string + credentialSecretName: + type: string + credentialSecretUid: + type: string + secretKeyKey: + type: string + type: + type: string + required: + - type + type: object + endpoint: + type: string + keyId: + type: string + region: + type: string + required: + - auth + - endpoint + - keyId + - region + type: object + provider: + type: string + rootKeyRef: + properties: + key: + type: string + resolvedPath: + type: string + secretName: + type: string + secretUid: + type: string + required: + - key + - resolvedPath + - secretName + - secretUid + type: object + required: + - provider + type: object + feChecks: + items: + properties: + configConsistent: + type: boolean + materialReady: + type: boolean + podName: + type: string + podReady: + type: boolean + required: + - configConsistent + - materialReady + - podName + - podReady + type: object + type: array + metadataInitialized: + type: boolean + observedGeneration: + format: int64 + type: integer + operation: + properties: + commitJournalId: + type: string + lastTransitionTime: + format: date-time + type: string + recovery: + properties: + decision: + type: string + requestId: + type: string + resolvedAt: + format: date-time + type: string + required: + - decision + - requestId + - resolvedAt + type: object + requestId: + type: string + source: + properties: + defaultAlgorithm: + type: string + kms: + properties: + auth: + properties: + accessKeyKey: + type: string + credentialSecretName: + type: string + credentialSecretUid: + type: string + secretKeyKey: + type: string + type: + type: string + required: + - type + type: object + endpoint: + type: string + keyId: + type: string + region: + type: string + required: + - auth + - endpoint + - keyId + - region + type: object + provider: + type: string + rootKeyRef: + properties: + key: + type: string + resolvedPath: + type: string + secretName: + type: string + secretUid: + type: string + required: + - key + - resolvedPath + - secretName + - secretUid + type: object + required: + - provider + type: object + specGeneration: + format: int64 + type: integer + sqlState: + type: string + stage: + type: string + startedAt: + format: date-time + type: string + target: + properties: + defaultAlgorithm: + type: string + kms: + properties: + auth: + properties: + accessKeyKey: + type: string + credentialSecretName: + type: string + credentialSecretUid: + type: string + secretKeyKey: + type: string + type: + type: string + required: + - type + type: object + endpoint: + type: string + keyId: + type: string + region: + type: string + required: + - auth + - endpoint + - keyId + - region + type: object + provider: + type: string + rootKeyRef: + properties: + key: + type: string + resolvedPath: + type: string + secretName: + type: string + secretUid: + type: string + required: + - key + - resolvedPath + - secretName + - secretUid + type: object + required: + - provider + type: object + type: + type: string + required: + - lastTransitionTime + - specGeneration + - stage + - startedAt + - type + type: object + state: + type: string + usedRequestIds: + items: + type: string + type: array + type: object type: object type: object served: true diff --git a/config/crd/bases/disaggregated.cluster.doris.com_dorisdisaggregatedclusters.yaml b/config/crd/bases/disaggregated.cluster.doris.com_dorisdisaggregatedclusters.yaml index 0547f69d..d3b77921 100644 --- a/config/crd/bases/disaggregated.cluster.doris.com_dorisdisaggregatedclusters.yaml +++ b/config/crd/bases/disaggregated.cluster.doris.com_dorisdisaggregatedclusters.yaml @@ -7047,6 +7047,137 @@ spec: type: object type: array type: object + tde: + description: TDE declares cluster-wide transparent data encryption + management. + properties: + credentialRotation: + properties: + requestId: + minLength: 1 + type: string + required: + - requestId + type: object + defaultAlgorithm: + enum: + - PLAINTEXT + - AES256 + - SM4 + type: string + managementPolicy: + enum: + - Managed + - ObserveOnly + type: string + masterKeyRotation: + properties: + checkIntervalMs: + format: int64 + minimum: 1 + type: integer + rotateIntervalMs: + format: int64 + minimum: 1 + type: integer + required: + - checkIntervalMs + - rotateIntervalMs + type: object + provider: + properties: + kms: + properties: + auth: + properties: + credentialSecretRef: + properties: + accessKeyKey: + type: string + name: + type: string + secretKeyKey: + type: string + required: + - accessKeyKey + - name + - secretKeyKey + type: object + type: + enum: + - EnvironmentSecret + - InstanceRole + type: string + required: + - type + type: object + endpoint: + type: string + keyId: + type: string + region: + type: string + required: + - auth + - endpoint + - keyId + - region + type: object + local: + properties: + secretKeyRef: + properties: + key: + type: string + name: + type: string + required: + - key + - name + type: object + required: + - secretKeyRef + type: object + type: + enum: + - Local + - AwsKms + - AliyunKms + type: string + required: + - type + type: object + recovery: + properties: + decision: + enum: + - ConfirmApplied + - ConfirmNotApplied + type: string + requestId: + minLength: 1 + type: string + rotationRequestId: + minLength: 1 + type: string + required: + - decision + - requestId + - rotationRequestId + type: object + rootKeyRotation: + properties: + requestId: + minLength: 1 + type: string + required: + - requestId + type: object + required: + - defaultAlgorithm + - managementPolicy + - provider + type: object type: object status: properties: @@ -7254,6 +7385,305 @@ spec: description: is the most recent generation observed for DorisDisaggregatedCluster format: int64 type: integer + tde: + description: TDE reports the observed encryption configuration and + operation state. + properties: + activeConfigHash: + type: string + activeFeSpecHash: + type: string + conditions: + items: + description: Condition contains details for one aspect of the + current state of this API Resource. + properties: + lastTransitionTime: + description: |- + lastTransitionTime is the last time the condition transitioned from one status to another. + This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. + format: date-time + type: string + message: + description: |- + message is a human readable message indicating details about the transition. + This may be an empty string. + maxLength: 32768 + type: string + observedGeneration: + description: |- + observedGeneration represents the .metadata.generation that the condition was set based upon. + For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date + with respect to the current state of the instance. + format: int64 + minimum: 0 + type: integer + reason: + description: |- + reason contains a programmatic identifier indicating the reason for the condition's last transition. + Producers of specific condition types may define expected values and meanings for this field, + and whether the values are considered a guaranteed API. + The value should be a CamelCase string. + This field may not be empty. + maxLength: 1024 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ + type: string + status: + description: status of the condition, one of True, False, + Unknown. + enum: + - "True" + - "False" + - Unknown + type: string + type: + description: type of condition in CamelCase or in foo.example.com/CamelCase. + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + required: + - lastTransitionTime + - message + - reason + - status + - type + type: object + type: array + current: + properties: + defaultAlgorithm: + type: string + kms: + properties: + auth: + properties: + accessKeyKey: + type: string + credentialSecretName: + type: string + credentialSecretUid: + type: string + secretKeyKey: + type: string + type: + type: string + required: + - type + type: object + endpoint: + type: string + keyId: + type: string + region: + type: string + required: + - auth + - endpoint + - keyId + - region + type: object + provider: + type: string + rootKeyRef: + properties: + key: + type: string + resolvedPath: + type: string + secretName: + type: string + secretUid: + type: string + required: + - key + - resolvedPath + - secretName + - secretUid + type: object + required: + - provider + type: object + feChecks: + items: + properties: + configConsistent: + type: boolean + materialReady: + type: boolean + podName: + type: string + podReady: + type: boolean + required: + - configConsistent + - materialReady + - podName + - podReady + type: object + type: array + metadataInitialized: + type: boolean + observedGeneration: + format: int64 + type: integer + operation: + properties: + commitJournalId: + type: string + lastTransitionTime: + format: date-time + type: string + recovery: + properties: + decision: + type: string + requestId: + type: string + resolvedAt: + format: date-time + type: string + required: + - decision + - requestId + - resolvedAt + type: object + requestId: + type: string + source: + properties: + defaultAlgorithm: + type: string + kms: + properties: + auth: + properties: + accessKeyKey: + type: string + credentialSecretName: + type: string + credentialSecretUid: + type: string + secretKeyKey: + type: string + type: + type: string + required: + - type + type: object + endpoint: + type: string + keyId: + type: string + region: + type: string + required: + - auth + - endpoint + - keyId + - region + type: object + provider: + type: string + rootKeyRef: + properties: + key: + type: string + resolvedPath: + type: string + secretName: + type: string + secretUid: + type: string + required: + - key + - resolvedPath + - secretName + - secretUid + type: object + required: + - provider + type: object + specGeneration: + format: int64 + type: integer + sqlState: + type: string + stage: + type: string + startedAt: + format: date-time + type: string + target: + properties: + defaultAlgorithm: + type: string + kms: + properties: + auth: + properties: + accessKeyKey: + type: string + credentialSecretName: + type: string + credentialSecretUid: + type: string + secretKeyKey: + type: string + type: + type: string + required: + - type + type: object + endpoint: + type: string + keyId: + type: string + region: + type: string + required: + - auth + - endpoint + - keyId + - region + type: object + provider: + type: string + rootKeyRef: + properties: + key: + type: string + resolvedPath: + type: string + secretName: + type: string + secretUid: + type: string + required: + - key + - resolvedPath + - secretName + - secretUid + type: object + required: + - provider + type: object + type: + type: string + required: + - lastTransitionTime + - specGeneration + - stage + - startedAt + - type + type: object + state: + type: string + usedRequestIds: + items: + type: string + type: array + type: object type: object type: object served: true diff --git a/config/crd/bases/doris.apache.com_dorisclusters.yaml b/config/crd/bases/doris.apache.com_dorisclusters.yaml index eb9b96e6..87ae4bab 100644 --- a/config/crd/bases/doris.apache.com_dorisclusters.yaml +++ b/config/crd/bases/doris.apache.com_dorisclusters.yaml @@ -9263,6 +9263,137 @@ spec: type: array type: object type: array + tde: + description: TDE declares cluster-wide transparent data encryption + management. + properties: + credentialRotation: + properties: + requestId: + minLength: 1 + type: string + required: + - requestId + type: object + defaultAlgorithm: + enum: + - PLAINTEXT + - AES256 + - SM4 + type: string + managementPolicy: + enum: + - Managed + - ObserveOnly + type: string + masterKeyRotation: + properties: + checkIntervalMs: + format: int64 + minimum: 1 + type: integer + rotateIntervalMs: + format: int64 + minimum: 1 + type: integer + required: + - checkIntervalMs + - rotateIntervalMs + type: object + provider: + properties: + kms: + properties: + auth: + properties: + credentialSecretRef: + properties: + accessKeyKey: + type: string + name: + type: string + secretKeyKey: + type: string + required: + - accessKeyKey + - name + - secretKeyKey + type: object + type: + enum: + - EnvironmentSecret + - InstanceRole + type: string + required: + - type + type: object + endpoint: + type: string + keyId: + type: string + region: + type: string + required: + - auth + - endpoint + - keyId + - region + type: object + local: + properties: + secretKeyRef: + properties: + key: + type: string + name: + type: string + required: + - key + - name + type: object + required: + - secretKeyRef + type: object + type: + enum: + - Local + - AwsKms + - AliyunKms + type: string + required: + - type + type: object + recovery: + properties: + decision: + enum: + - ConfirmApplied + - ConfirmNotApplied + type: string + requestId: + minLength: 1 + type: string + rotationRequestId: + minLength: 1 + type: string + required: + - decision + - requestId + - rotationRequestId + type: object + rootKeyRotation: + properties: + requestId: + minLength: 1 + type: string + required: + - requestId + type: object + required: + - defaultAlgorithm + - managementPolicy + - provider + type: object type: object status: description: DorisClusterStatus defines the observed state of DorisCluster @@ -9489,6 +9620,305 @@ spec: required: - componentCondition type: object + tde: + description: TDE reports the observed encryption configuration and + operation state. + properties: + activeConfigHash: + type: string + activeFeSpecHash: + type: string + conditions: + items: + description: Condition contains details for one aspect of the + current state of this API Resource. + properties: + lastTransitionTime: + description: |- + lastTransitionTime is the last time the condition transitioned from one status to another. + This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. + format: date-time + type: string + message: + description: |- + message is a human readable message indicating details about the transition. + This may be an empty string. + maxLength: 32768 + type: string + observedGeneration: + description: |- + observedGeneration represents the .metadata.generation that the condition was set based upon. + For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date + with respect to the current state of the instance. + format: int64 + minimum: 0 + type: integer + reason: + description: |- + reason contains a programmatic identifier indicating the reason for the condition's last transition. + Producers of specific condition types may define expected values and meanings for this field, + and whether the values are considered a guaranteed API. + The value should be a CamelCase string. + This field may not be empty. + maxLength: 1024 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ + type: string + status: + description: status of the condition, one of True, False, + Unknown. + enum: + - "True" + - "False" + - Unknown + type: string + type: + description: type of condition in CamelCase or in foo.example.com/CamelCase. + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + required: + - lastTransitionTime + - message + - reason + - status + - type + type: object + type: array + current: + properties: + defaultAlgorithm: + type: string + kms: + properties: + auth: + properties: + accessKeyKey: + type: string + credentialSecretName: + type: string + credentialSecretUid: + type: string + secretKeyKey: + type: string + type: + type: string + required: + - type + type: object + endpoint: + type: string + keyId: + type: string + region: + type: string + required: + - auth + - endpoint + - keyId + - region + type: object + provider: + type: string + rootKeyRef: + properties: + key: + type: string + resolvedPath: + type: string + secretName: + type: string + secretUid: + type: string + required: + - key + - resolvedPath + - secretName + - secretUid + type: object + required: + - provider + type: object + feChecks: + items: + properties: + configConsistent: + type: boolean + materialReady: + type: boolean + podName: + type: string + podReady: + type: boolean + required: + - configConsistent + - materialReady + - podName + - podReady + type: object + type: array + metadataInitialized: + type: boolean + observedGeneration: + format: int64 + type: integer + operation: + properties: + commitJournalId: + type: string + lastTransitionTime: + format: date-time + type: string + recovery: + properties: + decision: + type: string + requestId: + type: string + resolvedAt: + format: date-time + type: string + required: + - decision + - requestId + - resolvedAt + type: object + requestId: + type: string + source: + properties: + defaultAlgorithm: + type: string + kms: + properties: + auth: + properties: + accessKeyKey: + type: string + credentialSecretName: + type: string + credentialSecretUid: + type: string + secretKeyKey: + type: string + type: + type: string + required: + - type + type: object + endpoint: + type: string + keyId: + type: string + region: + type: string + required: + - auth + - endpoint + - keyId + - region + type: object + provider: + type: string + rootKeyRef: + properties: + key: + type: string + resolvedPath: + type: string + secretName: + type: string + secretUid: + type: string + required: + - key + - resolvedPath + - secretName + - secretUid + type: object + required: + - provider + type: object + specGeneration: + format: int64 + type: integer + sqlState: + type: string + stage: + type: string + startedAt: + format: date-time + type: string + target: + properties: + defaultAlgorithm: + type: string + kms: + properties: + auth: + properties: + accessKeyKey: + type: string + credentialSecretName: + type: string + credentialSecretUid: + type: string + secretKeyKey: + type: string + type: + type: string + required: + - type + type: object + endpoint: + type: string + keyId: + type: string + region: + type: string + required: + - auth + - endpoint + - keyId + - region + type: object + provider: + type: string + rootKeyRef: + properties: + key: + type: string + resolvedPath: + type: string + secretName: + type: string + secretUid: + type: string + required: + - key + - resolvedPath + - secretName + - secretUid + type: object + required: + - provider + type: object + type: + type: string + required: + - lastTransitionTime + - specGeneration + - stage + - startedAt + - type + type: object + state: + type: string + usedRequestIds: + items: + type: string + type: array + type: object type: object type: object served: true diff --git a/config/crd/bases/doris.selectdb.com_dorisclusters.yaml b/config/crd/bases/doris.selectdb.com_dorisclusters.yaml index eb9b96e6..87ae4bab 100644 --- a/config/crd/bases/doris.selectdb.com_dorisclusters.yaml +++ b/config/crd/bases/doris.selectdb.com_dorisclusters.yaml @@ -9263,6 +9263,137 @@ spec: type: array type: object type: array + tde: + description: TDE declares cluster-wide transparent data encryption + management. + properties: + credentialRotation: + properties: + requestId: + minLength: 1 + type: string + required: + - requestId + type: object + defaultAlgorithm: + enum: + - PLAINTEXT + - AES256 + - SM4 + type: string + managementPolicy: + enum: + - Managed + - ObserveOnly + type: string + masterKeyRotation: + properties: + checkIntervalMs: + format: int64 + minimum: 1 + type: integer + rotateIntervalMs: + format: int64 + minimum: 1 + type: integer + required: + - checkIntervalMs + - rotateIntervalMs + type: object + provider: + properties: + kms: + properties: + auth: + properties: + credentialSecretRef: + properties: + accessKeyKey: + type: string + name: + type: string + secretKeyKey: + type: string + required: + - accessKeyKey + - name + - secretKeyKey + type: object + type: + enum: + - EnvironmentSecret + - InstanceRole + type: string + required: + - type + type: object + endpoint: + type: string + keyId: + type: string + region: + type: string + required: + - auth + - endpoint + - keyId + - region + type: object + local: + properties: + secretKeyRef: + properties: + key: + type: string + name: + type: string + required: + - key + - name + type: object + required: + - secretKeyRef + type: object + type: + enum: + - Local + - AwsKms + - AliyunKms + type: string + required: + - type + type: object + recovery: + properties: + decision: + enum: + - ConfirmApplied + - ConfirmNotApplied + type: string + requestId: + minLength: 1 + type: string + rotationRequestId: + minLength: 1 + type: string + required: + - decision + - requestId + - rotationRequestId + type: object + rootKeyRotation: + properties: + requestId: + minLength: 1 + type: string + required: + - requestId + type: object + required: + - defaultAlgorithm + - managementPolicy + - provider + type: object type: object status: description: DorisClusterStatus defines the observed state of DorisCluster @@ -9489,6 +9620,305 @@ spec: required: - componentCondition type: object + tde: + description: TDE reports the observed encryption configuration and + operation state. + properties: + activeConfigHash: + type: string + activeFeSpecHash: + type: string + conditions: + items: + description: Condition contains details for one aspect of the + current state of this API Resource. + properties: + lastTransitionTime: + description: |- + lastTransitionTime is the last time the condition transitioned from one status to another. + This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. + format: date-time + type: string + message: + description: |- + message is a human readable message indicating details about the transition. + This may be an empty string. + maxLength: 32768 + type: string + observedGeneration: + description: |- + observedGeneration represents the .metadata.generation that the condition was set based upon. + For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date + with respect to the current state of the instance. + format: int64 + minimum: 0 + type: integer + reason: + description: |- + reason contains a programmatic identifier indicating the reason for the condition's last transition. + Producers of specific condition types may define expected values and meanings for this field, + and whether the values are considered a guaranteed API. + The value should be a CamelCase string. + This field may not be empty. + maxLength: 1024 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ + type: string + status: + description: status of the condition, one of True, False, + Unknown. + enum: + - "True" + - "False" + - Unknown + type: string + type: + description: type of condition in CamelCase or in foo.example.com/CamelCase. + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + required: + - lastTransitionTime + - message + - reason + - status + - type + type: object + type: array + current: + properties: + defaultAlgorithm: + type: string + kms: + properties: + auth: + properties: + accessKeyKey: + type: string + credentialSecretName: + type: string + credentialSecretUid: + type: string + secretKeyKey: + type: string + type: + type: string + required: + - type + type: object + endpoint: + type: string + keyId: + type: string + region: + type: string + required: + - auth + - endpoint + - keyId + - region + type: object + provider: + type: string + rootKeyRef: + properties: + key: + type: string + resolvedPath: + type: string + secretName: + type: string + secretUid: + type: string + required: + - key + - resolvedPath + - secretName + - secretUid + type: object + required: + - provider + type: object + feChecks: + items: + properties: + configConsistent: + type: boolean + materialReady: + type: boolean + podName: + type: string + podReady: + type: boolean + required: + - configConsistent + - materialReady + - podName + - podReady + type: object + type: array + metadataInitialized: + type: boolean + observedGeneration: + format: int64 + type: integer + operation: + properties: + commitJournalId: + type: string + lastTransitionTime: + format: date-time + type: string + recovery: + properties: + decision: + type: string + requestId: + type: string + resolvedAt: + format: date-time + type: string + required: + - decision + - requestId + - resolvedAt + type: object + requestId: + type: string + source: + properties: + defaultAlgorithm: + type: string + kms: + properties: + auth: + properties: + accessKeyKey: + type: string + credentialSecretName: + type: string + credentialSecretUid: + type: string + secretKeyKey: + type: string + type: + type: string + required: + - type + type: object + endpoint: + type: string + keyId: + type: string + region: + type: string + required: + - auth + - endpoint + - keyId + - region + type: object + provider: + type: string + rootKeyRef: + properties: + key: + type: string + resolvedPath: + type: string + secretName: + type: string + secretUid: + type: string + required: + - key + - resolvedPath + - secretName + - secretUid + type: object + required: + - provider + type: object + specGeneration: + format: int64 + type: integer + sqlState: + type: string + stage: + type: string + startedAt: + format: date-time + type: string + target: + properties: + defaultAlgorithm: + type: string + kms: + properties: + auth: + properties: + accessKeyKey: + type: string + credentialSecretName: + type: string + credentialSecretUid: + type: string + secretKeyKey: + type: string + type: + type: string + required: + - type + type: object + endpoint: + type: string + keyId: + type: string + region: + type: string + required: + - auth + - endpoint + - keyId + - region + type: object + provider: + type: string + rootKeyRef: + properties: + key: + type: string + resolvedPath: + type: string + secretName: + type: string + secretUid: + type: string + required: + - key + - resolvedPath + - secretName + - secretUid + type: object + required: + - provider + type: object + type: + type: string + required: + - lastTransitionTime + - specGeneration + - stage + - startedAt + - type + type: object + state: + type: string + usedRequestIds: + items: + type: string + type: array + type: object type: object type: object served: true diff --git a/config/operator/disaggregated-operator.yaml b/config/operator/disaggregated-operator.yaml index 6641442d..0f9374ad 100644 --- a/config/operator/disaggregated-operator.yaml +++ b/config/operator/disaggregated-operator.yaml @@ -354,7 +354,7 @@ webhooks: name: doris-operator-service namespace: doris path: /validate-doris-selectdb-com-v1-doriscluster - failurePolicy: Ignore + failurePolicy: Fail name: vdoriscluster.kb.io rules: - apiGroups: @@ -374,7 +374,7 @@ webhooks: name: doris-operator-service namespace: doris path: /validate-disaggregated-cluster-doris-com-v1-dorisdisaggregatedcluster - failurePolicy: Ignore + failurePolicy: Fail name: vdorisdisaggregatedcluster.kb.io rules: - apiGroups: diff --git a/config/operator/operator.yaml b/config/operator/operator.yaml index 6adf837e..95f7fc6f 100644 --- a/config/operator/operator.yaml +++ b/config/operator/operator.yaml @@ -346,7 +346,7 @@ webhooks: name: doris-operator-service namespace: doris path: /validate-doris-selectdb-com-v1-doriscluster - failurePolicy: Ignore + failurePolicy: Fail name: vdoriscluster.kb.io rules: - apiGroups: @@ -366,7 +366,7 @@ webhooks: name: doris-operator-service namespace: doris path: /validate-disaggregated-cluster-doris-com-v1-dorisdisaggregatedcluster - failurePolicy: Ignore + failurePolicy: Fail name: vdorisdisaggregatedcluster.kb.io rules: - apiGroups: diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml index d5016a6c..1ad9ec55 100644 --- a/config/rbac/role.yaml +++ b/config/rbac/role.yaml @@ -57,8 +57,11 @@ rules: resources: - configmaps verbs: + - create - get - list + - patch + - update - watch - apiGroups: - "" diff --git a/go.mod b/go.mod index 85cd0253..fc302a10 100644 --- a/go.mod +++ b/go.mod @@ -8,6 +8,13 @@ require ( github.com/DATA-DOG/go-sqlmock v1.5.2 github.com/FoundationDB/fdb-kubernetes-operator v1.36.0 github.com/MakeNowJust/heredoc v1.0.0 + github.com/alibabacloud-go/darabonba-openapi/v2 v2.1.13 + github.com/alibabacloud-go/kms-20160120/v3 v3.4.0 + github.com/alibabacloud-go/tea v1.3.13 + github.com/aws/aws-sdk-go-v2 v1.41.0 + github.com/aws/aws-sdk-go-v2/config v1.32.5 + github.com/aws/aws-sdk-go-v2/credentials v1.19.5 + github.com/aws/aws-sdk-go-v2/service/kms v1.49.4 github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc github.com/go-sql-driver/mysql v1.8.1 github.com/jmoiron/sqlx v1.4.0 @@ -29,8 +36,32 @@ require ( require ( filippo.io/edwards25519 v1.1.0 // indirect + github.com/alibabacloud-go/alibabacloud-gateway-pop v0.0.8 // indirect + github.com/alibabacloud-go/alibabacloud-gateway-spi v0.0.5 // indirect + github.com/alibabacloud-go/darabonba-array v0.1.0 // indirect + github.com/alibabacloud-go/darabonba-encode-util v0.0.2 // indirect + github.com/alibabacloud-go/darabonba-map v0.0.2 // indirect + github.com/alibabacloud-go/darabonba-signature-util v0.0.7 // indirect + github.com/alibabacloud-go/darabonba-string v1.0.2 // indirect + github.com/alibabacloud-go/debug v1.0.1 // indirect + github.com/alibabacloud-go/endpoint-util v1.1.0 // indirect + github.com/alibabacloud-go/openapi-util v0.1.1 // indirect + github.com/alibabacloud-go/tea-utils/v2 v2.0.7 // indirect + github.com/aliyun/credentials-go v1.4.5 // indirect + github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.16 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.16 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.16 // indirect + github.com/aws/aws-sdk-go-v2/internal/ini v1.8.4 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.4 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.16 // indirect + github.com/aws/aws-sdk-go-v2/service/signin v1.0.4 // indirect + github.com/aws/aws-sdk-go-v2/service/sso v1.30.7 // indirect + github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.12 // indirect + github.com/aws/aws-sdk-go-v2/service/sts v1.41.5 // indirect + github.com/aws/smithy-go v1.24.0 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect + github.com/clbanning/mxj/v2 v2.7.0 // indirect github.com/emicklei/go-restful/v3 v3.12.1 // indirect github.com/evanphx/json-patch v5.9.0+incompatible // indirect github.com/evanphx/json-patch/v5 v5.9.0 // indirect @@ -76,6 +107,7 @@ require ( github.com/subosito/gotenv v1.4.2 // indirect github.com/tidwall/match v1.1.1 // indirect github.com/tidwall/pretty v1.2.0 // indirect + github.com/tjfoc/gmsm v1.4.1 // indirect github.com/x448/float16 v0.8.4 // indirect go.uber.org/multierr v1.11.0 // indirect go.uber.org/zap v1.27.0 // indirect diff --git a/go.sum b/go.sum index 59644269..03fef88f 100644 --- a/go.sum +++ b/go.sum @@ -46,8 +46,86 @@ github.com/FoundationDB/fdb-kubernetes-operator v1.36.0 h1:YVWcLOv+msXxTx4UvgSGy github.com/FoundationDB/fdb-kubernetes-operator v1.36.0/go.mod h1:NkiJsjHSkK9R7p5OxJbrnZOgl/wKb75ZkgkY/YWqK44= github.com/MakeNowJust/heredoc v1.0.0 h1:cXCdzVdstXyiTqTvfqk9SDHpKNjxuom+DOlyEeQ4pzQ= github.com/MakeNowJust/heredoc v1.0.0/go.mod h1:mG5amYoWBHf8vpLOuehzbGGw0EHxpZZ6lCpQ4fNJ8LE= +github.com/alibabacloud-go/alibabacloud-gateway-pop v0.0.6/go.mod h1:4EUIoxs/do24zMOGGqYVWgw0s9NtiylnJglOeEB5UJo= +github.com/alibabacloud-go/alibabacloud-gateway-pop v0.0.8 h1:ViQyUFKBVnhzsODcNzJK/uz1WXqzX+3xeQsEDy610PA= +github.com/alibabacloud-go/alibabacloud-gateway-pop v0.0.8/go.mod h1:e3etxyckfZ4sHJsmA2uBz07BUMKQWyPeZNP0dqi/5kw= +github.com/alibabacloud-go/alibabacloud-gateway-spi v0.0.4/go.mod h1:sCavSAvdzOjul4cEqeVtvlSaSScfNsTQ+46HwlTL1hc= +github.com/alibabacloud-go/alibabacloud-gateway-spi v0.0.5 h1:zE8vH9C7JiZLNJJQ5OwjU9mSi4T9ef9u3BURT6LCLC8= +github.com/alibabacloud-go/alibabacloud-gateway-spi v0.0.5/go.mod h1:tWnyE9AjF8J8qqLk645oUmVUnFybApTQWklQmi5tY6g= +github.com/alibabacloud-go/darabonba-array v0.1.0 h1:vR8s7b1fWAQIjEjWnuF0JiKsCvclSRTfDzZHTYqfufY= +github.com/alibabacloud-go/darabonba-array v0.1.0/go.mod h1:BLKxr0brnggqOJPqT09DFJ8g3fsDshapUD3C3aOEFaI= +github.com/alibabacloud-go/darabonba-encode-util v0.0.2 h1:1uJGrbsGEVqWcWxrS9MyC2NG0Ax+GpOM5gtupki31XE= +github.com/alibabacloud-go/darabonba-encode-util v0.0.2/go.mod h1:JiW9higWHYXm7F4PKuMgEUETNZasrDM6vqVr/Can7H8= +github.com/alibabacloud-go/darabonba-map v0.0.2 h1:qvPnGB4+dJbJIxOOfawxzF3hzMnIpjmafa0qOTp6udc= +github.com/alibabacloud-go/darabonba-map v0.0.2/go.mod h1:28AJaX8FOE/ym8OUFWga+MtEzBunJwQGceGQlvaPGPc= +github.com/alibabacloud-go/darabonba-openapi/v2 v2.1.13 h1:Q00FU3H94Ts0ZIHDmY+fYGgB7dV9D/YX6FGsgorQPgw= +github.com/alibabacloud-go/darabonba-openapi/v2 v2.1.13/go.mod h1:lxFGfobinVsQ49ntjpgWghXmIF0/Sm4+wvBJ1h5RtaE= +github.com/alibabacloud-go/darabonba-signature-util v0.0.7 h1:UzCnKvsjPFzApvODDNEYqBHMFt1w98wC7FOo0InLyxg= +github.com/alibabacloud-go/darabonba-signature-util v0.0.7/go.mod h1:oUzCYV2fcCH797xKdL6BDH8ADIHlzrtKVjeRtunBNTQ= +github.com/alibabacloud-go/darabonba-string v1.0.2 h1:E714wms5ibdzCqGeYJ9JCFywE5nDyvIXIIQbZVFkkqo= +github.com/alibabacloud-go/darabonba-string v1.0.2/go.mod h1:93cTfV3vuPhhEwGGpKKqhVW4jLe7tDpo3LUM0i0g6mA= +github.com/alibabacloud-go/debug v0.0.0-20190504072949-9472017b5c68/go.mod h1:6pb/Qy8c+lqua8cFpEy7g39NRRqOWc3rOwAy8m5Y2BY= +github.com/alibabacloud-go/debug v1.0.0/go.mod h1:8gfgZCCAC3+SCzjWtY053FrOcd4/qlH6IHTI4QyICOc= +github.com/alibabacloud-go/debug v1.0.1 h1:MsW9SmUtbb1Fnt3ieC6NNZi6aEwrXfDksD4QA6GSbPg= +github.com/alibabacloud-go/debug v1.0.1/go.mod h1:8gfgZCCAC3+SCzjWtY053FrOcd4/qlH6IHTI4QyICOc= +github.com/alibabacloud-go/endpoint-util v1.1.0 h1:r/4D3VSw888XGaeNpP994zDUaxdgTSHBbVfZlzf6b5Q= +github.com/alibabacloud-go/endpoint-util v1.1.0/go.mod h1:O5FuCALmCKs2Ff7JFJMudHs0I5EBgecXXxZRyswlEjE= +github.com/alibabacloud-go/kms-20160120/v3 v3.4.0 h1:rPxSs0VNCrpD7Ksus33376t/1K+WjAzX9iqWUwbkXpQ= +github.com/alibabacloud-go/kms-20160120/v3 v3.4.0/go.mod h1:5jyc6B9XWw2g2E/0ln2+qWmYrJA3/+KR912dOreBy/w= +github.com/alibabacloud-go/openapi-util v0.1.0/go.mod h1:sQuElr4ywwFRlCCberQwKRFhRzIyG4QTP/P4y1CJ6Ws= +github.com/alibabacloud-go/openapi-util v0.1.1 h1:ujGErJjG8ncRW6XtBBMphzHTvCxn4DjrVw4m04HsS28= +github.com/alibabacloud-go/openapi-util v0.1.1/go.mod h1:/UehBSE2cf1gYT43GV4E+RxTdLRzURImCYY0aRmlXpw= +github.com/alibabacloud-go/tea v1.1.0/go.mod h1:IkGyUSX4Ba1V+k4pCtJUc6jDpZLFph9QMy2VUPTwukg= +github.com/alibabacloud-go/tea v1.1.7/go.mod h1:/tmnEaQMyb4Ky1/5D+SE1BAsa5zj/KeGOFfwYm3N/p4= +github.com/alibabacloud-go/tea v1.1.8/go.mod h1:/tmnEaQMyb4Ky1/5D+SE1BAsa5zj/KeGOFfwYm3N/p4= +github.com/alibabacloud-go/tea v1.1.11/go.mod h1:/tmnEaQMyb4Ky1/5D+SE1BAsa5zj/KeGOFfwYm3N/p4= +github.com/alibabacloud-go/tea v1.1.17/go.mod h1:nXxjm6CIFkBhwW4FQkNrolwbfon8Svy6cujmKFUq98A= +github.com/alibabacloud-go/tea v1.1.20/go.mod h1:nXxjm6CIFkBhwW4FQkNrolwbfon8Svy6cujmKFUq98A= +github.com/alibabacloud-go/tea v1.2.2/go.mod h1:CF3vOzEMAG+bR4WOql8gc2G9H3EkH3ZLAQdpmpXMgwk= +github.com/alibabacloud-go/tea v1.3.13 h1:WhGy6LIXaMbBM6VBYcsDCz6K/TPsT1Ri2hPmmZffZ94= +github.com/alibabacloud-go/tea v1.3.13/go.mod h1:A560v/JTQ1n5zklt2BEpurJzZTI8TUT+Psg2drWlxRg= +github.com/alibabacloud-go/tea-utils v1.3.1/go.mod h1:EI/o33aBfj3hETm4RLiAxF/ThQdSngxrpF8rKUDJjPE= +github.com/alibabacloud-go/tea-utils/v2 v2.0.5/go.mod h1:dL6vbUT35E4F4bFTHL845eUloqaerYBYPsdWR2/jhe4= +github.com/alibabacloud-go/tea-utils/v2 v2.0.6/go.mod h1:qxn986l+q33J5VkialKMqT/TTs3E+U9MJpd001iWQ9I= +github.com/alibabacloud-go/tea-utils/v2 v2.0.7 h1:WDx5qW3Xa5ZgJ1c8NfqJkF6w+AU5wB8835UdhPr6Ax0= +github.com/alibabacloud-go/tea-utils/v2 v2.0.7/go.mod h1:qxn986l+q33J5VkialKMqT/TTs3E+U9MJpd001iWQ9I= +github.com/aliyun/credentials-go v1.1.2/go.mod h1:ozcZaMR5kLM7pwtCMEpVmQ242suV6qTJya2bDq4X1Tw= +github.com/aliyun/credentials-go v1.3.1/go.mod h1:8jKYhQuDawt8x2+fusqa1Y6mPxemTsBEN04dgcAcYz0= +github.com/aliyun/credentials-go v1.3.6/go.mod h1:1LxUuX7L5YrZUWzBrRyk0SwSdH4OmPrib8NVePL3fxM= +github.com/aliyun/credentials-go v1.4.5 h1:O76WYKgdy1oQYYiJkERjlA2dxGuvLRrzuO2ScrtGWSk= +github.com/aliyun/credentials-go v1.4.5/go.mod h1:Jm6d+xIgwJVLVWT561vy67ZRP4lPTQxMbEYRuT2Ti1U= github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio= github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs= +github.com/aws/aws-sdk-go-v2 v1.41.0 h1:tNvqh1s+v0vFYdA1xq0aOJH+Y5cRyZ5upu6roPgPKd4= +github.com/aws/aws-sdk-go-v2 v1.41.0/go.mod h1:MayyLB8y+buD9hZqkCW3kX1AKq07Y5pXxtgB+rRFhz0= +github.com/aws/aws-sdk-go-v2/config v1.32.5 h1:pz3duhAfUgnxbtVhIK39PGF/AHYyrzGEyRD9Og0QrE8= +github.com/aws/aws-sdk-go-v2/config v1.32.5/go.mod h1:xmDjzSUs/d0BB7ClzYPAZMmgQdrodNjPPhd6bGASwoE= +github.com/aws/aws-sdk-go-v2/credentials v1.19.5 h1:xMo63RlqP3ZZydpJDMBsH9uJ10hgHYfQFIk1cHDXrR4= +github.com/aws/aws-sdk-go-v2/credentials v1.19.5/go.mod h1:hhbH6oRcou+LpXfA/0vPElh/e0M3aFeOblE1sssAAEk= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.16 h1:80+uETIWS1BqjnN9uJ0dBUaETh+P1XwFy5vwHwK5r9k= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.16/go.mod h1:wOOsYuxYuB/7FlnVtzeBYRcjSRtQpAW0hCP7tIULMwo= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.16 h1:rgGwPzb82iBYSvHMHXc8h9mRoOUBZIGFgKb9qniaZZc= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.16/go.mod h1:L/UxsGeKpGoIj6DxfhOWHWQ/kGKcd4I1VncE4++IyKA= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.16 h1:1jtGzuV7c82xnqOVfx2F0xmJcOw5374L7N6juGW6x6U= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.16/go.mod h1:M2E5OQf+XLe+SZGmmpaI2yy+J326aFf6/+54PoxSANc= +github.com/aws/aws-sdk-go-v2/internal/ini v1.8.4 h1:WKuaxf++XKWlHWu9ECbMlha8WOEGm0OUEZqm4K/Gcfk= +github.com/aws/aws-sdk-go-v2/internal/ini v1.8.4/go.mod h1:ZWy7j6v1vWGmPReu0iSGvRiise4YI5SkR3OHKTZ6Wuc= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.4 h1:0ryTNEdJbzUCEWkVXEXoqlXV72J5keC1GvILMOuD00E= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.4/go.mod h1:HQ4qwNZh32C3CBeO6iJLQlgtMzqeG17ziAA/3KDJFow= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.16 h1:oHjJHeUy0ImIV0bsrX0X91GkV5nJAyv1l1CC9lnO0TI= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.16/go.mod h1:iRSNGgOYmiYwSCXxXaKb9HfOEj40+oTKn8pTxMlYkRM= +github.com/aws/aws-sdk-go-v2/service/kms v1.49.4 h1:2gom8MohxN0SnhHZBYAC4S8jHG+ENEnXjyJ5xKe3vLc= +github.com/aws/aws-sdk-go-v2/service/kms v1.49.4/go.mod h1:HO31s0qt0lso/ADvZQyzKs8js/ku0fMHsfyXW8OPVYc= +github.com/aws/aws-sdk-go-v2/service/signin v1.0.4 h1:HpI7aMmJ+mm1wkSHIA2t5EaFFv5EFYXePW30p1EIrbQ= +github.com/aws/aws-sdk-go-v2/service/signin v1.0.4/go.mod h1:C5RdGMYGlfM0gYq/tifqgn4EbyX99V15P2V3R+VHbQU= +github.com/aws/aws-sdk-go-v2/service/sso v1.30.7 h1:eYnlt6QxnFINKzwxP5/Ucs1vkG7VT3Iezmvfgc2waUw= +github.com/aws/aws-sdk-go-v2/service/sso v1.30.7/go.mod h1:+fWt2UHSb4kS7Pu8y+BMBvJF0EWx+4H0hzNwtDNRTrg= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.12 h1:AHDr0DaHIAo8c9t1emrzAlVDFp+iMMKnPdYy6XO4MCE= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.12/go.mod h1:GQ73XawFFiWxyWXMHWfhiomvP3tXtdNar/fi8z18sx0= +github.com/aws/aws-sdk-go-v2/service/sts v1.41.5 h1:SciGFVNZ4mHdm7gpD1dgZYnCuVdX1s+lFTg4+4DOy70= +github.com/aws/aws-sdk-go-v2/service/sts v1.41.5/go.mod h1:iW40X4QBmUxdP+fZNOpfmkdMZqsovezbAeO+Ubiv2pk= +github.com/aws/smithy-go v1.24.0 h1:LpilSUItNPFr1eY85RYgTIg5eIEPtvFbskaFcmmIUnk= +github.com/aws/smithy-go v1.24.0/go.mod h1:LEj2LM3rBRQJxPZTB4KuzZkaZYnZPnvgIhb4pu07mx0= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= @@ -56,6 +134,8 @@ github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XL github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= +github.com/clbanning/mxj/v2 v2.7.0 h1:WA/La7UGCanFe5NpHF0Q3DNtnCsVoxbPKuyBNHWRyME= +github.com/clbanning/mxj/v2 v2.7.0/go.mod h1:hNiWqW14h+kc+MdF9C6/YoRfjEJoR3ou6tn/Qo+ve2s= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= @@ -173,6 +253,8 @@ github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+ github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= github.com/googleapis/google-cloud-go-testing v0.0.0-20200911160855-bcd43fbb19e8/go.mod h1:dvDLG8qkwmyD9a/MJJN3XJcT3xFxOKAvTZGvuZmac9g= +github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= +github.com/gopherjs/gopherjs v0.0.0-20200217142428-fce0ec30dd00/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= github.com/gorilla/websocket v1.5.0 h1:PPwGk2jz7EePpoHN/+ClbZu8SPxiqlu12wZP/3sWmnc= github.com/gorilla/websocket v1.5.0/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= @@ -187,10 +269,12 @@ github.com/jmoiron/sqlx v1.4.0 h1:1PLqN7S1UYp5t4SrVVnt4nUVNemrDAtxlulVe+Qgm3o= github.com/jmoiron/sqlx v1.4.0/go.mod h1:ZrZ7UsYB/weZdl2Bxg6jCRO9c3YHl8r3ahlKmRT4JLY= github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= +github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= +github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/kisielk/sqlstruct v0.0.0-20201105191214-5f3e10d3ab46/go.mod h1:yyMNCyc/Ib3bDTKd379tNMpB/7/H5TjM2Y9QJ5THLbE= @@ -217,12 +301,15 @@ github.com/moby/spdystream v0.5.0/go.mod h1:xBAYlnt/ay+11ShkdFKNAG7LsyK/tmNBVvVO github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f h1:y5//uYreIhSUg3J1GEMiLbxo1LJaP8RfCpH6pymGZus= github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+o7JKHSa8/e818NopupXU1YMK5fe1lsApnBw= +github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= github.com/onsi/ginkgo/v2 v2.21.0 h1:7rg/4f3rB88pb5obDgNZrNHrQ4e6WpjonchcpuBRnZM= github.com/onsi/ginkgo/v2 v2.21.0/go.mod h1:7Du3c42kxCUegi0IImZ1wUQzMBVecgIHjR1C+NkhLQo= github.com/onsi/gomega v1.35.1 h1:Cwbd75ZBPxFSuZ6T+rN/WCb/gOc6YgFBXLlZLhC7Ds4= @@ -251,6 +338,9 @@ github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4= github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= +github.com/smartystreets/assertions v1.1.0/go.mod h1:tcbTF8ujkAEcZ8TElKY+i30BzYlVhC/LOxJk7iOWnoo= +github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= github.com/spf13/afero v1.9.5 h1:stMpOSZFs//0Lv29HduCmli3GUfpFoF3Y1Q/aXj/wVM= github.com/spf13/afero v1.9.5/go.mod h1:UBogFpq8E9Hx+xc5CNTTEpTnuHVmXDwZcZcE1eb/UhQ= github.com/spf13/cast v1.5.1 h1:R+kOtfhWQE6TVQzY+4D7wJLBgkdVasCEFxSUBYBYIlA= @@ -264,6 +354,7 @@ github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An github.com/spf13/viper v1.16.0 h1:rGGH0XDZhdUOryiDWjmIvUSWpbNqisK8Wk0Vyefw8hc= github.com/spf13/viper v1.16.0/go.mod h1:yg78JgCJcbrQOvV9YLXgkLaZqUidkY9K+Dd1FofRzQg= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= @@ -284,12 +375,17 @@ github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA= github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM= github.com/tidwall/pretty v1.2.0 h1:RWIZEg2iJ8/g6fDDYzMpobmaoGh5OLl4AXtGUGPcqCs= github.com/tidwall/pretty v1.2.0/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= +github.com/tjfoc/gmsm v1.3.2/go.mod h1:HaUcFuY0auTiaHB9MHFGCPx5IaLhTUd2atbCFBQXn9w= +github.com/tjfoc/gmsm v1.4.1 h1:aMe1GlZb+0bLjn+cKTPEvvn9oUEBlJitaZiiBwsbgho= +github.com/tjfoc/gmsm v1.4.1/go.mod h1:j4INPkHWMrhJb38G+J6W4Tw0AbuN8Thu3PbdVYhVcTE= github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM= github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg= github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.1.30/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= @@ -306,9 +402,20 @@ golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACk golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20191219195013-becbf705a915/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20200510223506-06a226fb4e37/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20201012173705-84dcc777aaee/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= +golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= +golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliYc= +golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4= +golang.org/x/crypto v0.18.0/go.mod h1:R0j02AL6hcrfOiy9T4ZYp/rcWeMxM3L6QYxlOuEG1mg= +golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU= +golang.org/x/crypto v0.21.0/go.mod h1:0BP7YvVV9gBbVKyeTG0Gyn+gZm94bibOW5BjDEYAOMs= +golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8= +golang.org/x/crypto v0.24.0/go.mod h1:Z1PMYSOR5nyMcyAVAIQSKCDwalqy85Aqn1x3Ws4L5DM= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -342,6 +449,11 @@ golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= +golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/mod v0.15.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= +golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= golang.org/x/mod v0.21.0 h1:vvrHzRwRfVKSiLrG+d4FMl/Qi4ukBCE6kZlTUkDYRT0= golang.org/x/mod v0.21.0/go.mod h1:6SkKJ3Xj0I0BrPOZoBy3bdMptDDU9oJrpohJ3eWZ1fY= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -370,12 +482,24 @@ golang.org/x/net v0.0.0-20200520182314-0ba52f642ac2/go.mod h1:qpuaurCH72eLCgpAm/ golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20201010224723-4f7140c49acb/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201031054903-ff519b6c9102/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201209123823-ac852fbbde11/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20201224014010-6772e930b67b/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= +golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= +golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= +golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= +golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk= +golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE= +golang.org/x/net v0.20.0/go.mod h1:z8BVo6PvndSri0LbOE3hAn0apkU+1YvI6E70E9jsnvY= +golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44= +golang.org/x/net v0.23.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg= +golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM= +golang.org/x/net v0.26.0/go.mod h1:5YKkiSynbBIh3p6iOc/vibscux0x38BZDkn8sCUPxHE= golang.org/x/net v0.33.0 h1:74SYHlV8BIgHIFC/LrYkOGIwL19eTYXQ5wc6TBuO36I= golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= @@ -399,6 +523,11 @@ golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= +golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sync v0.10.0 h1:3NQrjDixjgGwUOCaF8w2+VYHv0Ve/vGYSbdkTa98gmQ= golang.org/x/sync v0.10.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -422,6 +551,7 @@ golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200331124033-c3d80250170d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200501052902-10377860bb8e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200509044756-6aff5f38e54f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200511232937-7e40ca221e25/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200515095857-1151b9dac4a9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -436,9 +566,31 @@ golang.org/x/sys v0.0.0-20210225134936-a50acf3fe073/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423185535-09eb48e85fd7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.16.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.28.0 h1:Fksou7UEQUWlKvIdsqzJmUmCX3cZuD2+P3XyyzwMhlA= golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/telemetry v0.0.0-20240228155512-f48c80bd79b2/go.mod h1:TeRTkGYfJXctD9OcfyVLyj2J3IxLnKwHJR8f4D8a3YE= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= +golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= +golang.org/x/term v0.12.0/go.mod h1:owVbMEjm3cBLCHdkQu9b1opXd4ETQWc3BhuQGKgXgvU= +golang.org/x/term v0.13.0/go.mod h1:LTmsnFJwVN6bCy1rVCoS+qHT1HhALEFxKncY3WNNh4U= +golang.org/x/term v0.16.0/go.mod h1:yn7UURbUtPyrVJPGPq404EukNFxcm/foM+bV/bfcDsY= +golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk= +golang.org/x/term v0.18.0/go.mod h1:ILwASektA3OnRv7amZ1xhE/KTR+u50pbXfZ03+6Nx58= +golang.org/x/term v0.20.0/go.mod h1:8UkIAJTvZgivsXaD6/pH6U9ecQzZ45awqEOzuCvwpFY= +golang.org/x/term v0.21.0/go.mod h1:ooXLefLobQVslOqselCNF4SxFAaoS6KujMbsGzSDmX0= golang.org/x/term v0.27.0 h1:WP60Sv1nlK1T6SupCHbXzSaN0b9wUmsPoRS9b61A23Q= golang.org/x/term v0.27.0/go.mod h1:iMsnZpn0cago0GOrHO2+Y7u7JPn5AylBrcoWkElMTSM= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -449,6 +601,12 @@ golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= +golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= +golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= +golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= +golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI= golang.org/x/text v0.21.0 h1:zyQAAkrwaneQ066sspRyJaG9VNi/YJ1NfzcGB3hZ/qo= golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -462,6 +620,7 @@ golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3 golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= @@ -490,6 +649,7 @@ golang.org/x/tools v0.0.0-20200304193943-95d2e580d8eb/go.mod h1:o4KQGtdN14AW+yjs golang.org/x/tools v0.0.0-20200312045724-11d5b4c81c7d/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= golang.org/x/tools v0.0.0-20200331025713-a30bf2db82d4/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8= golang.org/x/tools v0.0.0-20200501065659-ab2804fb9c9d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200509030707-2212a7e161a5/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200512131952-2bc93b1c0c88/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200515010526-7d3b6ebf133d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200618134242-20370b0cb4b2/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= @@ -505,6 +665,10 @@ golang.org/x/tools v0.0.0-20210105154028-b0ab187a4818/go.mod h1:emZCQorbCU4vsT4f golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20210108195828-e2f9c7f1fc8e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= +golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= +golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= +golang.org/x/tools v0.13.0/go.mod h1:HvlwmtVNQAhOuCjW7xxvovg8wbNq7LwfXh/k7wXUl58= +golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk= golang.org/x/tools v0.26.0 h1:v/60pFQmzmT9ExmjDv2gGIfi3OqfKoEP6I5+umXlbnQ= golang.org/x/tools v0.26.0/go.mod h1:TPVVj70c7JJ3WCazhD8OdXcZg/og+b9+tH/KxylGwH0= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -605,6 +769,7 @@ google.golang.org/protobuf v1.35.1 h1:m3LfL6/Ca+fqnjnlqQXNpFPABW1UD7mjh8KO2mKFyt google.golang.org/protobuf v1.35.1/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= @@ -612,9 +777,11 @@ gopkg.in/evanphx/json-patch.v4 v4.12.0 h1:n6jtcsulIzXPJaxegRbvFNNrZDjbij7ny3gmSP gopkg.in/evanphx/json-patch.v4 v4.12.0/go.mod h1:p8EYWUEYMpynmqDbY58zCKCFZw8pRWMG4EsWvDvM72M= gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= +gopkg.in/ini.v1 v1.56.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA= gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/helm-charts/doris-operator/crds/disaggregated.cluster.doris.com_dorisdisaggregatedclusters.yaml b/helm-charts/doris-operator/crds/disaggregated.cluster.doris.com_dorisdisaggregatedclusters.yaml index 0547f69d..d3b77921 100644 --- a/helm-charts/doris-operator/crds/disaggregated.cluster.doris.com_dorisdisaggregatedclusters.yaml +++ b/helm-charts/doris-operator/crds/disaggregated.cluster.doris.com_dorisdisaggregatedclusters.yaml @@ -7047,6 +7047,137 @@ spec: type: object type: array type: object + tde: + description: TDE declares cluster-wide transparent data encryption + management. + properties: + credentialRotation: + properties: + requestId: + minLength: 1 + type: string + required: + - requestId + type: object + defaultAlgorithm: + enum: + - PLAINTEXT + - AES256 + - SM4 + type: string + managementPolicy: + enum: + - Managed + - ObserveOnly + type: string + masterKeyRotation: + properties: + checkIntervalMs: + format: int64 + minimum: 1 + type: integer + rotateIntervalMs: + format: int64 + minimum: 1 + type: integer + required: + - checkIntervalMs + - rotateIntervalMs + type: object + provider: + properties: + kms: + properties: + auth: + properties: + credentialSecretRef: + properties: + accessKeyKey: + type: string + name: + type: string + secretKeyKey: + type: string + required: + - accessKeyKey + - name + - secretKeyKey + type: object + type: + enum: + - EnvironmentSecret + - InstanceRole + type: string + required: + - type + type: object + endpoint: + type: string + keyId: + type: string + region: + type: string + required: + - auth + - endpoint + - keyId + - region + type: object + local: + properties: + secretKeyRef: + properties: + key: + type: string + name: + type: string + required: + - key + - name + type: object + required: + - secretKeyRef + type: object + type: + enum: + - Local + - AwsKms + - AliyunKms + type: string + required: + - type + type: object + recovery: + properties: + decision: + enum: + - ConfirmApplied + - ConfirmNotApplied + type: string + requestId: + minLength: 1 + type: string + rotationRequestId: + minLength: 1 + type: string + required: + - decision + - requestId + - rotationRequestId + type: object + rootKeyRotation: + properties: + requestId: + minLength: 1 + type: string + required: + - requestId + type: object + required: + - defaultAlgorithm + - managementPolicy + - provider + type: object type: object status: properties: @@ -7254,6 +7385,305 @@ spec: description: is the most recent generation observed for DorisDisaggregatedCluster format: int64 type: integer + tde: + description: TDE reports the observed encryption configuration and + operation state. + properties: + activeConfigHash: + type: string + activeFeSpecHash: + type: string + conditions: + items: + description: Condition contains details for one aspect of the + current state of this API Resource. + properties: + lastTransitionTime: + description: |- + lastTransitionTime is the last time the condition transitioned from one status to another. + This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. + format: date-time + type: string + message: + description: |- + message is a human readable message indicating details about the transition. + This may be an empty string. + maxLength: 32768 + type: string + observedGeneration: + description: |- + observedGeneration represents the .metadata.generation that the condition was set based upon. + For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date + with respect to the current state of the instance. + format: int64 + minimum: 0 + type: integer + reason: + description: |- + reason contains a programmatic identifier indicating the reason for the condition's last transition. + Producers of specific condition types may define expected values and meanings for this field, + and whether the values are considered a guaranteed API. + The value should be a CamelCase string. + This field may not be empty. + maxLength: 1024 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ + type: string + status: + description: status of the condition, one of True, False, + Unknown. + enum: + - "True" + - "False" + - Unknown + type: string + type: + description: type of condition in CamelCase or in foo.example.com/CamelCase. + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + required: + - lastTransitionTime + - message + - reason + - status + - type + type: object + type: array + current: + properties: + defaultAlgorithm: + type: string + kms: + properties: + auth: + properties: + accessKeyKey: + type: string + credentialSecretName: + type: string + credentialSecretUid: + type: string + secretKeyKey: + type: string + type: + type: string + required: + - type + type: object + endpoint: + type: string + keyId: + type: string + region: + type: string + required: + - auth + - endpoint + - keyId + - region + type: object + provider: + type: string + rootKeyRef: + properties: + key: + type: string + resolvedPath: + type: string + secretName: + type: string + secretUid: + type: string + required: + - key + - resolvedPath + - secretName + - secretUid + type: object + required: + - provider + type: object + feChecks: + items: + properties: + configConsistent: + type: boolean + materialReady: + type: boolean + podName: + type: string + podReady: + type: boolean + required: + - configConsistent + - materialReady + - podName + - podReady + type: object + type: array + metadataInitialized: + type: boolean + observedGeneration: + format: int64 + type: integer + operation: + properties: + commitJournalId: + type: string + lastTransitionTime: + format: date-time + type: string + recovery: + properties: + decision: + type: string + requestId: + type: string + resolvedAt: + format: date-time + type: string + required: + - decision + - requestId + - resolvedAt + type: object + requestId: + type: string + source: + properties: + defaultAlgorithm: + type: string + kms: + properties: + auth: + properties: + accessKeyKey: + type: string + credentialSecretName: + type: string + credentialSecretUid: + type: string + secretKeyKey: + type: string + type: + type: string + required: + - type + type: object + endpoint: + type: string + keyId: + type: string + region: + type: string + required: + - auth + - endpoint + - keyId + - region + type: object + provider: + type: string + rootKeyRef: + properties: + key: + type: string + resolvedPath: + type: string + secretName: + type: string + secretUid: + type: string + required: + - key + - resolvedPath + - secretName + - secretUid + type: object + required: + - provider + type: object + specGeneration: + format: int64 + type: integer + sqlState: + type: string + stage: + type: string + startedAt: + format: date-time + type: string + target: + properties: + defaultAlgorithm: + type: string + kms: + properties: + auth: + properties: + accessKeyKey: + type: string + credentialSecretName: + type: string + credentialSecretUid: + type: string + secretKeyKey: + type: string + type: + type: string + required: + - type + type: object + endpoint: + type: string + keyId: + type: string + region: + type: string + required: + - auth + - endpoint + - keyId + - region + type: object + provider: + type: string + rootKeyRef: + properties: + key: + type: string + resolvedPath: + type: string + secretName: + type: string + secretUid: + type: string + required: + - key + - resolvedPath + - secretName + - secretUid + type: object + required: + - provider + type: object + type: + type: string + required: + - lastTransitionTime + - specGeneration + - stage + - startedAt + - type + type: object + state: + type: string + usedRequestIds: + items: + type: string + type: array + type: object type: object type: object served: true diff --git a/helm-charts/doris-operator/crds/doris.apache.com_dorisclusters.yaml b/helm-charts/doris-operator/crds/doris.apache.com_dorisclusters.yaml index eb9b96e6..87ae4bab 100644 --- a/helm-charts/doris-operator/crds/doris.apache.com_dorisclusters.yaml +++ b/helm-charts/doris-operator/crds/doris.apache.com_dorisclusters.yaml @@ -9263,6 +9263,137 @@ spec: type: array type: object type: array + tde: + description: TDE declares cluster-wide transparent data encryption + management. + properties: + credentialRotation: + properties: + requestId: + minLength: 1 + type: string + required: + - requestId + type: object + defaultAlgorithm: + enum: + - PLAINTEXT + - AES256 + - SM4 + type: string + managementPolicy: + enum: + - Managed + - ObserveOnly + type: string + masterKeyRotation: + properties: + checkIntervalMs: + format: int64 + minimum: 1 + type: integer + rotateIntervalMs: + format: int64 + minimum: 1 + type: integer + required: + - checkIntervalMs + - rotateIntervalMs + type: object + provider: + properties: + kms: + properties: + auth: + properties: + credentialSecretRef: + properties: + accessKeyKey: + type: string + name: + type: string + secretKeyKey: + type: string + required: + - accessKeyKey + - name + - secretKeyKey + type: object + type: + enum: + - EnvironmentSecret + - InstanceRole + type: string + required: + - type + type: object + endpoint: + type: string + keyId: + type: string + region: + type: string + required: + - auth + - endpoint + - keyId + - region + type: object + local: + properties: + secretKeyRef: + properties: + key: + type: string + name: + type: string + required: + - key + - name + type: object + required: + - secretKeyRef + type: object + type: + enum: + - Local + - AwsKms + - AliyunKms + type: string + required: + - type + type: object + recovery: + properties: + decision: + enum: + - ConfirmApplied + - ConfirmNotApplied + type: string + requestId: + minLength: 1 + type: string + rotationRequestId: + minLength: 1 + type: string + required: + - decision + - requestId + - rotationRequestId + type: object + rootKeyRotation: + properties: + requestId: + minLength: 1 + type: string + required: + - requestId + type: object + required: + - defaultAlgorithm + - managementPolicy + - provider + type: object type: object status: description: DorisClusterStatus defines the observed state of DorisCluster @@ -9489,6 +9620,305 @@ spec: required: - componentCondition type: object + tde: + description: TDE reports the observed encryption configuration and + operation state. + properties: + activeConfigHash: + type: string + activeFeSpecHash: + type: string + conditions: + items: + description: Condition contains details for one aspect of the + current state of this API Resource. + properties: + lastTransitionTime: + description: |- + lastTransitionTime is the last time the condition transitioned from one status to another. + This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. + format: date-time + type: string + message: + description: |- + message is a human readable message indicating details about the transition. + This may be an empty string. + maxLength: 32768 + type: string + observedGeneration: + description: |- + observedGeneration represents the .metadata.generation that the condition was set based upon. + For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date + with respect to the current state of the instance. + format: int64 + minimum: 0 + type: integer + reason: + description: |- + reason contains a programmatic identifier indicating the reason for the condition's last transition. + Producers of specific condition types may define expected values and meanings for this field, + and whether the values are considered a guaranteed API. + The value should be a CamelCase string. + This field may not be empty. + maxLength: 1024 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ + type: string + status: + description: status of the condition, one of True, False, + Unknown. + enum: + - "True" + - "False" + - Unknown + type: string + type: + description: type of condition in CamelCase or in foo.example.com/CamelCase. + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + required: + - lastTransitionTime + - message + - reason + - status + - type + type: object + type: array + current: + properties: + defaultAlgorithm: + type: string + kms: + properties: + auth: + properties: + accessKeyKey: + type: string + credentialSecretName: + type: string + credentialSecretUid: + type: string + secretKeyKey: + type: string + type: + type: string + required: + - type + type: object + endpoint: + type: string + keyId: + type: string + region: + type: string + required: + - auth + - endpoint + - keyId + - region + type: object + provider: + type: string + rootKeyRef: + properties: + key: + type: string + resolvedPath: + type: string + secretName: + type: string + secretUid: + type: string + required: + - key + - resolvedPath + - secretName + - secretUid + type: object + required: + - provider + type: object + feChecks: + items: + properties: + configConsistent: + type: boolean + materialReady: + type: boolean + podName: + type: string + podReady: + type: boolean + required: + - configConsistent + - materialReady + - podName + - podReady + type: object + type: array + metadataInitialized: + type: boolean + observedGeneration: + format: int64 + type: integer + operation: + properties: + commitJournalId: + type: string + lastTransitionTime: + format: date-time + type: string + recovery: + properties: + decision: + type: string + requestId: + type: string + resolvedAt: + format: date-time + type: string + required: + - decision + - requestId + - resolvedAt + type: object + requestId: + type: string + source: + properties: + defaultAlgorithm: + type: string + kms: + properties: + auth: + properties: + accessKeyKey: + type: string + credentialSecretName: + type: string + credentialSecretUid: + type: string + secretKeyKey: + type: string + type: + type: string + required: + - type + type: object + endpoint: + type: string + keyId: + type: string + region: + type: string + required: + - auth + - endpoint + - keyId + - region + type: object + provider: + type: string + rootKeyRef: + properties: + key: + type: string + resolvedPath: + type: string + secretName: + type: string + secretUid: + type: string + required: + - key + - resolvedPath + - secretName + - secretUid + type: object + required: + - provider + type: object + specGeneration: + format: int64 + type: integer + sqlState: + type: string + stage: + type: string + startedAt: + format: date-time + type: string + target: + properties: + defaultAlgorithm: + type: string + kms: + properties: + auth: + properties: + accessKeyKey: + type: string + credentialSecretName: + type: string + credentialSecretUid: + type: string + secretKeyKey: + type: string + type: + type: string + required: + - type + type: object + endpoint: + type: string + keyId: + type: string + region: + type: string + required: + - auth + - endpoint + - keyId + - region + type: object + provider: + type: string + rootKeyRef: + properties: + key: + type: string + resolvedPath: + type: string + secretName: + type: string + secretUid: + type: string + required: + - key + - resolvedPath + - secretName + - secretUid + type: object + required: + - provider + type: object + type: + type: string + required: + - lastTransitionTime + - specGeneration + - stage + - startedAt + - type + type: object + state: + type: string + usedRequestIds: + items: + type: string + type: array + type: object type: object type: object served: true diff --git a/helm-charts/doris-operator/templates/clusterrole.yaml b/helm-charts/doris-operator/templates/clusterrole.yaml index fe9b18db..4525eff2 100644 --- a/helm-charts/doris-operator/templates/clusterrole.yaml +++ b/helm-charts/doris-operator/templates/clusterrole.yaml @@ -56,8 +56,11 @@ rules: resources: - configmaps verbs: + - create - get - list + - patch + - update - watch - apiGroups: - "" diff --git a/pkg/controller/controllers_utils.go b/pkg/controller/controllers_utils.go index ac5bf4c4..8270a885 100644 --- a/pkg/controller/controllers_utils.go +++ b/pkg/controller/controllers_utils.go @@ -32,7 +32,8 @@ func inconsistentStatus(status *v1.DorisClusterStatus, dcr *v1.DorisCluster) boo return inconsistentFEStatus(status.FEStatus, dcr.Status.FEStatus) || inconsistentBEStatus(status.BEStatus, dcr.Status.BEStatus) || inconsistentCnStatus(status.CnStatus, dcr.Status.CnStatus) || - inconsistentBrokerStatus(status.BrokerStatus, dcr.Status.BrokerStatus) + inconsistentBrokerStatus(status.BrokerStatus, dcr.Status.BrokerStatus) || + !reflect.DeepEqual(status.TDE, dcr.Status.TDE) } func inconsistentCnStatus(eStatus *v1.CnStatus, nStatus *v1.CnStatus) bool { diff --git a/pkg/controller/controllers_utils_test.go b/pkg/controller/controllers_utils_test.go new file mode 100644 index 00000000..c187d228 --- /dev/null +++ b/pkg/controller/controllers_utils_test.go @@ -0,0 +1,36 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package controller + +import ( + "testing" + + dorisv1 "github.com/apache/doris-operator/api/doris/v1" + tdev1 "github.com/apache/doris-operator/api/tde" +) + +func TestInconsistentStatusIncludesTDE(t *testing.T) { + desired := dorisv1.DorisClusterStatus{TDE: &tdev1.TDEStatus{State: tdev1.StateReconciling}} + stored := &dorisv1.DorisCluster{Status: dorisv1.DorisClusterStatus{TDE: &tdev1.TDEStatus{State: tdev1.StateActive}}} + if !inconsistentStatus(&desired, stored) { + t.Fatal("TDE-only status change was ignored") + } + stored.Status.TDE = desired.TDE.DeepCopy() + if inconsistentStatus(&desired, stored) { + t.Fatal("equal TDE status was reported as inconsistent") + } +} diff --git a/pkg/controller/disaggregated_cluster_controller.go b/pkg/controller/disaggregated_cluster_controller.go index 69b753d6..e2f3865d 100644 --- a/pkg/controller/disaggregated_cluster_controller.go +++ b/pkg/controller/disaggregated_cluster_controller.go @@ -30,6 +30,7 @@ import ( dcgs "github.com/apache/doris-operator/pkg/controller/sub_controller/disaggregated_cluster/computegroups" dfe "github.com/apache/doris-operator/pkg/controller/sub_controller/disaggregated_cluster/disaggregated_fe" "github.com/apache/doris-operator/pkg/controller/sub_controller/disaggregated_cluster/metaservice" + "github.com/apache/doris-operator/pkg/tde" appv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" @@ -94,9 +95,25 @@ func (dc *DisaggregatedClusterReconciler) SetupWithManager(mgr ctrl.Manager) err builder := dc.resourceBuilder(ctrl.NewControllerManagedBy(mgr)) builder = dc.watchPodBuilder(builder) builder = dc.watchFDBConfigMapBuilder(builder) + builder = builder.Watches(&corev1.Secret{}, handler.EnqueueRequestsFromMapFunc(dc.mapSecretToDDCs)) return builder.Complete(dc) } +func (dc *DisaggregatedClusterReconciler) mapSecretToDDCs(ctx context.Context, object client.Object) []reconcile.Request { + var clusters dv1.DorisDisaggregatedClusterList + if err := dc.List(ctx, &clusters, client.InNamespace(object.GetNamespace())); err != nil { + return nil + } + requests := make([]reconcile.Request, 0) + for i := range clusters.Items { + cluster := &clusters.Items[i] + if tdeReferencesSecret(cluster.Spec.TDE, cluster.Status.TDE, object.GetName()) { + requests = append(requests, reconcile.Request{NamespacedName: client.ObjectKeyFromObject(cluster)}) + } + } + return requests +} + func (dc *DisaggregatedClusterReconciler) watchPodBuilder(builder *ctrl.Builder) *ctrl.Builder { mapFn := handler.EnqueueRequestsFromMapFunc( func(ctx context.Context, a client.Object) []reconcile.Request { @@ -220,6 +237,13 @@ func (dc *DisaggregatedClusterReconciler) Reconcile(ctx context.Context, req rec klog.Warningf("disaggreatedClusterReconciler not find resource DorisDisaggregatedCluster namespaceName %s", req.NamespacedName) return ctrl.Result{}, nil } + if ddc.DeletionTimestamp.IsZero() { + if blocked, gateErr := tde.EnforceDDCFELifecycleGate(ctx, dc.Client, &ddc); gateErr != nil { + return ctrl.Result{}, gateErr + } else if blocked { + return ctrl.Result{RequeueAfter: 10 * time.Second}, nil + } + } hv := hash.HashObject(ddc.Spec) var res ctrl.Result @@ -250,10 +274,21 @@ func (dc *DisaggregatedClusterReconciler) Reconcile(ctx context.Context, req rec if stsRes, stsErr = dc.reorganizeStatus(&ddc); stsErr != nil { return stsRes, stsErr } + tdeRes, tdeErr := tde.ReconcileDDC(ctx, dc.Client, &ddc) + if tdeErr != nil { + return tdeRes, tdeErr + } + if !tdeRes.IsZero() { + stsRes = tdeRes + } //update cr or status - if stsRes, stsErr = dc.updateObjectORStatus(ctx, &ddc, hv); stsErr != nil { - return stsRes, stsErr + updateRes, updateErr := dc.updateObjectORStatus(ctx, &ddc, hv) + if updateErr != nil { + return updateRes, updateErr + } + if !updateRes.IsZero() { + stsRes = updateRes } return stsRes, stsErr diff --git a/pkg/controller/doriscluster_controller.go b/pkg/controller/doriscluster_controller.go index a5961e04..1ee3fe25 100644 --- a/pkg/controller/doriscluster_controller.go +++ b/pkg/controller/doriscluster_controller.go @@ -43,6 +43,7 @@ import ( bk "github.com/apache/doris-operator/pkg/controller/sub_controller/broker" cn "github.com/apache/doris-operator/pkg/controller/sub_controller/cn" "github.com/apache/doris-operator/pkg/controller/sub_controller/fe" + "github.com/apache/doris-operator/pkg/tde" appv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/types" @@ -101,7 +102,7 @@ var ( //+kubebuilder:rbac:groups=autoscaling,resources=horizontalpodautoscalers,verbs=get;list;watch;create;update;patch;delete //+kubebuilder:rbac:groups=core,resources=services,verbs=get;list;watch;create;update;patch;delete //+kubebuilder:rbac:groups="core",resources=endpoints,verbs=get;watch;list -//+kubebuilder:rbac:groups=core,resources=configmaps,verbs=get;list;watch +//+kubebuilder:rbac:groups=core,resources=configmaps,verbs=get;list;watch;create;update;patch //+kubebuilder:rbac:groups=core,resources=persistentvolumeclaims,verbs=get;list;update;watch //+kubebuilder:rbac:groups=admissionregistration,resources=validatingwebhookconfigurations,verbs=get;list;update;watch @@ -134,6 +135,11 @@ func (r *DorisClusterReconciler) Reconcile(ctx context.Context, req ctrl.Request r.resourceClean(ctx, dcr) return ctrl.Result{}, nil } + if blocked, gateErr := tde.EnforceDCRFELifecycleGate(ctx, r.Client, dcr); gateErr != nil { + return requeueIfError(gateErr) + } else if blocked { + return ctrl.Result{RequeueAfter: 10 * time.Second}, nil + } if dcr.Spec.EnableRestartWhenConfigChange { coreConfigMaps := resource.GetDorisCoreConfigMapNames(dcr) @@ -151,6 +157,10 @@ func (r *DorisClusterReconciler) Reconcile(ctx context.Context, req ctrl.Request return requeueIfError(err) } } + tdeResult, err := tde.ReconcileDCR(ctx, r.Client, dcr) + if err != nil { + return requeueIfError(err) + } //generate the dcr status. r.clearNoEffectResources(ctx, dcr) @@ -169,7 +179,14 @@ func (r *DorisClusterReconciler) Reconcile(ctx context.Context, req ctrl.Request return requeueIfError(err) } - return r.updateDorisClusterStatus(ctx, dcr) + statusResult, err := r.updateDorisClusterStatus(ctx, dcr) + if err != nil { + return statusResult, err + } + if !tdeResult.IsZero() { + return tdeResult, nil + } + return statusResult, nil } // if cluster spec be reverted, doris operator should revert to old. @@ -352,9 +369,25 @@ func (r *DorisClusterReconciler) SetupWithManager(mgr ctrl.Manager) error { builder := r.resourceBuilder(ctrl.NewControllerManagedBy(mgr)) builder = r.watchPodBuilder(builder) builder = r.watchConfigMapBuilder(builder) + builder = builder.Watches(&corev1.Secret{}, handler.EnqueueRequestsFromMapFunc(r.mapSecretToDCRs)) return builder.Complete(r) } +func (r *DorisClusterReconciler) mapSecretToDCRs(ctx context.Context, object client.Object) []reconcile.Request { + var clusters dorisv1.DorisClusterList + if err := r.List(ctx, &clusters, client.InNamespace(object.GetNamespace())); err != nil { + return nil + } + requests := make([]reconcile.Request, 0) + for i := range clusters.Items { + cluster := &clusters.Items[i] + if tdeReferencesSecret(cluster.Spec.TDE, cluster.Status.TDE, object.GetName()) { + requests = append(requests, reconcile.Request{NamespacedName: client.ObjectKeyFromObject(cluster)}) + } + } + return requests +} + // Init initial the DorisClusterReconciler for reconcile. func (r *DorisClusterReconciler) Init(mgr ctrl.Manager, options *Options) { subcs := make(map[string]sub_controller.SubController) diff --git a/pkg/controller/sub_controller/disaggregated_cluster/disaggregated_fe/controller.go b/pkg/controller/sub_controller/disaggregated_cluster/disaggregated_fe/controller.go index f9c60d44..c06ba0ee 100644 --- a/pkg/controller/sub_controller/disaggregated_cluster/disaggregated_fe/controller.go +++ b/pkg/controller/sub_controller/disaggregated_cluster/disaggregated_fe/controller.go @@ -29,6 +29,7 @@ import ( "github.com/apache/doris-operator/pkg/common/utils/mysql" "github.com/apache/doris-operator/pkg/common/utils/resource" sc "github.com/apache/doris-operator/pkg/controller/sub_controller" + "github.com/apache/doris-operator/pkg/tde" appv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" @@ -81,12 +82,16 @@ func (dfc *DisaggregatedFEController) Sync(ctx context.Context, obj client.Objec klog.Errorf("disaggregatedFEController Sync disaggregatedDorisCluster namespace=%s,name=%s ,The number of disaggregated fe ElectionNumber(%d) is large than Replicas(%d), Replicas has been corrected to the correct minimum value", ddc.Namespace, ddc.Name, electionNumber, *(ddc.Spec.FeSpec.Replicas)) ddc.Spec.FeSpec.Replicas = &electionNumber } + workDDC, err := tde.PrepareDDCConfig(ctx, dfc.K8sclient, ddc) + if err != nil { + return err + } - confMap := dfc.GetConfigValuesFromConfigMaps(ddc.Namespace, resource.FE_RESOLVEKEY, ddc.Spec.FeSpec.ConfigMaps) - svcInternal := dfc.newInternalService(ddc, confMap) - svc := dfc.newService(ddc, confMap) + confMap := dfc.GetConfigValuesFromConfigMaps(ddc.Namespace, resource.FE_RESOLVEKEY, workDDC.Spec.FeSpec.ConfigMaps) + svcInternal := dfc.newInternalService(workDDC, confMap) + svc := dfc.newService(workDDC, confMap) - st := dfc.NewStatefulset(ddc, confMap) + st := dfc.NewStatefulset(workDDC, confMap) //initial fe status on start. in resource process step, may be use the status record the process. dfc.initialFEStatus(ddc) diff --git a/pkg/controller/sub_controller/disaggregated_cluster/disaggregated_fe/statefulset.go b/pkg/controller/sub_controller/disaggregated_cluster/disaggregated_fe/statefulset.go index b97cf44e..e19aa7ef 100644 --- a/pkg/controller/sub_controller/disaggregated_cluster/disaggregated_fe/statefulset.go +++ b/pkg/controller/sub_controller/disaggregated_cluster/disaggregated_fe/statefulset.go @@ -24,6 +24,7 @@ import ( v1 "github.com/apache/doris-operator/api/disaggregated/v1" "github.com/apache/doris-operator/pkg/common/utils/resource" sub "github.com/apache/doris-operator/pkg/controller/sub_controller" + "github.com/apache/doris-operator/pkg/tde" appv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -118,6 +119,7 @@ func (dfc *DisaggregatedFEController) NewPodTemplateSpec(ddc *v1.DorisDisaggrega } pts.Spec.Affinity = dfc.ConstructDefaultAffinity(v1.DorisDisaggregatedClusterName, labels[v1.DorisDisaggregatedClusterName], ddc.Spec.FeSpec.Affinity) + tde.ApplyPodOverlay(&pts, resource.DISAGGREGATED_FE_MAIN_CONTAINER_NAME, ddc.Spec.TDE, ddc.Status.TDE) return pts } diff --git a/pkg/controller/sub_controller/fe/controller.go b/pkg/controller/sub_controller/fe/controller.go index 6b10d8b0..97211f91 100644 --- a/pkg/controller/sub_controller/fe/controller.go +++ b/pkg/controller/sub_controller/fe/controller.go @@ -23,6 +23,7 @@ import ( "github.com/apache/doris-operator/pkg/common/utils/k8s" "github.com/apache/doris-operator/pkg/common/utils/resource" "github.com/apache/doris-operator/pkg/controller/sub_controller" + "github.com/apache/doris-operator/pkg/tde" appv1 "k8s.io/api/apps/v1" "k8s.io/client-go/tools/record" "k8s.io/klog/v2" @@ -88,12 +89,16 @@ func (fc *Controller) Sync(ctx context.Context, cluster *v1.DorisCluster) error oldStatus = *(cluster.Status.FEStatus.DeepCopy()) } fc.InitStatus(cluster, v1.Component_FE) + workCluster, err := tde.PrepareDCRConfig(ctx, fc.K8sclient, cluster) + if err != nil { + return err + } if cluster.Spec.EnableRestartWhenConfigChange { fc.CompareConfigmapAndTriggerRestart(cluster, oldStatus, v1.Component_FE) } - feSpec := cluster.Spec.FeSpec + feSpec := workCluster.Spec.FeSpec //get the fe configMap for resolve ports. config, err := fc.GetConfig(ctx, &feSpec.BaseSpec.ConfigMapInfo, cluster.Namespace, v1.Component_FE) if err != nil { @@ -106,9 +111,9 @@ func (fc *Controller) Sync(ctx context.Context, cluster *v1.DorisCluster) error fc.CheckSharedPVC(ctx, cluster) //generate new fe service. - svc := resource.BuildExternalService(cluster, v1.Component_FE, config) + svc := resource.BuildExternalService(workCluster, v1.Component_FE, config) //create or update fe external and domain search service, update the status of fe on src. - internalService := resource.BuildInternalService(cluster, v1.Component_FE, config) + internalService := resource.BuildInternalService(workCluster, v1.Component_FE, config) if err := k8s.ApplyService(ctx, fc.K8sclient, &internalService, resource.ServiceDeepEqual); err != nil { klog.Errorf("fe controller sync apply internalService name=%s, namespace=%s, clusterName=%s failed.message=%s.", internalService.Name, internalService.Namespace, cluster.Name, err.Error()) @@ -129,7 +134,7 @@ func (fc *Controller) Sync(ctx context.Context, cluster *v1.DorisCluster) error return err } - st := fc.buildFEStatefulSet(cluster, config) + st := fc.buildFEStatefulSet(workCluster, config) if err = k8s.ApplyStatefulSet(ctx, fc.K8sclient, &st, func(new *appv1.StatefulSet, old *appv1.StatefulSet) bool { fc.RestrictConditionsEqual(new, old) return resource.StatefulSetDeepEqual(new, old, false) diff --git a/pkg/controller/sub_controller/fe/pod.go b/pkg/controller/sub_controller/fe/pod.go index 541a34aa..6ecf4132 100644 --- a/pkg/controller/sub_controller/fe/pod.go +++ b/pkg/controller/sub_controller/fe/pod.go @@ -22,6 +22,7 @@ import ( v1 "github.com/apache/doris-operator/api/doris/v1" "github.com/apache/doris-operator/pkg/common/utils/resource" + "github.com/apache/doris-operator/pkg/tde" corev1 "k8s.io/api/core/v1" ) @@ -34,6 +35,7 @@ func (fc *Controller) buildFEPodTemplateSpec(dcr *v1.DorisCluster, config map[st containers = append(containers, feContainer) containers = resource.ApplySecurityContext(containers, dcr.Spec.FeSpec.ContainerSecurityContext) podTemplateSpec.Spec.Containers = containers + tde.ApplyPodOverlay(&podTemplateSpec, "fe", dcr.Spec.TDE, dcr.Status.TDE) return podTemplateSpec } diff --git a/pkg/controller/tde_watch.go b/pkg/controller/tde_watch.go new file mode 100644 index 00000000..cf5a2a8f --- /dev/null +++ b/pkg/controller/tde_watch.go @@ -0,0 +1,48 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 + +package controller + +import tdev1 "github.com/apache/doris-operator/api/tde" + +func tdeReferencesSecret(config *tdev1.TDEConfig, status *tdev1.TDEStatus, name string) bool { + if config == nil { + return statusReferencesSecret(status, name) + } + if config.Provider.Local != nil && config.Provider.Local.SecretKeyRef.Name == name { + return true + } + if config.Provider.Kms != nil && config.Provider.Kms.Auth.CredentialSecretRef != nil && config.Provider.Kms.Auth.CredentialSecretRef.Name == name { + return true + } + return statusReferencesSecret(status, name) +} + +func statusReferencesSecret(status *tdev1.TDEStatus, name string) bool { + if status == nil { + return false + } + providers := []*tdev1.ProviderStatus{status.Current} + if status.Operation != nil { + providers = append(providers, status.Operation.Source, status.Operation.Target) + } + for _, provider := range providers { + if provider == nil { + continue + } + if provider.RootKeyRef != nil && provider.RootKeyRef.SecretName == name { + return true + } + if provider.Kms != nil && provider.Kms.Auth.CredentialSecretName == name { + return true + } + } + return false +} diff --git a/pkg/tde/config.go b/pkg/tde/config.go new file mode 100644 index 00000000..21ff1dd9 --- /dev/null +++ b/pkg/tde/config.go @@ -0,0 +1,248 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 + +package tde + +import ( + "context" + "fmt" + "reflect" + "sort" + "strings" + + ddcv1 "github.com/apache/doris-operator/api/disaggregated/v1" + dorisv1 "github.com/apache/doris-operator/api/doris/v1" + tdev1 "github.com/apache/doris-operator/api/tde" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +const ( + configMountPath = "/etc/doris" + feConfigKey = "fe.conf" +) + +var managedConfigKeys = map[string]struct{}{ + "doris_tde_key_provider": {}, + "doris_tde_root_key_file": {}, + "doris_tde_key_id": {}, + "doris_tde_key_endpoint": {}, + "doris_tde_key_region": {}, + "doris_tde_algorithm": {}, + "doris_tde_rotate_master_key_interval_ms": {}, + "doris_tde_check_rotate_master_key_interval_ms": {}, +} + +func PrepareDCRConfig(ctx context.Context, c client.Client, dcr *dorisv1.DorisCluster) (*dorisv1.DorisCluster, error) { + if dcr.Spec.TDE == nil || dcr.Spec.TDE.ManagementPolicy != tdev1.ManagementPolicyManaged || dcr.Spec.FeSpec == nil { + return dcr, nil + } + working := dcr.DeepCopy() + baseName, sourceIndex, err := dcrCoreConfig(working.Spec.FeSpec.ConfigMapInfo) + if err != nil { + return nil, err + } + effectiveName := effectiveConfigMapName(dcr.Name, "dcr") + if err := ensureEffectiveConfigMap(ctx, c, dcr.Namespace, baseName, effectiveName, + metav1.NewControllerRef(dcr, schema.GroupVersionKind{Group: dorisv1.GroupVersion.Group, Version: dorisv1.GroupVersion.Version, Kind: "DorisCluster"}), + working.Spec.TDE, working.Status.TDE); err != nil { + return nil, err + } + if sourceIndex < 0 { + working.Spec.FeSpec.ConfigMapInfo.ConfigMapName = effectiveName + } else { + working.Spec.FeSpec.ConfigMapInfo.ConfigMaps[sourceIndex].ConfigMapName = effectiveName + } + return working, nil +} + +func PrepareDDCConfig(ctx context.Context, c client.Client, ddc *ddcv1.DorisDisaggregatedCluster) (*ddcv1.DorisDisaggregatedCluster, error) { + if ddc.Spec.TDE == nil || ddc.Spec.TDE.ManagementPolicy != tdev1.ManagementPolicyManaged { + return ddc, nil + } + working := ddc.DeepCopy() + baseName, index, err := ddcCoreConfig(working.Spec.FeSpec.ConfigMaps) + if err != nil { + return nil, err + } + effectiveName := effectiveConfigMapName(ddc.Name, "ddc") + if err := ensureEffectiveConfigMap(ctx, c, ddc.Namespace, baseName, effectiveName, + metav1.NewControllerRef(ddc, schema.GroupVersionKind{Group: ddcv1.GroupVersion.Group, Version: ddcv1.GroupVersion.Version, Kind: "DorisDisaggregatedCluster"}), + working.Spec.TDE, working.Status.TDE); err != nil { + return nil, err + } + working.Spec.FeSpec.ConfigMaps[index].Name = effectiveName + return working, nil +} + +func ensureEffectiveConfigMap(ctx context.Context, c client.Client, namespace, baseName, effectiveName string, + owner *metav1.OwnerReference, spec *tdev1.TDEConfig, status *tdev1.TDEStatus) error { + var base corev1.ConfigMap + if err := c.Get(ctx, types.NamespacedName{Namespace: namespace, Name: baseName}, &base); err != nil { + return fmt.Errorf("get FE configmap %s: %w", baseName, err) + } + baseConfig, ok := base.Data[feConfigKey] + if !ok { + return fmt.Errorf("FE configmap %s does not contain %s", baseName, feConfigKey) + } + provider := providerForRuntime(spec, status) + data := make(map[string]string, len(base.Data)) + for key, value := range base.Data { + data[key] = value + } + data[feConfigKey] = mergeFEConfig(baseConfig, provider, spec) + binaryData := make(map[string][]byte, len(base.BinaryData)) + for key, value := range base.BinaryData { + binaryData[key] = append([]byte(nil), value...) + } + + var current corev1.ConfigMap + key := types.NamespacedName{Namespace: namespace, Name: effectiveName} + err := c.Get(ctx, key, ¤t) + if apierrors.IsNotFound(err) { + current = corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: effectiveName, Namespace: namespace, OwnerReferences: []metav1.OwnerReference{*owner}}} + current.Data, current.BinaryData = data, binaryData + return c.Create(ctx, ¤t) + } + if err != nil { + return err + } + for i := range current.OwnerReferences { + if current.OwnerReferences[i].Controller != nil && *current.OwnerReferences[i].Controller && current.OwnerReferences[i].UID != owner.UID { + return fmt.Errorf("effective FE configmap %s is controlled by another object", effectiveName) + } + } + if reflect.DeepEqual(current.Data, data) && reflect.DeepEqual(current.BinaryData, binaryData) && len(current.OwnerReferences) > 0 { + return nil + } + current.Data, current.BinaryData = data, binaryData + if len(current.OwnerReferences) == 0 { + current.OwnerReferences = []metav1.OwnerReference{*owner} + } + return c.Update(ctx, ¤t) +} + +func dcrCoreConfig(info dorisv1.ConfigMapInfo) (string, int, error) { + if info.ConfigMapName != "" { + return info.ConfigMapName, -1, nil + } + for i := range info.ConfigMaps { + if info.ConfigMaps[i].MountPath == "" || info.ConfigMaps[i].MountPath == configMountPath { + return info.ConfigMaps[i].ConfigMapName, i, nil + } + } + return "", -1, fmt.Errorf("FE configmap mounted at %s is not configured", configMountPath) +} + +func ddcCoreConfig(configMaps []ddcv1.ConfigMap) (string, int, error) { + for i := range configMaps { + if configMaps[i].MountPath == "" || configMaps[i].MountPath == configMountPath { + return configMaps[i].Name, i, nil + } + } + return "", -1, fmt.Errorf("FE configmap mounted at %s is not configured", configMountPath) +} + +func effectiveConfigMapName(clusterName, clusterKind string) string { + suffix := "-fe-tde-" + clusterKind + if len(clusterName)+len(suffix) <= 63 { + return clusterName + suffix + } + return strings.TrimRight(clusterName[:63-len(suffix)], "-") + suffix +} + +func providerForRuntime(spec *tdev1.TDEConfig, status *tdev1.TDEStatus) *tdev1.ProviderStatus { + if status != nil && status.Current == nil && status.Operation != nil && status.Operation.Type == tdev1.OperationEnableTDE && + status.Operation.Target != nil { + return status.Operation.Target.DeepCopy() + } + if status != nil && status.Operation != nil && status.Operation.Type == tdev1.OperationRotateKmsCredential && + status.Operation.Stage != tdev1.StageCompleted && status.Operation.Target != nil { + return status.Operation.Target.DeepCopy() + } + if status != nil && status.Operation != nil && status.Operation.Type == tdev1.OperationRotateRootKey { + if status.Operation.SQLState == tdev1.SQLApplied && status.Operation.Stage != tdev1.StageWaitingForFEReplay && status.Operation.Target != nil { + return status.Operation.Target.DeepCopy() + } + if status.Operation.Source != nil { + return status.Operation.Source.DeepCopy() + } + } + if status != nil && status.Current != nil { + if !sameRootSpec(status.Current, spec) { + return status.Current.DeepCopy() + } + current := status.Current.DeepCopy() + current.DefaultAlgorithm = spec.DefaultAlgorithm + return current + } + return providerStatusFromSpec(spec, nil) +} + +func sameRootSpec(current *tdev1.ProviderStatus, spec *tdev1.TDEConfig) bool { + if current == nil || spec == nil || current.Provider != spec.Provider.Type { + return false + } + if current.Provider == tdev1.ProviderLocal { + return current.RootKeyRef != nil && spec.Provider.Local != nil && + current.RootKeyRef.SecretName == spec.Provider.Local.SecretKeyRef.Name && current.RootKeyRef.Key == spec.Provider.Local.SecretKeyRef.Key + } + return current.Kms != nil && spec.Provider.Kms != nil && current.Kms.KeyID == spec.Provider.Kms.KeyID && + current.Kms.Endpoint == spec.Provider.Kms.Endpoint && current.Kms.Region == spec.Provider.Kms.Region +} + +func mergeFEConfig(base string, provider *tdev1.ProviderStatus, spec *tdev1.TDEConfig) string { + lines := strings.Split(base, "\n") + out := make([]string, 0, len(lines)+8) + for _, line := range lines { + key := strings.TrimSpace(strings.SplitN(line, "=", 2)[0]) + if _, managed := managedConfigKeys[key]; !managed { + out = append(out, line) + } + } + values := map[string]string{ + "doris_tde_algorithm": provider.DefaultAlgorithm, + } + switch provider.Provider { + case tdev1.ProviderLocal: + values["doris_tde_key_provider"] = "local" + values["doris_tde_root_key_file"] = provider.RootKeyRef.ResolvedPath + case tdev1.ProviderAwsKms: + values["doris_tde_key_provider"] = "aws_kms" + case tdev1.ProviderAliyunKms: + values["doris_tde_key_provider"] = "aliyun_kms" + } + if provider.Kms != nil { + values["doris_tde_key_id"] = provider.Kms.KeyID + values["doris_tde_key_endpoint"] = provider.Kms.Endpoint + values["doris_tde_key_region"] = provider.Kms.Region + } + if spec.MasterKeyRotation != nil { + values["doris_tde_rotate_master_key_interval_ms"] = fmt.Sprintf("%d", spec.MasterKeyRotation.RotateIntervalMs) + values["doris_tde_check_rotate_master_key_interval_ms"] = fmt.Sprintf("%d", spec.MasterKeyRotation.CheckIntervalMs) + } + keys := make([]string, 0, len(values)) + for key := range values { + keys = append(keys, key) + } + sort.Strings(keys) + if len(out) > 0 && out[len(out)-1] != "" { + out = append(out, "") + } + out = append(out, "# Managed by doris-operator from spec.tde.") + for _, key := range keys { + out = append(out, key+"="+values[key]) + } + return strings.Join(out, "\n") +} diff --git a/pkg/tde/kms.go b/pkg/tde/kms.go new file mode 100644 index 00000000..e13b3fc2 --- /dev/null +++ b/pkg/tde/kms.go @@ -0,0 +1,194 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 + +package tde + +import ( + "bytes" + "context" + "fmt" + "net/url" + "sync" + "time" + + openapi "github.com/alibabacloud-go/darabonba-openapi/v2/utils" + aliyunkms "github.com/alibabacloud-go/kms-20160120/v3/client" + "github.com/alibabacloud-go/tea/dara" + "github.com/aws/aws-sdk-go-v2/aws" + awsconfig "github.com/aws/aws-sdk-go-v2/config" + "github.com/aws/aws-sdk-go-v2/credentials" + awskms "github.com/aws/aws-sdk-go-v2/service/kms" + awskmstypes "github.com/aws/aws-sdk-go-v2/service/kms/types" + + tdev1 "github.com/apache/doris-operator/api/tde" +) + +const kmsReadinessTTL = 5 * time.Minute + +type kmsReadinessEntry struct { + expiresAt time.Time +} + +var ( + kmsReadinessCache sync.Map + kmsReadinessCheck = validateKMSReadiness +) + +func validateKMSReadiness(ctx context.Context, provider tdev1.ProviderType, kms *tdev1.KmsProviderStatus, accessKey, secretKey []byte) error { + if kms == nil { + return fmt.Errorf("KMS configuration is missing") + } + cacheKey := fmt.Sprintf("%s\x00%s\x00%s\x00%s\x00%s", provider, kms.KeyID, kms.Endpoint, kms.Region, kms.Auth.CredentialSecretUID) + if cached, ok := kmsReadinessCache.Load(cacheKey); ok && time.Now().Before(cached.(kmsReadinessEntry).expiresAt) { + return nil + } + + checkCtx, cancel := context.WithTimeout(ctx, 15*time.Second) + defer cancel() + var err error + switch provider { + case tdev1.ProviderAwsKms: + err = validateAWSKMS(checkCtx, kms, accessKey, secretKey) + case tdev1.ProviderAliyunKms: + err = validateAliyunKMS(checkCtx, kms, accessKey, secretKey) + default: + return fmt.Errorf("unsupported KMS provider %s", provider) + } + if err != nil { + return err + } + kmsReadinessCache.Store(cacheKey, kmsReadinessEntry{expiresAt: time.Now().Add(kmsReadinessTTL)}) + return nil +} + +func validateAWSKMS(ctx context.Context, kms *tdev1.KmsProviderStatus, accessKey, secretKey []byte) error { + cfg, err := awsconfig.LoadDefaultConfig(ctx, + awsconfig.WithRegion(kms.Region), + awsconfig.WithCredentialsProvider(credentials.NewStaticCredentialsProvider(string(accessKey), string(secretKey), "")), + ) + if err != nil { + return fmt.Errorf("initialize AWS KMS client: %w", err) + } + client := awskms.NewFromConfig(cfg, func(options *awskms.Options) { + options.BaseEndpoint = aws.String(kms.Endpoint) + options.RetryMaxAttempts = 2 + }) + response, err := client.DescribeKey(ctx, &awskms.DescribeKeyInput{KeyId: aws.String(kms.KeyID)}) + if err != nil { + return fmt.Errorf("AWS KMS DescribeKey failed: %w", err) + } + if response.KeyMetadata == nil { + return fmt.Errorf("AWS KMS DescribeKey returned no key metadata") + } + if response.KeyMetadata.KeyState != awskmstypes.KeyStateEnabled { + return fmt.Errorf("AWS KMS key is not enabled: state=%s", response.KeyMetadata.KeyState) + } + probe := []byte("doris-operator-tde-readiness") + encrypted, err := client.Encrypt(ctx, &awskms.EncryptInput{KeyId: aws.String(kms.KeyID), Plaintext: probe}) + if err != nil { + return fmt.Errorf("AWS KMS Encrypt failed: %w", err) + } + if len(encrypted.CiphertextBlob) == 0 { + return fmt.Errorf("AWS KMS Encrypt returned no ciphertext") + } + decrypted, err := client.Decrypt(ctx, &awskms.DecryptInput{KeyId: aws.String(kms.KeyID), CiphertextBlob: encrypted.CiphertextBlob}) + if err != nil { + return fmt.Errorf("AWS KMS Decrypt failed: %w", err) + } + decryptedMatches := bytes.Equal(decrypted.Plaintext, probe) + clear(decrypted.Plaintext) + if !decryptedMatches { + return fmt.Errorf("AWS KMS Decrypt returned unexpected plaintext") + } + dataKey, err := client.GenerateDataKey(ctx, &awskms.GenerateDataKeyInput{KeyId: aws.String(kms.KeyID), KeySpec: awskmstypes.DataKeySpecAes256}) + if err != nil { + return fmt.Errorf("AWS KMS GenerateDataKey failed: %w", err) + } + if len(dataKey.Plaintext) == 0 || len(dataKey.CiphertextBlob) == 0 { + clear(dataKey.Plaintext) + return fmt.Errorf("AWS KMS GenerateDataKey returned incomplete key material") + } + clear(dataKey.Plaintext) + return nil +} + +func validateAliyunKMS(ctx context.Context, kms *tdev1.KmsProviderStatus, accessKey, secretKey []byte) error { + endpoint, err := url.Parse(kms.Endpoint) + if err != nil || endpoint.Host == "" { + return fmt.Errorf("Aliyun KMS endpoint is invalid") + } + connectTimeout, readTimeout := 5000, 10000 + config := &openapi.Config{ + AccessKeyId: dara.String(string(accessKey)), + AccessKeySecret: dara.String(string(secretKey)), + RegionId: dara.String(kms.Region), + Endpoint: dara.String(endpoint.Host), + Protocol: dara.String("https"), + ConnectTimeout: &connectTimeout, + ReadTimeout: &readTimeout, + } + client, err := aliyunkms.NewClient(config) + if err != nil { + return fmt.Errorf("initialize Aliyun KMS client: %w", err) + } + + resultCh := make(chan error, 1) + go func() { + response, callErr := client.DescribeKey(&aliyunkms.DescribeKeyRequest{KeyId: dara.String(kms.KeyID)}) + if callErr != nil { + resultCh <- fmt.Errorf("Aliyun KMS DescribeKey failed: %w", callErr) + return + } + if response == nil || response.Body == nil || response.Body.KeyMetadata == nil { + resultCh <- fmt.Errorf("Aliyun KMS DescribeKey returned no key metadata") + return + } + if dara.StringValue(response.Body.KeyMetadata.KeyState) != "Enabled" { + resultCh <- fmt.Errorf("Aliyun KMS key is not enabled: state=%s", dara.StringValue(response.Body.KeyMetadata.KeyState)) + return + } + + const encodedProbe = "ZG9yaXMtb3BlcmF0b3ItdGRlLXJlYWRpbmVzcw==" + encrypted, callErr := client.Encrypt(&aliyunkms.EncryptRequest{KeyId: dara.String(kms.KeyID), Plaintext: dara.String(encodedProbe)}) + if callErr != nil { + resultCh <- fmt.Errorf("Aliyun KMS Encrypt failed: %w", callErr) + return + } + if encrypted == nil || encrypted.Body == nil || dara.StringValue(encrypted.Body.CiphertextBlob) == "" { + resultCh <- fmt.Errorf("Aliyun KMS Encrypt returned no ciphertext") + return + } + decrypted, callErr := client.Decrypt(&aliyunkms.DecryptRequest{CiphertextBlob: encrypted.Body.CiphertextBlob}) + if callErr != nil { + resultCh <- fmt.Errorf("Aliyun KMS Decrypt failed: %w", callErr) + return + } + if decrypted == nil || decrypted.Body == nil || dara.StringValue(decrypted.Body.Plaintext) != encodedProbe { + resultCh <- fmt.Errorf("Aliyun KMS Decrypt returned unexpected plaintext") + return + } + dataKey, callErr := client.GenerateDataKey(&aliyunkms.GenerateDataKeyRequest{KeyId: dara.String(kms.KeyID), KeySpec: dara.String("AES_256")}) + if callErr != nil { + resultCh <- fmt.Errorf("Aliyun KMS GenerateDataKey failed: %w", callErr) + return + } + if dataKey == nil || dataKey.Body == nil || dara.StringValue(dataKey.Body.Plaintext) == "" || dara.StringValue(dataKey.Body.CiphertextBlob) == "" { + resultCh <- fmt.Errorf("Aliyun KMS GenerateDataKey returned incomplete key material") + return + } + resultCh <- nil + }() + select { + case <-ctx.Done(): + return fmt.Errorf("Aliyun KMS readiness check timed out: %w", ctx.Err()) + case callErr := <-resultCh: + return callErr + } +} diff --git a/pkg/tde/lifecycle.go b/pkg/tde/lifecycle.go new file mode 100644 index 00000000..817e7c13 --- /dev/null +++ b/pkg/tde/lifecycle.go @@ -0,0 +1,63 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 + +package tde + +import ( + "context" + + ddcv1 "github.com/apache/doris-operator/api/disaggregated/v1" + dorisv1 "github.com/apache/doris-operator/api/doris/v1" + tdev1 "github.com/apache/doris-operator/api/tde" + hashutil "github.com/apache/doris-operator/pkg/common/utils/hash" + "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +func EnforceDCRFELifecycleGate(ctx context.Context, c client.Client, dcr *dorisv1.DorisCluster) (bool, error) { + if !feLifecycleChangeBlocked(dcr.Spec.TDE, dcr.Spec.FeSpec, dcr.Status.TDE) { + return false, nil + } + status := dcr.Status.TDE.DeepCopy() + metaSetLifecycleBlocked(status, dcr.Generation) + dcr.Status.TDE = status + return true, persistDCRStatus(ctx, c, dcr, status) +} + +func EnforceDDCFELifecycleGate(ctx context.Context, c client.Client, ddc *ddcv1.DorisDisaggregatedCluster) (bool, error) { + if !feLifecycleChangeBlocked(ddc.Spec.TDE, ddc.Spec.FeSpec, ddc.Status.TDE) { + return false, nil + } + status := ddc.Status.TDE.DeepCopy() + metaSetLifecycleBlocked(status, ddc.Generation) + ddc.Status.TDE = status + return true, persistDDCStatus(ctx, c, ddc, status) +} + +func feLifecycleChangeBlocked(config *tdev1.TDEConfig, feSpec interface{}, status *tdev1.TDEStatus) bool { + if status == nil || status.Current == nil || status.ActiveConfigHash == "" || status.ActiveFESpecHash == "" { + return false + } + if hashutil.HashObject(feSpec) == status.ActiveFESpecHash { + return false + } + return tdev1.BlocksFELifecycle(status) || hashutil.HashObject(config) != status.ActiveConfigHash +} + +func metaSetLifecycleBlocked(status *tdev1.TDEStatus, generation int64) { + meta.SetStatusCondition(&status.Conditions, metav1.Condition{ + Type: conditionReconcileBlocked, + Status: metav1.ConditionTrue, + Reason: "FELifecycleChangeBlocked", + Message: "spec.feSpec cannot change while a TDE operation or configuration sync is pending, or in the same update as spec.tde", + ObservedGeneration: generation, + }) +} diff --git a/pkg/tde/pod.go b/pkg/tde/pod.go new file mode 100644 index 00000000..91dd5858 --- /dev/null +++ b/pkg/tde/pod.go @@ -0,0 +1,212 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 + +package tde + +import ( + "crypto/sha256" + "encoding/hex" + "encoding/json" + "fmt" + "path" + "strings" + + tdev1 "github.com/apache/doris-operator/api/tde" + corev1 "k8s.io/api/core/v1" +) + +const ( + keyMountBase = "/etc/selectdb/tde/keys" + accessKeyEnv = "DORIS_TDE_AK" + secretKeyEnv = "DORIS_TDE_SK" + materialAnnotation = "selectdb.com/tde-material" + configAnnotation = "selectdb.com/tde-config-hash" +) + +func ApplyPodOverlay(template *corev1.PodTemplateSpec, containerName string, spec *tdev1.TDEConfig, status *tdev1.TDEStatus) { + if spec == nil || spec.ManagementPolicy != tdev1.ManagementPolicyManaged { + return + } + providers := providersForPod(spec, status) + var materialVersions []string + for _, provider := range providers { + if provider == nil || provider.Provider != tdev1.ProviderLocal || provider.RootKeyRef == nil { + continue + } + volumeName := localVolumeName(provider.RootKeyRef.SecretName, provider.RootKeyRef.Key) + if !hasVolume(template.Spec.Volumes, volumeName) { + mode := int32(0440) + template.Spec.Volumes = append(template.Spec.Volumes, corev1.Volume{ + Name: volumeName, + VolumeSource: corev1.VolumeSource{Secret: &corev1.SecretVolumeSource{ + SecretName: provider.RootKeyRef.SecretName, + Items: []corev1.KeyToPath{{Key: provider.RootKeyRef.Key, Path: "root.key", Mode: &mode}}, + }}, + }) + } + for i := range template.Spec.Containers { + if template.Spec.Containers[i].Name != containerName { + continue + } + if !hasMount(template.Spec.Containers[i].VolumeMounts, volumeName) { + template.Spec.Containers[i].VolumeMounts = append(template.Spec.Containers[i].VolumeMounts, corev1.VolumeMount{ + Name: volumeName, MountPath: path.Dir(provider.RootKeyRef.ResolvedPath), ReadOnly: true, + }) + } + } + materialVersions = append(materialVersions, provider.RootKeyRef.SecretName+":"+provider.RootKeyRef.SecretUID) + } + active := providerForRuntime(spec, status) + for _, provider := range providers { + if provider != nil && provider.Provider != tdev1.ProviderLocal { + active = provider + } + } + if active != nil && active.Kms != nil && active.Kms.Auth.Type == tdev1.KmsAuthEnvironmentSecret { + for i := range template.Spec.Containers { + if template.Spec.Containers[i].Name != containerName { + continue + } + container := &template.Spec.Containers[i] + container.Env = removeEnv(container.Env, accessKeyEnv, secretKeyEnv) + container.Env = append(container.Env, + corev1.EnvVar{Name: accessKeyEnv, ValueFrom: &corev1.EnvVarSource{SecretKeyRef: &corev1.SecretKeySelector{ + LocalObjectReference: corev1.LocalObjectReference{Name: active.Kms.Auth.CredentialSecretName}, Key: active.Kms.Auth.AccessKeyKey, + }}}, + corev1.EnvVar{Name: secretKeyEnv, ValueFrom: &corev1.EnvVarSource{SecretKeyRef: &corev1.SecretKeySelector{ + LocalObjectReference: corev1.LocalObjectReference{Name: active.Kms.Auth.CredentialSecretName}, Key: active.Kms.Auth.SecretKeyKey, + }}}, + ) + } + materialVersions = append(materialVersions, active.Kms.Auth.CredentialSecretName+":"+active.Kms.Auth.CredentialSecretUID) + } + if template.Annotations == nil { + template.Annotations = map[string]string{} + } + template.Annotations[materialAnnotation] = strings.Join(materialVersions, ",") + template.Annotations[configAnnotation] = configID(providerForRuntime(spec, status), spec.MasterKeyRotation) +} + +func configID(provider *tdev1.ProviderStatus, rotation *tdev1.MasterKeyRotationSpec) string { + data, _ := json.Marshal(struct { + Provider *tdev1.ProviderStatus `json:"provider,omitempty"` + Rotation *tdev1.MasterKeyRotationSpec `json:"rotation,omitempty"` + }{Provider: provider, Rotation: rotation}) + return shortHash(string(data)) +} + +func providersForPod(spec *tdev1.TDEConfig, status *tdev1.TDEStatus) []*tdev1.ProviderStatus { + if status != nil && status.Operation != nil && status.Operation.Type == tdev1.OperationRotateRootKey && status.Operation.Stage != tdev1.StageCompleted { + if status.Operation.SQLState == tdev1.SQLApplied { + if status.Operation.Stage == tdev1.StageWaitingForFEReplay { + return []*tdev1.ProviderStatus{status.Operation.Source, status.Operation.Target} + } + return []*tdev1.ProviderStatus{status.Operation.Target} + } + if status.Operation.SQLState == tdev1.SQLNotApplied { + return []*tdev1.ProviderStatus{status.Operation.Source} + } + return []*tdev1.ProviderStatus{status.Operation.Source, status.Operation.Target} + } + return []*tdev1.ProviderStatus{providerForRuntime(spec, status)} +} + +func providerStatusFromSpec(spec *tdev1.TDEConfig, secretUIDs map[string]string) *tdev1.ProviderStatus { + if spec == nil { + return nil + } + status := &tdev1.ProviderStatus{Provider: spec.Provider.Type, DefaultAlgorithm: spec.DefaultAlgorithm} + switch spec.Provider.Type { + case tdev1.ProviderLocal: + if spec.Provider.Local == nil { + return status + } + ref := spec.Provider.Local.SecretKeyRef + status.RootKeyRef = &tdev1.RootKeyRefStatus{ + SecretName: ref.Name, Key: ref.Key, ResolvedPath: localKeyPath(ref.Name, ref.Key), SecretUID: secretUIDs[ref.Name], + } + case tdev1.ProviderAwsKms, tdev1.ProviderAliyunKms: + if spec.Provider.Kms == nil { + return status + } + kms := spec.Provider.Kms + status.Kms = &tdev1.KmsProviderStatus{KeyID: kms.KeyID, Endpoint: kms.Endpoint, Region: kms.Region, Auth: tdev1.KmsAuthStatus{Type: kms.Auth.Type}} + if kms.Auth.CredentialSecretRef != nil { + ref := kms.Auth.CredentialSecretRef + status.Kms.Auth.CredentialSecretName = ref.Name + status.Kms.Auth.CredentialSecretUID = secretUIDs[ref.Name] + status.Kms.Auth.AccessKeyKey = ref.AccessKeyKey + status.Kms.Auth.SecretKeyKey = ref.SecretKeyKey + } + } + return status +} + +func localKeyPath(secretName, key string) string { + return path.Join(keyMountBase, secretName+"-"+shortHash(key), "root.key") +} + +func localVolumeName(secretName, key string) string { + name := "tde-key-" + strings.ReplaceAll(secretName, ".", "-") + if len(name) > 48 { + name = name[:48] + } + return strings.TrimRight(name, "-") + "-" + shortHash(key) +} + +func shortHash(value string) string { + sum := sha256.Sum256([]byte(value)) + return hex.EncodeToString(sum[:4]) +} + +func hasVolume(volumes []corev1.Volume, name string) bool { + for i := range volumes { + if volumes[i].Name == name { + return true + } + } + return false +} + +func hasMount(mounts []corev1.VolumeMount, name string) bool { + for i := range mounts { + if mounts[i].Name == name { + return true + } + } + return false +} + +func removeEnv(envs []corev1.EnvVar, names ...string) []corev1.EnvVar { + remove := make(map[string]struct{}, len(names)) + for _, name := range names { + remove[name] = struct{}{} + } + result := envs[:0] + for _, env := range envs { + if _, ok := remove[env.Name]; !ok { + result = append(result, env) + } + } + return result +} + +func materialID(provider *tdev1.ProviderStatus) string { + if provider == nil { + return "" + } + if provider.RootKeyRef != nil { + return fmt.Sprintf("%s/%s@%s", provider.RootKeyRef.SecretName, provider.RootKeyRef.Key, provider.RootKeyRef.SecretUID) + } + if provider.Kms != nil { + return fmt.Sprintf("%s/%s/%s", provider.Kms.KeyID, provider.Kms.Region, provider.Kms.Auth.CredentialSecretUID) + } + return string(provider.Provider) +} diff --git a/pkg/tde/reconciler.go b/pkg/tde/reconciler.go new file mode 100644 index 00000000..cdd2f1e4 --- /dev/null +++ b/pkg/tde/reconciler.go @@ -0,0 +1,874 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 + +package tde + +import ( + "context" + "crypto/subtle" + "encoding/base64" + "errors" + "fmt" + "path" + "reflect" + "sort" + "strconv" + "strings" + "time" + + ddcv1 "github.com/apache/doris-operator/api/disaggregated/v1" + dorisv1 "github.com/apache/doris-operator/api/doris/v1" + tdev1 "github.com/apache/doris-operator/api/tde" + hashutil "github.com/apache/doris-operator/pkg/common/utils/hash" + "github.com/apache/doris-operator/pkg/common/utils/k8s" + "github.com/apache/doris-operator/pkg/common/utils/mysql" + "github.com/apache/doris-operator/pkg/common/utils/resource" + mysqldriver "github.com/go-sql-driver/mysql" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/util/retry" + "k8s.io/klog/v2" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +const ( + conditionMaterialReady = "MaterialReady" + conditionRotationOutcomeUnknown = "RotationOutcomeUnknown" + conditionRotationRejected = "RotationRejected" + conditionConfigSyncPending = "ConfigSyncPending" + conditionReconcileBlocked = "ReconcileBlocked" + submittingGracePeriod = time.Minute + metadataQueryTimeout = 15 * time.Second +) + +var rotationSQLTimeout = time.Minute + +type reconcileInput struct { + object client.Object + spec *tdev1.TDEConfig + status **tdev1.TDEStatus + generation int64 + statefulSetName string + connect func(context.Context) (*mysql.DB, error) + persistTDEStatus func(context.Context, *tdev1.TDEStatus) error + activeConfigHash string + activeFESpecHash string +} + +func ReconcileDCR(ctx context.Context, c client.Client, dcr *dorisv1.DorisCluster) (ctrl.Result, error) { + if dcr.Spec.TDE == nil { + return ctrl.Result{}, nil + } + return reconcile(ctx, c, reconcileInput{ + object: dcr, spec: dcr.Spec.TDE, status: &dcr.Status.TDE, generation: dcr.Generation, + statefulSetName: dorisv1.GenerateComponentStatefulSetName(dcr, dorisv1.Component_FE), + connect: func(ctx context.Context) (*mysql.DB, error) { return connectDCR(ctx, c, dcr) }, + persistTDEStatus: func(ctx context.Context, status *tdev1.TDEStatus) error { return persistDCRStatus(ctx, c, dcr, status) }, + activeConfigHash: hashutil.HashObject(dcr.Spec.TDE), activeFESpecHash: hashutil.HashObject(dcr.Spec.FeSpec), + }) +} + +func ReconcileDDC(ctx context.Context, c client.Client, ddc *ddcv1.DorisDisaggregatedCluster) (ctrl.Result, error) { + if ddc.Spec.TDE == nil { + return ctrl.Result{}, nil + } + return reconcile(ctx, c, reconcileInput{ + object: ddc, spec: ddc.Spec.TDE, status: &ddc.Status.TDE, generation: ddc.Generation, + statefulSetName: ddc.GetFEStatefulsetName(), + connect: func(ctx context.Context) (*mysql.DB, error) { return connectDDC(ctx, c, ddc) }, + persistTDEStatus: func(ctx context.Context, status *tdev1.TDEStatus) error { return persistDDCStatus(ctx, c, ddc, status) }, + activeConfigHash: hashutil.HashObject(ddc.Spec.TDE), activeFESpecHash: hashutil.HashObject(ddc.Spec.FeSpec), + }) +} + +func reconcile(ctx context.Context, c client.Client, in reconcileInput) (ctrl.Result, error) { + if in.spec.ManagementPolicy == tdev1.ManagementPolicyObserveOnly { + if *in.status == nil { + *in.status = &tdev1.TDEStatus{} + } + (*in.status).ObservedGeneration = in.generation + (*in.status).State = tdev1.StateUnknown + return ctrl.Result{}, nil + } + if errs := tdev1.Validate(in.spec); len(errs) != 0 { + return ctrl.Result{}, errors.Join(errs...) + } + if *in.status == nil { + *in.status = &tdev1.TDEStatus{} + } + status := *in.status + status.ObservedGeneration = in.generation + if in.spec.Recovery != nil && !tdev1.RotationOutcomeUnknown(status) && + (status.Operation == nil || status.Operation.Recovery == nil || status.Operation.Recovery.RequestID != in.spec.Recovery.RequestID) { + setRecoveryBlocked(status, in.generation, "UnexpectedRecoveryRequest", "recovery is only valid for an existing RotationOutcomeUnknown operation") + return ctrl.Result{}, nil + } + + target, targetKey, err := resolveProvider(ctx, c, in.object.GetNamespace(), in.spec) + if err != nil { + setMaterialFailure(status, in.spec.Provider.Type, err) + return ctrl.Result{RequeueAfter: 10 * time.Second}, nil + } + meta.RemoveStatusCondition(&status.Conditions, conditionMaterialReady) + if rootRotationCancellationRequested(status, target, in.spec) { + status.Operation.SQLState = tdev1.SQLNotApplied + completeOperation(status) + markActive(status, in) + status.ObservedGeneration = in.generation + meta.RemoveStatusCondition(&status.Conditions, conditionConfigSyncPending) + meta.RemoveStatusCondition(&status.Conditions, conditionReconcileBlocked) + return ctrl.Result{Requeue: true}, nil + } + + ready, checks, err := workloadReady(ctx, c, in.object.GetNamespace(), in.statefulSetName, in.spec, status) + status.FEChecks = checks + if err != nil && !apierrors.IsNotFound(err) { + return ctrl.Result{}, err + } + + if status.Current == nil { + newOperation := ensureOperation(status, tdev1.OperationEnableTDE, "", in.generation, nil, target) + status.State = tdev1.StateConfiguredUninitialized + if newOperation { + return ctrl.Result{Requeue: true}, nil + } + if !ready { + return ctrl.Result{RequeueAfter: 5 * time.Second}, nil + } + initialized, err := metadataInitialized(ctx, in.connect) + if err != nil || !initialized { + return ctrl.Result{RequeueAfter: 10 * time.Second}, nil + } + status.MetadataInitialized = true + status.Current = target.DeepCopy() + completeOperation(status) + markActive(status, in) + return ctrl.Result{}, nil + } + + sourceKey, err := validateCurrentProvider(ctx, c, in.object.GetNamespace(), status.Current) + if err != nil { + setMaterialFailure(status, status.Current.Provider, err) + return ctrl.Result{RequeueAfter: 10 * time.Second}, nil + } + if handled, result := reconcileRecovery(status, target, in.spec, ready, in.generation); handled { + if status.State == tdev1.StateActive { + markActive(status, in) + } + return result, nil + } + if err := validateTransition(status, target, in.spec); err != nil { + status.State = tdev1.StateReconciling + meta.SetStatusCondition(&status.Conditions, metav1.Condition{Type: conditionReconcileBlocked, Status: metav1.ConditionTrue, + Reason: "InvalidTransition", Message: err.Error(), ObservedGeneration: in.generation}) + return ctrl.Result{}, nil + } + meta.RemoveStatusCondition(&status.Conditions, conditionReconcileBlocked) + + if sameRootStatus(status.Current, target) { + if !sameAuthStatus(status.Current, target) { + requestID := "" + if in.spec.CredentialRotation != nil { + requestID = in.spec.CredentialRotation.RequestID + } + newOperation := ensureOperation(status, tdev1.OperationRotateKmsCredential, requestID, in.generation, status.Current, target) + status.State = tdev1.StateReconciling + if newOperation { + return ctrl.Result{Requeue: true}, nil + } + if !ready { + return ctrl.Result{RequeueAfter: 5 * time.Second}, nil + } + status.Current = target.DeepCopy() + completeOperation(status) + } + if status.Current.DefaultAlgorithm != target.DefaultAlgorithm { + newOperation := ensureOperation(status, tdev1.OperationUpdateAlgorithm, "", in.generation, status.Current, target) + status.State = tdev1.StateReconciling + if newOperation { + return ctrl.Result{Requeue: true}, nil + } + if !ready { + return ctrl.Result{RequeueAfter: 5 * time.Second}, nil + } + status.Current.DefaultAlgorithm = target.DefaultAlgorithm + completeOperation(status) + } + markActive(status, in) + status.ObservedGeneration = in.generation + return ctrl.Result{}, nil + } + + if sourceKey != nil && targetKey != nil && len(sourceKey) == len(targetKey) && subtle.ConstantTimeCompare(sourceKey, targetKey) == 1 { + setMaterialFailure(status, target.Provider, fmt.Errorf("source and target Local root keys must differ")) + return ctrl.Result{}, nil + } + + requestID := in.spec.RootKeyRotation.RequestID + newOperation := status.Operation == nil || status.Operation.Type != tdev1.OperationRotateRootKey || status.Operation.RequestID != requestID + if newOperation { + status.Operation = &tdev1.OperationStatus{ + Type: tdev1.OperationRotateRootKey, RequestID: requestID, SpecGeneration: in.generation, + Stage: tdev1.StageRollingOutMaterial, SQLState: tdev1.SQLNotStarted, + Source: status.Current.DeepCopy(), Target: target.DeepCopy(), StartedAt: metav1.Now(), LastTransitionTime: metav1.Now(), + } + recordRequestID(status, requestID) + meta.RemoveStatusCondition(&status.Conditions, conditionRotationRejected) + status.State = tdev1.StateReconciling + return ctrl.Result{Requeue: true}, nil + } + status.State = tdev1.StateReconciling + operation := status.Operation + if operation.SQLState == tdev1.SQLSubmitting && !newOperation { + remaining := submittingGraceRemaining(operation, time.Now()) + if remaining > 0 { + return ctrl.Result{RequeueAfter: remaining}, nil + } + operation.SQLState = tdev1.SQLOutcomeUnknown + operation.Stage = tdev1.StageFailed + operation.LastTransitionTime = metav1.Now() + meta.SetStatusCondition(&status.Conditions, metav1.Condition{Type: conditionRotationOutcomeUnknown, Status: metav1.ConditionTrue, + Reason: "OperatorRestartedWhileSubmitting", Message: "rotate SQL outcome is unknown; automatic retry is disabled", ObservedGeneration: in.generation}) + return ctrl.Result{}, nil + } + if operation.SQLState == tdev1.SQLOutcomeUnknown { + return ctrl.Result{}, nil + } + if operation.SQLState == tdev1.SQLNotStarted { + if !ready { + return ctrl.Result{RequeueAfter: 5 * time.Second}, nil + } + db, err := in.connect(ctx) + if err != nil { + operation.Stage = tdev1.StageRollingOutMaterial + operation.LastTransitionTime = metav1.Now() + return ctrl.Result{RequeueAfter: 10 * time.Second}, nil + } + defer db.Close() + operation.Stage = tdev1.StageRotatingRootKey + operation.SQLState = tdev1.SQLSubmitting + operation.LastTransitionTime = metav1.Now() + if err := in.persistTDEStatus(ctx, status.DeepCopy()); err != nil { + return ctrl.Result{}, err + } + err = executeRotateSQL(ctx, db, operation.Source, operation.Target) + if err != nil { + if message, rejected := confirmedSQLRejection(err); rejected { + operation.SQLState = tdev1.SQLRejected + operation.Stage = tdev1.StageFailed + operation.LastTransitionTime = metav1.Now() + meta.SetStatusCondition(&status.Conditions, metav1.Condition{Type: conditionRotationRejected, Status: metav1.ConditionTrue, + Reason: "RotateSQLRejected", Message: message, ObservedGeneration: in.generation}) + return ctrl.Result{}, nil + } + operation.SQLState = tdev1.SQLOutcomeUnknown + operation.Stage = tdev1.StageFailed + operation.LastTransitionTime = metav1.Now() + meta.SetStatusCondition(&status.Conditions, metav1.Condition{Type: conditionRotationOutcomeUnknown, Status: metav1.ConditionTrue, + Reason: "RotateSQLResultUnknown", Message: "rotate SQL did not return a confirmed success; automatic retry is disabled", ObservedGeneration: in.generation}) + return ctrl.Result{}, nil + } + operation.SQLState = tdev1.SQLApplied + operation.Stage = tdev1.StageWaitingForFEReplay + operation.LastTransitionTime = metav1.Now() + meta.SetStatusCondition(&status.Conditions, metav1.Condition{Type: conditionConfigSyncPending, Status: metav1.ConditionTrue, + Reason: "WaitingForFEReplay", Message: "root key was rotated; waiting for all FE nodes to replay the committed journal", ObservedGeneration: in.generation}) + return ctrl.Result{Requeue: true}, nil + } + if operation.SQLState == tdev1.SQLApplied { + if operation.Stage == tdev1.StageWaitingForFEReplay { + db, err := in.connect(ctx) + if err != nil { + return ctrl.Result{RequeueAfter: 10 * time.Second}, nil + } + allReplayed, err := allFrontendsReplayed(ctx, db, operation) + db.Close() + if err != nil || !allReplayed { + return ctrl.Result{RequeueAfter: 5 * time.Second}, nil + } + operation.Stage = tdev1.StageSyncingConfiguration + operation.LastTransitionTime = metav1.Now() + meta.SetStatusCondition(&status.Conditions, metav1.Condition{Type: conditionConfigSyncPending, Status: metav1.ConditionTrue, + Reason: "FEReplayCompleted", Message: "all FE nodes replayed the root key rotation; waiting for target configuration rollout", ObservedGeneration: in.generation}) + return ctrl.Result{Requeue: true}, nil + } + if !ready { + return ctrl.Result{RequeueAfter: 5 * time.Second}, nil + } + status.Current = operation.Target.DeepCopy() + status.MetadataInitialized = true + completeOperation(status) + meta.RemoveStatusCondition(&status.Conditions, conditionConfigSyncPending) + meta.RemoveStatusCondition(&status.Conditions, conditionRotationOutcomeUnknown) + meta.RemoveStatusCondition(&status.Conditions, conditionRotationRejected) + markActive(status, in) + } + return ctrl.Result{}, nil +} + +func rootRotationCancellationRequested(status *tdev1.TDEStatus, target *tdev1.ProviderStatus, spec *tdev1.TDEConfig) bool { + return status != nil && status.Current != nil && status.Operation != nil && + status.Operation.Type == tdev1.OperationRotateRootKey && status.Operation.Stage != tdev1.StageCompleted && + (status.Operation.SQLState == tdev1.SQLNotStarted || status.Operation.SQLState == tdev1.SQLRejected) && spec.RootKeyRotation == nil && + sameRootStatus(status.Current, target) && sameAuthStatus(status.Current, target) && + status.Current.DefaultAlgorithm == target.DefaultAlgorithm +} + +func reconcileRecovery(status *tdev1.TDEStatus, target *tdev1.ProviderStatus, spec *tdev1.TDEConfig, ready bool, generation int64) (bool, ctrl.Result) { + operation := status.Operation + if operation == nil || operation.Type != tdev1.OperationRotateRootKey { + return false, ctrl.Result{} + } + if tdev1.RotationOutcomeUnknown(status) { + recovery := spec.Recovery + if recovery == nil { + return true, ctrl.Result{} + } + if recovery.RotationRequestID != operation.RequestID || recovery.RequestID == "" { + setRecoveryBlocked(status, generation, "RecoveryRequestMismatch", "recovery request does not match the unknown root key rotation") + return true, ctrl.Result{} + } + if operation.Recovery != nil && operation.Recovery.RequestID == recovery.RequestID { + setRecoveryBlocked(status, generation, "RecoveryRequestReused", "recovery requestId was already used") + return true, ctrl.Result{} + } + if tdev1.RequestIDUsed(status, recovery.RequestID) { + setRecoveryBlocked(status, generation, "RecoveryRequestReused", "recovery requestId was already used") + return true, ctrl.Result{} + } + switch recovery.Decision { + case tdev1.RecoveryConfirmApplied: + if !sameRootStatus(operation.Target, target) || !sameAuthStatus(operation.Target, target) { + setRecoveryBlocked(status, generation, "RecoveryTargetMismatch", "ConfirmApplied requires the captured rotation target") + return true, ctrl.Result{} + } + operation.SQLState = tdev1.SQLApplied + case tdev1.RecoveryConfirmNotApplied: + if !sameRootStatus(operation.Source, target) || !sameAuthStatus(operation.Source, target) { + setRecoveryBlocked(status, generation, "RecoverySourceMismatch", "ConfirmNotApplied requires restoring the captured rotation source") + return true, ctrl.Result{} + } + operation.SQLState = tdev1.SQLNotApplied + status.Current = operation.Source.DeepCopy() + default: + setRecoveryBlocked(status, generation, "InvalidRecoveryDecision", "recovery decision must be ConfirmApplied or ConfirmNotApplied") + return true, ctrl.Result{} + } + operation.Recovery = &tdev1.RecoveryStatus{RequestID: recovery.RequestID, Decision: recovery.Decision, ResolvedAt: metav1.Now()} + recordRequestID(status, recovery.RequestID) + operation.Stage = tdev1.StageSyncingConfiguration + operation.LastTransitionTime = metav1.Now() + status.State = tdev1.StateReconciling + meta.RemoveStatusCondition(&status.Conditions, conditionRotationOutcomeUnknown) + meta.RemoveStatusCondition(&status.Conditions, conditionReconcileBlocked) + meta.SetStatusCondition(&status.Conditions, metav1.Condition{Type: conditionConfigSyncPending, Status: metav1.ConditionTrue, + Reason: "RotationOutcomeResolved", Message: "waiting for FE configuration rollout after manual recovery", ObservedGeneration: generation}) + return true, ctrl.Result{Requeue: true} + } + if operation.Recovery != nil && operation.Recovery.Decision == tdev1.RecoveryConfirmNotApplied && + operation.SQLState == tdev1.SQLNotApplied && operation.Stage == tdev1.StageSyncingConfiguration { + status.State = tdev1.StateReconciling + if !ready { + return true, ctrl.Result{RequeueAfter: 5 * time.Second} + } + completeOperation(status) + meta.RemoveStatusCondition(&status.Conditions, conditionConfigSyncPending) + status.State = tdev1.StateActive + return true, ctrl.Result{} + } + return false, ctrl.Result{} +} + +func markActive(status *tdev1.TDEStatus, in reconcileInput) { + status.State = tdev1.StateActive + status.ActiveConfigHash = in.activeConfigHash + status.ActiveFESpecHash = in.activeFESpecHash +} + +func setRecoveryBlocked(status *tdev1.TDEStatus, generation int64, reason, message string) { + status.State = tdev1.StateReconciling + meta.SetStatusCondition(&status.Conditions, metav1.Condition{Type: conditionReconcileBlocked, Status: metav1.ConditionTrue, + Reason: reason, Message: message, ObservedGeneration: generation}) +} + +func submittingGraceRemaining(operation *tdev1.OperationStatus, now time.Time) time.Duration { + return submittingGracePeriod - now.Sub(operation.LastTransitionTime.Time) +} + +func confirmedSQLRejection(err error) (string, bool) { + var serverError *mysqldriver.MySQLError + if !errors.As(err, &serverError) { + return "", false + } + return serverError.Message, true +} + +func resolveProvider(ctx context.Context, c client.Client, namespace string, spec *tdev1.TDEConfig) (*tdev1.ProviderStatus, []byte, error) { + uids := map[string]string{} + var key []byte + var accessKey, secretKey []byte + switch spec.Provider.Type { + case tdev1.ProviderLocal: + ref := spec.Provider.Local.SecretKeyRef + secret, decoded, err := getLocalKey(ctx, c, namespace, ref.Name, ref.Key, "") + if err != nil { + return nil, nil, err + } + uids[ref.Name] = string(secret.UID) + key = decoded + case tdev1.ProviderAwsKms, tdev1.ProviderAliyunKms: + if spec.Provider.Kms.Auth.Type == tdev1.KmsAuthEnvironmentSecret { + ref := spec.Provider.Kms.Auth.CredentialSecretRef + secret, err := getImmutableSecret(ctx, c, namespace, ref.Name) + if err != nil { + return nil, nil, err + } + if len(secret.Data[ref.AccessKeyKey]) == 0 || len(secret.Data[ref.SecretKeyKey]) == 0 { + return nil, nil, fmt.Errorf("KMS credential Secret %s is missing required keys", ref.Name) + } + uids[ref.Name] = string(secret.UID) + accessKey, secretKey = secret.Data[ref.AccessKeyKey], secret.Data[ref.SecretKeyKey] + } + } + provider := providerStatusFromSpec(spec, uids) + if provider.Kms != nil && provider.Kms.Auth.Type == tdev1.KmsAuthEnvironmentSecret { + if err := kmsReadinessCheck(ctx, provider.Provider, provider.Kms, accessKey, secretKey); err != nil { + return nil, nil, err + } + } + return provider, key, nil +} + +func validateCurrentProvider(ctx context.Context, c client.Client, namespace string, current *tdev1.ProviderStatus) ([]byte, error) { + if current == nil { + return nil, fmt.Errorf("current TDE provider is missing") + } + if current.Provider == tdev1.ProviderLocal { + if current.RootKeyRef == nil { + return nil, fmt.Errorf("current Local key reference is missing") + } + _, key, err := getLocalKey(ctx, c, namespace, current.RootKeyRef.SecretName, current.RootKeyRef.Key, current.RootKeyRef.SecretUID) + return key, err + } + if current.Kms != nil && current.Kms.Auth.Type == tdev1.KmsAuthEnvironmentSecret { + secret, err := getImmutableSecret(ctx, c, namespace, current.Kms.Auth.CredentialSecretName) + if err != nil { + return nil, err + } + if string(secret.UID) != current.Kms.Auth.CredentialSecretUID { + return nil, fmt.Errorf("current KMS credential Secret UID changed") + } + accessKey, secretKey := secret.Data[current.Kms.Auth.AccessKeyKey], secret.Data[current.Kms.Auth.SecretKeyKey] + if len(accessKey) == 0 || len(secretKey) == 0 { + return nil, fmt.Errorf("current KMS credential Secret is missing required keys") + } + if err := kmsReadinessCheck(ctx, current.Provider, current.Kms, accessKey, secretKey); err != nil { + return nil, err + } + } + return nil, nil +} + +func getLocalKey(ctx context.Context, c client.Client, namespace, name, key, expectedUID string) (*corev1.Secret, []byte, error) { + secret, err := getImmutableSecret(ctx, c, namespace, name) + if err != nil { + return nil, nil, err + } + if expectedUID != "" && string(secret.UID) != expectedUID { + return nil, nil, fmt.Errorf("Local root key Secret %s UID changed", name) + } + file, ok := secret.Data[key] + if !ok { + return nil, nil, fmt.Errorf("Local root key Secret %s does not contain key %s", name, key) + } + decoded, err := base64.StdEncoding.DecodeString(strings.TrimSpace(string(file))) + if err != nil { + return nil, nil, fmt.Errorf("Local root key Secret %s key %s is not valid Base64", name, key) + } + if len(decoded) != 16 && len(decoded) != 32 { + return nil, nil, fmt.Errorf("Local root key must decode to 16 or 32 bytes") + } + return secret, decoded, nil +} + +func getImmutableSecret(ctx context.Context, c client.Client, namespace, name string) (*corev1.Secret, error) { + var secret corev1.Secret + if err := c.Get(ctx, types.NamespacedName{Namespace: namespace, Name: name}, &secret); err != nil { + return nil, err + } + if secret.Immutable == nil || !*secret.Immutable { + return nil, fmt.Errorf("Secret %s must be immutable", name) + } + return &secret, nil +} + +func workloadReady(ctx context.Context, c client.Client, namespace, name string, spec *tdev1.TDEConfig, status *tdev1.TDEStatus) (bool, []tdev1.FECheck, error) { + var sts appsv1.StatefulSet + if err := c.Get(ctx, types.NamespacedName{Namespace: namespace, Name: name}, &sts); err != nil { + return false, nil, err + } + desired := int32(1) + if sts.Spec.Replicas != nil { + desired = *sts.Spec.Replicas + } + selector, err := metav1.LabelSelectorAsSelector(sts.Spec.Selector) + if err != nil { + return false, nil, err + } + expectedTemplate := corev1.PodTemplateSpec{Spec: corev1.PodSpec{Containers: []corev1.Container{{Name: "fe"}}}} + ApplyPodOverlay(&expectedTemplate, "fe", spec, status) + expectedMaterial := expectedTemplate.Annotations[materialAnnotation] + expectedConfig := expectedTemplate.Annotations[configAnnotation] + templateConsistent := sts.Spec.Template.Annotations[materialAnnotation] == expectedMaterial && + sts.Spec.Template.Annotations[configAnnotation] == expectedConfig + var pods corev1.PodList + if err := c.List(ctx, &pods, client.InNamespace(namespace), client.MatchingLabelsSelector{Selector: selector}); err != nil { + return false, nil, err + } + sort.Slice(pods.Items, func(i, j int) bool { return pods.Items[i].Name < pods.Items[j].Name }) + checks := make([]tdev1.FECheck, 0, len(pods.Items)) + for i := range pods.Items { + pod := &pods.Items[i] + ready := k8s.PodIsReady(&pod.Status) + consistent := templateConsistent && pod.Labels[appsv1.ControllerRevisionHashLabelKey] == sts.Status.UpdateRevision && + pod.Annotations[materialAnnotation] == expectedMaterial && pod.Annotations[configAnnotation] == expectedConfig + checks = append(checks, tdev1.FECheck{PodName: pod.Name, PodReady: ready, ConfigConsistent: consistent, MaterialReady: ready && consistent}) + } + ready := templateConsistent && sts.Status.ObservedGeneration >= sts.Generation && sts.Status.CurrentRevision != "" && sts.Status.CurrentRevision == sts.Status.UpdateRevision && + sts.Status.ReadyReplicas == desired && sts.Status.UpdatedReplicas == desired && int32(len(checks)) == desired + return ready, checks, nil +} + +func ensureOperation(status *tdev1.TDEStatus, operationType tdev1.OperationType, requestID string, generation int64, source, target *tdev1.ProviderStatus) bool { + if status.Operation != nil && status.Operation.Type == operationType && status.Operation.RequestID == requestID && status.Operation.SpecGeneration == generation { + return false + } + now := metav1.Now() + status.Operation = &tdev1.OperationStatus{Type: operationType, RequestID: requestID, SpecGeneration: generation, + Stage: tdev1.StageRollingOutMaterial, SQLState: tdev1.SQLNotStarted, Source: source.DeepCopy(), Target: target.DeepCopy(), StartedAt: now, LastTransitionTime: now} + recordRequestID(status, requestID) + return true +} + +func recordRequestID(status *tdev1.TDEStatus, requestID string) { + if status == nil || requestID == "" || tdev1.RequestIDUsed(status, requestID) { + return + } + status.UsedRequestIDs = append(status.UsedRequestIDs, requestID) +} + +func completeOperation(status *tdev1.TDEStatus) { + if status.Operation == nil { + return + } + status.Operation.Stage = tdev1.StageCompleted + status.Operation.LastTransitionTime = metav1.Now() +} + +func setMaterialFailure(status *tdev1.TDEStatus, provider tdev1.ProviderType, err error) { + if provider == tdev1.ProviderLocal { + status.State = tdev1.StateLocalKeyNotReady + } else { + status.State = tdev1.StateKmsNotReady + } + meta.SetStatusCondition(&status.Conditions, metav1.Condition{Type: conditionMaterialReady, Status: metav1.ConditionFalse, + Reason: "MaterialValidationFailed", Message: err.Error(), ObservedGeneration: status.ObservedGeneration}) +} + +func sameRootStatus(a, b *tdev1.ProviderStatus) bool { + if a == nil || b == nil || a.Provider != b.Provider { + return false + } + if a.Provider == tdev1.ProviderLocal { + return a.RootKeyRef != nil && b.RootKeyRef != nil && *a.RootKeyRef == *b.RootKeyRef + } + return a.Kms != nil && b.Kms != nil && a.Kms.KeyID == b.Kms.KeyID && a.Kms.Endpoint == b.Kms.Endpoint && a.Kms.Region == b.Kms.Region +} + +func sameAuthStatus(a, b *tdev1.ProviderStatus) bool { + if a == nil || b == nil || a.Provider == tdev1.ProviderLocal || b.Provider == tdev1.ProviderLocal { + return true + } + return a.Kms != nil && b.Kms != nil && reflect.DeepEqual(a.Kms.Auth, b.Kms.Auth) +} + +func validateTransition(status *tdev1.TDEStatus, target *tdev1.ProviderStatus, spec *tdev1.TDEConfig) error { + if status == nil || target == nil || spec == nil { + return nil + } + if tdev1.RotationOutcomeUnknown(status) { + return fmt.Errorf("the previous root key rotation outcome is unknown; resolve it before changing or retrying TDE configuration") + } + if operation := status.Operation; operation != nil && operation.Stage != tdev1.StageCompleted && operation.Stage != tdev1.StageFailed { + if !reflect.DeepEqual(operation.Target, target) { + return fmt.Errorf("spec.tde provider or algorithm cannot change while operation %q is in stage %q", operation.RequestID, operation.Stage) + } + switch operation.Type { + case tdev1.OperationRotateRootKey: + if spec.RootKeyRotation == nil || spec.RootKeyRotation.RequestID != operation.RequestID { + return fmt.Errorf("spec.tde.rootKeyRotation.requestId cannot change while operation %q is in stage %q", operation.RequestID, operation.Stage) + } + case tdev1.OperationRotateKmsCredential: + if spec.CredentialRotation == nil || spec.CredentialRotation.RequestID != operation.RequestID { + return fmt.Errorf("spec.tde.credentialRotation.requestId cannot change while operation %q is in stage %q", operation.RequestID, operation.Stage) + } + } + } + if status.Current == nil { + return nil + } + current := status.Current + rootChanged := !sameRootStatus(current, target) + authChanged := !sameAuthStatus(current, target) + if (current.Provider == tdev1.ProviderAwsKms && target.Provider == tdev1.ProviderAliyunKms) || + (current.Provider == tdev1.ProviderAliyunKms && target.Provider == tdev1.ProviderAwsKms) { + return fmt.Errorf("direct AwsKms/AliyunKms rotation is unsupported; rotate through Local using two requests") + } + if rootChanged && authChanged { + return fmt.Errorf("KMS credentials and root key cannot change in the same update") + } + if rootChanged { + if spec.RootKeyRotation == nil || spec.RootKeyRotation.RequestID == "" { + return fmt.Errorf("changing the TDE root key requires spec.tde.rootKeyRotation.requestId") + } + if operation := status.Operation; operation != nil && operation.Type == tdev1.OperationRotateRootKey && + operation.RequestID == spec.RootKeyRotation.RequestID && !sameRootStatus(operation.Target, target) { + return fmt.Errorf("spec.tde.rootKeyRotation.requestId %q was already used for a different target", spec.RootKeyRotation.RequestID) + } + if tdev1.RequestIDUsed(status, spec.RootKeyRotation.RequestID) && + (status.Operation == nil || status.Operation.Type != tdev1.OperationRotateRootKey || status.Operation.RequestID != spec.RootKeyRotation.RequestID) { + return fmt.Errorf("spec.tde.rootKeyRotation.requestId %q was already used", spec.RootKeyRotation.RequestID) + } + return nil + } + if authChanged { + if spec.CredentialRotation == nil || spec.CredentialRotation.RequestID == "" { + return fmt.Errorf("changing KMS credentials requires spec.tde.credentialRotation.requestId") + } + if operation := status.Operation; operation != nil && operation.Type == tdev1.OperationRotateKmsCredential && + operation.RequestID == spec.CredentialRotation.RequestID && !sameAuthStatus(operation.Target, target) { + return fmt.Errorf("spec.tde.credentialRotation.requestId %q was already used for different credentials", spec.CredentialRotation.RequestID) + } + if tdev1.RequestIDUsed(status, spec.CredentialRotation.RequestID) && + (status.Operation == nil || status.Operation.Type != tdev1.OperationRotateKmsCredential || status.Operation.RequestID != spec.CredentialRotation.RequestID) { + return fmt.Errorf("spec.tde.credentialRotation.requestId %q was already used", spec.CredentialRotation.RequestID) + } + } + return nil +} + +func metadataInitialized(ctx context.Context, connect func(context.Context) (*mysql.DB, error)) (bool, error) { + db, err := connect(ctx) + if err != nil { + return false, err + } + defer db.Close() + var count int + queryCtx, cancel := context.WithTimeout(ctx, metadataQueryTimeout) + defer cancel() + if err := db.GetContext(queryCtx, &count, "SELECT COUNT(*) FROM information_schema.encryption_keys"); err != nil { + return false, err + } + return count > 0, nil +} + +func executeRotateSQL(ctx context.Context, db *mysql.DB, source, target *tdev1.ProviderStatus) error { + sqlCtx, cancel := context.WithTimeout(ctx, rotationSQLTimeout) + defer cancel() + _, err := db.ExecContext(sqlCtx, BuildRotateSQL(source, target)) + return err +} + +func allFrontendsReplayed(ctx context.Context, db *mysql.DB, operation *tdev1.OperationStatus) (bool, error) { + queryCtx, cancel := context.WithTimeout(ctx, metadataQueryTimeout) + defer cancel() + var frontends []*mysql.Frontend + if err := db.DB.Unsafe().SelectContext(queryCtx, &frontends, "SHOW FRONTENDS"); err != nil { + klog.Errorf("TDE replay gate failed to query SHOW FRONTENDS: %v", err) + return false, err + } + if len(frontends) == 0 { + return false, nil + } + return frontendsHaveReplayed(frontends, operation) +} + +func frontendsHaveReplayed(frontends []*mysql.Frontend, operation *tdev1.OperationStatus) (bool, error) { + if operation.CommitJournalID == "" { + for _, frontend := range frontends { + if frontend.IsMaster { + if _, err := strconv.ParseInt(frontend.ReplayedJournalId, 10, 64); err != nil { + return false, fmt.Errorf("parse Master FE replayed journal id: %w", err) + } + operation.CommitJournalID = frontend.ReplayedJournalId + break + } + } + } + commitID, err := strconv.ParseInt(operation.CommitJournalID, 10, 64) + if err != nil { + return false, fmt.Errorf("parse root key rotation commit journal id: %w", err) + } + for _, frontend := range frontends { + if !frontend.Join || !frontend.Alive { + return false, nil + } + replayedID, err := strconv.ParseInt(frontend.ReplayedJournalId, 10, 64) + if err != nil || replayedID < commitID { + return false, nil + } + } + return true, nil +} + +func BuildRotateSQL(source, target *tdev1.ProviderStatus) string { + properties := map[string]string{} + switch target.Provider { + case tdev1.ProviderLocal: + properties["doris_tde_key_provider"] = "local" + properties["doris_tde_key_new_key_file"] = target.RootKeyRef.ResolvedPath + case tdev1.ProviderAwsKms: + properties["doris_tde_key_provider"] = "aws_kms" + case tdev1.ProviderAliyunKms: + properties["doris_tde_key_provider"] = "aliyun_kms" + } + if target.Kms != nil { + properties["doris_tde_key_id"] = target.Kms.KeyID + properties["doris_tde_key_endpoint"] = target.Kms.Endpoint + properties["doris_tde_key_region"] = target.Kms.Region + } + if source != nil && source.Provider == tdev1.ProviderLocal && source.RootKeyRef != nil { + properties["doris_tde_key_original_key_file"] = source.RootKeyRef.ResolvedPath + } + keys := make([]string, 0, len(properties)) + for key := range properties { + keys = append(keys, key) + } + sort.Strings(keys) + parts := make([]string, 0, len(keys)) + for _, key := range keys { + parts = append(parts, sqlString(key)+" = "+sqlString(properties[key])) + } + return "ADMIN ROTATE TDE ROOT KEY PROPERTIES(" + strings.Join(parts, ", ") + ")" +} + +func sqlString(value string) string { + return `"` + strings.NewReplacer(`\`, `\\`, `"`, `\"`).Replace(value) + `"` +} + +func persistDCRStatus(ctx context.Context, c client.Client, dcr *dorisv1.DorisCluster, status *tdev1.TDEStatus) error { + return retry.RetryOnConflict(retry.DefaultBackoff, func() error { + var latest dorisv1.DorisCluster + if err := c.Get(ctx, client.ObjectKeyFromObject(dcr), &latest); err != nil { + return err + } + latest.Status.TDE = status.DeepCopy() + return c.Status().Update(ctx, &latest) + }) +} + +func persistDDCStatus(ctx context.Context, c client.Client, ddc *ddcv1.DorisDisaggregatedCluster, status *tdev1.TDEStatus) error { + return retry.RetryOnConflict(retry.DefaultBackoff, func() error { + var latest ddcv1.DorisDisaggregatedCluster + if err := c.Get(ctx, client.ObjectKeyFromObject(ddc), &latest); err != nil { + return err + } + latest.Status.TDE = status.DeepCopy() + return c.Status().Update(ctx, &latest) + }) +} + +func connectDCR(ctx context.Context, c client.Client, dcr *dorisv1.DorisCluster) (*mysql.DB, error) { + secret, _ := k8s.GetSecret(ctx, c, dcr.Namespace, dcr.Spec.AuthSecret) + user, password := dorisv1.GetClusterSecret(dcr, secret) + config, err := k8s.GetConfig(ctx, c, &dcr.Spec.FeSpec.ConfigMapInfo, dcr.Namespace, dorisv1.Component_FE) + if err != nil { + return nil, err + } + tlsConfig, tlsSecret, err := dcrTLS(ctx, c, dcr, config) + if err != nil { + return nil, err + } + return mysql.NewDorisMasterSqlDB(mysql.DBConfig{User: user, Password: password, + Host: dorisv1.GenerateExternalServiceName(dcr, dorisv1.Component_FE) + "." + dcr.Namespace, + Port: strconv.Itoa(int(resource.GetPort(config, resource.QUERY_PORT))), Database: "mysql"}, tlsConfig, tlsSecret) +} + +func connectDDC(ctx context.Context, c client.Client, ddc *ddcv1.DorisDisaggregatedCluster) (*mysql.DB, error) { + user, password := "root", "" + if ddc.Spec.AuthSecret != "" { + secret, _ := k8s.GetSecret(ctx, c, ddc.Namespace, ddc.Spec.AuthSecret) + user, password = resource.GetDorisLoginInformation(secret) + } else if ddc.Spec.AdminUser != nil { + user, password = ddc.Spec.AdminUser.Name, ddc.Spec.AdminUser.Password + } + config, err := ddcFEConfig(ctx, c, ddc) + if err != nil { + return nil, err + } + tlsConfig, tlsSecret, err := ddcTLS(ctx, c, ddc, config) + if err != nil { + return nil, err + } + return mysql.NewDorisMasterSqlDB(mysql.DBConfig{User: user, Password: password, Host: ddc.GetFEVIPAddresss(), + Port: strconv.Itoa(int(resource.GetPort(config, resource.QUERY_PORT))), Database: "mysql"}, tlsConfig, tlsSecret) +} + +func ddcFEConfig(ctx context.Context, c client.Client, ddc *ddcv1.DorisDisaggregatedCluster) (map[string]interface{}, error) { + configMaps := make([]*corev1.ConfigMap, 0, len(ddc.Spec.FeSpec.ConfigMaps)) + for _, ref := range ddc.Spec.FeSpec.ConfigMaps { + var cm corev1.ConfigMap + if err := c.Get(ctx, types.NamespacedName{Namespace: ddc.Namespace, Name: ref.Name}, &cm); err != nil { + return nil, err + } + configMaps = append(configMaps, &cm) + } + return resource.ResolveConfigMaps(configMaps, dorisv1.Component_FE) +} + +func dcrTLS(ctx context.Context, c client.Client, dcr *dorisv1.DorisCluster, config map[string]interface{}) (*mysql.TLSConfig, *corev1.Secret, error) { + if resource.GetString(config, resource.ENABLE_TLS_KEY) == "" { + return nil, nil, nil + } + tlsConfig, dir := tlsConfigFromFE(config) + for _, ref := range dcr.Spec.FeSpec.Secrets { + if ref.MountPath == dir { + secret, err := k8s.GetSecret(ctx, c, dcr.Namespace, ref.SecretName) + return tlsConfig, secret, err + } + } + return nil, nil, fmt.Errorf("FE TLS Secret mounted at %s was not found", dir) +} + +func ddcTLS(ctx context.Context, c client.Client, ddc *ddcv1.DorisDisaggregatedCluster, config map[string]interface{}) (*mysql.TLSConfig, *corev1.Secret, error) { + if resource.GetString(config, resource.ENABLE_TLS_KEY) == "" { + return nil, nil, nil + } + tlsConfig, dir := tlsConfigFromFE(config) + for _, ref := range ddc.Spec.FeSpec.Secrets { + if ref.MountPath == dir { + secret, err := k8s.GetSecret(ctx, c, ddc.Namespace, ref.SecretName) + return tlsConfig, secret, err + } + } + return nil, nil, fmt.Errorf("FE TLS Secret mounted at %s was not found", dir) +} + +func tlsConfigFromFE(config map[string]interface{}) (*mysql.TLSConfig, string) { + ca := resource.GetString(config, resource.TLS_CA_CERTIFICATE_PATH_KEY) + cert := resource.GetString(config, resource.TLS_CERTIFICATE_PATH_KEY) + key := resource.GetString(config, resource.TLS_PRIVATE_KEY_PATH_KEY) + return &mysql.TLSConfig{CAFileName: path.Base(ca), ClientCertFileName: path.Base(cert), ClientKeyFileName: path.Base(key)}, path.Dir(ca) +} diff --git a/pkg/tde/tde_test.go b/pkg/tde/tde_test.go new file mode 100644 index 00000000..88bd6e2c --- /dev/null +++ b/pkg/tde/tde_test.go @@ -0,0 +1,624 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 + +package tde + +import ( + "context" + "database/sql/driver" + "fmt" + "regexp" + "strings" + "testing" + "time" + + "github.com/DATA-DOG/go-sqlmock" + dorisv1 "github.com/apache/doris-operator/api/doris/v1" + tdev1 "github.com/apache/doris-operator/api/tde" + hashutil "github.com/apache/doris-operator/pkg/common/utils/hash" + "github.com/apache/doris-operator/pkg/common/utils/mysql" + mysqldriver "github.com/go-sql-driver/mysql" + "github.com/jmoiron/sqlx" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client/fake" +) + +func TestBuildRotateSQLLocalToLocal(t *testing.T) { + source := localStatus("old-key", "old-uid") + target := localStatus("new-key", "new-uid") + sql := BuildRotateSQL(source, target) + for _, expected := range []string{"ADMIN ROTATE TDE ROOT KEY", `"doris_tde_key_provider" = "local"`, source.RootKeyRef.ResolvedPath, target.RootKeyRef.ResolvedPath} { + if !strings.Contains(sql, expected) { + t.Fatalf("SQL %q does not contain %q", sql, expected) + } + } +} + +func TestBuildRotateSQLLocalToAwsKms(t *testing.T) { + source := localStatus("old-key", "old-uid") + target := kmsStatus(tdev1.ProviderAwsKms, "aws-key", "credentials", "credentials-uid") + sql := BuildRotateSQL(source, target) + for _, expected := range []string{ + `"doris_tde_key_provider" = "aws_kms"`, + `"doris_tde_key_id" = "aws-key"`, + `"doris_tde_key_endpoint" = "https://kms.example.com"`, + `"doris_tde_key_region" = "region-1"`, + source.RootKeyRef.ResolvedPath, + } { + if !strings.Contains(sql, expected) { + t.Fatalf("SQL %q does not contain %q", sql, expected) + } + } + if strings.Contains(sql, "doris_tde_key_new_key_file") { + t.Fatalf("KMS rotate SQL unexpectedly contains a local target key: %s", sql) + } +} + +func TestMergeFEConfigReplacesManagedValues(t *testing.T) { + spec := localSpec("root-key") + merged := mergeFEConfig("query_port=9030\ndoris_tde_key_provider=aws_kms\ndoris_tde_algorithm=SM4\n", providerStatusFromSpec(spec, nil), spec) + if strings.Count(merged, "doris_tde_key_provider=") != 1 { + t.Fatalf("provider was not replaced: %s", merged) + } + if !strings.Contains(merged, "doris_tde_key_provider=local") || !strings.Contains(merged, "query_port=9030") { + t.Fatalf("unexpected merged config: %s", merged) + } +} + +func TestProviderForRuntimeUsesRotationTargetAfterSQLApplied(t *testing.T) { + spec := localSpec("root-v2") + status := &tdev1.TDEStatus{ + Current: localStatus("root-v1", "uid-1"), + Operation: &tdev1.OperationStatus{ + Type: tdev1.OperationRotateRootKey, + Stage: tdev1.StageSyncingConfiguration, + SQLState: tdev1.SQLApplied, + Source: localStatus("root-v1", "uid-1"), + Target: localStatus("root-v2", "uid-2"), + }, + } + + provider := providerForRuntime(spec, status) + if provider == nil || provider.RootKeyRef == nil || provider.RootKeyRef.SecretName != "root-v2" { + t.Fatalf("SQL-applied rotation must render the target provider, got %#v", provider) + } +} + +func TestProviderForRuntimeWaitsForFEReplayBeforeTargetConfig(t *testing.T) { + spec := localSpec("root-v2") + status := &tdev1.TDEStatus{Current: localStatus("root-v1", "uid-1"), Operation: &tdev1.OperationStatus{ + Type: tdev1.OperationRotateRootKey, Stage: tdev1.StageWaitingForFEReplay, SQLState: tdev1.SQLApplied, + Source: localStatus("root-v1", "uid-1"), Target: localStatus("root-v2", "uid-2"), + }} + + provider := providerForRuntime(spec, status) + if provider == nil || provider.RootKeyRef == nil || provider.RootKeyRef.SecretName != "root-v1" { + t.Fatalf("source config must remain active until FE replay completes, got %#v", provider) + } + providers := providersForPod(spec, status) + if len(providers) != 2 { + t.Fatalf("source and target materials must remain mounted while waiting for FE replay, got %#v", providers) + } +} + +func TestPrepareDCRConfigCreatesEffectiveConfigMap(t *testing.T) { + scheme := runtime.NewScheme() + if err := corev1.AddToScheme(scheme); err != nil { + t.Fatal(err) + } + if err := dorisv1.AddToScheme(scheme); err != nil { + t.Fatal(err) + } + base := &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: "fe-config", Namespace: "test"}, Data: map[string]string{"fe.conf": "query_port=9030\n"}} + replicas := int32(1) + dcr := &dorisv1.DorisCluster{ObjectMeta: metav1.ObjectMeta{Name: "demo", Namespace: "test", UID: types.UID("cluster-uid")}, + Spec: dorisv1.DorisClusterSpec{TDE: localSpec("root-key"), FeSpec: &dorisv1.FeSpec{BaseSpec: dorisv1.BaseSpec{Replicas: &replicas, + ConfigMapInfo: dorisv1.ConfigMapInfo{ConfigMapName: "fe-config"}}}}} + client := fake.NewClientBuilder().WithScheme(scheme).WithObjects(base, dcr).Build() + working, err := PrepareDCRConfig(context.Background(), client, dcr) + if err != nil { + t.Fatal(err) + } + if working.Spec.FeSpec.ConfigMapInfo.ConfigMapName == "fe-config" { + t.Fatal("effective ConfigMap was not selected") + } + var effective corev1.ConfigMap + if err := client.Get(context.Background(), types.NamespacedName{Name: working.Spec.FeSpec.ConfigMapInfo.ConfigMapName, Namespace: "test"}, &effective); err != nil { + t.Fatal(err) + } + if !strings.Contains(effective.Data["fe.conf"], "doris_tde_root_key_file=") { + t.Fatalf("TDE config missing: %s", effective.Data["fe.conf"]) + } +} + +func TestApplyPodOverlayKeepsBothLocalKeysBeforeRotate(t *testing.T) { + spec := localSpec("new-key") + status := &tdev1.TDEStatus{Operation: &tdev1.OperationStatus{Type: tdev1.OperationRotateRootKey, Stage: tdev1.StageRollingOutMaterial, + SQLState: tdev1.SQLNotStarted, Source: localStatus("old-key", "old-uid"), Target: localStatus("new-key", "new-uid")}} + template := &corev1.PodTemplateSpec{Spec: corev1.PodSpec{Containers: []corev1.Container{{Name: "fe"}}}} + ApplyPodOverlay(template, "fe", spec, status) + if len(template.Spec.Volumes) != 2 || len(template.Spec.Containers[0].VolumeMounts) != 2 { + t.Fatalf("expected source and target mounts, got %#v", template.Spec) + } +} + +func TestApplyPodOverlayHashIsStable(t *testing.T) { + spec := localSpec("root-key") + status := &tdev1.TDEStatus{Operation: &tdev1.OperationStatus{Type: tdev1.OperationEnableTDE, Target: localStatus("root-key", "root-uid")}} + var expected string + for i := 0; i < 20; i++ { + template := &corev1.PodTemplateSpec{Spec: corev1.PodSpec{Containers: []corev1.Container{{Name: "fe"}}}} + ApplyPodOverlay(template, "fe", spec.DeepCopy(), status.DeepCopy()) + if i == 0 { + expected = template.Annotations[configAnnotation] + } + if template.Annotations[configAnnotation] != expected { + t.Fatalf("TDE config hash changed: want %s, got %s", expected, template.Annotations[configAnnotation]) + } + if template.Annotations[materialAnnotation] != "root-key:root-uid" { + t.Fatalf("initial material UID missing: %q", template.Annotations[materialAnnotation]) + } + } +} + +func TestApplyPodOverlayInjectsKmsCredentials(t *testing.T) { + spec := kmsSpec(tdev1.ProviderAwsKms, "aws-key", "credentials") + status := &tdev1.TDEStatus{Current: kmsStatus(tdev1.ProviderAwsKms, "aws-key", "credentials", "credentials-uid")} + template := &corev1.PodTemplateSpec{Spec: corev1.PodSpec{Containers: []corev1.Container{{Name: "fe"}}}} + ApplyPodOverlay(template, "fe", spec, status) + + envs := template.Spec.Containers[0].Env + if len(envs) != 2 || envs[0].Name != accessKeyEnv || envs[0].ValueFrom.SecretKeyRef.Name != "credentials" || + envs[0].ValueFrom.SecretKeyRef.Key != "accessKey" || envs[1].Name != secretKeyEnv || + envs[1].ValueFrom.SecretKeyRef.Key != "secretKey" { + t.Fatalf("unexpected KMS environment: %#v", envs) + } + if template.Annotations[materialAnnotation] != "credentials:credentials-uid" { + t.Fatalf("unexpected material annotation: %q", template.Annotations[materialAnnotation]) + } +} + +func TestResolveProviderValidatesKmsEnvironmentSecret(t *testing.T) { + scheme := runtime.NewScheme() + if err := corev1.AddToScheme(scheme); err != nil { + t.Fatal(err) + } + immutable := true + secret := &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: "credentials", Namespace: "test", UID: types.UID("credentials-uid")}, + Immutable: &immutable, Data: map[string][]byte{"accessKey": []byte("test-ak"), "secretKey": []byte("test-sk")}} + c := fake.NewClientBuilder().WithScheme(scheme).WithObjects(secret).Build() + spec := kmsSpec(tdev1.ProviderAwsKms, "aws-key", "credentials") + + originalCheck := kmsReadinessCheck + defer func() { kmsReadinessCheck = originalCheck }() + called := false + kmsReadinessCheck = func(_ context.Context, provider tdev1.ProviderType, kms *tdev1.KmsProviderStatus, accessKey, secretKey []byte) error { + called = true + if provider != tdev1.ProviderAwsKms || kms.KeyID != "aws-key" || string(accessKey) != "test-ak" || string(secretKey) != "test-sk" { + t.Fatalf("unexpected KMS readiness input: provider=%s kms=%#v", provider, kms) + } + return nil + } + + provider, _, err := resolveProvider(context.Background(), c, "test", spec) + if err != nil { + t.Fatal(err) + } + if !called || provider.Kms.Auth.CredentialSecretUID != "credentials-uid" { + t.Fatalf("KMS readiness was not checked or UID was not captured: %#v", provider) + } +} + +func TestResolveProviderSkipsOperatorIdentityForInstanceRole(t *testing.T) { + scheme := runtime.NewScheme() + if err := corev1.AddToScheme(scheme); err != nil { + t.Fatal(err) + } + c := fake.NewClientBuilder().WithScheme(scheme).Build() + spec := kmsSpec(tdev1.ProviderAliyunKms, "aliyun-key", "") + spec.Provider.Kms.Auth = tdev1.KmsAuthSpec{Type: tdev1.KmsAuthInstanceRole} + + originalCheck := kmsReadinessCheck + defer func() { kmsReadinessCheck = originalCheck }() + kmsReadinessCheck = func(context.Context, tdev1.ProviderType, *tdev1.KmsProviderStatus, []byte, []byte) error { + t.Fatal("InstanceRole must not be validated with the Operator Pod identity") + return nil + } + + provider, _, err := resolveProvider(context.Background(), c, "test", spec) + if err != nil { + t.Fatal(err) + } + if provider.Kms.Auth.Type != tdev1.KmsAuthInstanceRole { + t.Fatalf("unexpected provider: %#v", provider) + } +} + +func TestLocalVolumeNameAcceptsDNSSubdomainSecret(t *testing.T) { + name := localVolumeName("root.key.v2", "root.key") + if strings.Contains(name, ".") { + t.Fatalf("volume name must be a DNS label: %q", name) + } +} + +func TestValidateTransitionRejectsUnsafeKmsChanges(t *testing.T) { + tests := []struct { + name string + status *tdev1.TDEStatus + target *tdev1.ProviderStatus + spec *tdev1.TDEConfig + message string + }{ + { + name: "direct cross KMS rotation", + status: &tdev1.TDEStatus{Current: kmsStatus(tdev1.ProviderAwsKms, "aws-key", "credentials", "uid-1")}, + target: kmsStatus(tdev1.ProviderAliyunKms, "aliyun-key", "credentials", "uid-1"), + spec: kmsSpec(tdev1.ProviderAliyunKms, "aliyun-key", "credentials"), + message: "direct AwsKms/AliyunKms", + }, + { + name: "credential change without request", + status: &tdev1.TDEStatus{Current: kmsStatus(tdev1.ProviderAwsKms, "aws-key", "credentials-v1", "uid-1")}, + target: kmsStatus(tdev1.ProviderAwsKms, "aws-key", "credentials-v2", "uid-2"), + spec: kmsSpec(tdev1.ProviderAwsKms, "aws-key", "credentials-v2"), + message: "credentialRotation.requestId", + }, + { + name: "reused root request", + status: &tdev1.TDEStatus{Current: localStatus("root-v1", "uid-1"), Operation: &tdev1.OperationStatus{ + Type: tdev1.OperationRotateRootKey, RequestID: "rotate-1", Stage: tdev1.StageCompleted, Target: localStatus("root-v2", "uid-2"), + }}, + target: localStatus("root-v3", "uid-3"), + spec: func() *tdev1.TDEConfig { + spec := localSpec("root-v3") + spec.RootKeyRotation = &tdev1.RotationRequest{RequestID: "rotate-1"} + return spec + }(), + message: "already used", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + err := validateTransition(tt.status, tt.target, tt.spec) + if err == nil || !strings.Contains(err.Error(), tt.message) { + t.Fatalf("expected error containing %q, got %v", tt.message, err) + } + }) + } +} + +func TestValidateTransitionRejectsActiveOperationMutation(t *testing.T) { + status := &tdev1.TDEStatus{Current: localStatus("root-v1", "uid-1"), Operation: &tdev1.OperationStatus{ + Type: tdev1.OperationRotateRootKey, RequestID: "rotate-1", Stage: tdev1.StageRollingOutMaterial, + Target: localStatus("root-v2", "uid-2"), + }} + spec := localSpec("root-v3") + spec.RootKeyRotation = &tdev1.RotationRequest{RequestID: "rotate-2"} + err := validateTransition(status, localStatus("root-v3", "uid-3"), spec) + if err == nil || !strings.Contains(err.Error(), "cannot change") { + t.Fatalf("expected active operation mutation to be rejected, got %v", err) + } +} + +func TestValidateTransitionRejectsRequestIDFromHistory(t *testing.T) { + status := &tdev1.TDEStatus{Current: localStatus("root-v2", "uid-2"), UsedRequestIDs: []string{"rotate-1"}} + spec := localSpec("root-v3") + spec.RootKeyRotation = &tdev1.RotationRequest{RequestID: "rotate-1"} + + err := validateTransition(status, localStatus("root-v3", "uid-3"), spec) + if err == nil || !strings.Contains(err.Error(), "already used") { + t.Fatalf("expected historical requestId reuse to be rejected, got %v", err) + } +} + +func TestValidateTransitionBlocksUnknownOutcomeRetry(t *testing.T) { + status := &tdev1.TDEStatus{ + Current: localStatus("root-v1", "uid-1"), + Operation: &tdev1.OperationStatus{ + Type: tdev1.OperationRotateRootKey, + RequestID: "rotate-1", + Stage: tdev1.StageFailed, + SQLState: tdev1.SQLOutcomeUnknown, + Target: localStatus("root-v2", "uid-2"), + }, + Conditions: []metav1.Condition{{Type: conditionRotationOutcomeUnknown, Status: metav1.ConditionTrue}}, + } + spec := localSpec("root-v2") + spec.RootKeyRotation = &tdev1.RotationRequest{RequestID: "rotate-2"} + err := validateTransition(status, localStatus("root-v2", "uid-2"), spec) + if err == nil || !strings.Contains(err.Error(), "outcome is unknown") { + t.Fatalf("expected unknown outcome to block a new request, got %v", err) + } +} + +func TestReconcileRecoveryConfirmApplied(t *testing.T) { + status := &tdev1.TDEStatus{ + State: tdev1.StateReconciling, + Current: localStatus("root-v1", "uid-1"), + Operation: &tdev1.OperationStatus{ + Type: tdev1.OperationRotateRootKey, RequestID: "rotate-1", Stage: tdev1.StageFailed, SQLState: tdev1.SQLOutcomeUnknown, + Source: localStatus("root-v1", "uid-1"), Target: localStatus("root-v2", "uid-2"), + }, + Conditions: []metav1.Condition{{Type: conditionRotationOutcomeUnknown, Status: metav1.ConditionTrue}}, + } + spec := localSpec("root-v2") + spec.RootKeyRotation = &tdev1.RotationRequest{RequestID: "rotate-1"} + spec.Recovery = &tdev1.RecoveryRequest{RequestID: "recover-1", RotationRequestID: "rotate-1", Decision: tdev1.RecoveryConfirmApplied} + + handled, result := reconcileRecovery(status, localStatus("root-v2", "uid-2"), spec, true, 3) + if !handled || !result.Requeue || status.Operation.SQLState != tdev1.SQLApplied || status.Operation.Stage != tdev1.StageSyncingConfiguration { + t.Fatalf("unexpected recovery result: handled=%v result=%+v status=%#v", handled, result, status.Operation) + } + if tdev1.RotationOutcomeUnknown(status) || status.Operation.Recovery == nil || status.Operation.Recovery.RequestID != "recover-1" { + t.Fatalf("unknown condition or recovery audit was not updated: %#v", status) + } +} + +func TestReconcileRecoveryConfirmNotApplied(t *testing.T) { + status := &tdev1.TDEStatus{ + State: tdev1.StateReconciling, + Current: localStatus("root-v1", "uid-1"), + Operation: &tdev1.OperationStatus{ + Type: tdev1.OperationRotateRootKey, RequestID: "rotate-1", Stage: tdev1.StageFailed, SQLState: tdev1.SQLOutcomeUnknown, + Source: localStatus("root-v1", "uid-1"), Target: localStatus("root-v2", "uid-2"), + }, + Conditions: []metav1.Condition{{Type: conditionRotationOutcomeUnknown, Status: metav1.ConditionTrue}}, + } + spec := localSpec("root-v1") + spec.Recovery = &tdev1.RecoveryRequest{RequestID: "recover-1", RotationRequestID: "rotate-1", Decision: tdev1.RecoveryConfirmNotApplied} + + handled, result := reconcileRecovery(status, localStatus("root-v1", "uid-1"), spec, false, 3) + if !handled || !result.Requeue || status.Operation.SQLState != tdev1.SQLNotApplied || status.Operation.Stage != tdev1.StageSyncingConfiguration { + t.Fatalf("unexpected recovery start: handled=%v result=%+v status=%#v", handled, result, status.Operation) + } + providers := providersForPod(spec, status) + if len(providers) != 1 || providers[0].RootKeyRef.SecretName != "root-v1" { + t.Fatalf("ConfirmNotApplied must remove target material, got %#v", providers) + } + handled, result = reconcileRecovery(status, localStatus("root-v1", "uid-1"), spec, true, 3) + if !handled || !result.IsZero() || status.Operation.Stage != tdev1.StageCompleted || status.State != tdev1.StateActive { + t.Fatalf("unexpected recovery completion: handled=%v result=%+v status=%#v", handled, result, status) + } +} + +func TestRootRotationCancellationCompletesWithoutSQL(t *testing.T) { + status := &tdev1.TDEStatus{Current: localStatus("root-v1", "uid-1"), Operation: &tdev1.OperationStatus{ + Type: tdev1.OperationRotateRootKey, RequestID: "rotate-1", Stage: tdev1.StageRollingOutMaterial, + SQLState: tdev1.SQLNotStarted, Source: localStatus("root-v1", "uid-1"), Target: localStatus("root-v2", "uid-2"), + }} + spec := localSpec("root-v1") + if !rootRotationCancellationRequested(status, localStatus("root-v1", "uid-1"), spec) { + t.Fatal("safe pre-SQL cancellation was not recognized") + } + status.Operation.SQLState = tdev1.SQLSubmitting + if rootRotationCancellationRequested(status, localStatus("root-v1", "uid-1"), spec) { + t.Fatal("cancellation was accepted after SQL submission started") + } +} + +func TestFELifecycleRuntimeGate(t *testing.T) { + spec := localSpec("root-v2") + feSpecV1 := &dorisv1.FeSpec{BaseSpec: dorisv1.BaseSpec{Image: "fe:v1"}} + feSpecV2 := feSpecV1.DeepCopy() + feSpecV2.Image = "fe:v2" + status := &tdev1.TDEStatus{ + Current: localStatus("root-v1", "uid-1"), + State: tdev1.StateActive, + ActiveConfigHash: hashutil.HashObject(localSpec("root-v1")), + ActiveFESpecHash: hashutil.HashObject(feSpecV1), + } + if !feLifecycleChangeBlocked(spec, feSpecV2, status) { + t.Fatal("simultaneous TDE and FE changes were not blocked") + } + if feLifecycleChangeBlocked(localSpec("root-v1"), feSpecV2, status) { + t.Fatal("an FE-only change while TDE is stable was blocked") + } + status.Operation = &tdev1.OperationStatus{Type: tdev1.OperationRotateRootKey, Stage: tdev1.StageRollingOutMaterial} + if !feLifecycleChangeBlocked(localSpec("root-v1"), feSpecV2, status) { + t.Fatal("an FE change during root key rotation was not blocked") + } +} + +func TestReconcileStartsRootRotationBeforeConnecting(t *testing.T) { + scheme := runtime.NewScheme() + if err := corev1.AddToScheme(scheme); err != nil { + t.Fatal(err) + } + if err := appsv1.AddToScheme(scheme); err != nil { + t.Fatal(err) + } + + immutable := true + oldSecret := &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: "old-key", Namespace: "test", UID: types.UID("old-uid")}, + Immutable: &immutable, Data: map[string][]byte{"root.key": []byte("MDEyMzQ1Njc4OWFiY2RlZg==")}} + newSecret := &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: "new-key", Namespace: "test", UID: types.UID("new-uid")}, + Immutable: &immutable, Data: map[string][]byte{"root.key": []byte("ZmVkY2JhOTg3NjU0MzIxMA==")}} + replicas := int32(1) + sts := &appsv1.StatefulSet{ + ObjectMeta: metav1.ObjectMeta{Name: "demo-fe", Namespace: "test", Generation: 1}, + Spec: appsv1.StatefulSetSpec{Replicas: &replicas, Selector: &metav1.LabelSelector{MatchLabels: map[string]string{"app": "fe"}}}, + Status: appsv1.StatefulSetStatus{ObservedGeneration: 1, CurrentRevision: "old", UpdateRevision: "old", ReadyReplicas: 1, UpdatedReplicas: 1}, + } + pod := &corev1.Pod{ObjectMeta: metav1.ObjectMeta{Name: "demo-fe-0", Namespace: "test", Labels: map[string]string{ + "app": "fe", appsv1.ControllerRevisionHashLabelKey: "old"}}, Status: corev1.PodStatus{Conditions: []corev1.PodCondition{{Type: corev1.PodReady, Status: corev1.ConditionTrue}}}} + c := fake.NewClientBuilder().WithScheme(scheme).WithObjects(oldSecret, newSecret, sts, pod).Build() + spec := localSpec("new-key") + spec.RootKeyRotation = &tdev1.RotationRequest{RequestID: "rotate-1"} + status := &tdev1.TDEStatus{State: tdev1.StateActive, Current: localStatus("old-key", "old-uid")} + connectCalled := false + + result, err := reconcile(context.Background(), c, reconcileInput{ + object: &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: "demo", Namespace: "test"}}, + spec: spec, status: &status, generation: 2, statefulSetName: "demo-fe", + connect: func(context.Context) (*mysql.DB, error) { + connectCalled = true + return nil, fmt.Errorf("must not connect before rollout") + }, + persistTDEStatus: func(context.Context, *tdev1.TDEStatus) error { return nil }, + }) + if err != nil { + t.Fatal(err) + } + if !result.Requeue || connectCalled { + t.Fatalf("new rotation must requeue before SQL connection: result=%+v connectCalled=%v", result, connectCalled) + } + if status.Operation == nil || status.Operation.Stage != tdev1.StageRollingOutMaterial || status.Operation.SQLState != tdev1.SQLNotStarted { + t.Fatalf("unexpected operation status: %#v", status.Operation) + } +} + +func TestWorkloadReadyRejectsStaleTDETemplate(t *testing.T) { + scheme := runtime.NewScheme() + if err := corev1.AddToScheme(scheme); err != nil { + t.Fatal(err) + } + if err := appsv1.AddToScheme(scheme); err != nil { + t.Fatal(err) + } + replicas := int32(1) + sts := &appsv1.StatefulSet{ + ObjectMeta: metav1.ObjectMeta{Name: "demo-fe", Namespace: "test", Generation: 1}, + Spec: appsv1.StatefulSetSpec{Replicas: &replicas, Selector: &metav1.LabelSelector{MatchLabels: map[string]string{"app": "fe"}}, + Template: corev1.PodTemplateSpec{ObjectMeta: metav1.ObjectMeta{Annotations: map[string]string{ + materialAnnotation: "root-v1:uid-1", configAnnotation: "old-config", + }}}}, + Status: appsv1.StatefulSetStatus{ObservedGeneration: 1, CurrentRevision: "ready", UpdateRevision: "ready", ReadyReplicas: 1, UpdatedReplicas: 1}, + } + pod := &corev1.Pod{ObjectMeta: metav1.ObjectMeta{Name: "demo-fe-0", Namespace: "test", Labels: map[string]string{ + "app": "fe", appsv1.ControllerRevisionHashLabelKey: "ready"}, Annotations: map[string]string{ + materialAnnotation: "root-v1:uid-1", configAnnotation: "old-config", + }}, Status: corev1.PodStatus{Conditions: []corev1.PodCondition{{Type: corev1.PodReady, Status: corev1.ConditionTrue}}}} + c := fake.NewClientBuilder().WithScheme(scheme).WithObjects(sts, pod).Build() + spec := localSpec("root-v2") + status := &tdev1.TDEStatus{Current: localStatus("root-v1", "uid-1"), Operation: &tdev1.OperationStatus{ + Type: tdev1.OperationRotateRootKey, RequestID: "rotate-1", Stage: tdev1.StageRollingOutMaterial, + SQLState: tdev1.SQLNotStarted, Source: localStatus("root-v1", "uid-1"), Target: localStatus("root-v2", "uid-2"), + }} + + ready, checks, err := workloadReady(context.Background(), c, "test", "demo-fe", spec, status) + if err != nil { + t.Fatal(err) + } + if ready || len(checks) != 1 || checks[0].ConfigConsistent || checks[0].MaterialReady { + t.Fatalf("stale TDE template was considered ready: ready=%v checks=%#v", ready, checks) + } +} + +func TestMySQLErrorIsConfirmedServerRejection(t *testing.T) { + message, rejected := confirmedSQLRejection(&mysqldriver.MySQLError{Number: 1105, Message: "key file missing"}) + if !rejected || message != "key file missing" { + t.Fatal("MySQL server error was not recognized as a confirmed rejection") + } +} + +func TestSubmittingGracePeriodDefersOutcomeUnknown(t *testing.T) { + now := time.Now() + operation := &tdev1.OperationStatus{LastTransitionTime: metav1.NewTime(now.Add(-10 * time.Second))} + remaining := submittingGraceRemaining(operation, now) + if remaining <= 0 || remaining > submittingGracePeriod { + t.Fatalf("unexpected submitting grace period remaining: %s", remaining) + } +} + +func TestExecuteRotateSQLHonorsTimeout(t *testing.T) { + rawDB, mock, err := sqlmock.New() + if err != nil { + t.Fatal(err) + } + defer rawDB.Close() + db := &mysql.DB{DB: sqlx.NewDb(rawDB, "sqlmock")} + source := localStatus("root-v1", "uid-1") + target := localStatus("root-v2", "uid-2") + mock.ExpectExec(regexp.QuoteMeta(BuildRotateSQL(source, target))).WillDelayFor(100 * time.Millisecond).WillReturnResult(sqlmock.NewResult(0, 1)) + + originalTimeout := rotationSQLTimeout + rotationSQLTimeout = 10 * time.Millisecond + defer func() { rotationSQLTimeout = originalTimeout }() + if err := executeRotateSQL(context.Background(), db, source, target); err == nil { + t.Fatal("rotate SQL did not honor its execution timeout") + } +} + +func TestFrontendsHaveReplayedCommitJournal(t *testing.T) { + operation := &tdev1.OperationStatus{} + frontends := []*mysql.Frontend{ + {IsMaster: true, Join: true, Alive: true, ReplayedJournalId: "102"}, + {Join: true, Alive: true, ReplayedJournalId: "101"}, + } + replayed, err := frontendsHaveReplayed(frontends, operation) + if err != nil { + t.Fatal(err) + } + if replayed || operation.CommitJournalID != "102" { + t.Fatalf("lagging follower was accepted or commit journal was not captured: replayed=%v operation=%#v", replayed, operation) + } + frontends[1].ReplayedJournalId = "102" + replayed, err = frontendsHaveReplayed(frontends, operation) + if err != nil || !replayed { + t.Fatalf("fully replayed FE set was rejected: replayed=%v err=%v", replayed, err) + } +} + +func TestAllFrontendsReplayedIgnoresFutureColumns(t *testing.T) { + rawDB, mock, err := sqlmock.New() + if err != nil { + t.Fatal(err) + } + defer rawDB.Close() + db := &mysql.DB{DB: sqlx.NewDb(rawDB, "sqlmock")} + columns := []string{"Name", "Host", "EditLogPort", "HttpPort", "QueryPort", "RpcPort", "ArrowFlightSqlPort", "Role", "IsMaster", + "ClusterId", "Join", "Alive", "ReplayedJournalId", "LastStartTime", "LastHeartbeat", "IsHelper", "ErrMsg", "Version", "CurrentConnected", + "LiveSince", "LocalResourceGroup"} + values := []driver.Value{"fe-0", "fe-0.internal.svc", 9010, 8030, 9030, 9020, -1, "FOLLOWER", true, + "1", true, true, "92", "2026-09-23 12:00:00", "2026-09-23 12:00:01", true, "", "doris", "Yes", + "2026-09-23 12:00:00", ""} + mock.ExpectQuery("SHOW FRONTENDS").WillReturnRows(sqlmock.NewRows(columns).AddRow(values...)) + + operation := &tdev1.OperationStatus{} + replayed, err := allFrontendsReplayed(context.Background(), db, operation) + if err != nil || !replayed || operation.CommitJournalID != "92" { + t.Fatalf("future SHOW FRONTENDS columns blocked replay detection: replayed=%v commit=%q err=%v", replayed, operation.CommitJournalID, err) + } + if err := mock.ExpectationsWereMet(); err != nil { + t.Fatal(err) + } +} + +func localSpec(secret string) *tdev1.TDEConfig { + return &tdev1.TDEConfig{ManagementPolicy: tdev1.ManagementPolicyManaged, DefaultAlgorithm: "AES256", + Provider: tdev1.ProviderSpec{Type: tdev1.ProviderLocal, Local: &tdev1.LocalProviderSpec{SecretKeyRef: tdev1.SecretKeyReference{Name: secret, Key: "root.key"}}}} +} + +func localStatus(secret, uid string) *tdev1.ProviderStatus { + return &tdev1.ProviderStatus{Provider: tdev1.ProviderLocal, DefaultAlgorithm: "AES256", RootKeyRef: &tdev1.RootKeyRefStatus{ + SecretName: secret, Key: "root.key", SecretUID: uid, ResolvedPath: localKeyPath(secret, "root.key")}} +} + +func kmsSpec(provider tdev1.ProviderType, keyID, secret string) *tdev1.TDEConfig { + return &tdev1.TDEConfig{ManagementPolicy: tdev1.ManagementPolicyManaged, DefaultAlgorithm: "AES256", Provider: tdev1.ProviderSpec{ + Type: provider, Kms: &tdev1.KmsProviderSpec{KeyID: keyID, Endpoint: "https://kms.example.com", Region: "region-1", + Auth: tdev1.KmsAuthSpec{Type: tdev1.KmsAuthEnvironmentSecret, CredentialSecretRef: &tdev1.KmsCredentialSecretReference{ + Name: secret, AccessKeyKey: "accessKey", SecretKeyKey: "secretKey", + }}}, + }} +} + +func kmsStatus(provider tdev1.ProviderType, keyID, secret, uid string) *tdev1.ProviderStatus { + return &tdev1.ProviderStatus{Provider: provider, DefaultAlgorithm: "AES256", Kms: &tdev1.KmsProviderStatus{ + KeyID: keyID, Endpoint: "https://kms.example.com", Region: "region-1", Auth: tdev1.KmsAuthStatus{ + Type: tdev1.KmsAuthEnvironmentSecret, CredentialSecretName: secret, CredentialSecretUID: uid, + AccessKeyKey: "accessKey", SecretKeyKey: "secretKey", + }, + }} +}