From c68281f04f15f60c76e3151139748ea7a91c01e1 Mon Sep 17 00:00:00 2001 From: Gianluca Mardente Date: Fri, 17 Jul 2026 20:35:27 +0200 Subject: [PATCH] Use shared clustercache from libsveltos; invalidate cache on auth errors Replaces the addon-controller-local controllers/clustercache package with the shared lib/clustercache now in libsveltos, so event-manager, healthcheck-manager, and classifier can reuse the same cache instead of each managing their own (or none at all). Calls InvalidateOnAuthError in handleDeployerError and processUndeployResult so a cached rest.Config/mapper/discovery-client (and the underlying workload-identity cache entry) gets evicted as soon as the API server rejects credentials with 401/403, rather than only on TTL expiry or an explicit cluster delete. --- api/v1beta1/zz_generated.deepcopy.go | 5 +- controllers/clustercache/cluster_cache.go | 371 ------------------ .../clustercache/cluster_cache_test.go | 146 ------- .../clustercache/clustercache_suite_test.go | 159 -------- controllers/clustercache/export_test.go | 44 --- .../clustercache/test_constants_test.go | 21 - controllers/clustersummary_deployer.go | 7 + controllers/clustersummary_transformations.go | 2 +- controllers/delete_checks.go | 2 +- controllers/drift_detection_upgrade.go | 2 +- controllers/handlers_helm.go | 2 +- controllers/handlers_kustomize.go | 2 +- controllers/handlers_resources.go | 2 +- controllers/handlers_utils.go | 2 +- controllers/handlers_utils_test.go | 2 +- controllers/resourcesummary.go | 2 +- controllers/utils.go | 2 +- controllers/utils_test.go | 2 +- go.mod | 2 +- go.sum | 4 +- 20 files changed, 23 insertions(+), 758 deletions(-) delete mode 100644 controllers/clustercache/cluster_cache.go delete mode 100644 controllers/clustercache/cluster_cache_test.go delete mode 100644 controllers/clustercache/clustercache_suite_test.go delete mode 100644 controllers/clustercache/export_test.go delete mode 100644 controllers/clustercache/test_constants_test.go diff --git a/api/v1beta1/zz_generated.deepcopy.go b/api/v1beta1/zz_generated.deepcopy.go index 9e31aec6..dd08190d 100644 --- a/api/v1beta1/zz_generated.deepcopy.go +++ b/api/v1beta1/zz_generated.deepcopy.go @@ -21,12 +21,11 @@ limitations under the License. package v1beta1 import ( + apiv1beta1 "github.com/projectsveltos/libsveltos/api/v1beta1" corev1 "k8s.io/api/core/v1" - v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/util/intstr" - - apiv1beta1 "github.com/projectsveltos/libsveltos/api/v1beta1" ) // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. diff --git a/controllers/clustercache/cluster_cache.go b/controllers/clustercache/cluster_cache.go deleted file mode 100644 index 5c2ea91d..00000000 --- a/controllers/clustercache/cluster_cache.go +++ /dev/null @@ -1,371 +0,0 @@ -/* -Copyright 2024. projectsveltos.io. All rights reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package clustercache - -import ( - "context" - "fmt" - "sync" - - "github.com/go-logr/logr" - corev1 "k8s.io/api/core/v1" - "k8s.io/client-go/discovery" - memory "k8s.io/client-go/discovery/cached" - "k8s.io/client-go/rest" - "k8s.io/client-go/restmapper" - "k8s.io/klog/v2/textlogger" - clusterv1 "sigs.k8s.io/cluster-api/api/core/v1beta2" - "sigs.k8s.io/cluster-api/util/secret" - "sigs.k8s.io/controller-runtime/pkg/client" - - libsveltosv1beta1 "github.com/projectsveltos/libsveltos/api/v1beta1" - "github.com/projectsveltos/libsveltos/lib/clusterproxy" - logs "github.com/projectsveltos/libsveltos/lib/logsettings" - libsveltosset "github.com/projectsveltos/libsveltos/lib/set" -) - -var ( - managerInstance *clusterCache - lock = &sync.Mutex{} -) - -type clusterCache struct { - rwMux sync.RWMutex - // Keeps cache of rest.Config for existing clusters - configs map[corev1.ObjectReference]*rest.Config - - mappers map[corev1.ObjectReference]*restmapper.DeferredDiscoveryRESTMapper - cachedDiscoveryClient map[corev1.ObjectReference]discovery.CachedDiscoveryInterface - - // key: cluster, value: Secret with kubeconfig - clusters map[corev1.ObjectReference]*corev1.ObjectReference - - // key: secret, value: set of clusters - // A secret can potentially contain kubeconfig for one or more clusters - secrets map[corev1.ObjectReference]*libsveltosset.Set -} - -// GetManager return manager instance -func GetManager() *clusterCache { - if managerInstance == nil { - lock.Lock() - defer lock.Unlock() - if managerInstance == nil { - managerInstance = &clusterCache{ - configs: make(map[corev1.ObjectReference]*rest.Config), - clusters: make(map[corev1.ObjectReference]*corev1.ObjectReference), - mappers: make(map[corev1.ObjectReference]*restmapper.DeferredDiscoveryRESTMapper), - cachedDiscoveryClient: make(map[corev1.ObjectReference]discovery.CachedDiscoveryInterface), - secrets: make(map[corev1.ObjectReference]*libsveltosset.Set), - rwMux: sync.RWMutex{}, - } - } - } - - return managerInstance -} - -// RemoveCluster removes restConfig cached data for the cluster -func (m *clusterCache) RemoveCluster(clusterNamespace, clusterName string, - clusterType libsveltosv1beta1.ClusterType) { - - cluster := getClusterObjectReference(clusterNamespace, clusterName, clusterType) - - m.rwMux.Lock() - defer m.rwMux.Unlock() - - // Remove from cache the restConfig for this cluster - delete(m.configs, *cluster) - - if sec, ok := m.clusters[*cluster]; ok { - m.updateSecretMap(sec, cluster) - } - - // Do not track this cluster anymore - delete(m.clusters, *cluster) - - delete(m.mappers, *cluster) - delete(m.cachedDiscoveryClient, *cluster) -} - -// RemoveSecret removes any in-memory data related to secret -func (m *clusterCache) RemoveSecret(sec *corev1.ObjectReference) { - m.rwMux.Lock() - defer m.rwMux.Unlock() - - v, ok := m.secrets[*sec] - if !ok { - return - } - - clusters := v.Items() - for i := range clusters { - delete(m.configs, clusters[i]) - delete(m.clusters, clusters[i]) - delete(m.cachedDiscoveryClient, clusters[i]) - delete(m.mappers, clusters[i]) - } -} - -// GetKubernetesRestConfig returns managed cluster restConfig. -// If result is cached, it will be returned immediately. Otherwise it will be built -// by fetching the Secret containing the cluster kubeconfig. -// Admins restConfig are never cached. -func (m *clusterCache) GetKubernetesRestConfig(ctx context.Context, mgmtClient client.Client, - clusterNamespace, clusterName, adminNamespace, adminName string, - clusterType libsveltosv1beta1.ClusterType, logger logr.Logger) (*rest.Config, error) { - - if adminNamespace != "" || adminName != "" { - // cluster configs for admins are not cached - return clusterproxy.GetKubernetesRestConfig(ctx, mgmtClient, clusterNamespace, clusterName, - adminNamespace, adminName, clusterType, logger) - } - - m.rwMux.Lock() - defer m.rwMux.Unlock() - - cluster := getClusterObjectReference(clusterNamespace, clusterName, clusterType) - - config, ok := m.configs[*cluster] - if ok { - if config != nil { - logger.V(logs.LogInfo).Info("remote restConfig cache hit") - } else { - logger.V(logs.LogInfo).Info("remote restConfig cache hit: cluster in pull mode") - } - return config, nil - } - - logger.V(logs.LogDebug).Info("remote restConfig cache miss") - remoteRestConfig, err := clusterproxy.GetKubernetesRestConfig(ctx, mgmtClient, clusterNamespace, clusterName, - adminNamespace, adminName, clusterType, logger) - if err != nil { - return nil, err - } - - var cachedDiscoveryClient discovery.CachedDiscoveryInterface - var mapper *restmapper.DeferredDiscoveryRESTMapper - if remoteRestConfig != nil { - dc, err := discovery.NewDiscoveryClientForConfig(remoteRestConfig) - if err != nil { - return nil, err - } - - cachedDiscoveryClient = memory.NewMemCacheClient(dc) - mapper = restmapper.NewDeferredDiscoveryRESTMapper(cachedDiscoveryClient) - } - - secretInfo, err := getSecretObjectReference(ctx, mgmtClient, clusterNamespace, clusterName, clusterType) - if err == nil { - // Either all internal structures are updated or none is - m.configs[*cluster] = remoteRestConfig - m.clusters[*cluster] = secretInfo - m.cachedDiscoveryClient[*cluster] = cachedDiscoveryClient - m.mappers[*cluster] = mapper - v, ok := m.secrets[*secretInfo] - if !ok { - v = &libsveltosset.Set{} - } - v.Insert(cluster) - m.secrets[*secretInfo] = v - } - - return remoteRestConfig, nil -} - -func (m *clusterCache) GetMapper(ctx context.Context, mgmtClient client.Client, - clusterNamespace, clusterName string, clusterType libsveltosv1beta1.ClusterType, - logger logr.Logger) (*restmapper.DeferredDiscoveryRESTMapper, error) { - - cluster := getClusterObjectReference(clusterNamespace, clusterName, clusterType) - - // 1. Use a Read Lock to check if it's already cached - m.rwMux.RLock() - mapper, ok := m.mappers[*cluster] - m.rwMux.RUnlock() - - if ok { - return mapper, nil - } - - // 2. Cache Miss: We need to initialize the config and mapper. - // Calling GetKubernetesRestConfig will populate m.mappers via the logic you wrote. - logger.V(logs.LogInfo).Info("mapper cache miss, initializing cluster config") - _, err := m.GetKubernetesRestConfig(ctx, mgmtClient, clusterNamespace, clusterName, - "", "", clusterType, logger) - if err != nil { - return nil, fmt.Errorf("failed to initialize cluster config for mapper: %w", err) - } - - // 3. Lock again to retrieve the newly created mapper - m.rwMux.RLock() - defer m.rwMux.RUnlock() - - if v, ok := m.mappers[*cluster]; ok { - return v, nil - } - - return nil, fmt.Errorf("mapper not found for cluster %s/%s after initialization", clusterNamespace, clusterName) -} - -func (m *clusterCache) GetCachedDiscoveryClient(ctx context.Context, mgmtClient client.Client, - clusterNamespace, clusterName string, clusterType libsveltosv1beta1.ClusterType, - logger logr.Logger) (discovery.CachedDiscoveryInterface, error) { - - cluster := getClusterObjectReference(clusterNamespace, clusterName, clusterType) - - // 1. Thread-safe check for existing cached client - m.rwMux.RLock() - dc, ok := m.cachedDiscoveryClient[*cluster] - m.rwMux.RUnlock() - - if ok { - return dc, nil - } - - // 2. Cache Miss: Initialize the cluster configuration - // This will populate m.cachedDiscoveryClient via GetKubernetesRestConfig - logger.V(logs.LogInfo).Info("discovery client cache miss, initializing cluster config") - _, err := m.GetKubernetesRestConfig(ctx, mgmtClient, clusterNamespace, clusterName, - "", "", clusterType, logger) - if err != nil { - return nil, fmt.Errorf("failed to initialize cluster config for discovery client: %w", err) - } - - // 3. Retrieve the newly created client - m.rwMux.RLock() - defer m.rwMux.RUnlock() - - if dc, ok := m.cachedDiscoveryClient[*cluster]; ok { - return dc, nil - } - - return nil, fmt.Errorf("cached discovery client not found for cluster %s/%s after initialization", - clusterNamespace, clusterName) -} - -func (m *clusterCache) ResetMapper(clusterNamespace, clusterName string, - clusterType libsveltosv1beta1.ClusterType) { - - m.rwMux.RLock() - defer m.rwMux.RUnlock() - - cluster := getClusterObjectReference(clusterNamespace, clusterName, clusterType) - if mapper, ok := m.mappers[*cluster]; ok { - mapper.Reset() - } -} - -func (m *clusterCache) InvalidateDiscoveryClient(clusterNamespace, clusterName string, - clusterType libsveltosv1beta1.ClusterType) { - - m.rwMux.RLock() - defer m.rwMux.RUnlock() - - cluster := getClusterObjectReference(clusterNamespace, clusterName, clusterType) - if dc, ok := m.cachedDiscoveryClient[*cluster]; ok { - dc.Invalidate() - } -} - -func (m *clusterCache) GetKubernetesClient(ctx context.Context, mgmtClient client.Client, - clusterNamespace, clusterName, adminNamespace, adminName string, - clusterType libsveltosv1beta1.ClusterType, logger logr.Logger) (client.Client, error) { - - if adminNamespace != "" || adminName != "" { - // cluster configs for admins are not cached - return clusterproxy.GetKubernetesClient(ctx, mgmtClient, clusterNamespace, clusterName, - adminNamespace, adminName, clusterType, logger) - } - - config, err := m.GetKubernetesRestConfig(ctx, mgmtClient, clusterNamespace, clusterName, - adminNamespace, adminName, clusterType, logger) - if err != nil { - return nil, err - } - - logger.V(logs.LogVerbose).Info("return new client") - return client.New(config, client.Options{Scheme: mgmtClient.Scheme()}) -} - -func (m *clusterCache) StoreRestConfig(clusterNamespace, clusterName string, - clusterType libsveltosv1beta1.ClusterType, config *rest.Config) { - - cluster := getClusterObjectReference(clusterNamespace, clusterName, clusterType) - - m.rwMux.Lock() - defer m.rwMux.Unlock() - - m.configs[*cluster] = config -} - -func (m *clusterCache) updateSecretMap(sec, cluster *corev1.ObjectReference) { - set, ok := m.secrets[*sec] - if ok { - set.Erase(cluster) - if set.Len() == 0 { - delete(m.secrets, *sec) - } - } -} - -func getClusterObjectReference(clusterNamespace, clusterName string, - clusterType libsveltosv1beta1.ClusterType) *corev1.ObjectReference { - - cluster := &corev1.ObjectReference{ - Namespace: clusterNamespace, - Name: clusterName, - Kind: clusterv1.ClusterKind, - APIVersion: clusterv1.GroupVersion.String(), - } - if clusterType == libsveltosv1beta1.ClusterTypeSveltos { - cluster.Kind = libsveltosv1beta1.SveltosClusterKind - cluster.APIVersion = libsveltosv1beta1.GroupVersion.String() - } - - return cluster -} - -func getSecretObjectReference(ctx context.Context, mgmtClient client.Client, - clusterNamespace, clusterName string, clusterType libsveltosv1beta1.ClusterType) (*corev1.ObjectReference, error) { - - secretKind := "Secret" - if clusterType == libsveltosv1beta1.ClusterTypeCapi { - return &corev1.ObjectReference{ - Namespace: clusterNamespace, - Name: getClusterAPISecretName(clusterName), - Kind: secretKind, - APIVersion: corev1.SchemeGroupVersion.String(), - }, nil - } - - logger := textlogger.NewLogger(textlogger.NewConfig()) - secretName, _, err := clusterproxy.GetSveltosSecretNameAndKey(ctx, logger, mgmtClient, clusterNamespace, clusterName) - if err != nil { - return nil, err - } - return &corev1.ObjectReference{ - Namespace: clusterNamespace, - Name: secretName, - Kind: secretKind, - APIVersion: corev1.SchemeGroupVersion.String(), - }, nil -} - -func getClusterAPISecretName(clusterName string) string { - return fmt.Sprintf("%s-%s", clusterName, secret.Kubeconfig) -} diff --git a/controllers/clustercache/cluster_cache_test.go b/controllers/clustercache/cluster_cache_test.go deleted file mode 100644 index d94c79ca..00000000 --- a/controllers/clustercache/cluster_cache_test.go +++ /dev/null @@ -1,146 +0,0 @@ -/* -Copyright 2024. projectsveltos.io. All rights reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package clustercache_test - -import ( - "context" - - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" - - "github.com/go-logr/logr" - corev1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/klog/v2/textlogger" - - "github.com/projectsveltos/addon-controller/controllers/clustercache" - libsveltosv1beta1 "github.com/projectsveltos/libsveltos/api/v1beta1" -) - -const ( - sveltosKubeconfigPostfix = "-sveltos-kubeconfig" -) - -var _ = Describe("Clustercache", func() { - var logger logr.Logger - var cluster *libsveltosv1beta1.SveltosCluster - - BeforeEach(func() { - logger = textlogger.NewLogger(textlogger.NewConfig()) - cluster = &libsveltosv1beta1.SveltosCluster{ - ObjectMeta: metav1.ObjectMeta{ - Name: "cache" + randomString(), - Namespace: "cache" + randomString(), - }, - } - }) - - It("GetKubernetesRestConfig stores in memory first time. RemoveCluster removes any entry associated to cluster", - func() { - secret := createClusterResources(cluster) - - cacheMgr := clustercache.GetManager() - _, err := cacheMgr.GetKubernetesRestConfig(context.TODO(), testEnv.Client, cluster.Namespace, - cluster.Name, "", "", libsveltosv1beta1.ClusterTypeSveltos, logger) - Expect(err).To(BeNil()) - - clusterObj := &corev1.ObjectReference{ - Namespace: cluster.Namespace, - Name: cluster.Name, - Kind: libsveltosv1beta1.SveltosClusterKind, - APIVersion: libsveltosv1beta1.GroupVersion.String(), - } - Expect(cacheMgr.GetConfigFromMap(clusterObj)).ToNot(BeNil()) - - storedSecret := cacheMgr.GetSecretForCluster(clusterObj) - Expect(storedSecret).ToNot(BeNil()) - Expect(storedSecret.Namespace).To(Equal(secret.Namespace)) - Expect(storedSecret.Name).To(Equal(secret.Name)) - - secretObj := &corev1.ObjectReference{ - Namespace: secret.Namespace, - Name: secret.Name, - Kind: testKindSecret, - APIVersion: corev1.SchemeGroupVersion.String(), - } - storedCluster := cacheMgr.GetClusterFromSecret(secretObj) - Expect(storedCluster).ToNot(BeNil()) - Expect(storedCluster.Namespace).To(Equal(cluster.Namespace)) - Expect(storedCluster.Name).To(Equal(cluster.Name)) - - cacheMgr.RemoveCluster(cluster.Namespace, clusterObj.Name, libsveltosv1beta1.ClusterTypeSveltos) - Expect(cacheMgr.GetConfigFromMap(clusterObj)).To(BeNil()) - Expect(cacheMgr.GetSecretForCluster(clusterObj)).To(BeNil()) - Expect(cacheMgr.GetClusterFromSecret(secretObj)).To(BeNil()) - }) - - It("RemoveSecret removes entries for all clusters using the modified secret", func() { - secret := createClusterResources(cluster) - - cacheMgr := clustercache.GetManager() - _, err := cacheMgr.GetKubernetesRestConfig(context.TODO(), testEnv.Client, cluster.Namespace, - cluster.Name, "", "", libsveltosv1beta1.ClusterTypeSveltos, logger) - Expect(err).To(BeNil()) - - clusterObj := &corev1.ObjectReference{ - Namespace: cluster.Namespace, - Name: cluster.Name, - Kind: libsveltosv1beta1.SveltosClusterKind, - APIVersion: libsveltosv1beta1.GroupVersion.String(), - } - Expect(cacheMgr.GetConfigFromMap(clusterObj)).ToNot(BeNil()) - - secretObj := &corev1.ObjectReference{ - Namespace: secret.Namespace, - Name: secret.Name, - Kind: testKindSecret, - APIVersion: corev1.SchemeGroupVersion.String(), - } - cacheMgr.RemoveSecret(secretObj) - Expect(cacheMgr.GetConfigFromMap(clusterObj)).To(BeNil()) - }) -}) - -func createClusterResources(cluster *libsveltosv1beta1.SveltosCluster) *corev1.Secret { - By("Create the cluster's namespace") - ns := &corev1.Namespace{ - ObjectMeta: metav1.ObjectMeta{ - Name: cluster.Namespace, - }, - } - - By("Create the secret with cluster kubeconfig") - secret := &corev1.Secret{ - ObjectMeta: metav1.ObjectMeta{ - Namespace: cluster.Namespace, - Name: cluster.Name + sveltosKubeconfigPostfix, - }, - Data: map[string][]byte{ - "value": testEnv.Kubeconfig, - }, - } - - Expect(testEnv.Create(context.TODO(), ns)).To(Succeed()) - Expect(waitForObject(context.TODO(), testEnv.Client, ns)).To(Succeed()) - - Expect(testEnv.Create(context.TODO(), cluster)).To(Succeed()) - Expect(testEnv.Create(context.TODO(), secret)).To(Succeed()) - - Expect(waitForObject(context.TODO(), testEnv.Client, cluster)).To(Succeed()) - Expect(waitForObject(context.TODO(), testEnv.Client, secret)).To(Succeed()) - return secret -} diff --git a/controllers/clustercache/clustercache_suite_test.go b/controllers/clustercache/clustercache_suite_test.go deleted file mode 100644 index 0cfd7485..00000000 --- a/controllers/clustercache/clustercache_suite_test.go +++ /dev/null @@ -1,159 +0,0 @@ -/* -Copyright 2024. projectsveltos.io. All rights reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package clustercache_test - -import ( - "context" - "fmt" - "path" - "testing" - "time" - - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" - - apierrors "k8s.io/apimachinery/pkg/api/errors" - "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" - "k8s.io/apimachinery/pkg/runtime" - "k8s.io/apimachinery/pkg/util/wait" - clientgoscheme "k8s.io/client-go/kubernetes/scheme" - "k8s.io/klog/v2" - "sigs.k8s.io/cluster-api/util" - ctrl "sigs.k8s.io/controller-runtime" - "sigs.k8s.io/controller-runtime/pkg/client" - - configv1beta1 "github.com/projectsveltos/addon-controller/api/v1beta1" - "github.com/projectsveltos/addon-controller/api/v1beta1/index" - "github.com/projectsveltos/addon-controller/internal/test/helpers" - libsveltosv1beta1 "github.com/projectsveltos/libsveltos/api/v1beta1" - libsveltoscrd "github.com/projectsveltos/libsveltos/lib/crd" - "github.com/projectsveltos/libsveltos/lib/k8s_utils" -) - -var ( - testEnv *helpers.TestEnvironment - cancel context.CancelFunc - ctx context.Context - scheme *runtime.Scheme -) - -var ( - cacheSyncBackoff = wait.Backoff{ - Duration: 100 * time.Millisecond, - Factor: 1.5, - Steps: 8, - Jitter: 0.4, - } -) - -func TestControllers(t *testing.T) { - RegisterFailHandler(Fail) - RunSpecs(t, "Controllers Suite") -} - -var _ = BeforeSuite(func() { - By("bootstrapping test environment") - - ctrl.SetLogger(klog.Background()) - - ctx, cancel = context.WithCancel(context.TODO()) - - var err error - scheme, err = setupScheme() - Expect(err).To(BeNil()) - - testEnvConfig := helpers.NewTestEnvironmentConfiguration([]string{ - path.Join("config", "crd", "bases"), - }, scheme) - testEnv, err = testEnvConfig.Build(scheme) - if err != nil { - panic(err) - } - - Expect(index.AddDefaultIndexes(ctx, testEnv.Manager)).To(Succeed()) - - go func() { - By("Starting the manager") - err = testEnv.StartManager(ctx) - if err != nil { - panic(fmt.Sprintf("Failed to start the envtest manager: %v", err)) - } - }() - - var sveltosCRD *unstructured.Unstructured - sveltosCRD, err = k8s_utils.GetUnstructured(libsveltoscrd.GetSveltosClusterCRDYAML()) - Expect(err).To(BeNil()) - Expect(testEnv.Create(context.TODO(), sveltosCRD)).To(Succeed()) - Expect(waitForObject(context.TODO(), testEnv, sveltosCRD)).To(Succeed()) - - // Wait for synchronization - // Sometimes we otherwise get "no matches for kind "..." in version "lib.projectsveltos.io/v1beta1" - time.Sleep(2 * time.Second) - - if synced := testEnv.GetCache().WaitForCacheSync(ctx); !synced { - time.Sleep(time.Second) - } -}) - -var _ = AfterSuite(func() { - cancel() - By("tearing down the test environment") - err := testEnv.Stop() - Expect(err).ToNot(HaveOccurred()) -}) - -func setupScheme() (*runtime.Scheme, error) { - s := runtime.NewScheme() - if err := configv1beta1.AddToScheme(s); err != nil { - return nil, err - } - if err := clientgoscheme.AddToScheme(s); err != nil { - return nil, err - } - if err := libsveltosv1beta1.AddToScheme(s); err != nil { - return nil, err - } - - return s, nil -} - -func randomString() string { - const length = 10 - return "a-" + util.RandomString(length) -} - -// waitForObject waits for the cache to be updated helps in preventing test flakes due to the cache sync delays. -func waitForObject(ctx context.Context, c client.Client, obj client.Object) error { - // Makes sure the cache is updated with the new object - objCopy := obj.DeepCopyObject().(client.Object) - key := client.ObjectKeyFromObject(obj) - if err := wait.ExponentialBackoff( - cacheSyncBackoff, - func() (done bool, err error) { - if err := c.Get(ctx, key, objCopy); err != nil { - if apierrors.IsNotFound(err) { - return false, nil - } - return false, err - } - return true, nil - }); err != nil { - return fmt.Errorf("object %s, %s is not being added to the testenv client cache: %w", - obj.GetObjectKind().GroupVersionKind().String(), key, err) - } - return nil -} diff --git a/controllers/clustercache/export_test.go b/controllers/clustercache/export_test.go deleted file mode 100644 index a88863a7..00000000 --- a/controllers/clustercache/export_test.go +++ /dev/null @@ -1,44 +0,0 @@ -/* -Copyright 2024. projectsveltos.io. All rights reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package clustercache - -import ( - corev1 "k8s.io/api/core/v1" - "k8s.io/client-go/rest" -) - -func (m *clusterCache) GetConfigFromMap(cluster *corev1.ObjectReference) *rest.Config { - return m.configs[*cluster] -} - -func (m *clusterCache) GetSecretForCluster(cluster *corev1.ObjectReference) *corev1.ObjectReference { - return m.clusters[*cluster] -} - -func (m *clusterCache) GetClusterFromSecret(secret *corev1.ObjectReference) *corev1.ObjectReference { - set := m.secrets[*secret] - if set == nil { - return nil - } - - if set.Len() == 0 { - return nil - } - - items := set.Items() - return &items[0] -} diff --git a/controllers/clustercache/test_constants_test.go b/controllers/clustercache/test_constants_test.go deleted file mode 100644 index 13781968..00000000 --- a/controllers/clustercache/test_constants_test.go +++ /dev/null @@ -1,21 +0,0 @@ -/* -Copyright 2025. projectsveltos.io. All rights reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package clustercache_test - -const ( - testKindSecret = "Secret" -) diff --git a/controllers/clustersummary_deployer.go b/controllers/clustersummary_deployer.go index 3e994bea..a4a1ee2e 100644 --- a/controllers/clustersummary_deployer.go +++ b/controllers/clustersummary_deployer.go @@ -34,6 +34,7 @@ import ( "github.com/projectsveltos/addon-controller/lib/clusterops" "github.com/projectsveltos/addon-controller/pkg/scope" libsveltosv1beta1 "github.com/projectsveltos/libsveltos/api/v1beta1" + "github.com/projectsveltos/libsveltos/lib/clustercache" "github.com/projectsveltos/libsveltos/lib/clusterproxy" "github.com/projectsveltos/libsveltos/lib/deployer" logs "github.com/projectsveltos/libsveltos/lib/logsettings" @@ -242,6 +243,9 @@ func (r *ClusterSummaryReconciler) handleDeployerError(deployerError error, clus clusterSummary := clusterSummaryScope.ClusterSummary + clustercache.GetManager().InvalidateOnAuthError(clusterSummary.Spec.ClusterNamespace, clusterSummary.Spec.ClusterName, + clusterSummary.Spec.ClusterType, deployerError) + // Check if error is a NonRetriableError type var nonRetriableError *configv1beta1.NonRetriableError if errors.As(deployerError, &nonRetriableError) { @@ -512,6 +516,9 @@ func (r *ClusterSummaryReconciler) processUndeployResult(ctx context.Context, cl return fmt.Errorf("feature is still being removed") } + clustercache.GetManager().InvalidateOnAuthError(clusterSummary.Spec.ClusterNamespace, clusterSummary.Spec.ClusterName, + clusterSummary.Spec.ClusterType, result.Err) + // Failure to undeploy because of missing permission is not treated as terminal: this Forbidden // error also covers admission webhooks denying the delete, so the reason is still surfaced via // FailureMessage even though the feature keeps retrying. diff --git a/controllers/clustersummary_transformations.go b/controllers/clustersummary_transformations.go index 4463f126..7a69c772 100644 --- a/controllers/clustersummary_transformations.go +++ b/controllers/clustersummary_transformations.go @@ -28,8 +28,8 @@ import ( sourcev1 "github.com/fluxcd/source-controller/api/v1" - "github.com/projectsveltos/addon-controller/controllers/clustercache" libsveltosv1beta1 "github.com/projectsveltos/libsveltos/api/v1beta1" + "github.com/projectsveltos/libsveltos/lib/clustercache" "github.com/projectsveltos/libsveltos/lib/clusterproxy" logs "github.com/projectsveltos/libsveltos/lib/logsettings" ) diff --git a/controllers/delete_checks.go b/controllers/delete_checks.go index 6440a99a..bb700715 100644 --- a/controllers/delete_checks.go +++ b/controllers/delete_checks.go @@ -22,9 +22,9 @@ import ( "github.com/go-logr/logr" configv1beta1 "github.com/projectsveltos/addon-controller/api/v1beta1" - "github.com/projectsveltos/addon-controller/controllers/clustercache" "github.com/projectsveltos/addon-controller/lib/clusterops" libsveltosv1beta1 "github.com/projectsveltos/libsveltos/api/v1beta1" + "github.com/projectsveltos/libsveltos/lib/clustercache" "github.com/projectsveltos/libsveltos/lib/clusterproxy" logs "github.com/projectsveltos/libsveltos/lib/logsettings" ) diff --git a/controllers/drift_detection_upgrade.go b/controllers/drift_detection_upgrade.go index 60e4ed12..97923dff 100644 --- a/controllers/drift_detection_upgrade.go +++ b/controllers/drift_detection_upgrade.go @@ -32,8 +32,8 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" configv1beta1 "github.com/projectsveltos/addon-controller/api/v1beta1" - "github.com/projectsveltos/addon-controller/controllers/clustercache" libsveltosv1beta1 "github.com/projectsveltos/libsveltos/api/v1beta1" + "github.com/projectsveltos/libsveltos/lib/clustercache" "github.com/projectsveltos/libsveltos/lib/clusterproxy" logs "github.com/projectsveltos/libsveltos/lib/logsettings" ) diff --git a/controllers/handlers_helm.go b/controllers/handlers_helm.go index 9f26bd8a..43206d12 100644 --- a/controllers/handlers_helm.go +++ b/controllers/handlers_helm.go @@ -82,9 +82,9 @@ import ( configv1beta1 "github.com/projectsveltos/addon-controller/api/v1beta1" "github.com/projectsveltos/addon-controller/controllers/chartmanager" - "github.com/projectsveltos/addon-controller/controllers/clustercache" "github.com/projectsveltos/addon-controller/lib/clusterops" libsveltosv1beta1 "github.com/projectsveltos/libsveltos/api/v1beta1" + "github.com/projectsveltos/libsveltos/lib/clustercache" "github.com/projectsveltos/libsveltos/lib/clusterproxy" "github.com/projectsveltos/libsveltos/lib/deployer" "github.com/projectsveltos/libsveltos/lib/k8s_utils" diff --git a/controllers/handlers_kustomize.go b/controllers/handlers_kustomize.go index 82344fa8..cf571460 100644 --- a/controllers/handlers_kustomize.go +++ b/controllers/handlers_kustomize.go @@ -49,9 +49,9 @@ import ( "sigs.k8s.io/yaml" configv1beta1 "github.com/projectsveltos/addon-controller/api/v1beta1" - "github.com/projectsveltos/addon-controller/controllers/clustercache" "github.com/projectsveltos/addon-controller/lib/clusterops" libsveltosv1beta1 "github.com/projectsveltos/libsveltos/api/v1beta1" + "github.com/projectsveltos/libsveltos/lib/clustercache" "github.com/projectsveltos/libsveltos/lib/clusterproxy" "github.com/projectsveltos/libsveltos/lib/deployer" "github.com/projectsveltos/libsveltos/lib/funcmap" diff --git a/controllers/handlers_resources.go b/controllers/handlers_resources.go index b6a3eb95..3c59982c 100644 --- a/controllers/handlers_resources.go +++ b/controllers/handlers_resources.go @@ -35,10 +35,10 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" configv1beta1 "github.com/projectsveltos/addon-controller/api/v1beta1" - "github.com/projectsveltos/addon-controller/controllers/clustercache" "github.com/projectsveltos/addon-controller/controllers/dependencymanager" "github.com/projectsveltos/addon-controller/lib/clusterops" libsveltosv1beta1 "github.com/projectsveltos/libsveltos/api/v1beta1" + "github.com/projectsveltos/libsveltos/lib/clustercache" "github.com/projectsveltos/libsveltos/lib/clusterproxy" "github.com/projectsveltos/libsveltos/lib/deployer" logs "github.com/projectsveltos/libsveltos/lib/logsettings" diff --git a/controllers/handlers_utils.go b/controllers/handlers_utils.go index bd4e5d4d..da1422be 100644 --- a/controllers/handlers_utils.go +++ b/controllers/handlers_utils.go @@ -50,9 +50,9 @@ import ( "sigs.k8s.io/yaml" configv1beta1 "github.com/projectsveltos/addon-controller/api/v1beta1" - "github.com/projectsveltos/addon-controller/controllers/clustercache" "github.com/projectsveltos/addon-controller/lib/clusterops" libsveltosv1beta1 "github.com/projectsveltos/libsveltos/api/v1beta1" + "github.com/projectsveltos/libsveltos/lib/clustercache" "github.com/projectsveltos/libsveltos/lib/clusterproxy" "github.com/projectsveltos/libsveltos/lib/deployer" "github.com/projectsveltos/libsveltos/lib/k8s_utils" diff --git a/controllers/handlers_utils_test.go b/controllers/handlers_utils_test.go index 55279b28..7c7ce92e 100644 --- a/controllers/handlers_utils_test.go +++ b/controllers/handlers_utils_test.go @@ -43,9 +43,9 @@ import ( configv1beta1 "github.com/projectsveltos/addon-controller/api/v1beta1" "github.com/projectsveltos/addon-controller/controllers" - "github.com/projectsveltos/addon-controller/controllers/clustercache" "github.com/projectsveltos/addon-controller/lib/clusterops" libsveltosv1beta1 "github.com/projectsveltos/libsveltos/api/v1beta1" + "github.com/projectsveltos/libsveltos/lib/clustercache" "github.com/projectsveltos/libsveltos/lib/deployer" "github.com/projectsveltos/libsveltos/lib/k8s_utils" "github.com/projectsveltos/libsveltos/lib/pullmode" diff --git a/controllers/resourcesummary.go b/controllers/resourcesummary.go index e717e2d6..2e56fe68 100644 --- a/controllers/resourcesummary.go +++ b/controllers/resourcesummary.go @@ -34,10 +34,10 @@ import ( "sigs.k8s.io/yaml" configv1beta1 "github.com/projectsveltos/addon-controller/api/v1beta1" - "github.com/projectsveltos/addon-controller/controllers/clustercache" "github.com/projectsveltos/addon-controller/lib/clusterops" driftdetection "github.com/projectsveltos/addon-controller/pkg/drift-detection" libsveltosv1beta1 "github.com/projectsveltos/libsveltos/api/v1beta1" + "github.com/projectsveltos/libsveltos/lib/clustercache" "github.com/projectsveltos/libsveltos/lib/clusterproxy" "github.com/projectsveltos/libsveltos/lib/crd" "github.com/projectsveltos/libsveltos/lib/deployer" diff --git a/controllers/utils.go b/controllers/utils.go index 12b8e69c..6285ceb3 100644 --- a/controllers/utils.go +++ b/controllers/utils.go @@ -41,8 +41,8 @@ import ( "sigs.k8s.io/yaml" configv1beta1 "github.com/projectsveltos/addon-controller/api/v1beta1" - "github.com/projectsveltos/addon-controller/controllers/clustercache" libsveltosv1beta1 "github.com/projectsveltos/libsveltos/api/v1beta1" + "github.com/projectsveltos/libsveltos/lib/clustercache" "github.com/projectsveltos/libsveltos/lib/clusterproxy" logs "github.com/projectsveltos/libsveltos/lib/logsettings" libsveltosset "github.com/projectsveltos/libsveltos/lib/set" diff --git a/controllers/utils_test.go b/controllers/utils_test.go index 9a7035f6..b188c8cd 100644 --- a/controllers/utils_test.go +++ b/controllers/utils_test.go @@ -42,9 +42,9 @@ import ( configv1beta1 "github.com/projectsveltos/addon-controller/api/v1beta1" "github.com/projectsveltos/addon-controller/controllers" - "github.com/projectsveltos/addon-controller/controllers/clustercache" "github.com/projectsveltos/addon-controller/lib/clusterops" libsveltosv1beta1 "github.com/projectsveltos/libsveltos/api/v1beta1" + "github.com/projectsveltos/libsveltos/lib/clustercache" "github.com/projectsveltos/libsveltos/lib/k8s_utils" "github.com/projectsveltos/libsveltos/lib/sveltos_upgrade" ) diff --git a/go.mod b/go.mod index afa0d624..3bf2ef4e 100644 --- a/go.mod +++ b/go.mod @@ -20,7 +20,7 @@ require ( github.com/onsi/gomega v1.42.1 github.com/opencontainers/image-spec v1.1.1 github.com/pkg/errors v0.9.1 - github.com/projectsveltos/libsveltos v1.12.1-0.20260715200227-148c08381bed + github.com/projectsveltos/libsveltos v1.12.1-0.20260717183230-6bffa4189087 github.com/prometheus/client_golang v1.23.2 github.com/robfig/cron v1.2.0 github.com/sigstore/cosign/v3 v3.1.1 diff --git a/go.sum b/go.sum index 04c0d4ac..ccbb7cdc 100644 --- a/go.sum +++ b/go.sum @@ -644,8 +644,8 @@ github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRI github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/poy/onpar v1.1.2 h1:QaNrNiZx0+Nar5dLgTVp5mXkyoVFIbepjyEoGSnhbAY= github.com/poy/onpar v1.1.2/go.mod h1:6X8FLNoxyr9kkmnlqpK6LSoiOtrO6MICtWwEuWkLjzg= -github.com/projectsveltos/libsveltos v1.12.1-0.20260715200227-148c08381bed h1:k5t1S+iaj2IqDUbL4za4ywEKoWJx0/gdEaXDTij5ZhQ= -github.com/projectsveltos/libsveltos v1.12.1-0.20260715200227-148c08381bed/go.mod h1:4/vcbYFCFE8uEGIHmltriAoHxdB8jgS2zI+9gruOfJ4= +github.com/projectsveltos/libsveltos v1.12.1-0.20260717183230-6bffa4189087 h1:uOHm6bX9SKfeR6htJHrg5pXK6d2fKmdeRrs9dkIvnkA= +github.com/projectsveltos/libsveltos v1.12.1-0.20260717183230-6bffa4189087/go.mod h1:4/vcbYFCFE8uEGIHmltriAoHxdB8jgS2zI+9gruOfJ4= github.com/projectsveltos/lua-utils/glua-json v0.0.0-20251212200258-2b3cdcb7c0f5 h1:khnc+994UszxZYu69J+R5FKiLA/Nk1JQj0EYAkwTWz0= github.com/projectsveltos/lua-utils/glua-json v0.0.0-20251212200258-2b3cdcb7c0f5/go.mod h1:yVL8KQFa9tmcxgwl9nwIMtKgtmIVC1zaFRSCfOwYvPY= github.com/projectsveltos/lua-utils/glua-runes v0.0.0-20251212200258-2b3cdcb7c0f5 h1:YbsebwRwTRhV8QacvEAdFqxcxHdeu7JTVtsBovbkgos=