From 1a2367bb78fd740e1db3204bbca6c6e0f08bb2b5 Mon Sep 17 00:00:00 2001 From: Felix Breuer Date: Thu, 13 Aug 2026 14:20:20 +0200 Subject: [PATCH 01/11] wip Signed-off-by: Felix Breuer --- .gitignore | 4 ++ go.mod | 3 + go.sum | 6 ++ test/kubetest2.go | 11 +++ test/kubetest2/deployer.go | 88 +++++++++++++++++++++++ test/kubetest2/env.go | 36 ++++++++++ test/kubetest2/flags.go | 88 +++++++++++++++++++++++ test/kubetest2/lifecycle.go | 121 ++++++++++++++++++++++++++++++++ test/kubetest2/naming.go | 15 ++++ test/kubetest2/payload.go | 131 +++++++++++++++++++++++++++++++++++ test/kubetest2/ske_client.go | 96 +++++++++++++++++++++++++ 11 files changed, 599 insertions(+) create mode 100644 test/kubetest2.go create mode 100644 test/kubetest2/deployer.go create mode 100644 test/kubetest2/env.go create mode 100644 test/kubetest2/flags.go create mode 100644 test/kubetest2/lifecycle.go create mode 100644 test/kubetest2/naming.go create mode 100644 test/kubetest2/payload.go create mode 100644 test/kubetest2/ske_client.go diff --git a/.gitignore b/.gitignore index 41e3565d..63550e4f 100644 --- a/.gitignore +++ b/.gitignore @@ -22,3 +22,7 @@ __debug_* test/e2e/inventory* test/e2e/sa-key* test/e2e/kubeconfig* + +# kubetest2 +_artifacts +_rundir diff --git a/go.mod b/go.mod index 90348018..02c6a4f8 100644 --- a/go.mod +++ b/go.mod @@ -17,6 +17,7 @@ require ( github.com/stackitcloud/stackit-sdk-go/core v0.26.0 github.com/stackitcloud/stackit-sdk-go/services/iaas v1.14.1 github.com/stackitcloud/stackit-sdk-go/services/loadbalancer v1.15.1 + github.com/stackitcloud/stackit-sdk-go/services/ske v1.6.0 go.uber.org/mock v0.6.0 golang.org/x/sync v0.22.0 golang.org/x/sys v0.47.0 @@ -31,6 +32,7 @@ require ( k8s.io/klog/v2 v2.140.0 k8s.io/mount-utils v0.36.3 k8s.io/utils v0.0.0-20260707023825-cf1189d6abe3 + sigs.k8s.io/kubetest2 v0.0.0-20260510045745-558f16b589d1 ) replace k8s.io/cloud-provider => github.com/stackitcloud/cloud-provider v0.36.0-ske-2 @@ -81,6 +83,7 @@ require ( github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.7 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/json-iterator/go v1.1.12 // indirect + github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 // indirect github.com/kylelemons/godebug v1.1.0 // indirect github.com/moby/sys/mountinfo v0.7.2 // indirect github.com/moby/term v0.5.2 // indirect diff --git a/go.sum b/go.sum index 85b2cbc1..c34c2bed 100644 --- a/go.sum +++ b/go.sum @@ -123,6 +123,8 @@ github.com/joshdk/go-junit v1.0.0 h1:S86cUKIdwBHWwA6xCmFlf3RTLfVXYQfvanM5Uh+K6GE github.com/joshdk/go-junit v1.0.0/go.mod h1:TiiV0PqkaNfFXjEiyjWM3XXrhVyCa1K4Zfga6W52ung= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= +github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 h1:Z9n2FFNUXsshfwJMBgNA0RU6/i7WVaAegv3PtuIHPMs= +github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51/go.mod h1:CzGEWj7cYgsdH8dAjBGEr58BoE7ScuLd+fwFZ44+/x8= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/klauspost/compress v1.19.1 h1:VsB4HPswih7mmZ8WleSFQ75c/Ui1M4trX5oAsJnhSlk= @@ -190,6 +192,8 @@ github.com/stackitcloud/stackit-sdk-go/services/loadbalancer v1.15.1 h1:+NT2okD8 github.com/stackitcloud/stackit-sdk-go/services/loadbalancer v1.15.1/go.mod h1:+Ld3dn648I+YKcBV3fEkYpDSr3fel421+LurJGywSBs= github.com/stackitcloud/stackit-sdk-go/services/resourcemanager v0.24.2 h1:4UxxJmCSCwV8q4bT4G+D1JH8F9Gm6BKaLGixX5DVcvI= github.com/stackitcloud/stackit-sdk-go/services/resourcemanager v0.24.2/go.mod h1:NEz3f+GV5G++BE9/MmZCsXJyCih7jtg0pZuSyG2sLEs= +github.com/stackitcloud/stackit-sdk-go/services/ske v1.6.0 h1:Dab1jzN0u9c67lvELoWf1RuagjO3eUBRytoX8SYL8Zs= +github.com/stackitcloud/stackit-sdk-go/services/ske v1.6.0/go.mod h1:NzcTU5GGlUF6Lys3Ra7ylRj4ZKxJr3f/29/yoE5tjPI= github.com/stoewer/go-strcase v1.3.1 h1:iS0MdW+kVTxgMoE1LAZyMiYJFKlOzLooE4MxjirtkAs= github.com/stoewer/go-strcase v1.3.1/go.mod h1:fAH5hQ5pehh+j3nZfvwdk2RgEgQjAoM8wodgtPmh1xo= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= @@ -369,6 +373,8 @@ sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.34.0 h1:hSfpvjjTQXQY2 sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.34.0/go.mod h1:Ve9uj1L+deCXFrPOk1LpFXqTg7LCFzFso6PA48q/XZw= sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730 h1:IpInykpT6ceI+QxKBbEflcR5EXP7sU1kvOlxwZh5txg= sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730/go.mod h1:mdzfpAEoE6DHQEN0uh9ZbOCuHbLK5wOm7dK4ctXE9Tg= +sigs.k8s.io/kubetest2 v0.0.0-20260510045745-558f16b589d1 h1:ttox7lK6ZgN/goexDpOxStOl1Q6j4CwprvK+rkS0c1M= +sigs.k8s.io/kubetest2 v0.0.0-20260510045745-558f16b589d1/go.mod h1:1IMaKFT+Htrbn6ixEZq6GTCk13CP/S0vwujFC5qs55U= sigs.k8s.io/randfill v1.0.0 h1:JfjMILfT8A6RbawdsK2JXGBR5AQVfd+9TbzrlneTyrU= sigs.k8s.io/randfill v1.0.0/go.mod h1:XeLlZ/jmk4i1HRopwe7/aU3H5n1zNUcX6TM94b3QxOY= sigs.k8s.io/structured-merge-diff/v6 v6.3.3 h1:u08YRbVUi59ri4YD6cg0UqNM4Dimn0sIl+wldcx5PYw= diff --git a/test/kubetest2.go b/test/kubetest2.go new file mode 100644 index 00000000..a8e9be55 --- /dev/null +++ b/test/kubetest2.go @@ -0,0 +1,11 @@ +package main + +import ( + "sigs.k8s.io/kubetest2/pkg/app" + + kubetest2stackit "github.com/stackitcloud/cloud-provider-stackit/test/kubetest2" +) + +func main() { + app.Main(kubetest2stackit.Name, kubetest2stackit.New) +} diff --git a/test/kubetest2/deployer.go b/test/kubetest2/deployer.go new file mode 100644 index 00000000..497b8c11 --- /dev/null +++ b/test/kubetest2/deployer.go @@ -0,0 +1,88 @@ +package kubetest2 + +import ( + "flag" + + "github.com/spf13/pflag" + "github.com/stackitcloud/cloud-provider-stackit/pkg/version" + "k8s.io/klog/v2" + "sigs.k8s.io/kubetest2/pkg/types" +) + +const ( + Name = "stackit" + + defaultRegion = "eu01" + defaultAvailabilityZone = "eu01-1" + defaultNodeCount int64 = 2 + defaultNodepoolName = "default" + defaultVolumeSizeGiB int64 = 100 + defaultKubeconfigExpiration int64 = 3600 + minKubeconfigExpiration int64 = 600 + maxKubeconfigExpiration int64 = 15552000 +) + +type Deployer struct { + options types.Options + + region string + kubernetesVersion string + availabilityZone string + machineType string + nodeImageName string + nodeImageVersion string + nodeCount int64 + nodepoolName string + volumeSizeGiB int64 + volumeType string + kubeconfigExpiresIn int64 + + projectID string + serviceAccount string + kubeconfigPath string + + skeClient skeClient +} + +var _ types.NewDeployer = New +var _ types.Deployer = &Deployer{} +var _ types.DeployerWithInit = &Deployer{} +var _ types.DeployerWithKubeconfig = &Deployer{} +var _ types.DeployerWithProvider = &Deployer{} +var _ types.DeployerWithVersion = &Deployer{} + +func New(opts types.Options) (types.Deployer, *pflag.FlagSet) { + d := &Deployer{ + options: opts, + region: defaultRegion, + availabilityZone: defaultAvailabilityZone, + nodeCount: defaultNodeCount, + nodepoolName: defaultNodepoolName, + volumeSizeGiB: defaultVolumeSizeGiB, + kubeconfigExpiresIn: defaultKubeconfigExpiration, + } + + fs := pflag.NewFlagSet(Name, pflag.ContinueOnError) + bindFlags(fs, d) + + klog.InitFlags(nil) + fs.AddGoFlagSet(flag.CommandLine) + + return d, fs +} + +func (d *Deployer) Provider() string { + return Name +} + +func (d *Deployer) Version() string { + return version.Version +} + +func (d *Deployer) Build() error { + return nil +} + +func (d *Deployer) DumpClusterLogs() error { + return nil +} diff --git a/test/kubetest2/env.go b/test/kubetest2/env.go new file mode 100644 index 00000000..a85c1fa1 --- /dev/null +++ b/test/kubetest2/env.go @@ -0,0 +1,36 @@ +package kubetest2 + +import ( + "os" + "path/filepath" + "strings" + + "k8s.io/klog/v2" +) + +func (d *Deployer) loadEnvironment() error { + klog.Infof("Loading STACKIT environment variables") + + serviceAccount, ok := os.LookupEnv("STACKIT_SERVICE_ACCOUNT") + if !ok || strings.TrimSpace(serviceAccount) == "" { + return incorrectUsagef("STACKIT_SERVICE_ACCOUNT environment variable is required") + } + + projectID, ok := os.LookupEnv("STACKIT_PROJECT_ID") + if !ok || strings.TrimSpace(projectID) == "" { + return incorrectUsagef("STACKIT_PROJECT_ID environment variable is required") + } + + d.serviceAccount = serviceAccount + d.projectID = projectID + d.kubeconfigPath = filepath.Join(d.options.RunDir(), "kubeconfig") + + klog.Infof( + "Loaded STACKIT environment: project_id=%q service_account_bytes=%d kubeconfig_path=%q", + d.projectID, + len(d.serviceAccount), + d.kubeconfigPath, + ) + + return nil +} diff --git a/test/kubetest2/flags.go b/test/kubetest2/flags.go new file mode 100644 index 00000000..530610d9 --- /dev/null +++ b/test/kubetest2/flags.go @@ -0,0 +1,88 @@ +package kubetest2 + +import ( + "fmt" + "strings" + + "github.com/spf13/pflag" + "k8s.io/klog/v2" + "sigs.k8s.io/kubetest2/pkg/types" +) + +func bindFlags(fs *pflag.FlagSet, d *Deployer) { + fs.StringVar(&d.region, "region", defaultRegion, "STACKIT region for the SKE cluster") + fs.StringVar(&d.kubernetesVersion, "kubernetes-version", "", "Kubernetes version for the SKE cluster") + fs.StringVar(&d.availabilityZone, "availability-zone", defaultAvailabilityZone, "Availability zone for the SKE nodepool") + fs.StringVar(&d.machineType, "machine-type", "", "Machine type for the SKE nodepool") + fs.StringVar(&d.nodeImageName, "node-image-name", "", "Node image name for the SKE nodepool") + fs.StringVar(&d.nodeImageVersion, "node-image-version", "", "Node image version for the SKE nodepool") + fs.Int64Var(&d.nodeCount, "node-count", defaultNodeCount, "Node count for the SKE nodepool") + fs.StringVar(&d.nodepoolName, "nodepool-name", defaultNodepoolName, "Nodepool name for the SKE cluster") + fs.Int64Var(&d.volumeSizeGiB, "volume-size", defaultVolumeSizeGiB, "Root volume size in GiB for the SKE nodepool") + fs.StringVar(&d.volumeType, "volume-type", "", "Root volume type for the SKE nodepool") + fs.Int64Var(&d.kubeconfigExpiresIn, "kubeconfig-expiration-seconds", defaultKubeconfigExpiration, "Admin kubeconfig expiration in seconds") +} + +func (d *Deployer) validate() error { + klog.Infof( + "Validating deployer configuration: run_id=%q region=%q kubernetes_version=%q availability_zone=%q machine_type=%q node_image_name=%q node_image_version=%q node_count=%d nodepool_name=%q volume_size=%d volume_type=%q kubeconfig_expiration_seconds=%d", + d.options.RunID(), + d.region, + d.kubernetesVersion, + d.availabilityZone, + d.machineType, + d.nodeImageName, + d.nodeImageVersion, + d.nodeCount, + d.nodepoolName, + d.volumeSizeGiB, + d.volumeType, + d.kubeconfigExpiresIn, + ) + + requiredFlags := map[string]string{ + "--region": d.region, + "--kubernetes-version": d.kubernetesVersion, + "--availability-zone": d.availabilityZone, + "--machine-type": d.machineType, + "--node-image-name": d.nodeImageName, + "--node-image-version": d.nodeImageVersion, + } + + for flagName, value := range requiredFlags { + if strings.TrimSpace(value) == "" { + return incorrectUsagef("%s is required", flagName) + } + } + + if strings.TrimSpace(d.nodepoolName) == "" { + return incorrectUsagef("--nodepool-name must not be empty") + } + if len(d.nodepoolName) > 15 { + return incorrectUsagef("--nodepool-name must be 15 characters or fewer") + } + if d.nodeCount < 1 { + return incorrectUsagef("--node-count must be greater than 0") + } + if d.volumeSizeGiB < 1 { + return incorrectUsagef("--volume-size must be greater than 0") + } + if d.kubeconfigExpiresIn < minKubeconfigExpiration || d.kubeconfigExpiresIn > maxKubeconfigExpiration { + return incorrectUsagef( + "--kubeconfig-expiration-seconds must be between %d and %d", + minKubeconfigExpiration, + maxKubeconfigExpiration, + ) + } + if strings.TrimSpace(d.options.RunID()) == "" { + return incorrectUsagef("kubetest2 run-id must not be empty") + } + + klog.Infof("Deployer configuration validation succeeded") + + return nil +} + +func incorrectUsagef(format string, args ...any) error { + return types.NewIncorrectUsage(fmt.Sprintf(format, args...)) +} diff --git a/test/kubetest2/lifecycle.go b/test/kubetest2/lifecycle.go new file mode 100644 index 00000000..5c123505 --- /dev/null +++ b/test/kubetest2/lifecycle.go @@ -0,0 +1,121 @@ +package kubetest2 + +import ( + "context" + "fmt" + "os" + + "github.com/stackitcloud/cloud-provider-stackit/pkg/stackit/stackiterrors" + "github.com/stackitcloud/stackit-sdk-go/services/ske" + "k8s.io/klog/v2" +) + +func (d *Deployer) Init() error { + klog.Infof("Initializing STACKIT kubetest2 deployer") + + if err := d.validate(); err != nil { + return err + } + if err := d.loadEnvironment(); err != nil { + return err + } + + client, err := newSKEClient(d.region, d.serviceAccount) + if err != nil { + return err + } + d.skeClient = client + + klog.Infof("STACKIT kubetest2 deployer initialized successfully") + + return nil +} + +func (d *Deployer) Up() error { + klog.Infof("Starting cluster up flow for cluster=%q", d.clusterName()) + + ctx := context.Background() + if err := d.validateProviderOptions(ctx); err != nil { + return err + } + + clusterName := d.clusterName() + payload := d.clusterPayload() + + klog.Infof("Submitting SKE create/update request for cluster=%q", clusterName) + if _, err := d.skeClient.CreateOrUpdateCluster(ctx, d.projectID, d.region, clusterName, payload); err != nil { + return fmt.Errorf("create or update SKE cluster %q: %w", clusterName, err) + } + klog.Infof("Submitted SKE create/update request for cluster=%q", clusterName) + + if _, err := d.skeClient.WaitForClusterReady(ctx, d.projectID, d.region, clusterName); err != nil { + return fmt.Errorf("wait for SKE cluster %q to become ready: %w", clusterName, err) + } + + klog.Infof("Creating kubeconfig for cluster=%q", clusterName) + kubeconfig, err := d.skeClient.CreateKubeconfig(ctx, d.projectID, d.region, clusterName, d.kubeconfigExpiresIn) + if err != nil { + return fmt.Errorf("create kubeconfig for SKE cluster %q: %w", clusterName, err) + } + klog.Infof("Writing kubeconfig for cluster=%q to %q", clusterName, d.kubeconfigPath) + if err := os.WriteFile(d.kubeconfigPath, []byte(kubeconfig.GetKubeconfig()), 0o600); err != nil { + return fmt.Errorf("write kubeconfig %q: %w", d.kubeconfigPath, err) + } + + klog.Infof("Cluster up flow completed successfully for cluster=%q", clusterName) + + return nil +} + +func (d *Deployer) Down() error { + ctx := context.Background() + clusterName := d.clusterName() + + klog.Infof("Starting cluster down flow for cluster=%q", clusterName) + + if err := d.skeClient.DeleteCluster(ctx, d.projectID, d.region, clusterName); err != nil { + if !stackiterrors.IsNotFound(err) { + return fmt.Errorf("delete SKE cluster %q: %w", clusterName, err) + } + klog.Infof("Cluster=%q already absent, treating delete as success", clusterName) + return nil + } + + if err := d.skeClient.WaitForClusterDeleted(ctx, d.projectID, d.region, clusterName); err != nil { + return fmt.Errorf("wait for SKE cluster %q deletion: %w", clusterName, err) + } + + klog.Infof("Cluster down flow completed successfully for cluster=%q", clusterName) + + return nil +} + +func (d *Deployer) IsUp() (bool, error) { + ctx := context.Background() + klog.Infof("Checking cluster state for cluster=%q", d.clusterName()) + cluster, err := d.skeClient.GetCluster(ctx, d.projectID, d.region, d.clusterName()) + if err != nil { + if stackiterrors.IsNotFound(err) { + klog.Infof("Cluster=%q not found during IsUp check", d.clusterName()) + return false, nil + } + return false, fmt.Errorf("get SKE cluster %q: %w", d.clusterName(), err) + } + + if cluster.Status == nil || cluster.Status.Aggregated == nil { + klog.Infof("Cluster=%q has no aggregated status yet", d.clusterName()) + return false, nil + } + + state := cluster.Status.GetAggregated() + klog.Infof("Cluster=%q current aggregated state=%q", d.clusterName(), state) + return state == ske.CLUSTERSTATUSSTATE_HEALTHY || state == ske.CLUSTERSTATUSSTATE_HIBERNATED, nil +} + +func (d *Deployer) Kubeconfig() (string, error) { + if d.kubeconfigPath == "" { + d.kubeconfigPath = d.options.RunDir() + "/kubeconfig" + } + klog.Infof("Returning kubeconfig path %q", d.kubeconfigPath) + return d.kubeconfigPath, nil +} diff --git a/test/kubetest2/naming.go b/test/kubetest2/naming.go new file mode 100644 index 00000000..0c8136ca --- /dev/null +++ b/test/kubetest2/naming.go @@ -0,0 +1,15 @@ +package kubetest2 + +import ( + "crypto/sha256" + "encoding/hex" +) + +func clusterNameForRun(runID string) string { + sum := sha256.Sum256([]byte(runID)) + return "kt2" + hex.EncodeToString(sum[:4]) +} + +func (d *Deployer) clusterName() string { + return clusterNameForRun(d.options.RunID()) +} diff --git a/test/kubetest2/payload.go b/test/kubetest2/payload.go new file mode 100644 index 00000000..c629c54e --- /dev/null +++ b/test/kubetest2/payload.go @@ -0,0 +1,131 @@ +package kubetest2 + +import ( + "context" + "fmt" + + "github.com/stackitcloud/stackit-sdk-go/services/ske" + "k8s.io/klog/v2" +) + +func (d *Deployer) validateProviderOptions(ctx context.Context) error { + klog.Infof("Validating SKE provider options for region=%q", d.region) + + options, err := d.skeClient.ListProviderOptions(ctx, d.region) + if err != nil { + return fmt.Errorf("list SKE provider options: %w", err) + } + + if !containsKubernetesVersion(options, d.kubernetesVersion) { + return incorrectUsagef("unsupported --kubernetes-version %q for region %q", d.kubernetesVersion, d.region) + } + if !containsAvailabilityZone(options, d.availabilityZone) { + return incorrectUsagef("unsupported --availability-zone %q for region %q", d.availabilityZone, d.region) + } + if !containsMachineType(options, d.machineType) { + return incorrectUsagef("unsupported --machine-type %q for region %q", d.machineType, d.region) + } + if !containsMachineImage(options, d.nodeImageName, d.nodeImageVersion) { + return incorrectUsagef( + "unsupported node image %q version %q for region %q", + d.nodeImageName, + d.nodeImageVersion, + d.region, + ) + } + if d.volumeType != "" && !containsVolumeType(options, d.volumeType) { + return incorrectUsagef("unsupported --volume-type %q for region %q", d.volumeType, d.region) + } + + klog.Infof("SKE provider option validation succeeded for region=%q", d.region) + + return nil +} + +func (d *Deployer) clusterPayload() ske.CreateOrUpdateClusterPayload { + klog.Infof( + "Building SKE cluster payload: cluster=%q kubernetes_version=%q availability_zone=%q machine_type=%q image=%q/%q node_count=%d nodepool=%q volume_size=%d volume_type=%q", + d.clusterName(), + d.kubernetesVersion, + d.availabilityZone, + d.machineType, + d.nodeImageName, + d.nodeImageVersion, + d.nodeCount, + d.nodepoolName, + d.volumeSizeGiB, + d.volumeType, + ) + + clusterKubernetes := ske.NewKubernetes(d.kubernetesVersion) + nodeImage := ske.NewImage(d.nodeImageName, d.nodeImageVersion) + nodeMachine := ske.NewMachine(*nodeImage, d.machineType) + nodeVolume := ske.NewVolume(d.volumeSizeGiB) + if d.volumeType != "" { + nodeVolume.SetType(d.volumeType) + } + + nodepool := ske.NewNodepool( + []string{d.availabilityZone}, + *nodeMachine, + d.nodeCount, + d.nodeCount, + d.nodepoolName, + *nodeVolume, + ) + nodepool.SetAllowSystemComponents(true) + + payload := ske.NewCreateOrUpdateClusterPayload(*clusterKubernetes, []ske.Nodepool{*nodepool}) + klog.Infof("Built SKE cluster payload for cluster=%q", d.clusterName()) + return *payload +} + +func containsKubernetesVersion(options *ske.ProviderOptions, version string) bool { + for _, item := range options.GetKubernetesVersions() { + if item.GetVersion() == version { + return true + } + } + return false +} + +func containsAvailabilityZone(options *ske.ProviderOptions, zone string) bool { + for _, item := range options.GetAvailabilityZones() { + if item.GetName() == zone { + return true + } + } + return false +} + +func containsMachineType(options *ske.ProviderOptions, machineType string) bool { + for _, item := range options.GetMachineTypes() { + if item.GetName() == machineType { + return true + } + } + return false +} + +func containsMachineImage(options *ske.ProviderOptions, imageName, imageVersion string) bool { + for _, image := range options.GetMachineImages() { + if image.GetName() != imageName { + continue + } + for _, version := range image.GetVersions() { + if version.GetVersion() == imageVersion { + return true + } + } + } + return false +} + +func containsVolumeType(options *ske.ProviderOptions, volumeType string) bool { + for _, item := range options.GetVolumeTypes() { + if item.GetName() == volumeType { + return true + } + } + return false +} diff --git a/test/kubetest2/ske_client.go b/test/kubetest2/ske_client.go new file mode 100644 index 00000000..83104ebc --- /dev/null +++ b/test/kubetest2/ske_client.go @@ -0,0 +1,96 @@ +package kubetest2 + +import ( + "context" + "fmt" + "strconv" + + "github.com/stackitcloud/cloud-provider-stackit/pkg/metrics" + sdkconfig "github.com/stackitcloud/stackit-sdk-go/core/config" + "github.com/stackitcloud/stackit-sdk-go/services/ske" + skewait "github.com/stackitcloud/stackit-sdk-go/services/ske/wait" + "k8s.io/klog/v2" +) + +type skeClient interface { + GetCluster(ctx context.Context, projectID, region, name string) (*ske.Cluster, error) + ListProviderOptions(ctx context.Context, region string) (*ske.ProviderOptions, error) + CreateOrUpdateCluster(ctx context.Context, projectID, region, name string, payload ske.CreateOrUpdateClusterPayload) (*ske.Cluster, error) + WaitForClusterReady(ctx context.Context, projectID, region, name string) (*ske.Cluster, error) + CreateKubeconfig(ctx context.Context, projectID, region, name string, expirationSeconds int64) (*ske.Kubeconfig, error) + DeleteCluster(ctx context.Context, projectID, region, name string) error + WaitForClusterDeleted(ctx context.Context, projectID, region, name string) error +} + +type sdkSKEClient struct { + api ske.DefaultApi +} + +func newSKEClient(region, serviceAccount string) (skeClient, error) { + klog.Infof("Creating SKE API client for region=%q with service_account_bytes=%d", region, len(serviceAccount)) + + httpClient := metrics.NewInstrumentedHTTPClient("ske") + apiClient, err := ske.NewAPIClient( + sdkconfig.WithServiceAccountKey(serviceAccount), + sdkconfig.WithHTTPClient(httpClient), + ) + if err != nil { + return nil, fmt.Errorf("create SKE client: %w", err) + } + + klog.Infof("Created SKE API client successfully") + + return &sdkSKEClient{api: apiClient}, nil +} + +func (c *sdkSKEClient) GetCluster(ctx context.Context, projectID, region, name string) (*ske.Cluster, error) { + klog.Infof("SKE GetCluster: project_id=%q region=%q cluster=%q", projectID, region, name) + return c.api.GetCluster(ctx, projectID, region, name).Execute() +} + +func (c *sdkSKEClient) ListProviderOptions(ctx context.Context, region string) (*ske.ProviderOptions, error) { + klog.Infof("SKE ListProviderOptions: region=%q", region) + return c.api.ListProviderOptions(ctx, region).Execute() +} + +func (c *sdkSKEClient) CreateOrUpdateCluster(ctx context.Context, projectID, region, name string, payload ske.CreateOrUpdateClusterPayload) (*ske.Cluster, error) { + klog.Infof("SKE CreateOrUpdateCluster: project_id=%q region=%q cluster=%q", projectID, region, name) + return c.api.CreateOrUpdateCluster(ctx, projectID, region, name).CreateOrUpdateClusterPayload(payload).Execute() +} + +func (c *sdkSKEClient) WaitForClusterReady(ctx context.Context, projectID, region, name string) (*ske.Cluster, error) { + klog.Infof("Waiting for SKE cluster to become ready: project_id=%q region=%q cluster=%q", projectID, region, name) + cluster, err := skewait.CreateOrUpdateClusterWaitHandler(ctx, c.api, projectID, region, name).WaitWithContext(ctx) + if err != nil { + return nil, err + } + if cluster != nil && cluster.Status != nil && cluster.Status.Aggregated != nil { + klog.Infof("SKE cluster is ready: cluster=%q state=%q", name, cluster.Status.GetAggregated()) + } + return cluster, nil +} + +func (c *sdkSKEClient) CreateKubeconfig(ctx context.Context, projectID, region, name string, expirationSeconds int64) (*ske.Kubeconfig, error) { + klog.Infof( + "SKE CreateKubeconfig: project_id=%q region=%q cluster=%q expiration_seconds=%d", + projectID, + region, + name, + expirationSeconds, + ) + payload := ske.NewCreateKubeconfigPayload() + payload.SetExpirationSeconds(strconv.FormatInt(expirationSeconds, 10)) + return c.api.CreateKubeconfig(ctx, projectID, region, name).CreateKubeconfigPayload(*payload).Execute() +} + +func (c *sdkSKEClient) DeleteCluster(ctx context.Context, projectID, region, name string) error { + klog.Infof("SKE DeleteCluster: project_id=%q region=%q cluster=%q", projectID, region, name) + _, err := c.api.DeleteCluster(ctx, projectID, region, name).Execute() + return err +} + +func (c *sdkSKEClient) WaitForClusterDeleted(ctx context.Context, projectID, region, name string) error { + klog.Infof("Waiting for SKE cluster deletion: project_id=%q region=%q cluster=%q", projectID, region, name) + _, err := skewait.DeleteClusterWaitHandler(ctx, c.api, projectID, region, name).WaitWithContext(ctx) + return err +} From c1273c4b840e23a9be00f23ec241433206e75373 Mon Sep 17 00:00:00 2001 From: Felix Breuer Date: Mon, 17 Aug 2026 15:10:57 +0200 Subject: [PATCH 02/11] add project wrapper Signed-off-by: Felix Breuer --- go.mod | 4 +- go.sum | 4 + test/kubetest2/bootstrap_clients.go | 168 ++++++ test/kubetest2/bootstrap_resources.go | 353 ++++++++++++ test/kubetest2/bootstrap_test.go | 761 ++++++++++++++++++++++++++ test/kubetest2/deployer.go | 19 +- test/kubetest2/env.go | 21 +- test/kubetest2/lifecycle.go | 57 +- test/kubetest2/naming.go | 25 + 9 files changed, 1389 insertions(+), 23 deletions(-) create mode 100644 test/kubetest2/bootstrap_clients.go create mode 100644 test/kubetest2/bootstrap_resources.go create mode 100644 test/kubetest2/bootstrap_test.go diff --git a/go.mod b/go.mod index 02c6a4f8..4fa03e25 100644 --- a/go.mod +++ b/go.mod @@ -15,8 +15,11 @@ require ( github.com/spf13/cobra v1.10.2 github.com/spf13/pflag v1.0.10 github.com/stackitcloud/stackit-sdk-go/core v0.26.0 + github.com/stackitcloud/stackit-sdk-go/services/authorization v0.15.1 github.com/stackitcloud/stackit-sdk-go/services/iaas v1.14.1 github.com/stackitcloud/stackit-sdk-go/services/loadbalancer v1.15.1 + github.com/stackitcloud/stackit-sdk-go/services/resourcemanager v0.24.2 + github.com/stackitcloud/stackit-sdk-go/services/serviceaccount v0.20.1 github.com/stackitcloud/stackit-sdk-go/services/ske v1.6.0 go.uber.org/mock v0.6.0 golang.org/x/sync v0.22.0 @@ -93,7 +96,6 @@ require ( github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect github.com/prometheus/common v0.70.1 // indirect github.com/prometheus/procfs v0.21.1 // indirect - github.com/stackitcloud/stackit-sdk-go/services/resourcemanager v0.24.2 // indirect github.com/stoewer/go-strcase v1.3.1 // indirect github.com/stretchr/objx v0.5.3 // indirect github.com/x448/float16 v0.8.4 // indirect diff --git a/go.sum b/go.sum index c34c2bed..dd7e9949 100644 --- a/go.sum +++ b/go.sum @@ -186,12 +186,16 @@ github.com/stackitcloud/cloud-provider v0.36.0-ske-2 h1:OCDooD0twsMDdcGbcEEofRa6 github.com/stackitcloud/cloud-provider v0.36.0-ske-2/go.mod h1:y/3sksoC0taJZR0PcAAYUqVyD6Jzu2X0lD4yCEPXPuI= github.com/stackitcloud/stackit-sdk-go/core v0.26.0 h1:jQEb9gkehfp6VCP6TcYk7BI10cz4l0KM2L6hqYBH2QA= github.com/stackitcloud/stackit-sdk-go/core v0.26.0/go.mod h1:WU1hhxnjXw2EV7CYa1nlEvNpMiRY6CvmIOaHuL3pOaA= +github.com/stackitcloud/stackit-sdk-go/services/authorization v0.15.1 h1:TtpVXxgCJ17AqSOIzooQru/H3QnC4Z2VIF7HbaeKeJs= +github.com/stackitcloud/stackit-sdk-go/services/authorization v0.15.1/go.mod h1:c9JimEdshtcIHAI2j2N388ybW+InO1fvYPvL6rxb2uY= github.com/stackitcloud/stackit-sdk-go/services/iaas v1.14.1 h1:P8XhOi8TLA7NbLHhxU8dCJjgQpha/65agos3pI59+vw= github.com/stackitcloud/stackit-sdk-go/services/iaas v1.14.1/go.mod h1:/QpOsaCOQjwGgQqG8y9xXqryp25AfkorPjInX9xIeOU= github.com/stackitcloud/stackit-sdk-go/services/loadbalancer v1.15.1 h1:+NT2okD80TlFNDTKSgaEvV/o7LevTrATqcDL0r4nuhU= github.com/stackitcloud/stackit-sdk-go/services/loadbalancer v1.15.1/go.mod h1:+Ld3dn648I+YKcBV3fEkYpDSr3fel421+LurJGywSBs= github.com/stackitcloud/stackit-sdk-go/services/resourcemanager v0.24.2 h1:4UxxJmCSCwV8q4bT4G+D1JH8F9Gm6BKaLGixX5DVcvI= github.com/stackitcloud/stackit-sdk-go/services/resourcemanager v0.24.2/go.mod h1:NEz3f+GV5G++BE9/MmZCsXJyCih7jtg0pZuSyG2sLEs= +github.com/stackitcloud/stackit-sdk-go/services/serviceaccount v0.20.1 h1:zVVA/Puge2PK3G8lrGGolnQzxYwwsNUgfMyyRUUQc44= +github.com/stackitcloud/stackit-sdk-go/services/serviceaccount v0.20.1/go.mod h1:xJNa86o89czK2o60+s46vz8+NSKDTbQduFGFaztJnsA= github.com/stackitcloud/stackit-sdk-go/services/ske v1.6.0 h1:Dab1jzN0u9c67lvELoWf1RuagjO3eUBRytoX8SYL8Zs= github.com/stackitcloud/stackit-sdk-go/services/ske v1.6.0/go.mod h1:NzcTU5GGlUF6Lys3Ra7ylRj4ZKxJr3f/29/yoE5tjPI= github.com/stoewer/go-strcase v1.3.1 h1:iS0MdW+kVTxgMoE1LAZyMiYJFKlOzLooE4MxjirtkAs= diff --git a/test/kubetest2/bootstrap_clients.go b/test/kubetest2/bootstrap_clients.go new file mode 100644 index 00000000..69f27656 --- /dev/null +++ b/test/kubetest2/bootstrap_clients.go @@ -0,0 +1,168 @@ +package kubetest2 + +import ( + "context" + "fmt" + + "github.com/stackitcloud/cloud-provider-stackit/pkg/metrics" + sdkconfig "github.com/stackitcloud/stackit-sdk-go/core/config" + "github.com/stackitcloud/stackit-sdk-go/services/authorization" + "github.com/stackitcloud/stackit-sdk-go/services/resourcemanager" + resourcemanagerwait "github.com/stackitcloud/stackit-sdk-go/services/resourcemanager/wait" + "github.com/stackitcloud/stackit-sdk-go/services/serviceaccount" +) + +const ( + projectLabelScopeKey = "scope" + projectLabelScopeValue = "PUBLIC" + projectLabelManagedKey = "kt2_managed" + projectLabelManagedValue = "true" + projectLabelRunIDKey = "kt2_run_id" + projectOwnerRole = "owner" + projectResourceType = "project" + childProjectRole = "ske.admin" + projectListPageSize float32 = 100 +) + +type projectClient interface { + ListProjects(ctx context.Context, parentContainerID string) ([]resourcemanager.Project, error) + CreateProject(ctx context.Context, parentContainerID, name, ownerEmail string, labels map[string]string) (*resourcemanager.Project, error) + WaitForProjectActive(ctx context.Context, containerID string) (*resourcemanager.GetProjectResponse, error) + DeleteProject(ctx context.Context, projectID string) error + WaitForProjectDeleted(ctx context.Context, projectID string) error +} + +type serviceAccountClient interface { + ListServiceAccounts(ctx context.Context, projectID string) ([]serviceaccount.ServiceAccount, error) + CreateServiceAccount(ctx context.Context, projectID, name string) (*serviceaccount.ServiceAccount, error) + CreateServiceAccountKey(ctx context.Context, projectID, serviceAccountEmail string) (*serviceaccount.CreateServiceAccountKeyResponse, error) +} + +type authorizationClient interface { + ListMembers(ctx context.Context, resourceType, resourceID string) ([]authorization.Member, error) + AddMembers(ctx context.Context, resourceID, resourceType string, members []authorization.Member) error +} + +type sdkProjectClient struct { + api *resourcemanager.APIClient +} + +type sdkServiceAccountClient struct { + api *serviceaccount.APIClient +} + +type sdkAuthorizationClient struct { + api *authorization.APIClient +} + +func newProjectClient(serviceAccountKey string) (projectClient, error) { + httpClient := metrics.NewInstrumentedHTTPClient("resourcemanager") + apiClient, err := resourcemanager.NewAPIClient( + sdkconfig.WithServiceAccountKey(serviceAccountKey), + sdkconfig.WithHTTPClient(httpClient), + ) + if err != nil { + return nil, fmt.Errorf("create Resource Manager client: %w", err) + } + return &sdkProjectClient{api: apiClient}, nil +} + +func newServiceAccountClient(serviceAccountKey string) (serviceAccountClient, error) { + httpClient := metrics.NewInstrumentedHTTPClient("serviceaccount") + apiClient, err := serviceaccount.NewAPIClient( + sdkconfig.WithServiceAccountKey(serviceAccountKey), + sdkconfig.WithHTTPClient(httpClient), + ) + if err != nil { + return nil, fmt.Errorf("create Service Account client: %w", err) + } + return &sdkServiceAccountClient{api: apiClient}, nil +} + +func newAuthorizationClient(serviceAccountKey string) (authorizationClient, error) { + httpClient := metrics.NewInstrumentedHTTPClient("authorization") + apiClient, err := authorization.NewAPIClient( + sdkconfig.WithServiceAccountKey(serviceAccountKey), + sdkconfig.WithHTTPClient(httpClient), + ) + if err != nil { + return nil, fmt.Errorf("create Authorization client: %w", err) + } + return &sdkAuthorizationClient{api: apiClient}, nil +} + +func (c *sdkProjectClient) ListProjects(ctx context.Context, parentContainerID string) ([]resourcemanager.Project, error) { + projects := make([]resourcemanager.Project, 0) + var offset float32 + for { + resp, err := c.api.ListProjects(ctx). + ContainerParentId(parentContainerID). + Offset(offset). + Limit(projectListPageSize). + Execute() + if err != nil { + return nil, err + } + items := resp.GetItems() + projects = append(projects, items...) + if len(items) < int(projectListPageSize) { + return projects, nil + } + offset += float32(len(items)) + } +} + +func (c *sdkProjectClient) CreateProject(ctx context.Context, parentContainerID, name, ownerEmail string, labels map[string]string) (*resourcemanager.Project, error) { + payload := resourcemanager.NewCreateProjectPayload( + parentContainerID, + []resourcemanager.Member{*resourcemanager.NewMember(projectOwnerRole, ownerEmail)}, + name, + ) + payload.SetLabels(labels) + return c.api.CreateProject(ctx).CreateProjectPayload(*payload).Execute() +} + +func (c *sdkProjectClient) WaitForProjectActive(ctx context.Context, containerID string) (*resourcemanager.GetProjectResponse, error) { + return resourcemanagerwait.CreateProjectWaitHandler(ctx, c.api, containerID).WaitWithContext(ctx) +} + +func (c *sdkProjectClient) DeleteProject(ctx context.Context, projectID string) error { + return c.api.DeleteProject(ctx, projectID).Execute() +} + +func (c *sdkProjectClient) WaitForProjectDeleted(ctx context.Context, projectID string) error { + _, err := resourcemanagerwait.DeleteProjectWaitHandler(ctx, c.api, projectID).WaitWithContext(ctx) + return err +} + +func (c *sdkServiceAccountClient) ListServiceAccounts(ctx context.Context, projectID string) ([]serviceaccount.ServiceAccount, error) { + resp, err := c.api.ListServiceAccounts(ctx, projectID).Execute() + if err != nil { + return nil, err + } + return resp.GetItems(), nil +} + +func (c *sdkServiceAccountClient) CreateServiceAccount(ctx context.Context, projectID, name string) (*serviceaccount.ServiceAccount, error) { + payload := serviceaccount.NewCreateServiceAccountPayload(name) + return c.api.CreateServiceAccount(ctx, projectID).CreateServiceAccountPayload(*payload).Execute() +} + +func (c *sdkServiceAccountClient) CreateServiceAccountKey(ctx context.Context, projectID, serviceAccountEmail string) (*serviceaccount.CreateServiceAccountKeyResponse, error) { + payload := serviceaccount.NewCreateServiceAccountKeyPayloadWithDefaults() + return c.api.CreateServiceAccountKey(ctx, projectID, serviceAccountEmail).CreateServiceAccountKeyPayload(*payload).Execute() +} + +func (c *sdkAuthorizationClient) ListMembers(ctx context.Context, resourceType, resourceID string) ([]authorization.Member, error) { + resp, err := c.api.ListMembers(ctx, resourceType, resourceID).Execute() + if err != nil { + return nil, err + } + return resp.GetMembers(), nil +} + +func (c *sdkAuthorizationClient) AddMembers(ctx context.Context, resourceID, resourceType string, members []authorization.Member) error { + payload := authorization.NewAddMembersPayload(members, resourceType) + _, err := c.api.AddMembers(ctx, resourceID).AddMembersPayload(*payload).Execute() + return err +} diff --git a/test/kubetest2/bootstrap_resources.go b/test/kubetest2/bootstrap_resources.go new file mode 100644 index 00000000..6c040364 --- /dev/null +++ b/test/kubetest2/bootstrap_resources.go @@ -0,0 +1,353 @@ +package kubetest2 + +import ( + "context" + "encoding/json" + "fmt" + "os" + "strings" + "time" + + "github.com/stackitcloud/stackit-sdk-go/services/authorization" + "github.com/stackitcloud/stackit-sdk-go/services/resourcemanager" + "github.com/stackitcloud/stackit-sdk-go/services/serviceaccount" + "k8s.io/klog/v2" +) + +type managedProject struct { + ContainerID string + ProjectID string + Name string + Labels map[string]string +} + +type managedServiceAccount struct { + Email string + ProjectID string +} + +type serviceAccountKeyFile struct { + Active bool `json:"active"` + CreatedAt time.Time `json:"createdAt"` + Credentials serviceAccountKeyCredentialsFile `json:"credentials"` + ID string `json:"id"` + KeyAlgorithm string `json:"keyAlgorithm"` + KeyOrigin string `json:"keyOrigin"` + KeyType string `json:"keyType"` + PublicKey string `json:"publicKey"` + ValidUntil *time.Time `json:"validUntil,omitempty"` +} + +type serviceAccountKeyCredentialsFile struct { + Aud string `json:"aud"` + Iss string `json:"iss"` + Kid string `json:"kid"` + PrivateKey *string `json:"privateKey,omitempty"` + Sub string `json:"sub"` + TokenEndpoint string `json:"tokenEndpoint"` +} + +func (d *Deployer) initializeBootstrapClients() error { + if d.projectClient == nil { + client, err := newProjectClient(d.serviceAccount) + if err != nil { + return err + } + d.projectClient = client + } + if d.serviceAccountClient == nil { + client, err := newServiceAccountClient(d.serviceAccount) + if err != nil { + return err + } + d.serviceAccountClient = client + } + if d.authorizationClient == nil { + client, err := newAuthorizationClient(d.serviceAccount) + if err != nil { + return err + } + d.authorizationClient = client + } + return nil +} + +func (d *Deployer) initializeSKEClient(serviceAccountKey string) error { + if d.skeClientFactory == nil { + d.skeClientFactory = newSKEClient + } + client, err := d.skeClientFactory(d.region, serviceAccountKey) + if err != nil { + return err + } + d.skeClient = client + return nil +} + +func (d *Deployer) ensureManagedClusterAccess(ctx context.Context) error { + project, err := d.resolveManagedProject(ctx) + if err != nil { + return err + } + d.projectID = project.ProjectID + + childServiceAccount, err := d.resolveManagedServiceAccount(ctx, project.ProjectID) + if err != nil { + return err + } + d.childServiceAccountEmail = childServiceAccount.Email + + if err := d.ensureProjectServiceAccountRole(ctx, project.ProjectID, childServiceAccount.Email); err != nil { + return err + } + + serviceAccountKey, err := d.ensureCachedChildServiceAccountKey(ctx, project.ProjectID, childServiceAccount.Email) + if err != nil { + return err + } + return d.initializeSKEClient(serviceAccountKey) +} + +func (d *Deployer) findManagedProject(ctx context.Context) (*managedProject, error) { + projects, err := d.projectClient.ListProjects(ctx, d.parentContainerID) + if err != nil { + return nil, fmt.Errorf("list STACKIT projects under parent container %q: %w", d.parentContainerID, err) + } + + matches := make([]managedProject, 0, 1) + for _, project := range projects { + if !d.matchesManagedProject(project) { + continue + } + matches = append(matches, managedProject{ + ContainerID: project.GetContainerId(), + ProjectID: project.GetProjectId(), + Name: project.GetName(), + Labels: project.GetLabels(), + }) + } + + switch len(matches) { + case 0: + return nil, nil + case 1: + return &matches[0], nil + default: + return nil, fmt.Errorf( + "found %d managed STACKIT projects for run token %q under parent container %q", + len(matches), + d.runToken(), + d.parentContainerID, + ) + } +} + +func (d *Deployer) resolveManagedProject(ctx context.Context) (*managedProject, error) { + project, err := d.findManagedProject(ctx) + if err != nil { + return nil, err + } + if project != nil { + klog.Infof("Reusing managed project=%q project_id=%q", project.Name, project.ProjectID) + return project, nil + } + + klog.Infof("Creating managed project=%q under parent_container_id=%q", d.projectName(), d.parentContainerID) + createdProject, err := d.projectClient.CreateProject( + ctx, + d.parentContainerID, + d.projectName(), + d.projectMemberEmail, + d.managedProjectLabels(), + ) + if err != nil { + return nil, fmt.Errorf("create STACKIT project %q: %w", d.projectName(), err) + } + + activeProject, err := d.projectClient.WaitForProjectActive(ctx, createdProject.GetContainerId()) + if err != nil { + return nil, fmt.Errorf("wait for STACKIT project %q to become active: %w", createdProject.GetProjectId(), err) + } + + return &managedProject{ + ContainerID: activeProject.GetContainerId(), + ProjectID: activeProject.GetProjectId(), + Name: activeProject.GetName(), + Labels: activeProject.GetLabels(), + }, nil +} + +func (d *Deployer) resolveManagedServiceAccount(ctx context.Context, projectID string) (*managedServiceAccount, error) { + serviceAccounts, err := d.serviceAccountClient.ListServiceAccounts(ctx, projectID) + if err != nil { + return nil, fmt.Errorf("list service accounts in STACKIT project %q: %w", projectID, err) + } + + matches := make([]managedServiceAccount, 0, 1) + for _, serviceAccount := range serviceAccounts { + if !d.matchesManagedServiceAccountEmail(serviceAccount.GetEmail()) { + continue + } + matches = append(matches, managedServiceAccount{ + Email: serviceAccount.GetEmail(), + ProjectID: serviceAccount.GetProjectId(), + }) + } + + switch len(matches) { + case 0: + klog.Infof("Creating managed service account=%q in project_id=%q", d.serviceAccountName(), projectID) + createdServiceAccount, err := d.serviceAccountClient.CreateServiceAccount(ctx, projectID, d.serviceAccountName()) + if err != nil { + return nil, fmt.Errorf("create service account %q in STACKIT project %q: %w", d.serviceAccountName(), projectID, err) + } + return &managedServiceAccount{ + Email: createdServiceAccount.GetEmail(), + ProjectID: createdServiceAccount.GetProjectId(), + }, nil + case 1: + klog.Infof("Reusing managed service account=%q in project_id=%q", matches[0].Email, projectID) + return &matches[0], nil + default: + return nil, fmt.Errorf( + "found %d managed service accounts for run token %q in project %q", + len(matches), + d.runToken(), + projectID, + ) + } +} + +func (d *Deployer) ensureProjectServiceAccountRole(ctx context.Context, projectID, serviceAccountEmail string) error { + members, err := d.authorizationClient.ListMembers(ctx, projectResourceType, projectID) + if err != nil { + return fmt.Errorf("list members for STACKIT project %q: %w", projectID, err) + } + for _, member := range members { + if member.GetSubject() == serviceAccountEmail && member.GetRole() == childProjectRole { + klog.Infof("Managed service account=%q already has role=%q in project_id=%q", serviceAccountEmail, childProjectRole, projectID) + return nil + } + } + + klog.Infof("Adding role=%q for managed service account=%q in project_id=%q", childProjectRole, serviceAccountEmail, projectID) + if err := d.authorizationClient.AddMembers( + ctx, + projectID, + projectResourceType, + []authorization.Member{*authorization.NewMember(childProjectRole, serviceAccountEmail)}, + ); err != nil { + return fmt.Errorf("add role %q for service account %q in STACKIT project %q: %w", childProjectRole, serviceAccountEmail, projectID, err) + } + return nil +} + +func (d *Deployer) ensureCachedChildServiceAccountKey(ctx context.Context, projectID, serviceAccountEmail string) (string, error) { + cachedKey, ok, err := d.readCachedChildServiceAccountKey() + if err != nil { + return "", fmt.Errorf("read cached child service-account key %q: %w", d.serviceAccountKeyPath, err) + } + if ok { + klog.Infof("Reusing cached child service-account key %q", d.serviceAccountKeyPath) + return cachedKey, nil + } + + klog.Infof("Creating child service-account key for service_account=%q in project_id=%q", serviceAccountEmail, projectID) + createdKey, err := d.serviceAccountClient.CreateServiceAccountKey(ctx, projectID, serviceAccountEmail) + if err != nil { + return "", fmt.Errorf("create service-account key for %q in STACKIT project %q: %w", serviceAccountEmail, projectID, err) + } + keyJSON, err := serviceAccountKeyJSON(createdKey) + if err != nil { + return "", fmt.Errorf("serialize service-account key for %q: %w", serviceAccountEmail, err) + } + if err := d.writeCachedChildServiceAccountKey(keyJSON); err != nil { + return "", fmt.Errorf("write cached child service-account key %q: %w", d.serviceAccountKeyPath, err) + } + return keyJSON, nil +} + +func (d *Deployer) readCachedChildServiceAccountKey() (string, bool, error) { + if strings.TrimSpace(d.serviceAccountKeyPath) == "" { + return "", false, nil + } + keyBytes, err := os.ReadFile(d.serviceAccountKeyPath) + if err != nil { + if os.IsNotExist(err) { + return "", false, nil + } + return "", false, err + } + return string(keyBytes), true, nil +} + +func (d *Deployer) writeCachedChildServiceAccountKey(serviceAccountKey string) error { + return os.WriteFile(d.serviceAccountKeyPath, []byte(serviceAccountKey), 0o600) +} + +func (d *Deployer) managedProjectLabels() map[string]string { + return map[string]string{ + projectLabelScopeKey: projectLabelScopeValue, + projectLabelManagedKey: projectLabelManagedValue, + projectLabelRunIDKey: d.runToken(), + } +} + +func (d *Deployer) matchesManagedProject(project resourcemanager.Project) bool { + if project.GetName() != d.projectName() { + return false + } + labels := project.GetLabels() + if labels == nil { + return false + } + return labels[projectLabelScopeKey] == projectLabelScopeValue && + labels[projectLabelManagedKey] == projectLabelManagedValue && + labels[projectLabelRunIDKey] == d.runToken() +} + +func (d *Deployer) matchesManagedServiceAccountEmail(email string) bool { + if email == "" { + return false + } + localPart, _, found := strings.Cut(email, "@") + if !found { + return false + } + return strings.HasPrefix(localPart, d.serviceAccountName()) +} + +func serviceAccountKeyJSON(createdKey *serviceaccount.CreateServiceAccountKeyResponse) (string, error) { + credentials := createdKey.GetCredentials() + privateKey, ok := credentials.GetPrivateKeyOk() + if !ok || strings.TrimSpace(privateKey) == "" { + return "", fmt.Errorf("service-account key response did not include a private key") + } + + serviceAccountKey := serviceAccountKeyFile{ + Active: createdKey.GetActive(), + CreatedAt: createdKey.GetCreatedAt(), + Credentials: serviceAccountKeyCredentialsFile{ + Aud: credentials.GetAud(), + Iss: credentials.GetIss(), + Kid: credentials.GetKid(), + PrivateKey: &privateKey, + Sub: credentials.GetSub(), + TokenEndpoint: credentials.GetTokenEndpoint(), + }, + ID: createdKey.GetId(), + KeyAlgorithm: string(createdKey.GetKeyAlgorithm()), + KeyOrigin: string(createdKey.GetKeyOrigin()), + KeyType: string(createdKey.GetKeyType()), + PublicKey: createdKey.GetPublicKey(), + } + if validUntil, ok := createdKey.GetValidUntilOk(); ok { + serviceAccountKey.ValidUntil = &validUntil + } + + keyJSON, err := json.Marshal(serviceAccountKey) + if err != nil { + return "", err + } + return string(keyJSON), nil +} diff --git a/test/kubetest2/bootstrap_test.go b/test/kubetest2/bootstrap_test.go new file mode 100644 index 00000000..eeebab41 --- /dev/null +++ b/test/kubetest2/bootstrap_test.go @@ -0,0 +1,761 @@ +package kubetest2 + +import ( + "context" + "net/http" + "os" + "path/filepath" + "reflect" + "strings" + "testing" + "time" + + oapierror "github.com/stackitcloud/stackit-sdk-go/core/oapierror" + "github.com/stackitcloud/stackit-sdk-go/services/authorization" + "github.com/stackitcloud/stackit-sdk-go/services/resourcemanager" + "github.com/stackitcloud/stackit-sdk-go/services/serviceaccount" + "github.com/stackitcloud/stackit-sdk-go/services/ske" + "sigs.k8s.io/kubetest2/pkg/types" +) + +type fakeOptions struct { + runID string + runDir string +} + +func (o fakeOptions) HelpRequested() bool { return false } +func (o fakeOptions) ShouldBuild() bool { return false } +func (o fakeOptions) ShouldUp() bool { return false } +func (o fakeOptions) ShouldDown() bool { return false } +func (o fakeOptions) ShouldTest() bool { return false } +func (o fakeOptions) SkipTestJUnitReport() bool { return false } +func (o fakeOptions) RunID() string { return o.runID } +func (o fakeOptions) RunDir() string { return o.runDir } +func (o fakeOptions) RundirInArtifacts() bool { return false } +func (o fakeOptions) PostTestCmd() []string { return nil } +func (o fakeOptions) PreTestCmd() []string { return nil } + +type fakeProjectClient struct { + listProjectsResult []resourcemanager.Project + listProjectsErr error + + createProjectResult *resourcemanager.Project + createProjectErr error + waitActiveResult *resourcemanager.GetProjectResponse + waitActiveErr error + + deleteErr error + waitDeletedErr error + + createCalls int + waitActiveCalls int + deleteCalls int + waitDeletedCalls int + + lastListParentContainerID string + lastCreateParentContainerID string + lastCreateName string + lastCreateOwnerEmail string + lastCreateLabels map[string]string + lastWaitActiveContainerID string + lastDeletedProjectID string + lastWaitDeletedProjectID string +} + +func (c *fakeProjectClient) ListProjects(_ context.Context, parentContainerID string) ([]resourcemanager.Project, error) { + c.lastListParentContainerID = parentContainerID + return c.listProjectsResult, c.listProjectsErr +} + +func (c *fakeProjectClient) CreateProject(_ context.Context, parentContainerID, name, ownerEmail string, labels map[string]string) (*resourcemanager.Project, error) { + c.createCalls++ + c.lastCreateParentContainerID = parentContainerID + c.lastCreateName = name + c.lastCreateOwnerEmail = ownerEmail + c.lastCreateLabels = labels + return c.createProjectResult, c.createProjectErr +} + +func (c *fakeProjectClient) WaitForProjectActive(_ context.Context, containerID string) (*resourcemanager.GetProjectResponse, error) { + c.waitActiveCalls++ + c.lastWaitActiveContainerID = containerID + return c.waitActiveResult, c.waitActiveErr +} + +func (c *fakeProjectClient) DeleteProject(_ context.Context, projectID string) error { + c.deleteCalls++ + c.lastDeletedProjectID = projectID + return c.deleteErr +} + +func (c *fakeProjectClient) WaitForProjectDeleted(_ context.Context, projectID string) error { + c.waitDeletedCalls++ + c.lastWaitDeletedProjectID = projectID + return c.waitDeletedErr +} + +type fakeServiceAccountClient struct { + listResult []serviceaccount.ServiceAccount + listErr error + + createResult *serviceaccount.ServiceAccount + createErr error + + createKeyResult *serviceaccount.CreateServiceAccountKeyResponse + createKeyErr error + + createCalls int + createKeyCalls int + + lastProjectIDForList string + lastProjectIDForCreate string + lastCreatedName string + lastProjectIDForCreateKey string + lastCreateKeyEmail string +} + +func (c *fakeServiceAccountClient) ListServiceAccounts(_ context.Context, projectID string) ([]serviceaccount.ServiceAccount, error) { + c.lastProjectIDForList = projectID + return c.listResult, c.listErr +} + +func (c *fakeServiceAccountClient) CreateServiceAccount(_ context.Context, projectID, name string) (*serviceaccount.ServiceAccount, error) { + c.createCalls++ + c.lastProjectIDForCreate = projectID + c.lastCreatedName = name + return c.createResult, c.createErr +} + +func (c *fakeServiceAccountClient) CreateServiceAccountKey(_ context.Context, projectID, serviceAccountEmail string) (*serviceaccount.CreateServiceAccountKeyResponse, error) { + c.createKeyCalls++ + c.lastProjectIDForCreateKey = projectID + c.lastCreateKeyEmail = serviceAccountEmail + return c.createKeyResult, c.createKeyErr +} + +type fakeAuthorizationClient struct { + listMembersResult []authorization.Member + listMembersErr error + addMembersErr error + + addCalls int + + lastResourceType string + lastResourceID string + lastAddedType string + lastAddedID string + lastAddedMembers []authorization.Member +} + +func (c *fakeAuthorizationClient) ListMembers(_ context.Context, resourceType, resourceID string) ([]authorization.Member, error) { + c.lastResourceType = resourceType + c.lastResourceID = resourceID + return c.listMembersResult, c.listMembersErr +} + +func (c *fakeAuthorizationClient) AddMembers(_ context.Context, resourceID, resourceType string, members []authorization.Member) error { + c.addCalls++ + c.lastAddedID = resourceID + c.lastAddedType = resourceType + c.lastAddedMembers = members + return c.addMembersErr +} + +type fakeSKEClient struct { + providerOptions *ske.ProviderOptions + providerOptionsErr error + getClusterResult *ske.Cluster + getClusterErr error + + createOrUpdateResult *ske.Cluster + createOrUpdateErr error + waitReadyResult *ske.Cluster + waitReadyErr error + kubeconfigResult *ske.Kubeconfig + kubeconfigErr error + + deleteClusterCalled bool + waitDeletedCalled bool + + lastGetProjectID string + lastCreateProjectID string + lastCreateRegion string + lastCreateClusterName string + lastKubeconfigProjectID string + lastExpirationSeconds int64 +} + +func (c *fakeSKEClient) GetCluster(_ context.Context, projectID, _, _ string) (*ske.Cluster, error) { + c.lastGetProjectID = projectID + return c.getClusterResult, c.getClusterErr +} + +func (c *fakeSKEClient) ListProviderOptions(_ context.Context, _ string) (*ske.ProviderOptions, error) { + return c.providerOptions, c.providerOptionsErr +} + +func (c *fakeSKEClient) CreateOrUpdateCluster(_ context.Context, projectID, region, name string, _ ske.CreateOrUpdateClusterPayload) (*ske.Cluster, error) { + c.lastCreateProjectID = projectID + c.lastCreateRegion = region + c.lastCreateClusterName = name + return c.createOrUpdateResult, c.createOrUpdateErr +} + +func (c *fakeSKEClient) WaitForClusterReady(_ context.Context, _, _, _ string) (*ske.Cluster, error) { + return c.waitReadyResult, c.waitReadyErr +} + +func (c *fakeSKEClient) CreateKubeconfig(_ context.Context, projectID, _, _ string, expirationSeconds int64) (*ske.Kubeconfig, error) { + c.lastKubeconfigProjectID = projectID + c.lastExpirationSeconds = expirationSeconds + return c.kubeconfigResult, c.kubeconfigErr +} + +func (c *fakeSKEClient) DeleteCluster(_ context.Context, _, _, _ string) error { + c.deleteClusterCalled = true + return nil +} + +func (c *fakeSKEClient) WaitForClusterDeleted(_ context.Context, _, _, _ string) error { + c.waitDeletedCalled = true + return nil +} + +func TestLoadEnvironmentValidation(t *testing.T) { + testCases := []struct { + name string + env map[string]string + wantErrContains string + }{ + { + name: "missing service account", + env: map[string]string{ + "STACKIT_PARENT_CONTAINER_ID": "parent-1", + "STACKIT_PROJECT_MEMBER_EMAIL": "owner@example.com", + }, + wantErrContains: "STACKIT_SERVICE_ACCOUNT", + }, + { + name: "missing parent container", + env: map[string]string{ + "STACKIT_SERVICE_ACCOUNT": "{}", + "STACKIT_PROJECT_MEMBER_EMAIL": "owner@example.com", + }, + wantErrContains: "STACKIT_PARENT_CONTAINER_ID", + }, + { + name: "missing project member email", + env: map[string]string{ + "STACKIT_SERVICE_ACCOUNT": "{}", + "STACKIT_PARENT_CONTAINER_ID": "parent-1", + }, + wantErrContains: "STACKIT_PROJECT_MEMBER_EMAIL", + }, + { + name: "project id no longer required", + env: map[string]string{ + "STACKIT_SERVICE_ACCOUNT": "{}", + "STACKIT_PARENT_CONTAINER_ID": "parent-1", + "STACKIT_PROJECT_MEMBER_EMAIL": "owner@example.com", + }, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + for _, key := range []string{ + "STACKIT_SERVICE_ACCOUNT", + "STACKIT_PARENT_CONTAINER_ID", + "STACKIT_PROJECT_MEMBER_EMAIL", + "STACKIT_PROJECT_ID", + } { + t.Setenv(key, "") + } + for key, value := range tc.env { + t.Setenv(key, value) + } + + runDir := t.TempDir() + d := &Deployer{options: fakeOptions{runID: "run-123", runDir: runDir}} + err := d.loadEnvironment() + if tc.wantErrContains != "" { + if err == nil || !strings.Contains(err.Error(), tc.wantErrContains) { + t.Fatalf("expected error containing %q, got %v", tc.wantErrContains, err) + } + return + } + if err != nil { + t.Fatalf("loadEnvironment() error = %v", err) + } + if d.parentContainerID != "parent-1" { + t.Fatalf("parentContainerID = %q, want %q", d.parentContainerID, "parent-1") + } + if d.projectMemberEmail != "owner@example.com" { + t.Fatalf("projectMemberEmail = %q, want %q", d.projectMemberEmail, "owner@example.com") + } + if d.projectID != "" { + t.Fatalf("projectID = %q, want empty", d.projectID) + } + if d.kubeconfigPath != filepath.Join(runDir, "kubeconfig") { + t.Fatalf("kubeconfigPath = %q", d.kubeconfigPath) + } + if d.serviceAccountKeyPath != filepath.Join(runDir, "service-account-key.json") { + t.Fatalf("serviceAccountKeyPath = %q", d.serviceAccountKeyPath) + } + }) + } +} + +func TestResolveManagedProjectCreatesWhenMissing(t *testing.T) { + d := newTestDeployer(t) + projectClient := &fakeProjectClient{ + createProjectResult: projectFixture(d.projectName(), "project-123", "container-123", d.managedProjectLabels()), + waitActiveResult: projectResponseFixture(d.projectName(), "project-123", "container-123", d.managedProjectLabels()), + } + d.projectClient = projectClient + + project, err := d.resolveManagedProject(context.Background()) + if err != nil { + t.Fatalf("resolveManagedProject() error = %v", err) + } + if project.ProjectID != "project-123" { + t.Fatalf("project id = %q, want %q", project.ProjectID, "project-123") + } + if projectClient.createCalls != 1 { + t.Fatalf("createCalls = %d, want 1", projectClient.createCalls) + } + if projectClient.waitActiveCalls != 1 { + t.Fatalf("waitActiveCalls = %d, want 1", projectClient.waitActiveCalls) + } + if projectClient.lastCreateParentContainerID != d.parentContainerID { + t.Fatalf("parent container = %q, want %q", projectClient.lastCreateParentContainerID, d.parentContainerID) + } + if projectClient.lastCreateName != d.projectName() { + t.Fatalf("created project name = %q, want %q", projectClient.lastCreateName, d.projectName()) + } + if projectClient.lastCreateOwnerEmail != d.projectMemberEmail { + t.Fatalf("created owner email = %q, want %q", projectClient.lastCreateOwnerEmail, d.projectMemberEmail) + } + if !reflect.DeepEqual(projectClient.lastCreateLabels, d.managedProjectLabels()) { + t.Fatalf("created labels = %#v, want %#v", projectClient.lastCreateLabels, d.managedProjectLabels()) + } +} + +func TestResolveManagedProjectErrorsOnMultipleMatches(t *testing.T) { + d := newTestDeployer(t) + d.projectClient = &fakeProjectClient{ + listProjectsResult: []resourcemanager.Project{ + *projectFixture(d.projectName(), "project-1", "container-1", d.managedProjectLabels()), + *projectFixture(d.projectName(), "project-2", "container-2", d.managedProjectLabels()), + }, + } + + _, err := d.findManagedProject(context.Background()) + if err == nil || !strings.Contains(err.Error(), "found 2 managed STACKIT projects") { + t.Fatalf("expected duplicate managed project error, got %v", err) + } +} + +func TestEnsureManagedClusterAccessReusesCachedKeyAndSkipsMembershipWrite(t *testing.T) { + d := newTestDeployer(t) + cachedKey := `{"credentials":{"privateKey":"cached"}}` + if err := os.WriteFile(d.serviceAccountKeyPath, []byte(cachedKey), 0o600); err != nil { + t.Fatalf("write cached key: %v", err) + } + + projectClient := &fakeProjectClient{ + listProjectsResult: []resourcemanager.Project{ + *projectFixture(d.projectName(), "project-123", "container-123", d.managedProjectLabels()), + }, + } + serviceAccountEmail := d.serviceAccountName() + "@sa.stackit.cloud" + serviceAccountClient := &fakeServiceAccountClient{ + listResult: []serviceaccount.ServiceAccount{ + *serviceAccountFixture(serviceAccountEmail, "project-123"), + }, + } + authorizationClient := &fakeAuthorizationClient{ + listMembersResult: []authorization.Member{ + *authorization.NewMember(childProjectRole, serviceAccountEmail), + }, + } + fakeSKE := &fakeSKEClient{} + var receivedKey string + + d.projectClient = projectClient + d.serviceAccountClient = serviceAccountClient + d.authorizationClient = authorizationClient + d.skeClientFactory = func(_, serviceAccount string) (skeClient, error) { + receivedKey = serviceAccount + return fakeSKE, nil + } + + if err := d.ensureManagedClusterAccess(context.Background()); err != nil { + t.Fatalf("ensureManagedClusterAccess() error = %v", err) + } + if receivedKey != cachedKey { + t.Fatalf("received key = %q, want cached key", receivedKey) + } + if authorizationClient.addCalls != 0 { + t.Fatalf("addCalls = %d, want 0", authorizationClient.addCalls) + } + if serviceAccountClient.createKeyCalls != 0 { + t.Fatalf("createKeyCalls = %d, want 0", serviceAccountClient.createKeyCalls) + } + if d.projectID != "project-123" { + t.Fatalf("projectID = %q, want %q", d.projectID, "project-123") + } +} + +func TestEnsureManagedClusterAccessCreatesKeyAndAddsMembership(t *testing.T) { + d := newTestDeployer(t) + + projectClient := &fakeProjectClient{ + listProjectsResult: []resourcemanager.Project{ + *projectFixture(d.projectName(), "project-123", "container-123", d.managedProjectLabels()), + }, + } + serviceAccountEmail := d.serviceAccountName() + "@sa.stackit.cloud" + serviceAccountClient := &fakeServiceAccountClient{ + listResult: []serviceaccount.ServiceAccount{ + *serviceAccountFixture(serviceAccountEmail, "project-123"), + }, + createKeyResult: createServiceAccountKeyResponseFixture(serviceAccountEmail), + } + authorizationClient := &fakeAuthorizationClient{} + var receivedKey string + + d.projectClient = projectClient + d.serviceAccountClient = serviceAccountClient + d.authorizationClient = authorizationClient + d.skeClientFactory = func(_, serviceAccount string) (skeClient, error) { + receivedKey = serviceAccount + return &fakeSKEClient{}, nil + } + + if err := d.ensureManagedClusterAccess(context.Background()); err != nil { + t.Fatalf("ensureManagedClusterAccess() error = %v", err) + } + if authorizationClient.addCalls != 1 { + t.Fatalf("addCalls = %d, want 1", authorizationClient.addCalls) + } + if serviceAccountClient.createKeyCalls != 1 { + t.Fatalf("createKeyCalls = %d, want 1", serviceAccountClient.createKeyCalls) + } + if authorizationClient.lastAddedType != projectResourceType || authorizationClient.lastAddedID != "project-123" { + t.Fatalf("AddMembers called with type=%q id=%q", authorizationClient.lastAddedType, authorizationClient.lastAddedID) + } + keyBytes, err := os.ReadFile(d.serviceAccountKeyPath) + if err != nil { + t.Fatalf("read cached key: %v", err) + } + if receivedKey != string(keyBytes) { + t.Fatalf("factory key mismatch") + } + if !strings.Contains(string(keyBytes), "\"privateKey\":\"PRIVATE\"") { + t.Fatalf("cached key did not contain serialized private key: %s", string(keyBytes)) + } + info, err := os.Stat(d.serviceAccountKeyPath) + if err != nil { + t.Fatalf("stat cached key: %v", err) + } + if info.Mode().Perm() != 0o600 { + t.Fatalf("cached key mode = %o, want 600", info.Mode().Perm()) + } +} + +func TestUpUsesDiscoveredProjectAndWritesKubeconfig(t *testing.T) { + d := newTestDeployer(t) + configureValidUpInputs(d) + + cachedKey := `{"credentials":{"privateKey":"cached"}}` + if err := os.WriteFile(d.serviceAccountKeyPath, []byte(cachedKey), 0o600); err != nil { + t.Fatalf("write cached key: %v", err) + } + + d.projectClient = &fakeProjectClient{ + listProjectsResult: []resourcemanager.Project{ + *projectFixture(d.projectName(), "project-123", "container-123", d.managedProjectLabels()), + }, + } + serviceAccountEmail := d.serviceAccountName() + "@sa.stackit.cloud" + d.serviceAccountClient = &fakeServiceAccountClient{ + listResult: []serviceaccount.ServiceAccount{ + *serviceAccountFixture(serviceAccountEmail, "project-123"), + }, + } + d.authorizationClient = &fakeAuthorizationClient{ + listMembersResult: []authorization.Member{ + *authorization.NewMember(childProjectRole, serviceAccountEmail), + }, + } + + fakeSKE := &fakeSKEClient{ + providerOptions: providerOptionsFixture(), + createOrUpdateResult: ske.NewClusterWithDefaults(), + waitReadyResult: ske.NewClusterWithDefaults(), + kubeconfigResult: func() *ske.Kubeconfig { + cfg := ske.NewKubeconfig() + cfg.SetKubeconfig("apiVersion: v1\n") + return cfg + }(), + } + d.skeClientFactory = func(_, serviceAccount string) (skeClient, error) { + if serviceAccount != cachedKey { + t.Fatalf("unexpected service account key passed to SKE client") + } + return fakeSKE, nil + } + + if err := d.Up(); err != nil { + t.Fatalf("Up() error = %v", err) + } + if fakeSKE.lastCreateProjectID != "project-123" { + t.Fatalf("CreateOrUpdate projectID = %q, want %q", fakeSKE.lastCreateProjectID, "project-123") + } + if fakeSKE.lastCreateClusterName != d.clusterName() { + t.Fatalf("cluster name = %q, want %q", fakeSKE.lastCreateClusterName, d.clusterName()) + } + kubeconfigBytes, err := os.ReadFile(d.kubeconfigPath) + if err != nil { + t.Fatalf("read kubeconfig: %v", err) + } + if string(kubeconfigBytes) != "apiVersion: v1\n" { + t.Fatalf("kubeconfig = %q", string(kubeconfigBytes)) + } +} + +func TestDownMissingProjectIsSuccess(t *testing.T) { + d := newTestDeployer(t) + projectClient := &fakeProjectClient{} + d.projectClient = projectClient + + if err := d.Down(); err != nil { + t.Fatalf("Down() error = %v", err) + } + if projectClient.deleteCalls != 0 { + t.Fatalf("deleteCalls = %d, want 0", projectClient.deleteCalls) + } +} + +func TestDownDeletesProjectWithoutTouchingClusterDelete(t *testing.T) { + d := newTestDeployer(t) + fakeSKE := &fakeSKEClient{} + d.skeClient = fakeSKE + d.projectClient = &fakeProjectClient{ + listProjectsResult: []resourcemanager.Project{ + *projectFixture(d.projectName(), "project-123", "container-123", d.managedProjectLabels()), + }, + } + + if err := d.Down(); err != nil { + t.Fatalf("Down() error = %v", err) + } + if d.projectClient.(*fakeProjectClient).deleteCalls != 1 { + t.Fatalf("deleteCalls = %d, want 1", d.projectClient.(*fakeProjectClient).deleteCalls) + } + if d.projectClient.(*fakeProjectClient).waitDeletedCalls != 1 { + t.Fatalf("waitDeletedCalls = %d, want 1", d.projectClient.(*fakeProjectClient).waitDeletedCalls) + } + if fakeSKE.deleteClusterCalled || fakeSKE.waitDeletedCalled { + t.Fatalf("cluster delete path was invoked unexpectedly") + } +} + +func TestIsUpNoProjectReturnsFalse(t *testing.T) { + d := newTestDeployer(t) + d.projectClient = &fakeProjectClient{} + + isUp, err := d.IsUp() + if err != nil { + t.Fatalf("IsUp() error = %v", err) + } + if isUp { + t.Fatalf("IsUp() = true, want false") + } +} + +func TestIsUpProjectWithoutCachedKeyReturnsError(t *testing.T) { + d := newTestDeployer(t) + d.projectClient = &fakeProjectClient{ + listProjectsResult: []resourcemanager.Project{ + *projectFixture(d.projectName(), "project-123", "container-123", d.managedProjectLabels()), + }, + } + + isUp, err := d.IsUp() + if err == nil || !strings.Contains(err.Error(), "child service-account key cache") { + t.Fatalf("expected missing key cache error, got up=%v err=%v", isUp, err) + } +} + +func TestIsUpProjectWithCachedKeyQueriesCluster(t *testing.T) { + d := newTestDeployer(t) + d.projectClient = &fakeProjectClient{ + listProjectsResult: []resourcemanager.Project{ + *projectFixture(d.projectName(), "project-123", "container-123", d.managedProjectLabels()), + }, + } + cachedKey := `{"credentials":{"privateKey":"cached"}}` + if err := os.WriteFile(d.serviceAccountKeyPath, []byte(cachedKey), 0o600); err != nil { + t.Fatalf("write cached key: %v", err) + } + fakeSKE := &fakeSKEClient{ + getClusterResult: healthyClusterFixture(), + } + d.skeClientFactory = func(_, serviceAccount string) (skeClient, error) { + if serviceAccount != cachedKey { + t.Fatalf("unexpected cached key content") + } + return fakeSKE, nil + } + + isUp, err := d.IsUp() + if err != nil { + t.Fatalf("IsUp() error = %v", err) + } + if !isUp { + t.Fatalf("IsUp() = false, want true") + } + if fakeSKE.lastGetProjectID != "project-123" { + t.Fatalf("GetCluster projectID = %q, want %q", fakeSKE.lastGetProjectID, "project-123") + } +} + +func newTestDeployer(t *testing.T) *Deployer { + t.Helper() + runDir := t.TempDir() + return &Deployer{ + options: fakeOptions{runID: "run-123", runDir: runDir}, + region: defaultRegion, + parentContainerID: "parent-123", + projectMemberEmail: "owner@example.com", + kubeconfigPath: filepath.Join(runDir, "kubeconfig"), + serviceAccountKeyPath: filepath.Join(runDir, "service-account-key.json"), + } +} + +func configureValidUpInputs(d *Deployer) { + d.kubernetesVersion = "1.31.0" + d.availabilityZone = defaultAvailabilityZone + d.machineType = "g1.2" + d.nodeImageName = "ubuntu" + d.nodeImageVersion = "v1" + d.nodepoolName = defaultNodepoolName + d.volumeType = "storage" +} + +func providerOptionsFixture() *ske.ProviderOptions { + kubernetesVersion := ske.NewKubernetesVersion() + kubernetesVersion.SetVersion("1.31.0") + + availabilityZone := ske.NewAvailabilityZone() + availabilityZone.SetName(defaultAvailabilityZone) + + machineType := ske.NewMachineType() + machineType.SetName("g1.2") + + imageVersion := ske.NewMachineImageVersion() + imageVersion.SetVersion("v1") + + machineImage := ske.NewMachineImage() + machineImage.SetName("ubuntu") + machineImage.SetVersions([]ske.MachineImageVersion{*imageVersion}) + + volumeType := ske.NewVolumeType() + volumeType.SetName("storage") + + providerOptions := ske.NewProviderOptions() + providerOptions.SetKubernetesVersions([]ske.KubernetesVersion{*kubernetesVersion}) + providerOptions.SetAvailabilityZones([]ske.AvailabilityZone{*availabilityZone}) + providerOptions.SetMachineTypes([]ske.MachineType{*machineType}) + providerOptions.SetMachineImages([]ske.MachineImage{*machineImage}) + providerOptions.SetVolumeTypes([]ske.VolumeType{*volumeType}) + return providerOptions +} + +func healthyClusterFixture() *ske.Cluster { + cluster := ske.NewClusterWithDefaults() + status := ske.NewClusterStatus() + status.SetAggregated(ske.CLUSTERSTATUSSTATE_HEALTHY) + cluster.SetStatus(*status) + return cluster +} + +func projectFixture(name, projectID, containerID string, labels map[string]string) *resourcemanager.Project { + project := resourcemanager.NewProjectWithDefaults() + project.SetName(name) + project.SetProjectId(projectID) + project.SetContainerId(containerID) + project.SetLabels(labels) + return project +} + +func projectResponseFixture(name, projectID, containerID string, labels map[string]string) *resourcemanager.GetProjectResponse { + project := resourcemanager.NewGetProjectResponseWithDefaults() + project.SetName(name) + project.SetProjectId(projectID) + project.SetContainerId(containerID) + project.SetLabels(labels) + return project +} + +func serviceAccountFixture(email, projectID string) *serviceaccount.ServiceAccount { + serviceAccountObject := serviceaccount.NewServiceAccountWithDefaults() + serviceAccountObject.SetEmail(email) + serviceAccountObject.SetProjectId(projectID) + serviceAccountObject.SetId("service-account-id") + serviceAccountObject.SetInternal(false) + return serviceAccountObject +} + +func createServiceAccountKeyResponseFixture(email string) *serviceaccount.CreateServiceAccountKeyResponse { + credentials := serviceaccount.NewCreateServiceAccountKeyResponseCredentials( + "https://accounts.stackit.cloud", + email, + "00000000-0000-0000-0000-000000000001", + "00000000-0000-0000-0000-000000000002", + ) + credentials.SetPrivateKey("PRIVATE") + credentials.SetTokenEndpoint("https://accounts.stackit.cloud/oauth/v2/token") + + return serviceaccount.NewCreateServiceAccountKeyResponse( + true, + time.Unix(0, 0).UTC(), + *credentials, + "00000000-0000-0000-0000-000000000003", + serviceaccount.CREATESERVICEACCOUNTKEYRESPONSEKEY_ALGORITHM__2048, + serviceaccount.CREATESERVICEACCOUNTKEYRESPONSEKEY_ORIGIN_GENERATED, + serviceaccount.CREATESERVICEACCOUNTKEYRESPONSEKEY_TYPE_USER_MANAGED, + "PUBLIC KEY", + ) +} + +func TestIsUpReturnsFalseWhenClusterMissing(t *testing.T) { + d := newTestDeployer(t) + d.projectClient = &fakeProjectClient{ + listProjectsResult: []resourcemanager.Project{ + *projectFixture(d.projectName(), "project-123", "container-123", d.managedProjectLabels()), + }, + } + cachedKey := `{"credentials":{"privateKey":"cached"}}` + if err := os.WriteFile(d.serviceAccountKeyPath, []byte(cachedKey), 0o600); err != nil { + t.Fatalf("write cached key: %v", err) + } + fakeSKE := &fakeSKEClient{ + getClusterErr: &oapierror.GenericOpenAPIError{StatusCode: http.StatusNotFound}, + } + d.skeClientFactory = func(_, _ string) (skeClient, error) { + return fakeSKE, nil + } + + isUp, err := d.IsUp() + if err != nil { + t.Fatalf("IsUp() error = %v", err) + } + if isUp { + t.Fatalf("IsUp() = true, want false") + } +} + +var _ types.Options = fakeOptions{} diff --git a/test/kubetest2/deployer.go b/test/kubetest2/deployer.go index 497b8c11..b861e7e9 100644 --- a/test/kubetest2/deployer.go +++ b/test/kubetest2/deployer.go @@ -37,11 +37,19 @@ type Deployer struct { volumeType string kubeconfigExpiresIn int64 - projectID string - serviceAccount string - kubeconfigPath string - - skeClient skeClient + projectID string + serviceAccount string + parentContainerID string + projectMemberEmail string + childServiceAccountEmail string + kubeconfigPath string + serviceAccountKeyPath string + + projectClient projectClient + serviceAccountClient serviceAccountClient + authorizationClient authorizationClient + skeClient skeClient + skeClientFactory func(region, serviceAccount string) (skeClient, error) } var _ types.NewDeployer = New @@ -60,6 +68,7 @@ func New(opts types.Options) (types.Deployer, *pflag.FlagSet) { nodepoolName: defaultNodepoolName, volumeSizeGiB: defaultVolumeSizeGiB, kubeconfigExpiresIn: defaultKubeconfigExpiration, + skeClientFactory: newSKEClient, } fs := pflag.NewFlagSet(Name, pflag.ContinueOnError) diff --git a/test/kubetest2/env.go b/test/kubetest2/env.go index a85c1fa1..12b2cae3 100644 --- a/test/kubetest2/env.go +++ b/test/kubetest2/env.go @@ -16,20 +16,29 @@ func (d *Deployer) loadEnvironment() error { return incorrectUsagef("STACKIT_SERVICE_ACCOUNT environment variable is required") } - projectID, ok := os.LookupEnv("STACKIT_PROJECT_ID") - if !ok || strings.TrimSpace(projectID) == "" { - return incorrectUsagef("STACKIT_PROJECT_ID environment variable is required") + parentContainerID, ok := os.LookupEnv("STACKIT_PARENT_CONTAINER_ID") + if !ok || strings.TrimSpace(parentContainerID) == "" { + return incorrectUsagef("STACKIT_PARENT_CONTAINER_ID environment variable is required") + } + + projectMemberEmail, ok := os.LookupEnv("STACKIT_PROJECT_MEMBER_EMAIL") + if !ok || strings.TrimSpace(projectMemberEmail) == "" { + return incorrectUsagef("STACKIT_PROJECT_MEMBER_EMAIL environment variable is required") } d.serviceAccount = serviceAccount - d.projectID = projectID + d.parentContainerID = parentContainerID + d.projectMemberEmail = projectMemberEmail d.kubeconfigPath = filepath.Join(d.options.RunDir(), "kubeconfig") + d.serviceAccountKeyPath = filepath.Join(d.options.RunDir(), "service-account-key.json") klog.Infof( - "Loaded STACKIT environment: project_id=%q service_account_bytes=%d kubeconfig_path=%q", - d.projectID, + "Loaded STACKIT environment: parent_container_id=%q project_member_email=%q service_account_bytes=%d kubeconfig_path=%q service_account_key_path=%q", + d.parentContainerID, + d.projectMemberEmail, len(d.serviceAccount), d.kubeconfigPath, + d.serviceAccountKeyPath, ) return nil diff --git a/test/kubetest2/lifecycle.go b/test/kubetest2/lifecycle.go index 5c123505..a30f4dd9 100644 --- a/test/kubetest2/lifecycle.go +++ b/test/kubetest2/lifecycle.go @@ -20,11 +20,9 @@ func (d *Deployer) Init() error { return err } - client, err := newSKEClient(d.region, d.serviceAccount) - if err != nil { + if err := d.initializeBootstrapClients(); err != nil { return err } - d.skeClient = client klog.Infof("STACKIT kubetest2 deployer initialized successfully") @@ -35,6 +33,9 @@ func (d *Deployer) Up() error { klog.Infof("Starting cluster up flow for cluster=%q", d.clusterName()) ctx := context.Background() + if err := d.ensureManagedClusterAccess(ctx); err != nil { + return err + } if err := d.validateProviderOptions(ctx); err != nil { return err } @@ -69,29 +70,63 @@ func (d *Deployer) Up() error { func (d *Deployer) Down() error { ctx := context.Background() - clusterName := d.clusterName() - klog.Infof("Starting cluster down flow for cluster=%q", clusterName) + klog.Infof("Starting cluster down flow for cluster=%q", d.clusterName()) + + project, err := d.findManagedProject(ctx) + if err != nil { + return err + } + if project == nil { + klog.Infof("Managed project for run_id=%q is already absent", d.options.RunID()) + return nil + } - if err := d.skeClient.DeleteCluster(ctx, d.projectID, d.region, clusterName); err != nil { + d.projectID = project.ProjectID + if err := d.projectClient.DeleteProject(ctx, project.ProjectID); err != nil { if !stackiterrors.IsNotFound(err) { - return fmt.Errorf("delete SKE cluster %q: %w", clusterName, err) + return fmt.Errorf("delete STACKIT project %q: %w", project.ProjectID, err) } - klog.Infof("Cluster=%q already absent, treating delete as success", clusterName) + klog.Infof("Project=%q already absent, treating delete as success", project.ProjectID) return nil } - if err := d.skeClient.WaitForClusterDeleted(ctx, d.projectID, d.region, clusterName); err != nil { - return fmt.Errorf("wait for SKE cluster %q deletion: %w", clusterName, err) + if err := d.projectClient.WaitForProjectDeleted(ctx, project.ProjectID); err != nil { + return fmt.Errorf("wait for STACKIT project %q deletion: %w", project.ProjectID, err) } - klog.Infof("Cluster down flow completed successfully for cluster=%q", clusterName) + klog.Infof("Cluster down flow completed successfully for project=%q", project.ProjectID) return nil } func (d *Deployer) IsUp() (bool, error) { ctx := context.Background() + project, err := d.findManagedProject(ctx) + if err != nil { + return false, err + } + if project == nil { + klog.Infof("Managed project for run_id=%q not found during IsUp check", d.options.RunID()) + return false, nil + } + + d.projectID = project.ProjectID + serviceAccountKey, ok, err := d.readCachedChildServiceAccountKey() + if err != nil { + return false, fmt.Errorf("read cached child service-account key %q: %w", d.serviceAccountKeyPath, err) + } + if !ok { + return false, fmt.Errorf( + "managed project %q exists but child service-account key cache %q is missing", + project.ProjectID, + d.serviceAccountKeyPath, + ) + } + if err := d.initializeSKEClient(serviceAccountKey); err != nil { + return false, err + } + klog.Infof("Checking cluster state for cluster=%q", d.clusterName()) cluster, err := d.skeClient.GetCluster(ctx, d.projectID, d.region, d.clusterName()) if err != nil { diff --git a/test/kubetest2/naming.go b/test/kubetest2/naming.go index 0c8136ca..749e76aa 100644 --- a/test/kubetest2/naming.go +++ b/test/kubetest2/naming.go @@ -10,6 +10,31 @@ func clusterNameForRun(runID string) string { return "kt2" + hex.EncodeToString(sum[:4]) } +func runTokenForRun(runID string) string { + sum := sha256.Sum256([]byte(runID)) + return hex.EncodeToString(sum[:4]) +} + +func projectNameForRun(runID string) string { + return "kt2-" + runTokenForRun(runID) +} + +func serviceAccountNameForRun(runID string) string { + return "kt2-" + runTokenForRun(runID) +} + func (d *Deployer) clusterName() string { return clusterNameForRun(d.options.RunID()) } + +func (d *Deployer) runToken() string { + return runTokenForRun(d.options.RunID()) +} + +func (d *Deployer) projectName() string { + return projectNameForRun(d.options.RunID()) +} + +func (d *Deployer) serviceAccountName() string { + return serviceAccountNameForRun(d.options.RunID()) +} From beb2bc7f5f865f438c5a117651525c15fa9db346 Mon Sep 17 00:00:00 2001 From: Robert Kaussow Date: Tue, 18 Aug 2026 12:46:06 +0200 Subject: [PATCH 03/11] migrate deprecated --- test/kubetest2/bootstrap_clients.go | 28 +++++++++++++-------------- test/kubetest2/bootstrap_resources.go | 19 +++++++++--------- test/kubetest2/bootstrap_test.go | 12 ++++++------ 3 files changed, 30 insertions(+), 29 deletions(-) diff --git a/test/kubetest2/bootstrap_clients.go b/test/kubetest2/bootstrap_clients.go index 69f27656..fe27169f 100644 --- a/test/kubetest2/bootstrap_clients.go +++ b/test/kubetest2/bootstrap_clients.go @@ -6,10 +6,10 @@ import ( "github.com/stackitcloud/cloud-provider-stackit/pkg/metrics" sdkconfig "github.com/stackitcloud/stackit-sdk-go/core/config" - "github.com/stackitcloud/stackit-sdk-go/services/authorization" - "github.com/stackitcloud/stackit-sdk-go/services/resourcemanager" - resourcemanagerwait "github.com/stackitcloud/stackit-sdk-go/services/resourcemanager/wait" - "github.com/stackitcloud/stackit-sdk-go/services/serviceaccount" + authorization "github.com/stackitcloud/stackit-sdk-go/services/authorization/v2api" + resourcemanager "github.com/stackitcloud/stackit-sdk-go/services/resourcemanager/v0api" + resourcemanagerwait "github.com/stackitcloud/stackit-sdk-go/services/resourcemanager/v0api/wait" + serviceaccount "github.com/stackitcloud/stackit-sdk-go/services/serviceaccount/v2api" ) const ( @@ -95,7 +95,7 @@ func (c *sdkProjectClient) ListProjects(ctx context.Context, parentContainerID s projects := make([]resourcemanager.Project, 0) var offset float32 for { - resp, err := c.api.ListProjects(ctx). + resp, err := c.api.DefaultAPI.ListProjects(ctx). ContainerParentId(parentContainerID). Offset(offset). Limit(projectListPageSize). @@ -119,24 +119,24 @@ func (c *sdkProjectClient) CreateProject(ctx context.Context, parentContainerID, name, ) payload.SetLabels(labels) - return c.api.CreateProject(ctx).CreateProjectPayload(*payload).Execute() + return c.api.DefaultAPI.CreateProject(ctx).CreateProjectPayload(*payload).Execute() } func (c *sdkProjectClient) WaitForProjectActive(ctx context.Context, containerID string) (*resourcemanager.GetProjectResponse, error) { - return resourcemanagerwait.CreateProjectWaitHandler(ctx, c.api, containerID).WaitWithContext(ctx) + return resourcemanagerwait.CreateProjectWaitHandler(ctx, c.api.DefaultAPI, containerID).WaitWithContext(ctx) } func (c *sdkProjectClient) DeleteProject(ctx context.Context, projectID string) error { - return c.api.DeleteProject(ctx, projectID).Execute() + return c.api.DefaultAPI.DeleteProject(ctx, projectID).Execute() } func (c *sdkProjectClient) WaitForProjectDeleted(ctx context.Context, projectID string) error { - _, err := resourcemanagerwait.DeleteProjectWaitHandler(ctx, c.api, projectID).WaitWithContext(ctx) + _, err := resourcemanagerwait.DeleteProjectWaitHandler(ctx, c.api.DefaultAPI, projectID).WaitWithContext(ctx) return err } func (c *sdkServiceAccountClient) ListServiceAccounts(ctx context.Context, projectID string) ([]serviceaccount.ServiceAccount, error) { - resp, err := c.api.ListServiceAccounts(ctx, projectID).Execute() + resp, err := c.api.DefaultAPI.ListServiceAccounts(ctx, projectID).Execute() if err != nil { return nil, err } @@ -145,16 +145,16 @@ func (c *sdkServiceAccountClient) ListServiceAccounts(ctx context.Context, proje func (c *sdkServiceAccountClient) CreateServiceAccount(ctx context.Context, projectID, name string) (*serviceaccount.ServiceAccount, error) { payload := serviceaccount.NewCreateServiceAccountPayload(name) - return c.api.CreateServiceAccount(ctx, projectID).CreateServiceAccountPayload(*payload).Execute() + return c.api.DefaultAPI.CreateServiceAccount(ctx, projectID).CreateServiceAccountPayload(*payload).Execute() } func (c *sdkServiceAccountClient) CreateServiceAccountKey(ctx context.Context, projectID, serviceAccountEmail string) (*serviceaccount.CreateServiceAccountKeyResponse, error) { payload := serviceaccount.NewCreateServiceAccountKeyPayloadWithDefaults() - return c.api.CreateServiceAccountKey(ctx, projectID, serviceAccountEmail).CreateServiceAccountKeyPayload(*payload).Execute() + return c.api.DefaultAPI.CreateServiceAccountKey(ctx, projectID, serviceAccountEmail).CreateServiceAccountKeyPayload(*payload).Execute() } func (c *sdkAuthorizationClient) ListMembers(ctx context.Context, resourceType, resourceID string) ([]authorization.Member, error) { - resp, err := c.api.ListMembers(ctx, resourceType, resourceID).Execute() + resp, err := c.api.DefaultAPI.ListMembers(ctx, resourceType, resourceID).Execute() if err != nil { return nil, err } @@ -163,6 +163,6 @@ func (c *sdkAuthorizationClient) ListMembers(ctx context.Context, resourceType, func (c *sdkAuthorizationClient) AddMembers(ctx context.Context, resourceID, resourceType string, members []authorization.Member) error { payload := authorization.NewAddMembersPayload(members, resourceType) - _, err := c.api.AddMembers(ctx, resourceID).AddMembersPayload(*payload).Execute() + _, err := c.api.DefaultAPI.AddMembers(ctx, resourceID).AddMembersPayload(*payload).Execute() return err } diff --git a/test/kubetest2/bootstrap_resources.go b/test/kubetest2/bootstrap_resources.go index 6c040364..564b4feb 100644 --- a/test/kubetest2/bootstrap_resources.go +++ b/test/kubetest2/bootstrap_resources.go @@ -8,9 +8,9 @@ import ( "strings" "time" - "github.com/stackitcloud/stackit-sdk-go/services/authorization" - "github.com/stackitcloud/stackit-sdk-go/services/resourcemanager" - "github.com/stackitcloud/stackit-sdk-go/services/serviceaccount" + authorization "github.com/stackitcloud/stackit-sdk-go/services/authorization/v2api" + resourcemanager "github.com/stackitcloud/stackit-sdk-go/services/resourcemanager/v0api" + serviceaccount "github.com/stackitcloud/stackit-sdk-go/services/serviceaccount/v2api" "k8s.io/klog/v2" ) @@ -115,7 +115,8 @@ func (d *Deployer) findManagedProject(ctx context.Context) (*managedProject, err } matches := make([]managedProject, 0, 1) - for _, project := range projects { + for i := range projects { + project := &projects[i] if !d.matchesManagedProject(project) { continue } @@ -267,7 +268,7 @@ func (d *Deployer) ensureCachedChildServiceAccountKey(ctx context.Context, proje return keyJSON, nil } -func (d *Deployer) readCachedChildServiceAccountKey() (string, bool, error) { +func (d *Deployer) readCachedChildServiceAccountKey() (key string, ok bool, err error) { if strings.TrimSpace(d.serviceAccountKeyPath) == "" { return "", false, nil } @@ -293,7 +294,7 @@ func (d *Deployer) managedProjectLabels() map[string]string { } } -func (d *Deployer) matchesManagedProject(project resourcemanager.Project) bool { +func (d *Deployer) matchesManagedProject(project *resourcemanager.Project) bool { if project.GetName() != d.projectName() { return false } @@ -320,7 +321,7 @@ func (d *Deployer) matchesManagedServiceAccountEmail(email string) bool { func serviceAccountKeyJSON(createdKey *serviceaccount.CreateServiceAccountKeyResponse) (string, error) { credentials := createdKey.GetCredentials() privateKey, ok := credentials.GetPrivateKeyOk() - if !ok || strings.TrimSpace(privateKey) == "" { + if !ok || strings.TrimSpace(*privateKey) == "" { return "", fmt.Errorf("service-account key response did not include a private key") } @@ -331,7 +332,7 @@ func serviceAccountKeyJSON(createdKey *serviceaccount.CreateServiceAccountKeyRes Aud: credentials.GetAud(), Iss: credentials.GetIss(), Kid: credentials.GetKid(), - PrivateKey: &privateKey, + PrivateKey: privateKey, Sub: credentials.GetSub(), TokenEndpoint: credentials.GetTokenEndpoint(), }, @@ -342,7 +343,7 @@ func serviceAccountKeyJSON(createdKey *serviceaccount.CreateServiceAccountKeyRes PublicKey: createdKey.GetPublicKey(), } if validUntil, ok := createdKey.GetValidUntilOk(); ok { - serviceAccountKey.ValidUntil = &validUntil + serviceAccountKey.ValidUntil = validUntil } keyJSON, err := json.Marshal(serviceAccountKey) diff --git a/test/kubetest2/bootstrap_test.go b/test/kubetest2/bootstrap_test.go index eeebab41..e9440208 100644 --- a/test/kubetest2/bootstrap_test.go +++ b/test/kubetest2/bootstrap_test.go @@ -11,9 +11,9 @@ import ( "time" oapierror "github.com/stackitcloud/stackit-sdk-go/core/oapierror" - "github.com/stackitcloud/stackit-sdk-go/services/authorization" - "github.com/stackitcloud/stackit-sdk-go/services/resourcemanager" - "github.com/stackitcloud/stackit-sdk-go/services/serviceaccount" + authorization "github.com/stackitcloud/stackit-sdk-go/services/authorization/v2api" + resourcemanager "github.com/stackitcloud/stackit-sdk-go/services/resourcemanager/v0api" + serviceaccount "github.com/stackitcloud/stackit-sdk-go/services/serviceaccount/v2api" "github.com/stackitcloud/stackit-sdk-go/services/ske" "sigs.k8s.io/kubetest2/pkg/types" ) @@ -724,9 +724,9 @@ func createServiceAccountKeyResponseFixture(email string) *serviceaccount.Create time.Unix(0, 0).UTC(), *credentials, "00000000-0000-0000-0000-000000000003", - serviceaccount.CREATESERVICEACCOUNTKEYRESPONSEKEY_ALGORITHM__2048, - serviceaccount.CREATESERVICEACCOUNTKEYRESPONSEKEY_ORIGIN_GENERATED, - serviceaccount.CREATESERVICEACCOUNTKEYRESPONSEKEY_TYPE_USER_MANAGED, + serviceaccount.CREATESERVICEACCOUNTKEYRESPONSEKEYALGORITHM_RSA_2048, + serviceaccount.CREATESERVICEACCOUNTKEYRESPONSEKEYORIGIN_GENERATED, + serviceaccount.CREATESERVICEACCOUNTKEYRESPONSEKEYTYPE_USER_MANAGED, "PUBLIC KEY", ) } From 3b44cfa5f97a69c9595e35eb111c3976b4da9fa7 Mon Sep 17 00:00:00 2001 From: Robert Kaussow Date: Tue, 18 Aug 2026 13:10:47 +0200 Subject: [PATCH 04/11] allow api endpoint overwrites --- test/kubetest2/bootstrap_clients.go | 33 ++++++++-------- test/kubetest2/bootstrap_resources.go | 8 ++-- test/kubetest2/bootstrap_test.go | 54 ++++++++++++++++++++++++--- test/kubetest2/deployer.go | 7 +++- test/kubetest2/env.go | 12 ++++++ test/kubetest2/ske_client.go | 10 +---- 6 files changed, 89 insertions(+), 35 deletions(-) diff --git a/test/kubetest2/bootstrap_clients.go b/test/kubetest2/bootstrap_clients.go index fe27169f..fa50d5d5 100644 --- a/test/kubetest2/bootstrap_clients.go +++ b/test/kubetest2/bootstrap_clients.go @@ -55,36 +55,35 @@ type sdkAuthorizationClient struct { api *authorization.APIClient } -func newProjectClient(serviceAccountKey string) (projectClient, error) { - httpClient := metrics.NewInstrumentedHTTPClient("resourcemanager") - apiClient, err := resourcemanager.NewAPIClient( +func apiClientOptions(serviceAccountKey, endpoint, apiName string) []sdkconfig.ConfigurationOption { + opts := []sdkconfig.ConfigurationOption{ sdkconfig.WithServiceAccountKey(serviceAccountKey), - sdkconfig.WithHTTPClient(httpClient), - ) + sdkconfig.WithHTTPClient(metrics.NewInstrumentedHTTPClient(apiName)), + } + if endpoint != "" { + opts = append(opts, sdkconfig.WithEndpoint(endpoint)) + } + return opts +} + +func newProjectClient(serviceAccountKey, endpoint string) (projectClient, error) { + apiClient, err := resourcemanager.NewAPIClient(apiClientOptions(serviceAccountKey, endpoint, "resourcemanager")...) if err != nil { return nil, fmt.Errorf("create Resource Manager client: %w", err) } return &sdkProjectClient{api: apiClient}, nil } -func newServiceAccountClient(serviceAccountKey string) (serviceAccountClient, error) { - httpClient := metrics.NewInstrumentedHTTPClient("serviceaccount") - apiClient, err := serviceaccount.NewAPIClient( - sdkconfig.WithServiceAccountKey(serviceAccountKey), - sdkconfig.WithHTTPClient(httpClient), - ) +func newServiceAccountClient(serviceAccountKey, endpoint string) (serviceAccountClient, error) { + apiClient, err := serviceaccount.NewAPIClient(apiClientOptions(serviceAccountKey, endpoint, "serviceaccount")...) if err != nil { return nil, fmt.Errorf("create Service Account client: %w", err) } return &sdkServiceAccountClient{api: apiClient}, nil } -func newAuthorizationClient(serviceAccountKey string) (authorizationClient, error) { - httpClient := metrics.NewInstrumentedHTTPClient("authorization") - apiClient, err := authorization.NewAPIClient( - sdkconfig.WithServiceAccountKey(serviceAccountKey), - sdkconfig.WithHTTPClient(httpClient), - ) +func newAuthorizationClient(serviceAccountKey, endpoint string) (authorizationClient, error) { + apiClient, err := authorization.NewAPIClient(apiClientOptions(serviceAccountKey, endpoint, "authorization")...) if err != nil { return nil, fmt.Errorf("create Authorization client: %w", err) } diff --git a/test/kubetest2/bootstrap_resources.go b/test/kubetest2/bootstrap_resources.go index 564b4feb..35ce4585 100644 --- a/test/kubetest2/bootstrap_resources.go +++ b/test/kubetest2/bootstrap_resources.go @@ -49,21 +49,21 @@ type serviceAccountKeyCredentialsFile struct { func (d *Deployer) initializeBootstrapClients() error { if d.projectClient == nil { - client, err := newProjectClient(d.serviceAccount) + client, err := newProjectClient(d.serviceAccount, d.resourceManagerEndpoint) if err != nil { return err } d.projectClient = client } if d.serviceAccountClient == nil { - client, err := newServiceAccountClient(d.serviceAccount) + client, err := newServiceAccountClient(d.serviceAccount, d.serviceAccountEndpoint) if err != nil { return err } d.serviceAccountClient = client } if d.authorizationClient == nil { - client, err := newAuthorizationClient(d.serviceAccount) + client, err := newAuthorizationClient(d.serviceAccount, d.authorizationEndpoint) if err != nil { return err } @@ -76,7 +76,7 @@ func (d *Deployer) initializeSKEClient(serviceAccountKey string) error { if d.skeClientFactory == nil { d.skeClientFactory = newSKEClient } - client, err := d.skeClientFactory(d.region, serviceAccountKey) + client, err := d.skeClientFactory(d.region, serviceAccountKey, d.skeEndpoint) if err != nil { return err } diff --git a/test/kubetest2/bootstrap_test.go b/test/kubetest2/bootstrap_test.go index e9440208..aa609f4f 100644 --- a/test/kubetest2/bootstrap_test.go +++ b/test/kubetest2/bootstrap_test.go @@ -268,6 +268,10 @@ func TestLoadEnvironmentValidation(t *testing.T) { "STACKIT_PARENT_CONTAINER_ID", "STACKIT_PROJECT_MEMBER_EMAIL", "STACKIT_PROJECT_ID", + "STACKIT_RESOURCE_MANAGER_ENDPOINT", + "STACKIT_SERVICE_ACCOUNT_ENDPOINT", + "STACKIT_AUTHORIZATION_ENDPOINT", + "STACKIT_SKE_ENDPOINT", } { t.Setenv(key, "") } @@ -306,6 +310,46 @@ func TestLoadEnvironmentValidation(t *testing.T) { } } +func TestLoadEnvironmentReadsOptionalEndpoints(t *testing.T) { + for _, key := range []string{ + "STACKIT_SERVICE_ACCOUNT", + "STACKIT_PARENT_CONTAINER_ID", + "STACKIT_PROJECT_MEMBER_EMAIL", + "STACKIT_RESOURCE_MANAGER_ENDPOINT", + "STACKIT_SERVICE_ACCOUNT_ENDPOINT", + "STACKIT_AUTHORIZATION_ENDPOINT", + "STACKIT_SKE_ENDPOINT", + } { + t.Setenv(key, "") + } + t.Setenv("STACKIT_SERVICE_ACCOUNT", "{}") + t.Setenv("STACKIT_PARENT_CONTAINER_ID", "parent-1") + t.Setenv("STACKIT_PROJECT_MEMBER_EMAIL", "owner@example.com") + t.Setenv("STACKIT_RESOURCE_MANAGER_ENDPOINT", "https://resource-manager.example.com") + t.Setenv("STACKIT_SERVICE_ACCOUNT_ENDPOINT", "https://service-account.example.com") + t.Setenv("STACKIT_AUTHORIZATION_ENDPOINT", "https://authorization.example.com") + t.Setenv("STACKIT_SKE_ENDPOINT", "https://ske.example.com") + + runDir := t.TempDir() + d := &Deployer{options: fakeOptions{runID: "run-123", runDir: runDir}} + if err := d.loadEnvironment(); err != nil { + t.Fatalf("loadEnvironment() error = %v", err) + } + + if d.resourceManagerEndpoint != "https://resource-manager.example.com" { + t.Fatalf("resourceManagerEndpoint = %q", d.resourceManagerEndpoint) + } + if d.serviceAccountEndpoint != "https://service-account.example.com" { + t.Fatalf("serviceAccountEndpoint = %q", d.serviceAccountEndpoint) + } + if d.authorizationEndpoint != "https://authorization.example.com" { + t.Fatalf("authorizationEndpoint = %q", d.authorizationEndpoint) + } + if d.skeEndpoint != "https://ske.example.com" { + t.Fatalf("skeEndpoint = %q", d.skeEndpoint) + } +} + func TestResolveManagedProjectCreatesWhenMissing(t *testing.T) { d := newTestDeployer(t) projectClient := &fakeProjectClient{ @@ -385,7 +429,7 @@ func TestEnsureManagedClusterAccessReusesCachedKeyAndSkipsMembershipWrite(t *tes d.projectClient = projectClient d.serviceAccountClient = serviceAccountClient d.authorizationClient = authorizationClient - d.skeClientFactory = func(_, serviceAccount string) (skeClient, error) { + d.skeClientFactory = func(_, serviceAccount, _ string) (skeClient, error) { receivedKey = serviceAccount return fakeSKE, nil } @@ -428,7 +472,7 @@ func TestEnsureManagedClusterAccessCreatesKeyAndAddsMembership(t *testing.T) { d.projectClient = projectClient d.serviceAccountClient = serviceAccountClient d.authorizationClient = authorizationClient - d.skeClientFactory = func(_, serviceAccount string) (skeClient, error) { + d.skeClientFactory = func(_, serviceAccount, _ string) (skeClient, error) { receivedKey = serviceAccount return &fakeSKEClient{}, nil } @@ -500,7 +544,7 @@ func TestUpUsesDiscoveredProjectAndWritesKubeconfig(t *testing.T) { return cfg }(), } - d.skeClientFactory = func(_, serviceAccount string) (skeClient, error) { + d.skeClientFactory = func(_, serviceAccount, _ string) (skeClient, error) { if serviceAccount != cachedKey { t.Fatalf("unexpected service account key passed to SKE client") } @@ -603,7 +647,7 @@ func TestIsUpProjectWithCachedKeyQueriesCluster(t *testing.T) { fakeSKE := &fakeSKEClient{ getClusterResult: healthyClusterFixture(), } - d.skeClientFactory = func(_, serviceAccount string) (skeClient, error) { + d.skeClientFactory = func(_, serviceAccount, _ string) (skeClient, error) { if serviceAccount != cachedKey { t.Fatalf("unexpected cached key content") } @@ -745,7 +789,7 @@ func TestIsUpReturnsFalseWhenClusterMissing(t *testing.T) { fakeSKE := &fakeSKEClient{ getClusterErr: &oapierror.GenericOpenAPIError{StatusCode: http.StatusNotFound}, } - d.skeClientFactory = func(_, _ string) (skeClient, error) { + d.skeClientFactory = func(_, _, _ string) (skeClient, error) { return fakeSKE, nil } diff --git a/test/kubetest2/deployer.go b/test/kubetest2/deployer.go index b861e7e9..02825608 100644 --- a/test/kubetest2/deployer.go +++ b/test/kubetest2/deployer.go @@ -45,11 +45,16 @@ type Deployer struct { kubeconfigPath string serviceAccountKeyPath string + resourceManagerEndpoint string + serviceAccountEndpoint string + authorizationEndpoint string + skeEndpoint string + projectClient projectClient serviceAccountClient serviceAccountClient authorizationClient authorizationClient skeClient skeClient - skeClientFactory func(region, serviceAccount string) (skeClient, error) + skeClientFactory func(region, serviceAccount, endpoint string) (skeClient, error) } var _ types.NewDeployer = New diff --git a/test/kubetest2/env.go b/test/kubetest2/env.go index 12b2cae3..44f058a4 100644 --- a/test/kubetest2/env.go +++ b/test/kubetest2/env.go @@ -29,6 +29,10 @@ func (d *Deployer) loadEnvironment() error { d.serviceAccount = serviceAccount d.parentContainerID = parentContainerID d.projectMemberEmail = projectMemberEmail + d.resourceManagerEndpoint = strings.TrimSpace(os.Getenv("STACKIT_RESOURCE_MANAGER_ENDPOINT")) + d.serviceAccountEndpoint = strings.TrimSpace(os.Getenv("STACKIT_SERVICE_ACCOUNT_ENDPOINT")) + d.authorizationEndpoint = strings.TrimSpace(os.Getenv("STACKIT_AUTHORIZATION_ENDPOINT")) + d.skeEndpoint = strings.TrimSpace(os.Getenv("STACKIT_SKE_ENDPOINT")) d.kubeconfigPath = filepath.Join(d.options.RunDir(), "kubeconfig") d.serviceAccountKeyPath = filepath.Join(d.options.RunDir(), "service-account-key.json") @@ -41,5 +45,13 @@ func (d *Deployer) loadEnvironment() error { d.serviceAccountKeyPath, ) + klog.Infof( + "STACKIT API endpoint overrides: resource_manager=%q service_account=%q authorization=%q ske=%q", + d.resourceManagerEndpoint, + d.serviceAccountEndpoint, + d.authorizationEndpoint, + d.skeEndpoint, + ) + return nil } diff --git a/test/kubetest2/ske_client.go b/test/kubetest2/ske_client.go index 83104ebc..d56060f6 100644 --- a/test/kubetest2/ske_client.go +++ b/test/kubetest2/ske_client.go @@ -5,8 +5,6 @@ import ( "fmt" "strconv" - "github.com/stackitcloud/cloud-provider-stackit/pkg/metrics" - sdkconfig "github.com/stackitcloud/stackit-sdk-go/core/config" "github.com/stackitcloud/stackit-sdk-go/services/ske" skewait "github.com/stackitcloud/stackit-sdk-go/services/ske/wait" "k8s.io/klog/v2" @@ -26,14 +24,10 @@ type sdkSKEClient struct { api ske.DefaultApi } -func newSKEClient(region, serviceAccount string) (skeClient, error) { +func newSKEClient(region, serviceAccount, endpoint string) (skeClient, error) { klog.Infof("Creating SKE API client for region=%q with service_account_bytes=%d", region, len(serviceAccount)) - httpClient := metrics.NewInstrumentedHTTPClient("ske") - apiClient, err := ske.NewAPIClient( - sdkconfig.WithServiceAccountKey(serviceAccount), - sdkconfig.WithHTTPClient(httpClient), - ) + apiClient, err := ske.NewAPIClient(apiClientOptions(serviceAccount, endpoint, "ske")...) if err != nil { return nil, fmt.Errorf("create SKE client: %w", err) } From 8bd730bb4b19d30c40e49d92539bb96cab8bc9dd Mon Sep 17 00:00:00 2001 From: Robert Kaussow Date: Tue, 18 Aug 2026 14:33:34 +0200 Subject: [PATCH 05/11] use email from owner service account --- test/kubetest2/bootstrap_clients.go | 13 ++++++++++++- test/kubetest2/bootstrap_test.go | 25 ++++++++++++------------- test/kubetest2/env.go | 20 +++++++++++++++++--- 3 files changed, 41 insertions(+), 17 deletions(-) diff --git a/test/kubetest2/bootstrap_clients.go b/test/kubetest2/bootstrap_clients.go index fa50d5d5..c0e6115c 100644 --- a/test/kubetest2/bootstrap_clients.go +++ b/test/kubetest2/bootstrap_clients.go @@ -66,6 +66,13 @@ func apiClientOptions(serviceAccountKey, endpoint, apiName string) []sdkconfig.C return opts } +func apiEndpointURL(cfg *sdkconfig.Configuration) string { + if cfg == nil || len(cfg.Servers) == 0 { + return "unknown" + } + return cfg.Servers[0].URL +} + func newProjectClient(serviceAccountKey, endpoint string) (projectClient, error) { apiClient, err := resourcemanager.NewAPIClient(apiClientOptions(serviceAccountKey, endpoint, "resourcemanager")...) if err != nil { @@ -149,7 +156,11 @@ func (c *sdkServiceAccountClient) CreateServiceAccount(ctx context.Context, proj func (c *sdkServiceAccountClient) CreateServiceAccountKey(ctx context.Context, projectID, serviceAccountEmail string) (*serviceaccount.CreateServiceAccountKeyResponse, error) { payload := serviceaccount.NewCreateServiceAccountKeyPayloadWithDefaults() - return c.api.DefaultAPI.CreateServiceAccountKey(ctx, projectID, serviceAccountEmail).CreateServiceAccountKeyPayload(*payload).Execute() + resp, err := c.api.DefaultAPI.CreateServiceAccountKey(ctx, projectID, serviceAccountEmail).CreateServiceAccountKeyPayload(*payload).Execute() + if err != nil { + return nil, fmt.Errorf("%w (endpoint: %s)", err, apiEndpointURL(c.api.GetConfig())) + } + return resp, nil } func (c *sdkAuthorizationClient) ListMembers(ctx context.Context, resourceType, resourceID string) ([]authorization.Member, error) { diff --git a/test/kubetest2/bootstrap_test.go b/test/kubetest2/bootstrap_test.go index aa609f4f..69e0a490 100644 --- a/test/kubetest2/bootstrap_test.go +++ b/test/kubetest2/bootstrap_test.go @@ -18,6 +18,8 @@ import ( "sigs.k8s.io/kubetest2/pkg/types" ) +const validServiceAccountKey = `{"credentials":{"iss":"owner@example.com"}}` + type fakeOptions struct { runID string runDir string @@ -230,33 +232,30 @@ func TestLoadEnvironmentValidation(t *testing.T) { { name: "missing service account", env: map[string]string{ - "STACKIT_PARENT_CONTAINER_ID": "parent-1", - "STACKIT_PROJECT_MEMBER_EMAIL": "owner@example.com", + "STACKIT_PARENT_CONTAINER_ID": "parent-1", }, wantErrContains: "STACKIT_SERVICE_ACCOUNT", }, { name: "missing parent container", env: map[string]string{ - "STACKIT_SERVICE_ACCOUNT": "{}", - "STACKIT_PROJECT_MEMBER_EMAIL": "owner@example.com", + "STACKIT_SERVICE_ACCOUNT": validServiceAccountKey, }, wantErrContains: "STACKIT_PARENT_CONTAINER_ID", }, { - name: "missing project member email", + name: "invalid service account key", env: map[string]string{ "STACKIT_SERVICE_ACCOUNT": "{}", "STACKIT_PARENT_CONTAINER_ID": "parent-1", }, - wantErrContains: "STACKIT_PROJECT_MEMBER_EMAIL", + wantErrContains: "invalid STACKIT_SERVICE_ACCOUNT", }, { name: "project id no longer required", env: map[string]string{ - "STACKIT_SERVICE_ACCOUNT": "{}", - "STACKIT_PARENT_CONTAINER_ID": "parent-1", - "STACKIT_PROJECT_MEMBER_EMAIL": "owner@example.com", + "STACKIT_SERVICE_ACCOUNT": validServiceAccountKey, + "STACKIT_PARENT_CONTAINER_ID": "parent-1", }, }, } @@ -266,7 +265,6 @@ func TestLoadEnvironmentValidation(t *testing.T) { for _, key := range []string{ "STACKIT_SERVICE_ACCOUNT", "STACKIT_PARENT_CONTAINER_ID", - "STACKIT_PROJECT_MEMBER_EMAIL", "STACKIT_PROJECT_ID", "STACKIT_RESOURCE_MANAGER_ENDPOINT", "STACKIT_SERVICE_ACCOUNT_ENDPOINT", @@ -314,7 +312,6 @@ func TestLoadEnvironmentReadsOptionalEndpoints(t *testing.T) { for _, key := range []string{ "STACKIT_SERVICE_ACCOUNT", "STACKIT_PARENT_CONTAINER_ID", - "STACKIT_PROJECT_MEMBER_EMAIL", "STACKIT_RESOURCE_MANAGER_ENDPOINT", "STACKIT_SERVICE_ACCOUNT_ENDPOINT", "STACKIT_AUTHORIZATION_ENDPOINT", @@ -322,9 +319,8 @@ func TestLoadEnvironmentReadsOptionalEndpoints(t *testing.T) { } { t.Setenv(key, "") } - t.Setenv("STACKIT_SERVICE_ACCOUNT", "{}") + t.Setenv("STACKIT_SERVICE_ACCOUNT", validServiceAccountKey) t.Setenv("STACKIT_PARENT_CONTAINER_ID", "parent-1") - t.Setenv("STACKIT_PROJECT_MEMBER_EMAIL", "owner@example.com") t.Setenv("STACKIT_RESOURCE_MANAGER_ENDPOINT", "https://resource-manager.example.com") t.Setenv("STACKIT_SERVICE_ACCOUNT_ENDPOINT", "https://service-account.example.com") t.Setenv("STACKIT_AUTHORIZATION_ENDPOINT", "https://authorization.example.com") @@ -336,6 +332,9 @@ func TestLoadEnvironmentReadsOptionalEndpoints(t *testing.T) { t.Fatalf("loadEnvironment() error = %v", err) } + if d.projectMemberEmail != "owner@example.com" { + t.Fatalf("projectMemberEmail = %q, want %q", d.projectMemberEmail, "owner@example.com") + } if d.resourceManagerEndpoint != "https://resource-manager.example.com" { t.Fatalf("resourceManagerEndpoint = %q", d.resourceManagerEndpoint) } diff --git a/test/kubetest2/env.go b/test/kubetest2/env.go index 44f058a4..271e1366 100644 --- a/test/kubetest2/env.go +++ b/test/kubetest2/env.go @@ -1,6 +1,8 @@ package kubetest2 import ( + "encoding/json" + "fmt" "os" "path/filepath" "strings" @@ -21,9 +23,9 @@ func (d *Deployer) loadEnvironment() error { return incorrectUsagef("STACKIT_PARENT_CONTAINER_ID environment variable is required") } - projectMemberEmail, ok := os.LookupEnv("STACKIT_PROJECT_MEMBER_EMAIL") - if !ok || strings.TrimSpace(projectMemberEmail) == "" { - return incorrectUsagef("STACKIT_PROJECT_MEMBER_EMAIL environment variable is required") + projectMemberEmail, err := extractServiceAccountEmail(serviceAccount) + if err != nil { + return incorrectUsagef("invalid STACKIT_SERVICE_ACCOUNT: %v", err) } d.serviceAccount = serviceAccount @@ -55,3 +57,15 @@ func (d *Deployer) loadEnvironment() error { return nil } + +func extractServiceAccountEmail(serviceAccountKey string) (string, error) { + var key serviceAccountKeyFile + if err := json.Unmarshal([]byte(serviceAccountKey), &key); err != nil { + return "", fmt.Errorf("parse service account key: %w", err) + } + email := strings.TrimSpace(key.Credentials.Iss) + if email == "" { + return "", fmt.Errorf("service account key has no email in credentials.iss") + } + return email, nil +} From fb4c5daa6bc636c11d1a1107cfda72566e3c445e Mon Sep 17 00:00:00 2001 From: Robert Kaussow Date: Tue, 18 Aug 2026 16:16:02 +0200 Subject: [PATCH 06/11] add ske sevice enablement --- go.mod | 1 + go.sum | 2 + test/kubetest2/bootstrap_clients.go | 34 ++++++ test/kubetest2/bootstrap_resources.go | 79 ++++++++++++- test/kubetest2/bootstrap_test.go | 163 ++++++++++++++++++++++++++ test/kubetest2/deployer.go | 22 ++-- test/kubetest2/env.go | 4 +- 7 files changed, 293 insertions(+), 12 deletions(-) diff --git a/go.mod b/go.mod index 4fa03e25..8974e3e3 100644 --- a/go.mod +++ b/go.mod @@ -96,6 +96,7 @@ require ( github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect github.com/prometheus/common v0.70.1 // indirect github.com/prometheus/procfs v0.21.1 // indirect + github.com/stackitcloud/stackit-sdk-go/services/serviceenablement v1.7.0 // indirect github.com/stoewer/go-strcase v1.3.1 // indirect github.com/stretchr/objx v0.5.3 // indirect github.com/x448/float16 v0.8.4 // indirect diff --git a/go.sum b/go.sum index dd7e9949..56f24f10 100644 --- a/go.sum +++ b/go.sum @@ -196,6 +196,8 @@ github.com/stackitcloud/stackit-sdk-go/services/resourcemanager v0.24.2 h1:4UxxJ github.com/stackitcloud/stackit-sdk-go/services/resourcemanager v0.24.2/go.mod h1:NEz3f+GV5G++BE9/MmZCsXJyCih7jtg0pZuSyG2sLEs= github.com/stackitcloud/stackit-sdk-go/services/serviceaccount v0.20.1 h1:zVVA/Puge2PK3G8lrGGolnQzxYwwsNUgfMyyRUUQc44= github.com/stackitcloud/stackit-sdk-go/services/serviceaccount v0.20.1/go.mod h1:xJNa86o89czK2o60+s46vz8+NSKDTbQduFGFaztJnsA= +github.com/stackitcloud/stackit-sdk-go/services/serviceenablement v1.7.0 h1:TNZHrunhsXRbuqZcucLs2Gqy1sEyvabufM7pB5Tscmo= +github.com/stackitcloud/stackit-sdk-go/services/serviceenablement v1.7.0/go.mod h1:fXq3TmVLb4JMSve989NFFViMFoYa83s7M3hJWgN6mdQ= github.com/stackitcloud/stackit-sdk-go/services/ske v1.6.0 h1:Dab1jzN0u9c67lvELoWf1RuagjO3eUBRytoX8SYL8Zs= github.com/stackitcloud/stackit-sdk-go/services/ske v1.6.0/go.mod h1:NzcTU5GGlUF6Lys3Ra7ylRj4ZKxJr3f/29/yoE5tjPI= github.com/stoewer/go-strcase v1.3.1 h1:iS0MdW+kVTxgMoE1LAZyMiYJFKlOzLooE4MxjirtkAs= diff --git a/test/kubetest2/bootstrap_clients.go b/test/kubetest2/bootstrap_clients.go index c0e6115c..d70eea6c 100644 --- a/test/kubetest2/bootstrap_clients.go +++ b/test/kubetest2/bootstrap_clients.go @@ -10,6 +10,8 @@ import ( resourcemanager "github.com/stackitcloud/stackit-sdk-go/services/resourcemanager/v0api" resourcemanagerwait "github.com/stackitcloud/stackit-sdk-go/services/resourcemanager/v0api/wait" serviceaccount "github.com/stackitcloud/stackit-sdk-go/services/serviceaccount/v2api" + serviceenablement "github.com/stackitcloud/stackit-sdk-go/services/serviceenablement/v2api" + serviceenablementwait "github.com/stackitcloud/stackit-sdk-go/services/serviceenablement/v2api/wait" ) const ( @@ -22,6 +24,7 @@ const ( projectResourceType = "project" childProjectRole = "ske.admin" projectListPageSize float32 = 100 + skeServiceID = "cloud.stackit.ske" ) type projectClient interface { @@ -43,6 +46,12 @@ type authorizationClient interface { AddMembers(ctx context.Context, resourceID, resourceType string, members []authorization.Member) error } +type serviceEnablementClient interface { + GetServiceStatus(ctx context.Context, region, projectID, serviceID string) (*serviceenablement.ServiceStatus, error) + EnableService(ctx context.Context, region, projectID, serviceID string) error + WaitForServiceEnabled(ctx context.Context, region, projectID, serviceID string) error +} + type sdkProjectClient struct { api *resourcemanager.APIClient } @@ -55,6 +64,10 @@ type sdkAuthorizationClient struct { api *authorization.APIClient } +type sdkServiceEnablementClient struct { + api *serviceenablement.APIClient +} + func apiClientOptions(serviceAccountKey, endpoint, apiName string) []sdkconfig.ConfigurationOption { opts := []sdkconfig.ConfigurationOption{ sdkconfig.WithServiceAccountKey(serviceAccountKey), @@ -97,6 +110,14 @@ func newAuthorizationClient(serviceAccountKey, endpoint string) (authorizationCl return &sdkAuthorizationClient{api: apiClient}, nil } +func newServiceEnablementClient(serviceAccountKey, endpoint string) (serviceEnablementClient, error) { + apiClient, err := serviceenablement.NewAPIClient(apiClientOptions(serviceAccountKey, endpoint, "serviceenablement")...) + if err != nil { + return nil, fmt.Errorf("create Service Enablement client: %w", err) + } + return &sdkServiceEnablementClient{api: apiClient}, nil +} + func (c *sdkProjectClient) ListProjects(ctx context.Context, parentContainerID string) ([]resourcemanager.Project, error) { projects := make([]resourcemanager.Project, 0) var offset float32 @@ -176,3 +197,16 @@ func (c *sdkAuthorizationClient) AddMembers(ctx context.Context, resourceID, res _, err := c.api.DefaultAPI.AddMembers(ctx, resourceID).AddMembersPayload(*payload).Execute() return err } + +func (c *sdkServiceEnablementClient) GetServiceStatus(ctx context.Context, region, projectID, serviceID string) (*serviceenablement.ServiceStatus, error) { + return c.api.DefaultAPI.GetServiceStatusRegional(ctx, region, projectID, serviceID).Execute() +} + +func (c *sdkServiceEnablementClient) EnableService(ctx context.Context, region, projectID, serviceID string) error { + return c.api.DefaultAPI.EnableServiceRegional(ctx, region, projectID, serviceID).Execute() +} + +func (c *sdkServiceEnablementClient) WaitForServiceEnabled(ctx context.Context, region, projectID, serviceID string) error { + _, err := serviceenablementwait.EnableServiceWaitHandler(ctx, c.api.DefaultAPI, region, projectID, serviceID).WaitWithContext(ctx) + return err +} diff --git a/test/kubetest2/bootstrap_resources.go b/test/kubetest2/bootstrap_resources.go index 35ce4585..e8f0b64a 100644 --- a/test/kubetest2/bootstrap_resources.go +++ b/test/kubetest2/bootstrap_resources.go @@ -3,17 +3,28 @@ package kubetest2 import ( "context" "encoding/json" + "errors" "fmt" + "net/http" "os" "strings" "time" + oapierror "github.com/stackitcloud/stackit-sdk-go/core/oapierror" authorization "github.com/stackitcloud/stackit-sdk-go/services/authorization/v2api" resourcemanager "github.com/stackitcloud/stackit-sdk-go/services/resourcemanager/v0api" serviceaccount "github.com/stackitcloud/stackit-sdk-go/services/serviceaccount/v2api" + serviceenablement "github.com/stackitcloud/stackit-sdk-go/services/serviceenablement/v2api" + "k8s.io/apimachinery/pkg/util/wait" "k8s.io/klog/v2" ) +var childKeyCreationRetryBackoff = wait.Backoff{ + Duration: 3 * time.Second, + Factor: 2.0, + Steps: 5, +} + type managedProject struct { ContainerID string ProjectID string @@ -69,6 +80,13 @@ func (d *Deployer) initializeBootstrapClients() error { } d.authorizationClient = client } + if d.serviceEnablementClient == nil { + client, err := newServiceEnablementClient(d.serviceAccount, d.serviceEnablementEndpoint) + if err != nil { + return err + } + d.serviceEnablementClient = client + } return nil } @@ -91,6 +109,10 @@ func (d *Deployer) ensureManagedClusterAccess(ctx context.Context) error { } d.projectID = project.ProjectID + if err := d.ensureSKEServiceEnabled(ctx, project.ProjectID); err != nil { + return err + } + childServiceAccount, err := d.resolveManagedServiceAccount(ctx, project.ProjectID) if err != nil { return err @@ -108,6 +130,36 @@ func (d *Deployer) ensureManagedClusterAccess(ctx context.Context) error { return d.initializeSKEClient(serviceAccountKey) } +// ensureSKEServiceEnabled idempotently enables the SKE (Kubernetes Engine) +// service for the managed project and waits until it is enabled. +func (d *Deployer) ensureSKEServiceEnabled(ctx context.Context, projectID string) error { + status, err := d.serviceEnablementClient.GetServiceStatus(ctx, d.region, projectID, skeServiceID) + if err != nil { + if !isNotFound(err) { + return fmt.Errorf("get SKE service status for project %q: %w", projectID, err) + } + klog.Infof("SKE service not yet enabled for project_id=%q", projectID) + } else if status.GetState() == serviceenablement.SERVICESTATUSSTATE_ENABLED { + klog.Infof("SKE service already enabled for project_id=%q", projectID) + return nil + } else { + klog.Infof("SKE service in state %q for project_id=%q, enabling", status.GetState(), projectID) + } + + if err := d.serviceEnablementClient.EnableService(ctx, d.region, projectID, skeServiceID); err != nil { + return fmt.Errorf("enable SKE service for project %q: %w", projectID, err) + } + if err := d.serviceEnablementClient.WaitForServiceEnabled(ctx, d.region, projectID, skeServiceID); err != nil { + return fmt.Errorf("wait for SKE service enablement for project %q: %w", projectID, err) + } + return nil +} + +func isNotFound(err error) bool { + var oapiErr *oapierror.GenericOpenAPIError + return errors.As(err, &oapiErr) && oapiErr.StatusCode == http.StatusNotFound +} + func (d *Deployer) findManagedProject(ctx context.Context) (*managedProject, error) { projects, err := d.projectClient.ListProjects(ctx, d.parentContainerID) if err != nil { @@ -254,7 +306,9 @@ func (d *Deployer) ensureCachedChildServiceAccountKey(ctx context.Context, proje } klog.Infof("Creating child service-account key for service_account=%q in project_id=%q", serviceAccountEmail, projectID) - createdKey, err := d.serviceAccountClient.CreateServiceAccountKey(ctx, projectID, serviceAccountEmail) + createdKey, err := retryWithBackoff(ctx, childKeyCreationRetryBackoff, func() (*serviceaccount.CreateServiceAccountKeyResponse, error) { + return d.serviceAccountClient.CreateServiceAccountKey(ctx, projectID, serviceAccountEmail) + }) if err != nil { return "", fmt.Errorf("create service-account key for %q in STACKIT project %q: %w", serviceAccountEmail, projectID, err) } @@ -268,6 +322,29 @@ func (d *Deployer) ensureCachedChildServiceAccountKey(ctx context.Context, proje return keyJSON, nil } +// retryWithBackoff retries fn until it succeeds or the backoff is exhausted. +// It is used for operations that may fail transiently, e.g. while a freshly +// created service account is still propagating through the STACKIT IAM and is +// not yet ready to have a key created for it. +func retryWithBackoff[T any](ctx context.Context, backoff wait.Backoff, fn func() (T, error)) (T, error) { + var result T + var lastErr error + + waitErr := wait.ExponentialBackoffWithContext(ctx, backoff, func(_ context.Context) (bool, error) { + val, err := fn() + if err != nil { + lastErr = err + return false, nil + } + result = val + return true, nil + }) + if waitErr != nil { + return result, fmt.Errorf("backoff failed: %w, last error: %v", waitErr, lastErr) + } + return result, nil +} + func (d *Deployer) readCachedChildServiceAccountKey() (key string, ok bool, err error) { if strings.TrimSpace(d.serviceAccountKeyPath) == "" { return "", false, nil diff --git a/test/kubetest2/bootstrap_test.go b/test/kubetest2/bootstrap_test.go index 69e0a490..264ff045 100644 --- a/test/kubetest2/bootstrap_test.go +++ b/test/kubetest2/bootstrap_test.go @@ -2,6 +2,7 @@ package kubetest2 import ( "context" + "errors" "net/http" "os" "path/filepath" @@ -14,7 +15,9 @@ import ( authorization "github.com/stackitcloud/stackit-sdk-go/services/authorization/v2api" resourcemanager "github.com/stackitcloud/stackit-sdk-go/services/resourcemanager/v0api" serviceaccount "github.com/stackitcloud/stackit-sdk-go/services/serviceaccount/v2api" + serviceenablement "github.com/stackitcloud/stackit-sdk-go/services/serviceenablement/v2api" "github.com/stackitcloud/stackit-sdk-go/services/ske" + "k8s.io/apimachinery/pkg/util/wait" "sigs.k8s.io/kubetest2/pkg/types" ) @@ -163,6 +166,49 @@ func (c *fakeAuthorizationClient) AddMembers(_ context.Context, resourceID, reso return c.addMembersErr } +type fakeServiceEnablementClient struct { + getStatusResult *serviceenablement.ServiceStatus + getStatusErr error + enableErr error + waitErr error + + enableCalls int + waitCalls int + + lastGetStatusRegion string + lastGetStatusProjectID string + lastGetStatusServiceID string + lastEnableRegion string + lastEnableProjectID string + lastEnableServiceID string + lastWaitRegion string + lastWaitProjectID string + lastWaitServiceID string +} + +func (c *fakeServiceEnablementClient) GetServiceStatus(_ context.Context, region, projectID, serviceID string) (*serviceenablement.ServiceStatus, error) { + c.lastGetStatusRegion = region + c.lastGetStatusProjectID = projectID + c.lastGetStatusServiceID = serviceID + return c.getStatusResult, c.getStatusErr +} + +func (c *fakeServiceEnablementClient) EnableService(_ context.Context, region, projectID, serviceID string) error { + c.enableCalls++ + c.lastEnableRegion = region + c.lastEnableProjectID = projectID + c.lastEnableServiceID = serviceID + return c.enableErr +} + +func (c *fakeServiceEnablementClient) WaitForServiceEnabled(_ context.Context, region, projectID, serviceID string) error { + c.waitCalls++ + c.lastWaitRegion = region + c.lastWaitProjectID = projectID + c.lastWaitServiceID = serviceID + return c.waitErr +} + type fakeSKEClient struct { providerOptions *ske.ProviderOptions providerOptionsErr error @@ -428,6 +474,7 @@ func TestEnsureManagedClusterAccessReusesCachedKeyAndSkipsMembershipWrite(t *tes d.projectClient = projectClient d.serviceAccountClient = serviceAccountClient d.authorizationClient = authorizationClient + d.serviceEnablementClient = &fakeServiceEnablementClient{getStatusResult: serviceenablement.NewServiceStatus()} d.skeClientFactory = func(_, serviceAccount, _ string) (skeClient, error) { receivedKey = serviceAccount return fakeSKE, nil @@ -471,6 +518,7 @@ func TestEnsureManagedClusterAccessCreatesKeyAndAddsMembership(t *testing.T) { d.projectClient = projectClient d.serviceAccountClient = serviceAccountClient d.authorizationClient = authorizationClient + d.serviceEnablementClient = &fakeServiceEnablementClient{getStatusResult: serviceenablement.NewServiceStatus()} d.skeClientFactory = func(_, serviceAccount, _ string) (skeClient, error) { receivedKey = serviceAccount return &fakeSKEClient{}, nil @@ -507,6 +555,120 @@ func TestEnsureManagedClusterAccessCreatesKeyAndAddsMembership(t *testing.T) { } } +func TestEnsureSKEServiceEnabledSkipsWhenAlreadyEnabled(t *testing.T) { + d := newTestDeployer(t) + client := &fakeServiceEnablementClient{getStatusResult: serviceenablement.NewServiceStatus()} + d.serviceEnablementClient = client + + if err := d.ensureSKEServiceEnabled(context.Background(), "project-123"); err != nil { + t.Fatalf("ensureSKEServiceEnabled() error = %v", err) + } + if client.enableCalls != 0 { + t.Fatalf("enableCalls = %d, want 0", client.enableCalls) + } + if client.waitCalls != 0 { + t.Fatalf("waitCalls = %d, want 0", client.waitCalls) + } + if client.lastGetStatusServiceID != skeServiceID { + t.Fatalf("service ID = %q, want %q", client.lastGetStatusServiceID, skeServiceID) + } +} + +func TestEnsureSKEServiceEnabledEnablesWhenNotFound(t *testing.T) { + d := newTestDeployer(t) + client := &fakeServiceEnablementClient{ + getStatusErr: &oapierror.GenericOpenAPIError{StatusCode: http.StatusNotFound}, + } + d.serviceEnablementClient = client + + if err := d.ensureSKEServiceEnabled(context.Background(), "project-123"); err != nil { + t.Fatalf("ensureSKEServiceEnabled() error = %v", err) + } + if client.enableCalls != 1 { + t.Fatalf("enableCalls = %d, want 1", client.enableCalls) + } + if client.waitCalls != 1 { + t.Fatalf("waitCalls = %d, want 1", client.waitCalls) + } + if client.lastEnableProjectID != "project-123" || client.lastEnableServiceID != skeServiceID { + t.Fatalf("EnableService called with project_id=%q service_id=%q", client.lastEnableProjectID, client.lastEnableServiceID) + } +} + +func TestEnsureSKEServiceEnabledEnablesWhenDisabled(t *testing.T) { + d := newTestDeployer(t) + client := &fakeServiceEnablementClient{ + getStatusResult: disabledServiceStatusFixture(), + } + d.serviceEnablementClient = client + + if err := d.ensureSKEServiceEnabled(context.Background(), "project-123"); err != nil { + t.Fatalf("ensureSKEServiceEnabled() error = %v", err) + } + if client.enableCalls != 1 { + t.Fatalf("enableCalls = %d, want 1", client.enableCalls) + } + if client.waitCalls != 1 { + t.Fatalf("waitCalls = %d, want 1", client.waitCalls) + } +} + +func TestEnsureSKEServiceEnabledFailsOnGetStatusError(t *testing.T) { + d := newTestDeployer(t) + client := &fakeServiceEnablementClient{ + getStatusErr: &oapierror.GenericOpenAPIError{StatusCode: http.StatusForbidden}, + } + d.serviceEnablementClient = client + + if err := d.ensureSKEServiceEnabled(context.Background(), "project-123"); err == nil { + t.Fatal("ensureSKEServiceEnabled() error = nil, want non-nil") + } + if client.enableCalls != 0 { + t.Fatalf("enableCalls = %d, want 0", client.enableCalls) + } +} + +func disabledServiceStatusFixture() *serviceenablement.ServiceStatus { + status := serviceenablement.NewServiceStatus() + state := serviceenablement.SERVICESTATUSSTATE_DISABLED + status.State = &state + return status +} + +func TestRetryWithBackoffRetriesUntilSuccess(t *testing.T) { + calls := 0 + result, err := retryWithBackoff(context.Background(), wait.Backoff{Duration: 0, Factor: 1, Steps: 3}, func() (string, error) { + calls++ + if calls < 2 { + return "", errors.New("transient") + } + return "done", nil + }) + if err != nil { + t.Fatalf("retryWithBackoff() error = %v", err) + } + if result != "done" { + t.Fatalf("result = %q, want %q", result, "done") + } + if calls != 2 { + t.Fatalf("calls = %d, want 2", calls) + } +} + +func TestRetryWithBackoffReturnsLastErrorWhenExhausted(t *testing.T) { + calls := 0 + _, err := retryWithBackoff(context.Background(), wait.Backoff{Duration: 0, Factor: 1, Steps: 3}, func() (int, error) { + calls++ + return 0, errors.New("always fails") + }) + if err == nil { + t.Fatal("retryWithBackoff() error = nil, want non-nil") + } + if calls != 3 { + t.Fatalf("calls = %d, want 3", calls) + } +} + func TestUpUsesDiscoveredProjectAndWritesKubeconfig(t *testing.T) { d := newTestDeployer(t) configureValidUpInputs(d) @@ -532,6 +694,7 @@ func TestUpUsesDiscoveredProjectAndWritesKubeconfig(t *testing.T) { *authorization.NewMember(childProjectRole, serviceAccountEmail), }, } + d.serviceEnablementClient = &fakeServiceEnablementClient{getStatusResult: serviceenablement.NewServiceStatus()} fakeSKE := &fakeSKEClient{ providerOptions: providerOptionsFixture(), diff --git a/test/kubetest2/deployer.go b/test/kubetest2/deployer.go index 02825608..9725dae6 100644 --- a/test/kubetest2/deployer.go +++ b/test/kubetest2/deployer.go @@ -45,16 +45,18 @@ type Deployer struct { kubeconfigPath string serviceAccountKeyPath string - resourceManagerEndpoint string - serviceAccountEndpoint string - authorizationEndpoint string - skeEndpoint string - - projectClient projectClient - serviceAccountClient serviceAccountClient - authorizationClient authorizationClient - skeClient skeClient - skeClientFactory func(region, serviceAccount, endpoint string) (skeClient, error) + resourceManagerEndpoint string + serviceAccountEndpoint string + authorizationEndpoint string + serviceEnablementEndpoint string + skeEndpoint string + + projectClient projectClient + serviceAccountClient serviceAccountClient + authorizationClient authorizationClient + serviceEnablementClient serviceEnablementClient + skeClient skeClient + skeClientFactory func(region, serviceAccount, endpoint string) (skeClient, error) } var _ types.NewDeployer = New diff --git a/test/kubetest2/env.go b/test/kubetest2/env.go index 271e1366..51bfbcfb 100644 --- a/test/kubetest2/env.go +++ b/test/kubetest2/env.go @@ -34,6 +34,7 @@ func (d *Deployer) loadEnvironment() error { d.resourceManagerEndpoint = strings.TrimSpace(os.Getenv("STACKIT_RESOURCE_MANAGER_ENDPOINT")) d.serviceAccountEndpoint = strings.TrimSpace(os.Getenv("STACKIT_SERVICE_ACCOUNT_ENDPOINT")) d.authorizationEndpoint = strings.TrimSpace(os.Getenv("STACKIT_AUTHORIZATION_ENDPOINT")) + d.serviceEnablementEndpoint = strings.TrimSpace(os.Getenv("STACKIT_SERVICE_ENABLEMENT_ENDPOINT")) d.skeEndpoint = strings.TrimSpace(os.Getenv("STACKIT_SKE_ENDPOINT")) d.kubeconfigPath = filepath.Join(d.options.RunDir(), "kubeconfig") d.serviceAccountKeyPath = filepath.Join(d.options.RunDir(), "service-account-key.json") @@ -48,10 +49,11 @@ func (d *Deployer) loadEnvironment() error { ) klog.Infof( - "STACKIT API endpoint overrides: resource_manager=%q service_account=%q authorization=%q ske=%q", + "STACKIT API endpoint overrides: resource_manager=%q service_account=%q authorization=%q service_enablement=%q ske=%q", d.resourceManagerEndpoint, d.serviceAccountEndpoint, d.authorizationEndpoint, + d.serviceEnablementEndpoint, d.skeEndpoint, ) From e8f1e5974731d8cc6cb692d19f7413b62920fceb Mon Sep 17 00:00:00 2001 From: Robert Kaussow Date: Tue, 18 Aug 2026 17:11:41 +0200 Subject: [PATCH 07/11] migrate tests to ginkgo --- Makefile | 2 +- test/kubetest2/bootstrap_resources_test.go | 65 ++ test/kubetest2/bootstrap_test.go | 926 +++++++++------------ test/kubetest2/env_test.go | 24 + test/kubetest2/flags_test.go | 42 + test/kubetest2/payload_test.go | 87 ++ test/kubetest2/suite_test.go | 13 + 7 files changed, 616 insertions(+), 543 deletions(-) create mode 100644 test/kubetest2/bootstrap_resources_test.go create mode 100644 test/kubetest2/env_test.go create mode 100644 test/kubetest2/flags_test.go create mode 100644 test/kubetest2/payload_test.go create mode 100644 test/kubetest2/suite_test.go diff --git a/Makefile b/Makefile index 18f94467..4b1bc2d9 100644 --- a/Makefile +++ b/Makefile @@ -65,7 +65,7 @@ modules: ## Runs go mod to ensure modules are up to date. .PHONY: test test: ## Run tests. - ./hack/test.sh ./cmd/... ./pkg/... + ./hack/test.sh ./cmd/... ./pkg/... ./test/... .PHONY: test-cover test-cover: ## Run tests with coverage. diff --git a/test/kubetest2/bootstrap_resources_test.go b/test/kubetest2/bootstrap_resources_test.go new file mode 100644 index 00000000..ca1f0597 --- /dev/null +++ b/test/kubetest2/bootstrap_resources_test.go @@ -0,0 +1,65 @@ +package kubetest2 + +import ( + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = Describe("matchesManagedProject", func() { + It("matches a project with the expected name and labels", func() { + d := newTestDeployer() + project := projectFixture(d.projectName(), "project-1", "container-1", d.managedProjectLabels()) + Expect(d.matchesManagedProject(project)).To(BeTrue()) + }) + + It("rejects a project with a different name", func() { + d := newTestDeployer() + project := projectFixture("other-name", "project-1", "container-1", d.managedProjectLabels()) + Expect(d.matchesManagedProject(project)).To(BeFalse()) + }) + + It("rejects a project without labels", func() { + d := newTestDeployer() + project := projectFixture(d.projectName(), "project-1", "container-1", nil) + Expect(d.matchesManagedProject(project)).To(BeFalse()) + }) + + DescribeTable("rejects a project with mismatched labels", + func(labels map[string]string) { + d := newTestDeployer() + project := projectFixture(d.projectName(), "project-1", "container-1", labels) + Expect(d.matchesManagedProject(project)).To(BeFalse()) + }, + Entry("wrong scope", map[string]string{ + projectLabelScopeKey: "PRIVATE", + projectLabelManagedKey: projectLabelManagedValue, + projectLabelRunIDKey: runTokenForRun("run-123"), + }), + Entry("missing managed label", map[string]string{ + projectLabelScopeKey: projectLabelScopeValue, + projectLabelRunIDKey: runTokenForRun("run-123"), + }), + Entry("wrong run id", map[string]string{ + projectLabelScopeKey: projectLabelScopeValue, + projectLabelManagedKey: projectLabelManagedValue, + projectLabelRunIDKey: "deadbeef", + }), + ) +}) + +var _ = Describe("matchesManagedServiceAccountEmail", func() { + It("matches the managed service account prefix", func() { + d := newTestDeployer() + Expect(d.matchesManagedServiceAccountEmail(d.serviceAccountName() + "@sa.stackit.cloud")).To(BeTrue()) + }) + + DescribeTable("rejects non-matching emails", + func(email string) { + d := newTestDeployer() + Expect(d.matchesManagedServiceAccountEmail(email)).To(BeFalse()) + }, + Entry("empty email", ""), + Entry("missing at sign", "kt2-no-separator"), + Entry("different local prefix", "other-account@sa.stackit.cloud"), + ) +}) diff --git a/test/kubetest2/bootstrap_test.go b/test/kubetest2/bootstrap_test.go index 264ff045..44192f0d 100644 --- a/test/kubetest2/bootstrap_test.go +++ b/test/kubetest2/bootstrap_test.go @@ -6,11 +6,10 @@ import ( "net/http" "os" "path/filepath" - "reflect" - "strings" - "testing" "time" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" oapierror "github.com/stackitcloud/stackit-sdk-go/core/oapierror" authorization "github.com/stackitcloud/stackit-sdk-go/services/authorization/v2api" resourcemanager "github.com/stackitcloud/stackit-sdk-go/services/resourcemanager/v0api" @@ -269,568 +268,426 @@ func (c *fakeSKEClient) WaitForClusterDeleted(_ context.Context, _, _, _ string) return nil } -func TestLoadEnvironmentValidation(t *testing.T) { - testCases := []struct { - name string - env map[string]string - wantErrContains string - }{ - { - name: "missing service account", - env: map[string]string{ - "STACKIT_PARENT_CONTAINER_ID": "parent-1", - }, - wantErrContains: "STACKIT_SERVICE_ACCOUNT", - }, - { - name: "missing parent container", - env: map[string]string{ - "STACKIT_SERVICE_ACCOUNT": validServiceAccountKey, - }, - wantErrContains: "STACKIT_PARENT_CONTAINER_ID", - }, - { - name: "invalid service account key", - env: map[string]string{ - "STACKIT_SERVICE_ACCOUNT": "{}", - "STACKIT_PARENT_CONTAINER_ID": "parent-1", - }, - wantErrContains: "invalid STACKIT_SERVICE_ACCOUNT", - }, - { - name: "project id no longer required", - env: map[string]string{ - "STACKIT_SERVICE_ACCOUNT": validServiceAccountKey, - "STACKIT_PARENT_CONTAINER_ID": "parent-1", - }, - }, +var _ = Describe("loadEnvironment", func() { + envVarKeys := []string{ + "STACKIT_SERVICE_ACCOUNT", + "STACKIT_PARENT_CONTAINER_ID", + "STACKIT_PROJECT_ID", + "STACKIT_RESOURCE_MANAGER_ENDPOINT", + "STACKIT_SERVICE_ACCOUNT_ENDPOINT", + "STACKIT_AUTHORIZATION_ENDPOINT", + "STACKIT_SKE_ENDPOINT", } - for _, tc := range testCases { - t.Run(tc.name, func(t *testing.T) { - for _, key := range []string{ - "STACKIT_SERVICE_ACCOUNT", - "STACKIT_PARENT_CONTAINER_ID", - "STACKIT_PROJECT_ID", - "STACKIT_RESOURCE_MANAGER_ENDPOINT", - "STACKIT_SERVICE_ACCOUNT_ENDPOINT", - "STACKIT_AUTHORIZATION_ENDPOINT", - "STACKIT_SKE_ENDPOINT", - } { - t.Setenv(key, "") + DescribeTable("validates required environment variables", + func(env map[string]string, wantErrContains string) { + for _, key := range envVarKeys { + setEnvVar(key, "") } - for key, value := range tc.env { - t.Setenv(key, value) + for key, value := range env { + setEnvVar(key, value) } - runDir := t.TempDir() + runDir := GinkgoT().TempDir() d := &Deployer{options: fakeOptions{runID: "run-123", runDir: runDir}} err := d.loadEnvironment() - if tc.wantErrContains != "" { - if err == nil || !strings.Contains(err.Error(), tc.wantErrContains) { - t.Fatalf("expected error containing %q, got %v", tc.wantErrContains, err) - } + if wantErrContains != "" { + Expect(err).To(MatchError(ContainSubstring(wantErrContains))) return } - if err != nil { - t.Fatalf("loadEnvironment() error = %v", err) - } - if d.parentContainerID != "parent-1" { - t.Fatalf("parentContainerID = %q, want %q", d.parentContainerID, "parent-1") - } - if d.projectMemberEmail != "owner@example.com" { - t.Fatalf("projectMemberEmail = %q, want %q", d.projectMemberEmail, "owner@example.com") - } - if d.projectID != "" { - t.Fatalf("projectID = %q, want empty", d.projectID) - } - if d.kubeconfigPath != filepath.Join(runDir, "kubeconfig") { - t.Fatalf("kubeconfigPath = %q", d.kubeconfigPath) - } - if d.serviceAccountKeyPath != filepath.Join(runDir, "service-account-key.json") { - t.Fatalf("serviceAccountKeyPath = %q", d.serviceAccountKeyPath) - } - }) - } -} - -func TestLoadEnvironmentReadsOptionalEndpoints(t *testing.T) { - for _, key := range []string{ - "STACKIT_SERVICE_ACCOUNT", - "STACKIT_PARENT_CONTAINER_ID", - "STACKIT_RESOURCE_MANAGER_ENDPOINT", - "STACKIT_SERVICE_ACCOUNT_ENDPOINT", - "STACKIT_AUTHORIZATION_ENDPOINT", - "STACKIT_SKE_ENDPOINT", - } { - t.Setenv(key, "") - } - t.Setenv("STACKIT_SERVICE_ACCOUNT", validServiceAccountKey) - t.Setenv("STACKIT_PARENT_CONTAINER_ID", "parent-1") - t.Setenv("STACKIT_RESOURCE_MANAGER_ENDPOINT", "https://resource-manager.example.com") - t.Setenv("STACKIT_SERVICE_ACCOUNT_ENDPOINT", "https://service-account.example.com") - t.Setenv("STACKIT_AUTHORIZATION_ENDPOINT", "https://authorization.example.com") - t.Setenv("STACKIT_SKE_ENDPOINT", "https://ske.example.com") - - runDir := t.TempDir() - d := &Deployer{options: fakeOptions{runID: "run-123", runDir: runDir}} - if err := d.loadEnvironment(); err != nil { - t.Fatalf("loadEnvironment() error = %v", err) - } + Expect(err).NotTo(HaveOccurred()) + Expect(d.parentContainerID).To(Equal("parent-1")) + Expect(d.projectMemberEmail).To(Equal("owner@example.com")) + Expect(d.projectID).To(BeEmpty()) + Expect(d.kubeconfigPath).To(Equal(filepath.Join(runDir, "kubeconfig"))) + Expect(d.serviceAccountKeyPath).To(Equal(filepath.Join(runDir, "service-account-key.json"))) + }, + Entry("missing service account", map[string]string{ + "STACKIT_PARENT_CONTAINER_ID": "parent-1", + }, "STACKIT_SERVICE_ACCOUNT"), + Entry("missing parent container", map[string]string{ + "STACKIT_SERVICE_ACCOUNT": validServiceAccountKey, + }, "STACKIT_PARENT_CONTAINER_ID"), + Entry("invalid service account key", map[string]string{ + "STACKIT_SERVICE_ACCOUNT": "{}", + "STACKIT_PARENT_CONTAINER_ID": "parent-1", + }, "invalid STACKIT_SERVICE_ACCOUNT"), + Entry("project id no longer required", map[string]string{ + "STACKIT_SERVICE_ACCOUNT": validServiceAccountKey, + "STACKIT_PARENT_CONTAINER_ID": "parent-1", + }, ""), + ) - if d.projectMemberEmail != "owner@example.com" { - t.Fatalf("projectMemberEmail = %q, want %q", d.projectMemberEmail, "owner@example.com") - } - if d.resourceManagerEndpoint != "https://resource-manager.example.com" { - t.Fatalf("resourceManagerEndpoint = %q", d.resourceManagerEndpoint) - } - if d.serviceAccountEndpoint != "https://service-account.example.com" { - t.Fatalf("serviceAccountEndpoint = %q", d.serviceAccountEndpoint) - } - if d.authorizationEndpoint != "https://authorization.example.com" { - t.Fatalf("authorizationEndpoint = %q", d.authorizationEndpoint) - } - if d.skeEndpoint != "https://ske.example.com" { - t.Fatalf("skeEndpoint = %q", d.skeEndpoint) - } -} + It("reads optional endpoints", func() { + for _, key := range envVarKeys { + setEnvVar(key, "") + } + setEnvVar("STACKIT_SERVICE_ACCOUNT", validServiceAccountKey) + setEnvVar("STACKIT_PARENT_CONTAINER_ID", "parent-1") + setEnvVar("STACKIT_RESOURCE_MANAGER_ENDPOINT", "https://resource-manager.example.com") + setEnvVar("STACKIT_SERVICE_ACCOUNT_ENDPOINT", "https://service-account.example.com") + setEnvVar("STACKIT_AUTHORIZATION_ENDPOINT", "https://authorization.example.com") + setEnvVar("STACKIT_SKE_ENDPOINT", "https://ske.example.com") + + runDir := GinkgoT().TempDir() + d := &Deployer{options: fakeOptions{runID: "run-123", runDir: runDir}} + Expect(d.loadEnvironment()).To(Succeed()) + + Expect(d.projectMemberEmail).To(Equal("owner@example.com")) + Expect(d.resourceManagerEndpoint).To(Equal("https://resource-manager.example.com")) + Expect(d.serviceAccountEndpoint).To(Equal("https://service-account.example.com")) + Expect(d.authorizationEndpoint).To(Equal("https://authorization.example.com")) + Expect(d.skeEndpoint).To(Equal("https://ske.example.com")) + }) +}) + +var _ = Describe("resolveManagedProject", func() { + It("creates a project when missing", func() { + d := newTestDeployer() + projectClient := &fakeProjectClient{ + createProjectResult: projectFixture(d.projectName(), "project-123", "container-123", d.managedProjectLabels()), + waitActiveResult: projectResponseFixture(d.projectName(), "project-123", "container-123", d.managedProjectLabels()), + } + d.projectClient = projectClient + + project, err := d.resolveManagedProject(context.Background()) + Expect(err).NotTo(HaveOccurred()) + Expect(project.ProjectID).To(Equal("project-123")) + Expect(projectClient.createCalls).To(Equal(1)) + Expect(projectClient.waitActiveCalls).To(Equal(1)) + Expect(projectClient.lastCreateParentContainerID).To(Equal(d.parentContainerID)) + Expect(projectClient.lastCreateName).To(Equal(d.projectName())) + Expect(projectClient.lastCreateOwnerEmail).To(Equal(d.projectMemberEmail)) + Expect(projectClient.lastCreateLabels).To(Equal(d.managedProjectLabels())) + }) -func TestResolveManagedProjectCreatesWhenMissing(t *testing.T) { - d := newTestDeployer(t) - projectClient := &fakeProjectClient{ - createProjectResult: projectFixture(d.projectName(), "project-123", "container-123", d.managedProjectLabels()), - waitActiveResult: projectResponseFixture(d.projectName(), "project-123", "container-123", d.managedProjectLabels()), - } - d.projectClient = projectClient + It("errors on multiple matches", func() { + d := newTestDeployer() + d.projectClient = &fakeProjectClient{ + listProjectsResult: []resourcemanager.Project{ + *projectFixture(d.projectName(), "project-1", "container-1", d.managedProjectLabels()), + *projectFixture(d.projectName(), "project-2", "container-2", d.managedProjectLabels()), + }, + } - project, err := d.resolveManagedProject(context.Background()) - if err != nil { - t.Fatalf("resolveManagedProject() error = %v", err) - } - if project.ProjectID != "project-123" { - t.Fatalf("project id = %q, want %q", project.ProjectID, "project-123") - } - if projectClient.createCalls != 1 { - t.Fatalf("createCalls = %d, want 1", projectClient.createCalls) - } - if projectClient.waitActiveCalls != 1 { - t.Fatalf("waitActiveCalls = %d, want 1", projectClient.waitActiveCalls) - } - if projectClient.lastCreateParentContainerID != d.parentContainerID { - t.Fatalf("parent container = %q, want %q", projectClient.lastCreateParentContainerID, d.parentContainerID) - } - if projectClient.lastCreateName != d.projectName() { - t.Fatalf("created project name = %q, want %q", projectClient.lastCreateName, d.projectName()) - } - if projectClient.lastCreateOwnerEmail != d.projectMemberEmail { - t.Fatalf("created owner email = %q, want %q", projectClient.lastCreateOwnerEmail, d.projectMemberEmail) - } - if !reflect.DeepEqual(projectClient.lastCreateLabels, d.managedProjectLabels()) { - t.Fatalf("created labels = %#v, want %#v", projectClient.lastCreateLabels, d.managedProjectLabels()) - } -} + _, err := d.findManagedProject(context.Background()) + Expect(err).To(MatchError(ContainSubstring("found 2 managed STACKIT projects"))) + }) +}) -func TestResolveManagedProjectErrorsOnMultipleMatches(t *testing.T) { - d := newTestDeployer(t) - d.projectClient = &fakeProjectClient{ - listProjectsResult: []resourcemanager.Project{ - *projectFixture(d.projectName(), "project-1", "container-1", d.managedProjectLabels()), - *projectFixture(d.projectName(), "project-2", "container-2", d.managedProjectLabels()), - }, - } +var _ = Describe("ensureManagedClusterAccess", func() { + It("reuses cached key and skips membership write", func() { + d := newTestDeployer() + cachedKey := `{"credentials":{"privateKey":"cached"}}` + Expect(os.WriteFile(d.serviceAccountKeyPath, []byte(cachedKey), 0o600)).To(Succeed()) - _, err := d.findManagedProject(context.Background()) - if err == nil || !strings.Contains(err.Error(), "found 2 managed STACKIT projects") { - t.Fatalf("expected duplicate managed project error, got %v", err) - } -} + projectClient := &fakeProjectClient{ + listProjectsResult: []resourcemanager.Project{ + *projectFixture(d.projectName(), "project-123", "container-123", d.managedProjectLabels()), + }, + } + serviceAccountEmail := d.serviceAccountName() + "@sa.stackit.cloud" + serviceAccountClient := &fakeServiceAccountClient{ + listResult: []serviceaccount.ServiceAccount{ + *serviceAccountFixture(serviceAccountEmail, "project-123"), + }, + } + authorizationClient := &fakeAuthorizationClient{ + listMembersResult: []authorization.Member{ + *authorization.NewMember(childProjectRole, serviceAccountEmail), + }, + } + fakeSKE := &fakeSKEClient{} + var receivedKey string + + d.projectClient = projectClient + d.serviceAccountClient = serviceAccountClient + d.authorizationClient = authorizationClient + d.serviceEnablementClient = &fakeServiceEnablementClient{getStatusResult: serviceenablement.NewServiceStatus()} + d.skeClientFactory = func(_, serviceAccount, _ string) (skeClient, error) { + receivedKey = serviceAccount + return fakeSKE, nil + } -func TestEnsureManagedClusterAccessReusesCachedKeyAndSkipsMembershipWrite(t *testing.T) { - d := newTestDeployer(t) - cachedKey := `{"credentials":{"privateKey":"cached"}}` - if err := os.WriteFile(d.serviceAccountKeyPath, []byte(cachedKey), 0o600); err != nil { - t.Fatalf("write cached key: %v", err) - } + Expect(d.ensureManagedClusterAccess(context.Background())).To(Succeed()) + Expect(receivedKey).To(Equal(cachedKey)) + Expect(authorizationClient.addCalls).To(Equal(0)) + Expect(serviceAccountClient.createKeyCalls).To(Equal(0)) + Expect(d.projectID).To(Equal("project-123")) + }) - projectClient := &fakeProjectClient{ - listProjectsResult: []resourcemanager.Project{ - *projectFixture(d.projectName(), "project-123", "container-123", d.managedProjectLabels()), - }, - } - serviceAccountEmail := d.serviceAccountName() + "@sa.stackit.cloud" - serviceAccountClient := &fakeServiceAccountClient{ - listResult: []serviceaccount.ServiceAccount{ - *serviceAccountFixture(serviceAccountEmail, "project-123"), - }, - } - authorizationClient := &fakeAuthorizationClient{ - listMembersResult: []authorization.Member{ - *authorization.NewMember(childProjectRole, serviceAccountEmail), - }, - } - fakeSKE := &fakeSKEClient{} - var receivedKey string - - d.projectClient = projectClient - d.serviceAccountClient = serviceAccountClient - d.authorizationClient = authorizationClient - d.serviceEnablementClient = &fakeServiceEnablementClient{getStatusResult: serviceenablement.NewServiceStatus()} - d.skeClientFactory = func(_, serviceAccount, _ string) (skeClient, error) { - receivedKey = serviceAccount - return fakeSKE, nil - } + It("creates key and adds membership", func() { + d := newTestDeployer() - if err := d.ensureManagedClusterAccess(context.Background()); err != nil { - t.Fatalf("ensureManagedClusterAccess() error = %v", err) - } - if receivedKey != cachedKey { - t.Fatalf("received key = %q, want cached key", receivedKey) - } - if authorizationClient.addCalls != 0 { - t.Fatalf("addCalls = %d, want 0", authorizationClient.addCalls) - } - if serviceAccountClient.createKeyCalls != 0 { - t.Fatalf("createKeyCalls = %d, want 0", serviceAccountClient.createKeyCalls) - } - if d.projectID != "project-123" { - t.Fatalf("projectID = %q, want %q", d.projectID, "project-123") - } -} + projectClient := &fakeProjectClient{ + listProjectsResult: []resourcemanager.Project{ + *projectFixture(d.projectName(), "project-123", "container-123", d.managedProjectLabels()), + }, + } + serviceAccountEmail := d.serviceAccountName() + "@sa.stackit.cloud" + serviceAccountClient := &fakeServiceAccountClient{ + listResult: []serviceaccount.ServiceAccount{ + *serviceAccountFixture(serviceAccountEmail, "project-123"), + }, + createKeyResult: createServiceAccountKeyResponseFixture(serviceAccountEmail), + } + authorizationClient := &fakeAuthorizationClient{} + var receivedKey string + + d.projectClient = projectClient + d.serviceAccountClient = serviceAccountClient + d.authorizationClient = authorizationClient + d.serviceEnablementClient = &fakeServiceEnablementClient{getStatusResult: serviceenablement.NewServiceStatus()} + d.skeClientFactory = func(_, serviceAccount, _ string) (skeClient, error) { + receivedKey = serviceAccount + return &fakeSKEClient{}, nil + } -func TestEnsureManagedClusterAccessCreatesKeyAndAddsMembership(t *testing.T) { - d := newTestDeployer(t) + Expect(d.ensureManagedClusterAccess(context.Background())).To(Succeed()) + Expect(authorizationClient.addCalls).To(Equal(1)) + Expect(serviceAccountClient.createKeyCalls).To(Equal(1)) + Expect(authorizationClient.lastAddedType).To(Equal(projectResourceType)) + Expect(authorizationClient.lastAddedID).To(Equal("project-123")) - projectClient := &fakeProjectClient{ - listProjectsResult: []resourcemanager.Project{ - *projectFixture(d.projectName(), "project-123", "container-123", d.managedProjectLabels()), - }, - } - serviceAccountEmail := d.serviceAccountName() + "@sa.stackit.cloud" - serviceAccountClient := &fakeServiceAccountClient{ - listResult: []serviceaccount.ServiceAccount{ - *serviceAccountFixture(serviceAccountEmail, "project-123"), - }, - createKeyResult: createServiceAccountKeyResponseFixture(serviceAccountEmail), - } - authorizationClient := &fakeAuthorizationClient{} - var receivedKey string - - d.projectClient = projectClient - d.serviceAccountClient = serviceAccountClient - d.authorizationClient = authorizationClient - d.serviceEnablementClient = &fakeServiceEnablementClient{getStatusResult: serviceenablement.NewServiceStatus()} - d.skeClientFactory = func(_, serviceAccount, _ string) (skeClient, error) { - receivedKey = serviceAccount - return &fakeSKEClient{}, nil - } + keyBytes, err := os.ReadFile(d.serviceAccountKeyPath) + Expect(err).NotTo(HaveOccurred()) + Expect(receivedKey).To(Equal(string(keyBytes))) + Expect(string(keyBytes)).To(ContainSubstring(`"privateKey":"PRIVATE"`)) - if err := d.ensureManagedClusterAccess(context.Background()); err != nil { - t.Fatalf("ensureManagedClusterAccess() error = %v", err) - } - if authorizationClient.addCalls != 1 { - t.Fatalf("addCalls = %d, want 1", authorizationClient.addCalls) - } - if serviceAccountClient.createKeyCalls != 1 { - t.Fatalf("createKeyCalls = %d, want 1", serviceAccountClient.createKeyCalls) - } - if authorizationClient.lastAddedType != projectResourceType || authorizationClient.lastAddedID != "project-123" { - t.Fatalf("AddMembers called with type=%q id=%q", authorizationClient.lastAddedType, authorizationClient.lastAddedID) - } - keyBytes, err := os.ReadFile(d.serviceAccountKeyPath) - if err != nil { - t.Fatalf("read cached key: %v", err) - } - if receivedKey != string(keyBytes) { - t.Fatalf("factory key mismatch") - } - if !strings.Contains(string(keyBytes), "\"privateKey\":\"PRIVATE\"") { - t.Fatalf("cached key did not contain serialized private key: %s", string(keyBytes)) - } - info, err := os.Stat(d.serviceAccountKeyPath) - if err != nil { - t.Fatalf("stat cached key: %v", err) - } - if info.Mode().Perm() != 0o600 { - t.Fatalf("cached key mode = %o, want 600", info.Mode().Perm()) - } -} + info, err := os.Stat(d.serviceAccountKeyPath) + Expect(err).NotTo(HaveOccurred()) + Expect(info.Mode().Perm()).To(Equal(os.FileMode(0o600))) + }) +}) + +var _ = Describe("ensureSKEServiceEnabled", func() { + It("skips when already enabled", func() { + d := newTestDeployer() + client := &fakeServiceEnablementClient{getStatusResult: serviceenablement.NewServiceStatus()} + d.serviceEnablementClient = client + + Expect(d.ensureSKEServiceEnabled(context.Background(), "project-123")).To(Succeed()) + Expect(client.enableCalls).To(Equal(0)) + Expect(client.waitCalls).To(Equal(0)) + Expect(client.lastGetStatusServiceID).To(Equal(skeServiceID)) + }) -func TestEnsureSKEServiceEnabledSkipsWhenAlreadyEnabled(t *testing.T) { - d := newTestDeployer(t) - client := &fakeServiceEnablementClient{getStatusResult: serviceenablement.NewServiceStatus()} - d.serviceEnablementClient = client + It("enables when not found", func() { + d := newTestDeployer() + client := &fakeServiceEnablementClient{ + getStatusErr: &oapierror.GenericOpenAPIError{StatusCode: http.StatusNotFound}, + } + d.serviceEnablementClient = client - if err := d.ensureSKEServiceEnabled(context.Background(), "project-123"); err != nil { - t.Fatalf("ensureSKEServiceEnabled() error = %v", err) - } - if client.enableCalls != 0 { - t.Fatalf("enableCalls = %d, want 0", client.enableCalls) - } - if client.waitCalls != 0 { - t.Fatalf("waitCalls = %d, want 0", client.waitCalls) - } - if client.lastGetStatusServiceID != skeServiceID { - t.Fatalf("service ID = %q, want %q", client.lastGetStatusServiceID, skeServiceID) - } -} + Expect(d.ensureSKEServiceEnabled(context.Background(), "project-123")).To(Succeed()) + Expect(client.enableCalls).To(Equal(1)) + Expect(client.waitCalls).To(Equal(1)) + Expect(client.lastEnableProjectID).To(Equal("project-123")) + Expect(client.lastEnableServiceID).To(Equal(skeServiceID)) + }) -func TestEnsureSKEServiceEnabledEnablesWhenNotFound(t *testing.T) { - d := newTestDeployer(t) - client := &fakeServiceEnablementClient{ - getStatusErr: &oapierror.GenericOpenAPIError{StatusCode: http.StatusNotFound}, - } - d.serviceEnablementClient = client + It("enables when disabled", func() { + d := newTestDeployer() + client := &fakeServiceEnablementClient{ + getStatusResult: disabledServiceStatusFixture(), + } + d.serviceEnablementClient = client - if err := d.ensureSKEServiceEnabled(context.Background(), "project-123"); err != nil { - t.Fatalf("ensureSKEServiceEnabled() error = %v", err) - } - if client.enableCalls != 1 { - t.Fatalf("enableCalls = %d, want 1", client.enableCalls) - } - if client.waitCalls != 1 { - t.Fatalf("waitCalls = %d, want 1", client.waitCalls) - } - if client.lastEnableProjectID != "project-123" || client.lastEnableServiceID != skeServiceID { - t.Fatalf("EnableService called with project_id=%q service_id=%q", client.lastEnableProjectID, client.lastEnableServiceID) - } -} + Expect(d.ensureSKEServiceEnabled(context.Background(), "project-123")).To(Succeed()) + Expect(client.enableCalls).To(Equal(1)) + Expect(client.waitCalls).To(Equal(1)) + }) -func TestEnsureSKEServiceEnabledEnablesWhenDisabled(t *testing.T) { - d := newTestDeployer(t) - client := &fakeServiceEnablementClient{ - getStatusResult: disabledServiceStatusFixture(), - } - d.serviceEnablementClient = client + It("fails on get status error", func() { + d := newTestDeployer() + client := &fakeServiceEnablementClient{ + getStatusErr: &oapierror.GenericOpenAPIError{StatusCode: http.StatusForbidden}, + } + d.serviceEnablementClient = client - if err := d.ensureSKEServiceEnabled(context.Background(), "project-123"); err != nil { - t.Fatalf("ensureSKEServiceEnabled() error = %v", err) - } - if client.enableCalls != 1 { - t.Fatalf("enableCalls = %d, want 1", client.enableCalls) - } - if client.waitCalls != 1 { - t.Fatalf("waitCalls = %d, want 1", client.waitCalls) - } -} + err := d.ensureSKEServiceEnabled(context.Background(), "project-123") + Expect(err).To(HaveOccurred()) + Expect(client.enableCalls).To(Equal(0)) + }) +}) + +var _ = Describe("retryWithBackoff", func() { + It("retries until success", func() { + calls := 0 + result, err := retryWithBackoff(context.Background(), wait.Backoff{Duration: 0, Factor: 1, Steps: 3}, func() (string, error) { + calls++ + if calls < 2 { + return "", errors.New("transient") + } + return "done", nil + }) + Expect(err).NotTo(HaveOccurred()) + Expect(result).To(Equal("done")) + Expect(calls).To(Equal(2)) + }) -func TestEnsureSKEServiceEnabledFailsOnGetStatusError(t *testing.T) { - d := newTestDeployer(t) - client := &fakeServiceEnablementClient{ - getStatusErr: &oapierror.GenericOpenAPIError{StatusCode: http.StatusForbidden}, - } - d.serviceEnablementClient = client + It("returns last error when exhausted", func() { + calls := 0 + _, err := retryWithBackoff(context.Background(), wait.Backoff{Duration: 0, Factor: 1, Steps: 3}, func() (int, error) { + calls++ + return 0, errors.New("always fails") + }) + Expect(err).To(HaveOccurred()) + Expect(calls).To(Equal(3)) + }) +}) - if err := d.ensureSKEServiceEnabled(context.Background(), "project-123"); err == nil { - t.Fatal("ensureSKEServiceEnabled() error = nil, want non-nil") - } - if client.enableCalls != 0 { - t.Fatalf("enableCalls = %d, want 0", client.enableCalls) - } -} +var _ = Describe("Up", func() { + It("uses discovered project and writes kubeconfig", func() { + d := newTestDeployer() + configureValidUpInputs(d) -func disabledServiceStatusFixture() *serviceenablement.ServiceStatus { - status := serviceenablement.NewServiceStatus() - state := serviceenablement.SERVICESTATUSSTATE_DISABLED - status.State = &state - return status -} + cachedKey := `{"credentials":{"privateKey":"cached"}}` + Expect(os.WriteFile(d.serviceAccountKeyPath, []byte(cachedKey), 0o600)).To(Succeed()) -func TestRetryWithBackoffRetriesUntilSuccess(t *testing.T) { - calls := 0 - result, err := retryWithBackoff(context.Background(), wait.Backoff{Duration: 0, Factor: 1, Steps: 3}, func() (string, error) { - calls++ - if calls < 2 { - return "", errors.New("transient") + d.projectClient = &fakeProjectClient{ + listProjectsResult: []resourcemanager.Project{ + *projectFixture(d.projectName(), "project-123", "container-123", d.managedProjectLabels()), + }, } - return "done", nil - }) - if err != nil { - t.Fatalf("retryWithBackoff() error = %v", err) - } - if result != "done" { - t.Fatalf("result = %q, want %q", result, "done") - } - if calls != 2 { - t.Fatalf("calls = %d, want 2", calls) - } -} + serviceAccountEmail := d.serviceAccountName() + "@sa.stackit.cloud" + d.serviceAccountClient = &fakeServiceAccountClient{ + listResult: []serviceaccount.ServiceAccount{ + *serviceAccountFixture(serviceAccountEmail, "project-123"), + }, + } + d.authorizationClient = &fakeAuthorizationClient{ + listMembersResult: []authorization.Member{ + *authorization.NewMember(childProjectRole, serviceAccountEmail), + }, + } + d.serviceEnablementClient = &fakeServiceEnablementClient{getStatusResult: serviceenablement.NewServiceStatus()} + + fakeSKE := &fakeSKEClient{ + providerOptions: providerOptionsFixture(), + createOrUpdateResult: ske.NewClusterWithDefaults(), + waitReadyResult: ske.NewClusterWithDefaults(), + kubeconfigResult: func() *ske.Kubeconfig { + cfg := ske.NewKubeconfig() + cfg.SetKubeconfig("apiVersion: v1\n") + return cfg + }(), + } + d.skeClientFactory = func(_, serviceAccount, _ string) (skeClient, error) { + Expect(serviceAccount).To(Equal(cachedKey)) + return fakeSKE, nil + } + + Expect(d.Up()).To(Succeed()) + Expect(fakeSKE.lastCreateProjectID).To(Equal("project-123")) + Expect(fakeSKE.lastCreateClusterName).To(Equal(d.clusterName())) -func TestRetryWithBackoffReturnsLastErrorWhenExhausted(t *testing.T) { - calls := 0 - _, err := retryWithBackoff(context.Background(), wait.Backoff{Duration: 0, Factor: 1, Steps: 3}, func() (int, error) { - calls++ - return 0, errors.New("always fails") + kubeconfigBytes, err := os.ReadFile(d.kubeconfigPath) + Expect(err).NotTo(HaveOccurred()) + Expect(string(kubeconfigBytes)).To(Equal("apiVersion: v1\n")) }) - if err == nil { - t.Fatal("retryWithBackoff() error = nil, want non-nil") - } - if calls != 3 { - t.Fatalf("calls = %d, want 3", calls) - } -} +}) -func TestUpUsesDiscoveredProjectAndWritesKubeconfig(t *testing.T) { - d := newTestDeployer(t) - configureValidUpInputs(d) +var _ = Describe("Down", func() { + It("succeeds when project is missing", func() { + d := newTestDeployer() + projectClient := &fakeProjectClient{} + d.projectClient = projectClient - cachedKey := `{"credentials":{"privateKey":"cached"}}` - if err := os.WriteFile(d.serviceAccountKeyPath, []byte(cachedKey), 0o600); err != nil { - t.Fatalf("write cached key: %v", err) - } + Expect(d.Down()).To(Succeed()) + Expect(projectClient.deleteCalls).To(Equal(0)) + }) - d.projectClient = &fakeProjectClient{ - listProjectsResult: []resourcemanager.Project{ - *projectFixture(d.projectName(), "project-123", "container-123", d.managedProjectLabels()), - }, - } - serviceAccountEmail := d.serviceAccountName() + "@sa.stackit.cloud" - d.serviceAccountClient = &fakeServiceAccountClient{ - listResult: []serviceaccount.ServiceAccount{ - *serviceAccountFixture(serviceAccountEmail, "project-123"), - }, - } - d.authorizationClient = &fakeAuthorizationClient{ - listMembersResult: []authorization.Member{ - *authorization.NewMember(childProjectRole, serviceAccountEmail), - }, - } - d.serviceEnablementClient = &fakeServiceEnablementClient{getStatusResult: serviceenablement.NewServiceStatus()} - - fakeSKE := &fakeSKEClient{ - providerOptions: providerOptionsFixture(), - createOrUpdateResult: ske.NewClusterWithDefaults(), - waitReadyResult: ske.NewClusterWithDefaults(), - kubeconfigResult: func() *ske.Kubeconfig { - cfg := ske.NewKubeconfig() - cfg.SetKubeconfig("apiVersion: v1\n") - return cfg - }(), - } - d.skeClientFactory = func(_, serviceAccount, _ string) (skeClient, error) { - if serviceAccount != cachedKey { - t.Fatalf("unexpected service account key passed to SKE client") + It("deletes project without touching cluster delete", func() { + d := newTestDeployer() + fakeSKE := &fakeSKEClient{} + d.skeClient = fakeSKE + d.projectClient = &fakeProjectClient{ + listProjectsResult: []resourcemanager.Project{ + *projectFixture(d.projectName(), "project-123", "container-123", d.managedProjectLabels()), + }, } - return fakeSKE, nil - } - if err := d.Up(); err != nil { - t.Fatalf("Up() error = %v", err) - } - if fakeSKE.lastCreateProjectID != "project-123" { - t.Fatalf("CreateOrUpdate projectID = %q, want %q", fakeSKE.lastCreateProjectID, "project-123") - } - if fakeSKE.lastCreateClusterName != d.clusterName() { - t.Fatalf("cluster name = %q, want %q", fakeSKE.lastCreateClusterName, d.clusterName()) - } - kubeconfigBytes, err := os.ReadFile(d.kubeconfigPath) - if err != nil { - t.Fatalf("read kubeconfig: %v", err) - } - if string(kubeconfigBytes) != "apiVersion: v1\n" { - t.Fatalf("kubeconfig = %q", string(kubeconfigBytes)) - } -} + Expect(d.Down()).To(Succeed()) + projectClient := d.projectClient.(*fakeProjectClient) + Expect(projectClient.deleteCalls).To(Equal(1)) + Expect(projectClient.waitDeletedCalls).To(Equal(1)) + Expect(fakeSKE.deleteClusterCalled).To(BeFalse()) + Expect(fakeSKE.waitDeletedCalled).To(BeFalse()) + }) +}) -func TestDownMissingProjectIsSuccess(t *testing.T) { - d := newTestDeployer(t) - projectClient := &fakeProjectClient{} - d.projectClient = projectClient +var _ = Describe("IsUp", func() { + It("returns false when no project", func() { + d := newTestDeployer() + d.projectClient = &fakeProjectClient{} - if err := d.Down(); err != nil { - t.Fatalf("Down() error = %v", err) - } - if projectClient.deleteCalls != 0 { - t.Fatalf("deleteCalls = %d, want 0", projectClient.deleteCalls) - } -} + isUp, err := d.IsUp() + Expect(err).NotTo(HaveOccurred()) + Expect(isUp).To(BeFalse()) + }) -func TestDownDeletesProjectWithoutTouchingClusterDelete(t *testing.T) { - d := newTestDeployer(t) - fakeSKE := &fakeSKEClient{} - d.skeClient = fakeSKE - d.projectClient = &fakeProjectClient{ - listProjectsResult: []resourcemanager.Project{ - *projectFixture(d.projectName(), "project-123", "container-123", d.managedProjectLabels()), - }, - } + It("returns error when project without cached key", func() { + d := newTestDeployer() + d.projectClient = &fakeProjectClient{ + listProjectsResult: []resourcemanager.Project{ + *projectFixture(d.projectName(), "project-123", "container-123", d.managedProjectLabels()), + }, + } - if err := d.Down(); err != nil { - t.Fatalf("Down() error = %v", err) - } - if d.projectClient.(*fakeProjectClient).deleteCalls != 1 { - t.Fatalf("deleteCalls = %d, want 1", d.projectClient.(*fakeProjectClient).deleteCalls) - } - if d.projectClient.(*fakeProjectClient).waitDeletedCalls != 1 { - t.Fatalf("waitDeletedCalls = %d, want 1", d.projectClient.(*fakeProjectClient).waitDeletedCalls) - } - if fakeSKE.deleteClusterCalled || fakeSKE.waitDeletedCalled { - t.Fatalf("cluster delete path was invoked unexpectedly") - } -} + _, err := d.IsUp() + Expect(err).To(MatchError(ContainSubstring("child service-account key cache"))) + }) -func TestIsUpNoProjectReturnsFalse(t *testing.T) { - d := newTestDeployer(t) - d.projectClient = &fakeProjectClient{} + It("queries cluster when project with cached key", func() { + d := newTestDeployer() + d.projectClient = &fakeProjectClient{ + listProjectsResult: []resourcemanager.Project{ + *projectFixture(d.projectName(), "project-123", "container-123", d.managedProjectLabels()), + }, + } + cachedKey := `{"credentials":{"privateKey":"cached"}}` + Expect(os.WriteFile(d.serviceAccountKeyPath, []byte(cachedKey), 0o600)).To(Succeed()) - isUp, err := d.IsUp() - if err != nil { - t.Fatalf("IsUp() error = %v", err) - } - if isUp { - t.Fatalf("IsUp() = true, want false") - } -} + fakeSKE := &fakeSKEClient{ + getClusterResult: healthyClusterFixture(), + } + d.skeClientFactory = func(_, serviceAccount, _ string) (skeClient, error) { + Expect(serviceAccount).To(Equal(cachedKey)) + return fakeSKE, nil + } -func TestIsUpProjectWithoutCachedKeyReturnsError(t *testing.T) { - d := newTestDeployer(t) - d.projectClient = &fakeProjectClient{ - listProjectsResult: []resourcemanager.Project{ - *projectFixture(d.projectName(), "project-123", "container-123", d.managedProjectLabels()), - }, - } + isUp, err := d.IsUp() + Expect(err).NotTo(HaveOccurred()) + Expect(isUp).To(BeTrue()) + Expect(fakeSKE.lastGetProjectID).To(Equal("project-123")) + }) - isUp, err := d.IsUp() - if err == nil || !strings.Contains(err.Error(), "child service-account key cache") { - t.Fatalf("expected missing key cache error, got up=%v err=%v", isUp, err) - } -} + It("returns false when cluster missing", func() { + d := newTestDeployer() + d.projectClient = &fakeProjectClient{ + listProjectsResult: []resourcemanager.Project{ + *projectFixture(d.projectName(), "project-123", "container-123", d.managedProjectLabels()), + }, + } + cachedKey := `{"credentials":{"privateKey":"cached"}}` + Expect(os.WriteFile(d.serviceAccountKeyPath, []byte(cachedKey), 0o600)).To(Succeed()) -func TestIsUpProjectWithCachedKeyQueriesCluster(t *testing.T) { - d := newTestDeployer(t) - d.projectClient = &fakeProjectClient{ - listProjectsResult: []resourcemanager.Project{ - *projectFixture(d.projectName(), "project-123", "container-123", d.managedProjectLabels()), - }, - } - cachedKey := `{"credentials":{"privateKey":"cached"}}` - if err := os.WriteFile(d.serviceAccountKeyPath, []byte(cachedKey), 0o600); err != nil { - t.Fatalf("write cached key: %v", err) - } - fakeSKE := &fakeSKEClient{ - getClusterResult: healthyClusterFixture(), - } - d.skeClientFactory = func(_, serviceAccount, _ string) (skeClient, error) { - if serviceAccount != cachedKey { - t.Fatalf("unexpected cached key content") + fakeSKE := &fakeSKEClient{ + getClusterErr: &oapierror.GenericOpenAPIError{StatusCode: http.StatusNotFound}, + } + d.skeClientFactory = func(_, _, _ string) (skeClient, error) { + return fakeSKE, nil } - return fakeSKE, nil - } - isUp, err := d.IsUp() - if err != nil { - t.Fatalf("IsUp() error = %v", err) - } - if !isUp { - t.Fatalf("IsUp() = false, want true") - } - if fakeSKE.lastGetProjectID != "project-123" { - t.Fatalf("GetCluster projectID = %q, want %q", fakeSKE.lastGetProjectID, "project-123") - } -} + isUp, err := d.IsUp() + Expect(err).NotTo(HaveOccurred()) + Expect(isUp).To(BeFalse()) + }) +}) -func newTestDeployer(t *testing.T) *Deployer { - t.Helper() - runDir := t.TempDir() +func newTestDeployer() *Deployer { + runDir := GinkgoT().TempDir() return &Deployer{ options: fakeOptions{runID: "run-123", runDir: runDir}, region: defaultRegion, @@ -851,6 +708,11 @@ func configureValidUpInputs(d *Deployer) { d.volumeType = "storage" } +func setEnvVar(key, value string) { + Expect(os.Setenv(key, value)).To(Succeed()) + DeferCleanup(os.Unsetenv, key) +} + func providerOptionsFixture() *ske.ProviderOptions { kubernetesVersion := ske.NewKubernetesVersion() kubernetesVersion.SetVersion("1.31.0") @@ -888,6 +750,13 @@ func healthyClusterFixture() *ske.Cluster { return cluster } +func disabledServiceStatusFixture() *serviceenablement.ServiceStatus { + status := serviceenablement.NewServiceStatus() + state := serviceenablement.SERVICESTATUSSTATE_DISABLED + status.State = &state + return status +} + func projectFixture(name, projectID, containerID string, labels map[string]string) *resourcemanager.Project { project := resourcemanager.NewProjectWithDefaults() project.SetName(name) @@ -937,31 +806,4 @@ func createServiceAccountKeyResponseFixture(email string) *serviceaccount.Create ) } -func TestIsUpReturnsFalseWhenClusterMissing(t *testing.T) { - d := newTestDeployer(t) - d.projectClient = &fakeProjectClient{ - listProjectsResult: []resourcemanager.Project{ - *projectFixture(d.projectName(), "project-123", "container-123", d.managedProjectLabels()), - }, - } - cachedKey := `{"credentials":{"privateKey":"cached"}}` - if err := os.WriteFile(d.serviceAccountKeyPath, []byte(cachedKey), 0o600); err != nil { - t.Fatalf("write cached key: %v", err) - } - fakeSKE := &fakeSKEClient{ - getClusterErr: &oapierror.GenericOpenAPIError{StatusCode: http.StatusNotFound}, - } - d.skeClientFactory = func(_, _, _ string) (skeClient, error) { - return fakeSKE, nil - } - - isUp, err := d.IsUp() - if err != nil { - t.Fatalf("IsUp() error = %v", err) - } - if isUp { - t.Fatalf("IsUp() = true, want false") - } -} - var _ types.Options = fakeOptions{} diff --git a/test/kubetest2/env_test.go b/test/kubetest2/env_test.go new file mode 100644 index 00000000..267acdae --- /dev/null +++ b/test/kubetest2/env_test.go @@ -0,0 +1,24 @@ +package kubetest2 + +import ( + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = Describe("extractServiceAccountEmail", func() { + It("extracts the email from the credentials.iss field", func() { + email, err := extractServiceAccountEmail(validServiceAccountKey) + Expect(err).NotTo(HaveOccurred()) + Expect(email).To(Equal("owner@example.com")) + }) + + DescribeTable("rejects invalid service account keys", + func(key string, wantErrContains string) { + _, err := extractServiceAccountEmail(key) + Expect(err).To(MatchError(ContainSubstring(wantErrContains))) + }, + Entry("malformed JSON", "not-json", "parse service account key"), + Entry("missing credentials.iss", `{"credentials":{"aud":"x"}}`, "no email in credentials.iss"), + Entry("blank credentials.iss", `{"credentials":{"iss":" "}}`, "no email in credentials.iss"), + ) +}) diff --git a/test/kubetest2/flags_test.go b/test/kubetest2/flags_test.go new file mode 100644 index 00000000..b0e9f083 --- /dev/null +++ b/test/kubetest2/flags_test.go @@ -0,0 +1,42 @@ +package kubetest2 + +import ( + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +func newValidTestDeployer() *Deployer { + d := newTestDeployer() + configureValidUpInputs(d) + d.nodeCount = defaultNodeCount + d.volumeSizeGiB = defaultVolumeSizeGiB + d.kubeconfigExpiresIn = defaultKubeconfigExpiration + return d +} + +var _ = Describe("validate", func() { + It("accepts a valid configuration", func() { + Expect(newValidTestDeployer().validate()).To(Succeed()) + }) + + DescribeTable("rejects invalid configuration", + func(mutate func(*Deployer), wantErrContains string) { + d := newValidTestDeployer() + mutate(d) + Expect(d.validate()).To(MatchError(ContainSubstring(wantErrContains))) + }, + Entry("missing region", func(d *Deployer) { d.region = "" }, "--region is required"), + Entry("missing kubernetes version", func(d *Deployer) { d.kubernetesVersion = "" }, "--kubernetes-version is required"), + Entry("missing availability zone", func(d *Deployer) { d.availabilityZone = "" }, "--availability-zone is required"), + Entry("missing machine type", func(d *Deployer) { d.machineType = "" }, "--machine-type is required"), + Entry("missing node image name", func(d *Deployer) { d.nodeImageName = "" }, "--node-image-name is required"), + Entry("missing node image version", func(d *Deployer) { d.nodeImageVersion = "" }, "--node-image-version is required"), + Entry("empty nodepool name", func(d *Deployer) { d.nodepoolName = "" }, "--nodepool-name must not be empty"), + Entry("nodepool name too long", func(d *Deployer) { d.nodepoolName = "this-name-is-way-too-long" }, "--nodepool-name must be 15 characters or fewer"), + Entry("zero node count", func(d *Deployer) { d.nodeCount = 0 }, "--node-count must be greater than 0"), + Entry("zero volume size", func(d *Deployer) { d.volumeSizeGiB = 0 }, "--volume-size must be greater than 0"), + Entry("kubeconfig expiration too small", func(d *Deployer) { d.kubeconfigExpiresIn = minKubeconfigExpiration - 1 }, "--kubeconfig-expiration-seconds must be between"), + Entry("kubeconfig expiration too large", func(d *Deployer) { d.kubeconfigExpiresIn = maxKubeconfigExpiration + 1 }, "--kubeconfig-expiration-seconds must be between"), + Entry("empty run id", func(d *Deployer) { d.options = fakeOptions{runID: "", runDir: d.options.RunDir()} }, "run-id must not be empty"), + ) +}) diff --git a/test/kubetest2/payload_test.go b/test/kubetest2/payload_test.go new file mode 100644 index 00000000..32b84a68 --- /dev/null +++ b/test/kubetest2/payload_test.go @@ -0,0 +1,87 @@ +package kubetest2 + +import ( + "context" + "errors" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = Describe("validateProviderOptions", func() { + It("accepts supported options", func() { + d := newValidTestDeployer() + d.skeClient = &fakeSKEClient{providerOptions: providerOptionsFixture()} + Expect(d.validateProviderOptions(context.Background())).To(Succeed()) + }) + + DescribeTable("rejects unsupported options", + func(mutate func(*Deployer), wantErrContains string) { + d := newValidTestDeployer() + d.skeClient = &fakeSKEClient{providerOptions: providerOptionsFixture()} + mutate(d) + Expect(d.validateProviderOptions(context.Background())).To(MatchError(ContainSubstring(wantErrContains))) + }, + Entry("unsupported kubernetes version", func(d *Deployer) { d.kubernetesVersion = "9.9.9" }, "unsupported --kubernetes-version"), + Entry("unsupported availability zone", func(d *Deployer) { d.availabilityZone = "eu99-9" }, "unsupported --availability-zone"), + Entry("unsupported machine type", func(d *Deployer) { d.machineType = "x9.9" }, "unsupported --machine-type"), + Entry("unsupported node image name", func(d *Deployer) { d.nodeImageName = "fedora" }, "unsupported node image"), + Entry("unsupported node image version", func(d *Deployer) { d.nodeImageVersion = "v99" }, "unsupported node image"), + Entry("unsupported volume type", func(d *Deployer) { d.volumeType = "nvme" }, "unsupported --volume-type"), + ) + + It("fails when listing provider options errors", func() { + d := newValidTestDeployer() + d.skeClient = &fakeSKEClient{providerOptionsErr: errors.New("boom")} + Expect(d.validateProviderOptions(context.Background())).To(MatchError(ContainSubstring("list SKE provider options"))) + }) + + It("allows an empty volume type", func() { + d := newValidTestDeployer() + d.volumeType = "" + d.skeClient = &fakeSKEClient{providerOptions: providerOptionsFixture()} + Expect(d.validateProviderOptions(context.Background())).To(Succeed()) + }) +}) + +var _ = Describe("clusterPayload", func() { + It("builds the expected payload", func() { + d := newValidTestDeployer() + payload := d.clusterPayload() + + kubernetes := payload.GetKubernetes() + Expect(kubernetes.GetVersion()).To(Equal("1.31.0")) + + nodepools := payload.GetNodepools() + Expect(nodepools).To(HaveLen(1)) + + nodepool := nodepools[0] + Expect(nodepool.GetName()).To(Equal(defaultNodepoolName)) + Expect(nodepool.GetAvailabilityZones()).To(ConsistOf(defaultAvailabilityZone)) + + machine := nodepool.GetMachine() + Expect(machine.GetType()).To(Equal("g1.2")) + image := machine.GetImage() + Expect(image.GetName()).To(Equal("ubuntu")) + Expect(image.GetVersion()).To(Equal("v1")) + + Expect(nodepool.GetMinimum()).To(Equal(d.nodeCount)) + Expect(nodepool.GetMaximum()).To(Equal(d.nodeCount)) + + volume := nodepool.GetVolume() + Expect(volume.GetSize()).To(Equal(d.volumeSizeGiB)) + Expect(volume.GetType()).To(Equal("storage")) + + Expect(nodepool.GetAllowSystemComponents()).To(BeTrue()) + }) + + It("omits the volume type when unset", func() { + d := newValidTestDeployer() + d.volumeType = "" + payload := d.clusterPayload() + + Expect(payload.GetNodepools()).To(HaveLen(1)) + volume := payload.GetNodepools()[0].GetVolume() + Expect(volume.GetType()).To(BeEmpty()) + }) +}) diff --git a/test/kubetest2/suite_test.go b/test/kubetest2/suite_test.go new file mode 100644 index 00000000..52d96754 --- /dev/null +++ b/test/kubetest2/suite_test.go @@ -0,0 +1,13 @@ +package kubetest2 + +import ( + "testing" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +func TestKubetest2(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "Kubetest2 Suite") +} From dcf6dcc753cf5cf28f79dbc97286d149265ff9ea Mon Sep 17 00:00:00 2001 From: Robert Kaussow Date: Wed, 19 Aug 2026 16:58:21 +0200 Subject: [PATCH 08/11] refactor file structure --- go.mod | 2 +- test/kubetest2/bootstrap_csi.go | 15 + test/kubetest2/bootstrap_project.go | 146 ++++ test/kubetest2/bootstrap_project_test.go | 157 ++++ test/kubetest2/bootstrap_resources_test.go | 65 -- ...sources.go => bootstrap_serviceaccount.go} | 204 +---- .../bootstrap_serviceaccount_test.go | 128 +++ .../{payload.go => bootstrap_ske.go} | 39 +- ...{payload_test.go => bootstrap_ske_test.go} | 0 test/kubetest2/bootstrap_test.go | 809 ------------------ .../{bootstrap_clients.go => clients.go} | 89 +- test/kubetest2/deployer.go | 19 +- test/kubetest2/env_test.go | 75 ++ test/kubetest2/lifecycle.go | 50 +- test/kubetest2/lifecycle_test.go | 160 ++++ test/kubetest2/naming.go | 23 +- test/kubetest2/ske_client.go | 16 +- test/kubetest2/suite_test.go | 368 ++++++++ 18 files changed, 1199 insertions(+), 1166 deletions(-) create mode 100644 test/kubetest2/bootstrap_csi.go create mode 100644 test/kubetest2/bootstrap_project.go create mode 100644 test/kubetest2/bootstrap_project_test.go delete mode 100644 test/kubetest2/bootstrap_resources_test.go rename test/kubetest2/{bootstrap_resources.go => bootstrap_serviceaccount.go} (57%) create mode 100644 test/kubetest2/bootstrap_serviceaccount_test.go rename test/kubetest2/{payload.go => bootstrap_ske.go} (68%) rename test/kubetest2/{payload_test.go => bootstrap_ske_test.go} (100%) delete mode 100644 test/kubetest2/bootstrap_test.go rename test/kubetest2/{bootstrap_clients.go => clients.go} (77%) create mode 100644 test/kubetest2/lifecycle_test.go diff --git a/go.mod b/go.mod index 8974e3e3..52e5256b 100644 --- a/go.mod +++ b/go.mod @@ -20,6 +20,7 @@ require ( github.com/stackitcloud/stackit-sdk-go/services/loadbalancer v1.15.1 github.com/stackitcloud/stackit-sdk-go/services/resourcemanager v0.24.2 github.com/stackitcloud/stackit-sdk-go/services/serviceaccount v0.20.1 + github.com/stackitcloud/stackit-sdk-go/services/serviceenablement v1.7.0 github.com/stackitcloud/stackit-sdk-go/services/ske v1.6.0 go.uber.org/mock v0.6.0 golang.org/x/sync v0.22.0 @@ -96,7 +97,6 @@ require ( github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect github.com/prometheus/common v0.70.1 // indirect github.com/prometheus/procfs v0.21.1 // indirect - github.com/stackitcloud/stackit-sdk-go/services/serviceenablement v1.7.0 // indirect github.com/stoewer/go-strcase v1.3.1 // indirect github.com/stretchr/objx v0.5.3 // indirect github.com/x448/float16 v0.8.4 // indirect diff --git a/test/kubetest2/bootstrap_csi.go b/test/kubetest2/bootstrap_csi.go new file mode 100644 index 00000000..728257ac --- /dev/null +++ b/test/kubetest2/bootstrap_csi.go @@ -0,0 +1,15 @@ +package kubetest2 + +import ( + "context" + + "k8s.io/klog/v2" +) + +// ensureCSI deploys the STACKIT CSI driver into the freshly provisioned +// cluster as the last bootstrap step. Not yet implemented; will install the +// CSI Helm chart here. +func (d *Deployer) ensureCSI(_ context.Context) error { + klog.Infof("CSI deployment not yet implemented, skipping") + return nil +} diff --git a/test/kubetest2/bootstrap_project.go b/test/kubetest2/bootstrap_project.go new file mode 100644 index 00000000..cf3a568e --- /dev/null +++ b/test/kubetest2/bootstrap_project.go @@ -0,0 +1,146 @@ +package kubetest2 + +import ( + "context" + "fmt" + + "github.com/stackitcloud/cloud-provider-stackit/pkg/stackit/stackiterrors" + resourcemanager "github.com/stackitcloud/stackit-sdk-go/services/resourcemanager/v0api" + serviceenablement "github.com/stackitcloud/stackit-sdk-go/services/serviceenablement/v2api" + "k8s.io/klog/v2" +) + +type managedProject struct { + ContainerID string + ProjectID string + Name string + Labels map[string]string +} + +// ensureProject idempotently resolves (or creates) the managed STACKIT project +// and enables the SKE service for it. Sets d.projectID on success. +func (d *Deployer) ensureProject(ctx context.Context) error { + project, err := d.resolveManagedProject(ctx) + if err != nil { + return err + } + d.projectID = project.ProjectID + + return d.ensureSKEServiceEnabled(ctx, project.ProjectID) +} + +func (d *Deployer) findManagedProject(ctx context.Context) (*managedProject, error) { + projects, err := d.projectClient.ListProjects(ctx, d.parentContainerID) + if err != nil { + return nil, fmt.Errorf("list STACKIT projects under parent container %q: %w", d.parentContainerID, err) + } + + matches := make([]managedProject, 0, 1) + for i := range projects { + project := &projects[i] + if !d.matchesManagedProject(project) { + continue + } + matches = append(matches, managedProject{ + ContainerID: project.GetContainerId(), + ProjectID: project.GetProjectId(), + Name: project.GetName(), + Labels: project.GetLabels(), + }) + } + + switch len(matches) { + case 0: + return nil, nil + case 1: + return &matches[0], nil + default: + return nil, fmt.Errorf( + "found %d managed STACKIT projects for run token %q under parent container %q", + len(matches), + d.runToken(), + d.parentContainerID, + ) + } +} + +func (d *Deployer) resolveManagedProject(ctx context.Context) (*managedProject, error) { + project, err := d.findManagedProject(ctx) + if err != nil { + return nil, err + } + if project != nil { + klog.Infof("Reusing managed project=%q project_id=%q", project.Name, project.ProjectID) + return project, nil + } + + klog.Infof("Creating managed project=%q under parent_container_id=%q", d.projectName(), d.parentContainerID) + createdProject, err := d.projectClient.CreateProject( + ctx, + d.parentContainerID, + d.projectName(), + d.projectMemberEmail, + d.managedProjectLabels(), + ) + if err != nil { + return nil, fmt.Errorf("create STACKIT project %q: %w", d.projectName(), err) + } + + activeProject, err := d.projectClient.WaitForProjectActive(ctx, createdProject.GetContainerId()) + if err != nil { + return nil, fmt.Errorf("wait for STACKIT project %q to become active: %w", createdProject.GetProjectId(), err) + } + + return &managedProject{ + ContainerID: activeProject.GetContainerId(), + ProjectID: activeProject.GetProjectId(), + Name: activeProject.GetName(), + Labels: activeProject.GetLabels(), + }, nil +} + +func (d *Deployer) managedProjectLabels() map[string]string { + return map[string]string{ + projectLabelScopeKey: projectLabelScopeValue, + projectLabelManagedKey: projectLabelManagedValue, + projectLabelRunIDKey: d.runToken(), + } +} + +func (d *Deployer) matchesManagedProject(project *resourcemanager.Project) bool { + if project.GetName() != d.projectName() { + return false + } + labels := project.GetLabels() + if labels == nil { + return false + } + return labels[projectLabelScopeKey] == projectLabelScopeValue && + labels[projectLabelManagedKey] == projectLabelManagedValue && + labels[projectLabelRunIDKey] == d.runToken() +} + +// ensureSKEServiceEnabled idempotently enables the SKE (Kubernetes Engine) +// service for the managed project and waits until it is enabled. +func (d *Deployer) ensureSKEServiceEnabled(ctx context.Context, projectID string) error { + status, err := d.serviceEnablementClient.GetServiceStatus(ctx, d.region, projectID, skeServiceID) + if err != nil { + if !stackiterrors.IsNotFound(err) { + return fmt.Errorf("get SKE service status for project %q: %w", projectID, err) + } + klog.Infof("SKE service not yet enabled for project_id=%q", projectID) + } else if status.GetState() == serviceenablement.SERVICESTATUSSTATE_ENABLED { + klog.Infof("SKE service already enabled for project_id=%q", projectID) + return nil + } else { + klog.Infof("SKE service in state %q for project_id=%q, enabling", status.GetState(), projectID) + } + + if err := d.serviceEnablementClient.EnableService(ctx, d.region, projectID, skeServiceID); err != nil { + return fmt.Errorf("enable SKE service for project %q: %w", projectID, err) + } + if err := d.serviceEnablementClient.WaitForServiceEnabled(ctx, d.region, projectID, skeServiceID); err != nil { + return fmt.Errorf("wait for SKE service enablement for project %q: %w", projectID, err) + } + return nil +} diff --git a/test/kubetest2/bootstrap_project_test.go b/test/kubetest2/bootstrap_project_test.go new file mode 100644 index 00000000..901a6aa9 --- /dev/null +++ b/test/kubetest2/bootstrap_project_test.go @@ -0,0 +1,157 @@ +package kubetest2 + +import ( + "context" + "net/http" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + oapierror "github.com/stackitcloud/stackit-sdk-go/core/oapierror" + resourcemanager "github.com/stackitcloud/stackit-sdk-go/services/resourcemanager/v0api" + serviceenablement "github.com/stackitcloud/stackit-sdk-go/services/serviceenablement/v2api" +) + +var _ = Describe("matchesManagedProject", func() { + It("matches a project with the expected name and labels", func() { + d := newTestDeployer() + project := projectFixture(d.projectName(), "project-1", "container-1", d.managedProjectLabels()) + Expect(d.matchesManagedProject(project)).To(BeTrue()) + }) + + It("rejects a project with a different name", func() { + d := newTestDeployer() + project := projectFixture("other-name", "project-1", "container-1", d.managedProjectLabels()) + Expect(d.matchesManagedProject(project)).To(BeFalse()) + }) + + It("rejects a project without labels", func() { + d := newTestDeployer() + project := projectFixture(d.projectName(), "project-1", "container-1", nil) + Expect(d.matchesManagedProject(project)).To(BeFalse()) + }) + + DescribeTable("rejects a project with mismatched labels", + func(labels map[string]string) { + d := newTestDeployer() + project := projectFixture(d.projectName(), "project-1", "container-1", labels) + Expect(d.matchesManagedProject(project)).To(BeFalse()) + }, + Entry("wrong scope", map[string]string{ + projectLabelScopeKey: "PRIVATE", + projectLabelManagedKey: projectLabelManagedValue, + projectLabelRunIDKey: runTokenForRun("run-123"), + }), + Entry("missing managed label", map[string]string{ + projectLabelScopeKey: projectLabelScopeValue, + projectLabelRunIDKey: runTokenForRun("run-123"), + }), + Entry("wrong run id", map[string]string{ + projectLabelScopeKey: projectLabelScopeValue, + projectLabelManagedKey: projectLabelManagedValue, + projectLabelRunIDKey: "deadbeef", + }), + ) +}) + +var _ = Describe("resolveManagedProject", func() { + It("creates a project when missing", func() { + d := newTestDeployer() + projectClient := &fakeProjectClient{ + createProjectResult: projectFixture(d.projectName(), "project-123", "container-123", d.managedProjectLabels()), + waitActiveResult: projectResponseFixture(d.projectName(), "project-123", "container-123", d.managedProjectLabels()), + } + d.projectClient = projectClient + + project, err := d.resolveManagedProject(context.Background()) + Expect(err).NotTo(HaveOccurred()) + Expect(project.ProjectID).To(Equal("project-123")) + Expect(projectClient.createCalls).To(Equal(1)) + Expect(projectClient.waitActiveCalls).To(Equal(1)) + Expect(projectClient.lastCreateParentContainerID).To(Equal(d.parentContainerID)) + Expect(projectClient.lastCreateName).To(Equal(d.projectName())) + Expect(projectClient.lastCreateOwnerEmail).To(Equal(d.projectMemberEmail)) + Expect(projectClient.lastCreateLabels).To(Equal(d.managedProjectLabels())) + }) + + It("errors on multiple matches", func() { + d := newTestDeployer() + d.projectClient = &fakeProjectClient{ + listProjectsResult: []resourcemanager.Project{ + *projectFixture(d.projectName(), "project-1", "container-1", d.managedProjectLabels()), + *projectFixture(d.projectName(), "project-2", "container-2", d.managedProjectLabels()), + }, + } + + _, err := d.findManagedProject(context.Background()) + Expect(err).To(MatchError(ContainSubstring("found 2 managed STACKIT projects"))) + }) +}) + +var _ = Describe("ensureProject", func() { + It("resolves the managed project and enables the SKE service", func() { + d := newTestDeployer() + d.projectClient = &fakeProjectClient{ + listProjectsResult: []resourcemanager.Project{ + *projectFixture(d.projectName(), "project-123", "container-123", d.managedProjectLabels()), + }, + } + serviceEnablementClient := &fakeServiceEnablementClient{getStatusResult: serviceenablement.NewServiceStatus()} + d.serviceEnablementClient = serviceEnablementClient + + Expect(d.ensureProject(context.Background())).To(Succeed()) + Expect(d.projectID).To(Equal("project-123")) + Expect(serviceEnablementClient.lastGetStatusProjectID).To(Equal("project-123")) + Expect(serviceEnablementClient.enableCalls).To(Equal(0)) + }) +}) + +var _ = Describe("ensureSKEServiceEnabled", func() { + It("skips when already enabled", func() { + d := newTestDeployer() + client := &fakeServiceEnablementClient{getStatusResult: serviceenablement.NewServiceStatus()} + d.serviceEnablementClient = client + + Expect(d.ensureSKEServiceEnabled(context.Background(), "project-123")).To(Succeed()) + Expect(client.enableCalls).To(Equal(0)) + Expect(client.waitCalls).To(Equal(0)) + Expect(client.lastGetStatusServiceID).To(Equal(skeServiceID)) + }) + + It("enables when not found", func() { + d := newTestDeployer() + client := &fakeServiceEnablementClient{ + getStatusErr: &oapierror.GenericOpenAPIError{StatusCode: http.StatusNotFound}, + } + d.serviceEnablementClient = client + + Expect(d.ensureSKEServiceEnabled(context.Background(), "project-123")).To(Succeed()) + Expect(client.enableCalls).To(Equal(1)) + Expect(client.waitCalls).To(Equal(1)) + Expect(client.lastEnableProjectID).To(Equal("project-123")) + Expect(client.lastEnableServiceID).To(Equal(skeServiceID)) + }) + + It("enables when disabled", func() { + d := newTestDeployer() + client := &fakeServiceEnablementClient{ + getStatusResult: disabledServiceStatusFixture(), + } + d.serviceEnablementClient = client + + Expect(d.ensureSKEServiceEnabled(context.Background(), "project-123")).To(Succeed()) + Expect(client.enableCalls).To(Equal(1)) + Expect(client.waitCalls).To(Equal(1)) + }) + + It("fails on get status error", func() { + d := newTestDeployer() + client := &fakeServiceEnablementClient{ + getStatusErr: &oapierror.GenericOpenAPIError{StatusCode: http.StatusForbidden}, + } + d.serviceEnablementClient = client + + err := d.ensureSKEServiceEnabled(context.Background(), "project-123") + Expect(err).To(HaveOccurred()) + Expect(client.enableCalls).To(Equal(0)) + }) +}) diff --git a/test/kubetest2/bootstrap_resources_test.go b/test/kubetest2/bootstrap_resources_test.go deleted file mode 100644 index ca1f0597..00000000 --- a/test/kubetest2/bootstrap_resources_test.go +++ /dev/null @@ -1,65 +0,0 @@ -package kubetest2 - -import ( - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" -) - -var _ = Describe("matchesManagedProject", func() { - It("matches a project with the expected name and labels", func() { - d := newTestDeployer() - project := projectFixture(d.projectName(), "project-1", "container-1", d.managedProjectLabels()) - Expect(d.matchesManagedProject(project)).To(BeTrue()) - }) - - It("rejects a project with a different name", func() { - d := newTestDeployer() - project := projectFixture("other-name", "project-1", "container-1", d.managedProjectLabels()) - Expect(d.matchesManagedProject(project)).To(BeFalse()) - }) - - It("rejects a project without labels", func() { - d := newTestDeployer() - project := projectFixture(d.projectName(), "project-1", "container-1", nil) - Expect(d.matchesManagedProject(project)).To(BeFalse()) - }) - - DescribeTable("rejects a project with mismatched labels", - func(labels map[string]string) { - d := newTestDeployer() - project := projectFixture(d.projectName(), "project-1", "container-1", labels) - Expect(d.matchesManagedProject(project)).To(BeFalse()) - }, - Entry("wrong scope", map[string]string{ - projectLabelScopeKey: "PRIVATE", - projectLabelManagedKey: projectLabelManagedValue, - projectLabelRunIDKey: runTokenForRun("run-123"), - }), - Entry("missing managed label", map[string]string{ - projectLabelScopeKey: projectLabelScopeValue, - projectLabelRunIDKey: runTokenForRun("run-123"), - }), - Entry("wrong run id", map[string]string{ - projectLabelScopeKey: projectLabelScopeValue, - projectLabelManagedKey: projectLabelManagedValue, - projectLabelRunIDKey: "deadbeef", - }), - ) -}) - -var _ = Describe("matchesManagedServiceAccountEmail", func() { - It("matches the managed service account prefix", func() { - d := newTestDeployer() - Expect(d.matchesManagedServiceAccountEmail(d.serviceAccountName() + "@sa.stackit.cloud")).To(BeTrue()) - }) - - DescribeTable("rejects non-matching emails", - func(email string) { - d := newTestDeployer() - Expect(d.matchesManagedServiceAccountEmail(email)).To(BeFalse()) - }, - Entry("empty email", ""), - Entry("missing at sign", "kt2-no-separator"), - Entry("different local prefix", "other-account@sa.stackit.cloud"), - ) -}) diff --git a/test/kubetest2/bootstrap_resources.go b/test/kubetest2/bootstrap_serviceaccount.go similarity index 57% rename from test/kubetest2/bootstrap_resources.go rename to test/kubetest2/bootstrap_serviceaccount.go index e8f0b64a..36683e23 100644 --- a/test/kubetest2/bootstrap_resources.go +++ b/test/kubetest2/bootstrap_serviceaccount.go @@ -3,18 +3,13 @@ package kubetest2 import ( "context" "encoding/json" - "errors" "fmt" - "net/http" "os" "strings" "time" - oapierror "github.com/stackitcloud/stackit-sdk-go/core/oapierror" authorization "github.com/stackitcloud/stackit-sdk-go/services/authorization/v2api" - resourcemanager "github.com/stackitcloud/stackit-sdk-go/services/resourcemanager/v0api" serviceaccount "github.com/stackitcloud/stackit-sdk-go/services/serviceaccount/v2api" - serviceenablement "github.com/stackitcloud/stackit-sdk-go/services/serviceenablement/v2api" "k8s.io/apimachinery/pkg/util/wait" "k8s.io/klog/v2" ) @@ -25,13 +20,6 @@ var childKeyCreationRetryBackoff = wait.Backoff{ Steps: 5, } -type managedProject struct { - ContainerID string - ProjectID string - Name string - Labels map[string]string -} - type managedServiceAccount struct { Email string ProjectID string @@ -58,178 +46,26 @@ type serviceAccountKeyCredentialsFile struct { TokenEndpoint string `json:"tokenEndpoint"` } -func (d *Deployer) initializeBootstrapClients() error { - if d.projectClient == nil { - client, err := newProjectClient(d.serviceAccount, d.resourceManagerEndpoint) - if err != nil { - return err - } - d.projectClient = client - } - if d.serviceAccountClient == nil { - client, err := newServiceAccountClient(d.serviceAccount, d.serviceAccountEndpoint) - if err != nil { - return err - } - d.serviceAccountClient = client - } - if d.authorizationClient == nil { - client, err := newAuthorizationClient(d.serviceAccount, d.authorizationEndpoint) - if err != nil { - return err - } - d.authorizationClient = client - } - if d.serviceEnablementClient == nil { - client, err := newServiceEnablementClient(d.serviceAccount, d.serviceEnablementEndpoint) - if err != nil { - return err - } - d.serviceEnablementClient = client - } - return nil -} - -func (d *Deployer) initializeSKEClient(serviceAccountKey string) error { - if d.skeClientFactory == nil { - d.skeClientFactory = newSKEClient - } - client, err := d.skeClientFactory(d.region, serviceAccountKey, d.skeEndpoint) - if err != nil { - return err - } - d.skeClient = client - return nil -} - -func (d *Deployer) ensureManagedClusterAccess(ctx context.Context) error { - project, err := d.resolveManagedProject(ctx) +// ensureServiceAccount idempotently resolves (or creates) the managed child +// service account, grants it the SKE admin role, caches a service-account key +// and initializes the SKE client from it. Depends on d.projectID being set. +func (d *Deployer) ensureServiceAccount(ctx context.Context) error { + childServiceAccount, err := d.resolveManagedServiceAccount(ctx, d.projectID) if err != nil { return err } - d.projectID = project.ProjectID - if err := d.ensureSKEServiceEnabled(ctx, project.ProjectID); err != nil { + if err := d.ensureProjectServiceAccountRole(ctx, d.projectID, childServiceAccount.Email); err != nil { return err } - childServiceAccount, err := d.resolveManagedServiceAccount(ctx, project.ProjectID) - if err != nil { - return err - } - d.childServiceAccountEmail = childServiceAccount.Email - - if err := d.ensureProjectServiceAccountRole(ctx, project.ProjectID, childServiceAccount.Email); err != nil { - return err - } - - serviceAccountKey, err := d.ensureCachedChildServiceAccountKey(ctx, project.ProjectID, childServiceAccount.Email) + serviceAccountKey, err := d.ensureCachedChildServiceAccountKey(ctx, d.projectID, childServiceAccount.Email) if err != nil { return err } return d.initializeSKEClient(serviceAccountKey) } -// ensureSKEServiceEnabled idempotently enables the SKE (Kubernetes Engine) -// service for the managed project and waits until it is enabled. -func (d *Deployer) ensureSKEServiceEnabled(ctx context.Context, projectID string) error { - status, err := d.serviceEnablementClient.GetServiceStatus(ctx, d.region, projectID, skeServiceID) - if err != nil { - if !isNotFound(err) { - return fmt.Errorf("get SKE service status for project %q: %w", projectID, err) - } - klog.Infof("SKE service not yet enabled for project_id=%q", projectID) - } else if status.GetState() == serviceenablement.SERVICESTATUSSTATE_ENABLED { - klog.Infof("SKE service already enabled for project_id=%q", projectID) - return nil - } else { - klog.Infof("SKE service in state %q for project_id=%q, enabling", status.GetState(), projectID) - } - - if err := d.serviceEnablementClient.EnableService(ctx, d.region, projectID, skeServiceID); err != nil { - return fmt.Errorf("enable SKE service for project %q: %w", projectID, err) - } - if err := d.serviceEnablementClient.WaitForServiceEnabled(ctx, d.region, projectID, skeServiceID); err != nil { - return fmt.Errorf("wait for SKE service enablement for project %q: %w", projectID, err) - } - return nil -} - -func isNotFound(err error) bool { - var oapiErr *oapierror.GenericOpenAPIError - return errors.As(err, &oapiErr) && oapiErr.StatusCode == http.StatusNotFound -} - -func (d *Deployer) findManagedProject(ctx context.Context) (*managedProject, error) { - projects, err := d.projectClient.ListProjects(ctx, d.parentContainerID) - if err != nil { - return nil, fmt.Errorf("list STACKIT projects under parent container %q: %w", d.parentContainerID, err) - } - - matches := make([]managedProject, 0, 1) - for i := range projects { - project := &projects[i] - if !d.matchesManagedProject(project) { - continue - } - matches = append(matches, managedProject{ - ContainerID: project.GetContainerId(), - ProjectID: project.GetProjectId(), - Name: project.GetName(), - Labels: project.GetLabels(), - }) - } - - switch len(matches) { - case 0: - return nil, nil - case 1: - return &matches[0], nil - default: - return nil, fmt.Errorf( - "found %d managed STACKIT projects for run token %q under parent container %q", - len(matches), - d.runToken(), - d.parentContainerID, - ) - } -} - -func (d *Deployer) resolveManagedProject(ctx context.Context) (*managedProject, error) { - project, err := d.findManagedProject(ctx) - if err != nil { - return nil, err - } - if project != nil { - klog.Infof("Reusing managed project=%q project_id=%q", project.Name, project.ProjectID) - return project, nil - } - - klog.Infof("Creating managed project=%q under parent_container_id=%q", d.projectName(), d.parentContainerID) - createdProject, err := d.projectClient.CreateProject( - ctx, - d.parentContainerID, - d.projectName(), - d.projectMemberEmail, - d.managedProjectLabels(), - ) - if err != nil { - return nil, fmt.Errorf("create STACKIT project %q: %w", d.projectName(), err) - } - - activeProject, err := d.projectClient.WaitForProjectActive(ctx, createdProject.GetContainerId()) - if err != nil { - return nil, fmt.Errorf("wait for STACKIT project %q to become active: %w", createdProject.GetProjectId(), err) - } - - return &managedProject{ - ContainerID: activeProject.GetContainerId(), - ProjectID: activeProject.GetProjectId(), - Name: activeProject.GetName(), - Labels: activeProject.GetLabels(), - }, nil -} - func (d *Deployer) resolveManagedServiceAccount(ctx context.Context, projectID string) (*managedServiceAccount, error) { serviceAccounts, err := d.serviceAccountClient.ListServiceAccounts(ctx, projectID) if err != nil { @@ -360,28 +196,10 @@ func (d *Deployer) readCachedChildServiceAccountKey() (key string, ok bool, err } func (d *Deployer) writeCachedChildServiceAccountKey(serviceAccountKey string) error { - return os.WriteFile(d.serviceAccountKeyPath, []byte(serviceAccountKey), 0o600) -} - -func (d *Deployer) managedProjectLabels() map[string]string { - return map[string]string{ - projectLabelScopeKey: projectLabelScopeValue, - projectLabelManagedKey: projectLabelManagedValue, - projectLabelRunIDKey: d.runToken(), - } -} - -func (d *Deployer) matchesManagedProject(project *resourcemanager.Project) bool { - if project.GetName() != d.projectName() { - return false - } - labels := project.GetLabels() - if labels == nil { - return false + if strings.TrimSpace(d.serviceAccountKeyPath) == "" { + return nil } - return labels[projectLabelScopeKey] == projectLabelScopeValue && - labels[projectLabelManagedKey] == projectLabelManagedValue && - labels[projectLabelRunIDKey] == d.runToken() + return os.WriteFile(d.serviceAccountKeyPath, []byte(serviceAccountKey), 0o600) } func (d *Deployer) matchesManagedServiceAccountEmail(email string) bool { @@ -392,7 +210,7 @@ func (d *Deployer) matchesManagedServiceAccountEmail(email string) bool { if !found { return false } - return strings.HasPrefix(localPart, d.serviceAccountName()) + return localPart == d.serviceAccountName() } func serviceAccountKeyJSON(createdKey *serviceaccount.CreateServiceAccountKeyResponse) (string, error) { diff --git a/test/kubetest2/bootstrap_serviceaccount_test.go b/test/kubetest2/bootstrap_serviceaccount_test.go new file mode 100644 index 00000000..99c23d25 --- /dev/null +++ b/test/kubetest2/bootstrap_serviceaccount_test.go @@ -0,0 +1,128 @@ +package kubetest2 + +import ( + "context" + "errors" + "os" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + authorization "github.com/stackitcloud/stackit-sdk-go/services/authorization/v2api" + serviceaccount "github.com/stackitcloud/stackit-sdk-go/services/serviceaccount/v2api" + "k8s.io/apimachinery/pkg/util/wait" +) + +var _ = Describe("matchesManagedServiceAccountEmail", func() { + It("matches the managed service account prefix", func() { + d := newTestDeployer() + Expect(d.matchesManagedServiceAccountEmail(d.serviceAccountName() + "@sa.stackit.cloud")).To(BeTrue()) + }) + + DescribeTable("rejects non-matching emails", + func(email string) { + d := newTestDeployer() + Expect(d.matchesManagedServiceAccountEmail(email)).To(BeFalse()) + }, + Entry("empty email", ""), + Entry("missing at sign", "kt2-no-separator"), + Entry("different local prefix", "other-account@sa.stackit.cloud"), + ) +}) + +var _ = Describe("ensureServiceAccount", func() { + It("reuses cached key and skips membership write", func() { + d := newTestDeployer() + d.projectID = "project-123" + cachedKey := `{"credentials":{"privateKey":"cached"}}` + Expect(os.WriteFile(d.serviceAccountKeyPath, []byte(cachedKey), 0o600)).To(Succeed()) + + serviceAccountEmail := d.serviceAccountName() + "@sa.stackit.cloud" + serviceAccountClient := &fakeServiceAccountClient{ + listResult: []serviceaccount.ServiceAccount{ + *serviceAccountFixture(serviceAccountEmail, "project-123"), + }, + } + authorizationClient := &fakeAuthorizationClient{ + listMembersResult: []authorization.Member{ + *authorization.NewMember(childProjectRole, serviceAccountEmail), + }, + } + fakeSKE := &fakeSKEClient{} + var receivedKey string + + d.serviceAccountClient = serviceAccountClient + d.authorizationClient = authorizationClient + d.skeClientFactory = func(_, serviceAccount, _ string) (skeClient, error) { + receivedKey = serviceAccount + return fakeSKE, nil + } + + Expect(d.ensureServiceAccount(context.Background())).To(Succeed()) + Expect(receivedKey).To(Equal(cachedKey)) + Expect(authorizationClient.addCalls).To(Equal(0)) + Expect(serviceAccountClient.createKeyCalls).To(Equal(0)) + }) + + It("creates key and adds membership", func() { + d := newTestDeployer() + d.projectID = "project-123" + + serviceAccountEmail := d.serviceAccountName() + "@sa.stackit.cloud" + serviceAccountClient := &fakeServiceAccountClient{ + listResult: []serviceaccount.ServiceAccount{ + *serviceAccountFixture(serviceAccountEmail, "project-123"), + }, + createKeyResult: createServiceAccountKeyResponseFixture(serviceAccountEmail), + } + authorizationClient := &fakeAuthorizationClient{} + var receivedKey string + + d.serviceAccountClient = serviceAccountClient + d.authorizationClient = authorizationClient + d.skeClientFactory = func(_, serviceAccount, _ string) (skeClient, error) { + receivedKey = serviceAccount + return &fakeSKEClient{}, nil + } + + Expect(d.ensureServiceAccount(context.Background())).To(Succeed()) + Expect(authorizationClient.addCalls).To(Equal(1)) + Expect(serviceAccountClient.createKeyCalls).To(Equal(1)) + Expect(authorizationClient.lastAddedType).To(Equal(projectResourceType)) + Expect(authorizationClient.lastAddedID).To(Equal("project-123")) + + keyBytes, err := os.ReadFile(d.serviceAccountKeyPath) + Expect(err).NotTo(HaveOccurred()) + Expect(receivedKey).To(Equal(string(keyBytes))) + Expect(string(keyBytes)).To(ContainSubstring(`"privateKey":"PRIVATE"`)) + + info, err := os.Stat(d.serviceAccountKeyPath) + Expect(err).NotTo(HaveOccurred()) + Expect(info.Mode().Perm()).To(Equal(os.FileMode(0o600))) + }) +}) + +var _ = Describe("retryWithBackoff", func() { + It("retries until success", func() { + calls := 0 + result, err := retryWithBackoff(context.Background(), wait.Backoff{Duration: 0, Factor: 1, Steps: 3}, func() (string, error) { + calls++ + if calls < 2 { + return "", errors.New("transient") + } + return "done", nil + }) + Expect(err).NotTo(HaveOccurred()) + Expect(result).To(Equal("done")) + Expect(calls).To(Equal(2)) + }) + + It("returns last error when exhausted", func() { + calls := 0 + _, err := retryWithBackoff(context.Background(), wait.Backoff{Duration: 0, Factor: 1, Steps: 3}, func() (int, error) { + calls++ + return 0, errors.New("always fails") + }) + Expect(err).To(HaveOccurred()) + Expect(calls).To(Equal(3)) + }) +}) diff --git a/test/kubetest2/payload.go b/test/kubetest2/bootstrap_ske.go similarity index 68% rename from test/kubetest2/payload.go rename to test/kubetest2/bootstrap_ske.go index c629c54e..e0d96e2c 100644 --- a/test/kubetest2/payload.go +++ b/test/kubetest2/bootstrap_ske.go @@ -3,11 +3,49 @@ package kubetest2 import ( "context" "fmt" + "os" "github.com/stackitcloud/stackit-sdk-go/services/ske" "k8s.io/klog/v2" ) +// ensureCluster idempotently validates provider options, then creates or +// updates the SKE cluster, waits until it is ready and writes its kubeconfig. +// Depends on d.projectID and d.skeClient being set. +func (d *Deployer) ensureCluster(ctx context.Context) error { + if err := d.validateProviderOptions(ctx); err != nil { + return err + } + + clusterName := d.clusterName() + payload := d.clusterPayload() + + klog.Infof("Submitting SKE create/update request for cluster=%q", clusterName) + if _, err := d.skeClient.CreateOrUpdateCluster(ctx, d.projectID, d.region, clusterName, payload); err != nil { + return fmt.Errorf("create or update SKE cluster %q: %w", clusterName, err) + } + klog.Infof("Submitted SKE create/update request for cluster=%q", clusterName) + + if _, err := d.skeClient.WaitForClusterReady(ctx, d.projectID, d.region, clusterName); err != nil { + return fmt.Errorf("wait for SKE cluster %q to become ready: %w", clusterName, err) + } + + return d.writeKubeconfig(ctx, clusterName) +} + +func (d *Deployer) writeKubeconfig(ctx context.Context, clusterName string) error { + klog.Infof("Creating kubeconfig for cluster=%q", clusterName) + kubeconfig, err := d.skeClient.CreateKubeconfig(ctx, d.projectID, d.region, clusterName, d.kubeconfigExpiresIn) + if err != nil { + return fmt.Errorf("create kubeconfig for SKE cluster %q: %w", clusterName, err) + } + klog.Infof("Writing kubeconfig for cluster=%q to %q", clusterName, d.kubeconfigPath) + if err := os.WriteFile(d.kubeconfigPath, []byte(kubeconfig.GetKubeconfig()), 0o600); err != nil { + return fmt.Errorf("write kubeconfig %q: %w", d.kubeconfigPath, err) + } + return nil +} + func (d *Deployer) validateProviderOptions(ctx context.Context) error { klog.Infof("Validating SKE provider options for region=%q", d.region) @@ -76,7 +114,6 @@ func (d *Deployer) clusterPayload() ske.CreateOrUpdateClusterPayload { nodepool.SetAllowSystemComponents(true) payload := ske.NewCreateOrUpdateClusterPayload(*clusterKubernetes, []ske.Nodepool{*nodepool}) - klog.Infof("Built SKE cluster payload for cluster=%q", d.clusterName()) return *payload } diff --git a/test/kubetest2/payload_test.go b/test/kubetest2/bootstrap_ske_test.go similarity index 100% rename from test/kubetest2/payload_test.go rename to test/kubetest2/bootstrap_ske_test.go diff --git a/test/kubetest2/bootstrap_test.go b/test/kubetest2/bootstrap_test.go deleted file mode 100644 index 44192f0d..00000000 --- a/test/kubetest2/bootstrap_test.go +++ /dev/null @@ -1,809 +0,0 @@ -package kubetest2 - -import ( - "context" - "errors" - "net/http" - "os" - "path/filepath" - "time" - - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" - oapierror "github.com/stackitcloud/stackit-sdk-go/core/oapierror" - authorization "github.com/stackitcloud/stackit-sdk-go/services/authorization/v2api" - resourcemanager "github.com/stackitcloud/stackit-sdk-go/services/resourcemanager/v0api" - serviceaccount "github.com/stackitcloud/stackit-sdk-go/services/serviceaccount/v2api" - serviceenablement "github.com/stackitcloud/stackit-sdk-go/services/serviceenablement/v2api" - "github.com/stackitcloud/stackit-sdk-go/services/ske" - "k8s.io/apimachinery/pkg/util/wait" - "sigs.k8s.io/kubetest2/pkg/types" -) - -const validServiceAccountKey = `{"credentials":{"iss":"owner@example.com"}}` - -type fakeOptions struct { - runID string - runDir string -} - -func (o fakeOptions) HelpRequested() bool { return false } -func (o fakeOptions) ShouldBuild() bool { return false } -func (o fakeOptions) ShouldUp() bool { return false } -func (o fakeOptions) ShouldDown() bool { return false } -func (o fakeOptions) ShouldTest() bool { return false } -func (o fakeOptions) SkipTestJUnitReport() bool { return false } -func (o fakeOptions) RunID() string { return o.runID } -func (o fakeOptions) RunDir() string { return o.runDir } -func (o fakeOptions) RundirInArtifacts() bool { return false } -func (o fakeOptions) PostTestCmd() []string { return nil } -func (o fakeOptions) PreTestCmd() []string { return nil } - -type fakeProjectClient struct { - listProjectsResult []resourcemanager.Project - listProjectsErr error - - createProjectResult *resourcemanager.Project - createProjectErr error - waitActiveResult *resourcemanager.GetProjectResponse - waitActiveErr error - - deleteErr error - waitDeletedErr error - - createCalls int - waitActiveCalls int - deleteCalls int - waitDeletedCalls int - - lastListParentContainerID string - lastCreateParentContainerID string - lastCreateName string - lastCreateOwnerEmail string - lastCreateLabels map[string]string - lastWaitActiveContainerID string - lastDeletedProjectID string - lastWaitDeletedProjectID string -} - -func (c *fakeProjectClient) ListProjects(_ context.Context, parentContainerID string) ([]resourcemanager.Project, error) { - c.lastListParentContainerID = parentContainerID - return c.listProjectsResult, c.listProjectsErr -} - -func (c *fakeProjectClient) CreateProject(_ context.Context, parentContainerID, name, ownerEmail string, labels map[string]string) (*resourcemanager.Project, error) { - c.createCalls++ - c.lastCreateParentContainerID = parentContainerID - c.lastCreateName = name - c.lastCreateOwnerEmail = ownerEmail - c.lastCreateLabels = labels - return c.createProjectResult, c.createProjectErr -} - -func (c *fakeProjectClient) WaitForProjectActive(_ context.Context, containerID string) (*resourcemanager.GetProjectResponse, error) { - c.waitActiveCalls++ - c.lastWaitActiveContainerID = containerID - return c.waitActiveResult, c.waitActiveErr -} - -func (c *fakeProjectClient) DeleteProject(_ context.Context, projectID string) error { - c.deleteCalls++ - c.lastDeletedProjectID = projectID - return c.deleteErr -} - -func (c *fakeProjectClient) WaitForProjectDeleted(_ context.Context, projectID string) error { - c.waitDeletedCalls++ - c.lastWaitDeletedProjectID = projectID - return c.waitDeletedErr -} - -type fakeServiceAccountClient struct { - listResult []serviceaccount.ServiceAccount - listErr error - - createResult *serviceaccount.ServiceAccount - createErr error - - createKeyResult *serviceaccount.CreateServiceAccountKeyResponse - createKeyErr error - - createCalls int - createKeyCalls int - - lastProjectIDForList string - lastProjectIDForCreate string - lastCreatedName string - lastProjectIDForCreateKey string - lastCreateKeyEmail string -} - -func (c *fakeServiceAccountClient) ListServiceAccounts(_ context.Context, projectID string) ([]serviceaccount.ServiceAccount, error) { - c.lastProjectIDForList = projectID - return c.listResult, c.listErr -} - -func (c *fakeServiceAccountClient) CreateServiceAccount(_ context.Context, projectID, name string) (*serviceaccount.ServiceAccount, error) { - c.createCalls++ - c.lastProjectIDForCreate = projectID - c.lastCreatedName = name - return c.createResult, c.createErr -} - -func (c *fakeServiceAccountClient) CreateServiceAccountKey(_ context.Context, projectID, serviceAccountEmail string) (*serviceaccount.CreateServiceAccountKeyResponse, error) { - c.createKeyCalls++ - c.lastProjectIDForCreateKey = projectID - c.lastCreateKeyEmail = serviceAccountEmail - return c.createKeyResult, c.createKeyErr -} - -type fakeAuthorizationClient struct { - listMembersResult []authorization.Member - listMembersErr error - addMembersErr error - - addCalls int - - lastResourceType string - lastResourceID string - lastAddedType string - lastAddedID string - lastAddedMembers []authorization.Member -} - -func (c *fakeAuthorizationClient) ListMembers(_ context.Context, resourceType, resourceID string) ([]authorization.Member, error) { - c.lastResourceType = resourceType - c.lastResourceID = resourceID - return c.listMembersResult, c.listMembersErr -} - -func (c *fakeAuthorizationClient) AddMembers(_ context.Context, resourceID, resourceType string, members []authorization.Member) error { - c.addCalls++ - c.lastAddedID = resourceID - c.lastAddedType = resourceType - c.lastAddedMembers = members - return c.addMembersErr -} - -type fakeServiceEnablementClient struct { - getStatusResult *serviceenablement.ServiceStatus - getStatusErr error - enableErr error - waitErr error - - enableCalls int - waitCalls int - - lastGetStatusRegion string - lastGetStatusProjectID string - lastGetStatusServiceID string - lastEnableRegion string - lastEnableProjectID string - lastEnableServiceID string - lastWaitRegion string - lastWaitProjectID string - lastWaitServiceID string -} - -func (c *fakeServiceEnablementClient) GetServiceStatus(_ context.Context, region, projectID, serviceID string) (*serviceenablement.ServiceStatus, error) { - c.lastGetStatusRegion = region - c.lastGetStatusProjectID = projectID - c.lastGetStatusServiceID = serviceID - return c.getStatusResult, c.getStatusErr -} - -func (c *fakeServiceEnablementClient) EnableService(_ context.Context, region, projectID, serviceID string) error { - c.enableCalls++ - c.lastEnableRegion = region - c.lastEnableProjectID = projectID - c.lastEnableServiceID = serviceID - return c.enableErr -} - -func (c *fakeServiceEnablementClient) WaitForServiceEnabled(_ context.Context, region, projectID, serviceID string) error { - c.waitCalls++ - c.lastWaitRegion = region - c.lastWaitProjectID = projectID - c.lastWaitServiceID = serviceID - return c.waitErr -} - -type fakeSKEClient struct { - providerOptions *ske.ProviderOptions - providerOptionsErr error - getClusterResult *ske.Cluster - getClusterErr error - - createOrUpdateResult *ske.Cluster - createOrUpdateErr error - waitReadyResult *ske.Cluster - waitReadyErr error - kubeconfigResult *ske.Kubeconfig - kubeconfigErr error - - deleteClusterCalled bool - waitDeletedCalled bool - - lastGetProjectID string - lastCreateProjectID string - lastCreateRegion string - lastCreateClusterName string - lastKubeconfigProjectID string - lastExpirationSeconds int64 -} - -func (c *fakeSKEClient) GetCluster(_ context.Context, projectID, _, _ string) (*ske.Cluster, error) { - c.lastGetProjectID = projectID - return c.getClusterResult, c.getClusterErr -} - -func (c *fakeSKEClient) ListProviderOptions(_ context.Context, _ string) (*ske.ProviderOptions, error) { - return c.providerOptions, c.providerOptionsErr -} - -func (c *fakeSKEClient) CreateOrUpdateCluster(_ context.Context, projectID, region, name string, _ ske.CreateOrUpdateClusterPayload) (*ske.Cluster, error) { - c.lastCreateProjectID = projectID - c.lastCreateRegion = region - c.lastCreateClusterName = name - return c.createOrUpdateResult, c.createOrUpdateErr -} - -func (c *fakeSKEClient) WaitForClusterReady(_ context.Context, _, _, _ string) (*ske.Cluster, error) { - return c.waitReadyResult, c.waitReadyErr -} - -func (c *fakeSKEClient) CreateKubeconfig(_ context.Context, projectID, _, _ string, expirationSeconds int64) (*ske.Kubeconfig, error) { - c.lastKubeconfigProjectID = projectID - c.lastExpirationSeconds = expirationSeconds - return c.kubeconfigResult, c.kubeconfigErr -} - -func (c *fakeSKEClient) DeleteCluster(_ context.Context, _, _, _ string) error { - c.deleteClusterCalled = true - return nil -} - -func (c *fakeSKEClient) WaitForClusterDeleted(_ context.Context, _, _, _ string) error { - c.waitDeletedCalled = true - return nil -} - -var _ = Describe("loadEnvironment", func() { - envVarKeys := []string{ - "STACKIT_SERVICE_ACCOUNT", - "STACKIT_PARENT_CONTAINER_ID", - "STACKIT_PROJECT_ID", - "STACKIT_RESOURCE_MANAGER_ENDPOINT", - "STACKIT_SERVICE_ACCOUNT_ENDPOINT", - "STACKIT_AUTHORIZATION_ENDPOINT", - "STACKIT_SKE_ENDPOINT", - } - - DescribeTable("validates required environment variables", - func(env map[string]string, wantErrContains string) { - for _, key := range envVarKeys { - setEnvVar(key, "") - } - for key, value := range env { - setEnvVar(key, value) - } - - runDir := GinkgoT().TempDir() - d := &Deployer{options: fakeOptions{runID: "run-123", runDir: runDir}} - err := d.loadEnvironment() - if wantErrContains != "" { - Expect(err).To(MatchError(ContainSubstring(wantErrContains))) - return - } - Expect(err).NotTo(HaveOccurred()) - Expect(d.parentContainerID).To(Equal("parent-1")) - Expect(d.projectMemberEmail).To(Equal("owner@example.com")) - Expect(d.projectID).To(BeEmpty()) - Expect(d.kubeconfigPath).To(Equal(filepath.Join(runDir, "kubeconfig"))) - Expect(d.serviceAccountKeyPath).To(Equal(filepath.Join(runDir, "service-account-key.json"))) - }, - Entry("missing service account", map[string]string{ - "STACKIT_PARENT_CONTAINER_ID": "parent-1", - }, "STACKIT_SERVICE_ACCOUNT"), - Entry("missing parent container", map[string]string{ - "STACKIT_SERVICE_ACCOUNT": validServiceAccountKey, - }, "STACKIT_PARENT_CONTAINER_ID"), - Entry("invalid service account key", map[string]string{ - "STACKIT_SERVICE_ACCOUNT": "{}", - "STACKIT_PARENT_CONTAINER_ID": "parent-1", - }, "invalid STACKIT_SERVICE_ACCOUNT"), - Entry("project id no longer required", map[string]string{ - "STACKIT_SERVICE_ACCOUNT": validServiceAccountKey, - "STACKIT_PARENT_CONTAINER_ID": "parent-1", - }, ""), - ) - - It("reads optional endpoints", func() { - for _, key := range envVarKeys { - setEnvVar(key, "") - } - setEnvVar("STACKIT_SERVICE_ACCOUNT", validServiceAccountKey) - setEnvVar("STACKIT_PARENT_CONTAINER_ID", "parent-1") - setEnvVar("STACKIT_RESOURCE_MANAGER_ENDPOINT", "https://resource-manager.example.com") - setEnvVar("STACKIT_SERVICE_ACCOUNT_ENDPOINT", "https://service-account.example.com") - setEnvVar("STACKIT_AUTHORIZATION_ENDPOINT", "https://authorization.example.com") - setEnvVar("STACKIT_SKE_ENDPOINT", "https://ske.example.com") - - runDir := GinkgoT().TempDir() - d := &Deployer{options: fakeOptions{runID: "run-123", runDir: runDir}} - Expect(d.loadEnvironment()).To(Succeed()) - - Expect(d.projectMemberEmail).To(Equal("owner@example.com")) - Expect(d.resourceManagerEndpoint).To(Equal("https://resource-manager.example.com")) - Expect(d.serviceAccountEndpoint).To(Equal("https://service-account.example.com")) - Expect(d.authorizationEndpoint).To(Equal("https://authorization.example.com")) - Expect(d.skeEndpoint).To(Equal("https://ske.example.com")) - }) -}) - -var _ = Describe("resolveManagedProject", func() { - It("creates a project when missing", func() { - d := newTestDeployer() - projectClient := &fakeProjectClient{ - createProjectResult: projectFixture(d.projectName(), "project-123", "container-123", d.managedProjectLabels()), - waitActiveResult: projectResponseFixture(d.projectName(), "project-123", "container-123", d.managedProjectLabels()), - } - d.projectClient = projectClient - - project, err := d.resolveManagedProject(context.Background()) - Expect(err).NotTo(HaveOccurred()) - Expect(project.ProjectID).To(Equal("project-123")) - Expect(projectClient.createCalls).To(Equal(1)) - Expect(projectClient.waitActiveCalls).To(Equal(1)) - Expect(projectClient.lastCreateParentContainerID).To(Equal(d.parentContainerID)) - Expect(projectClient.lastCreateName).To(Equal(d.projectName())) - Expect(projectClient.lastCreateOwnerEmail).To(Equal(d.projectMemberEmail)) - Expect(projectClient.lastCreateLabels).To(Equal(d.managedProjectLabels())) - }) - - It("errors on multiple matches", func() { - d := newTestDeployer() - d.projectClient = &fakeProjectClient{ - listProjectsResult: []resourcemanager.Project{ - *projectFixture(d.projectName(), "project-1", "container-1", d.managedProjectLabels()), - *projectFixture(d.projectName(), "project-2", "container-2", d.managedProjectLabels()), - }, - } - - _, err := d.findManagedProject(context.Background()) - Expect(err).To(MatchError(ContainSubstring("found 2 managed STACKIT projects"))) - }) -}) - -var _ = Describe("ensureManagedClusterAccess", func() { - It("reuses cached key and skips membership write", func() { - d := newTestDeployer() - cachedKey := `{"credentials":{"privateKey":"cached"}}` - Expect(os.WriteFile(d.serviceAccountKeyPath, []byte(cachedKey), 0o600)).To(Succeed()) - - projectClient := &fakeProjectClient{ - listProjectsResult: []resourcemanager.Project{ - *projectFixture(d.projectName(), "project-123", "container-123", d.managedProjectLabels()), - }, - } - serviceAccountEmail := d.serviceAccountName() + "@sa.stackit.cloud" - serviceAccountClient := &fakeServiceAccountClient{ - listResult: []serviceaccount.ServiceAccount{ - *serviceAccountFixture(serviceAccountEmail, "project-123"), - }, - } - authorizationClient := &fakeAuthorizationClient{ - listMembersResult: []authorization.Member{ - *authorization.NewMember(childProjectRole, serviceAccountEmail), - }, - } - fakeSKE := &fakeSKEClient{} - var receivedKey string - - d.projectClient = projectClient - d.serviceAccountClient = serviceAccountClient - d.authorizationClient = authorizationClient - d.serviceEnablementClient = &fakeServiceEnablementClient{getStatusResult: serviceenablement.NewServiceStatus()} - d.skeClientFactory = func(_, serviceAccount, _ string) (skeClient, error) { - receivedKey = serviceAccount - return fakeSKE, nil - } - - Expect(d.ensureManagedClusterAccess(context.Background())).To(Succeed()) - Expect(receivedKey).To(Equal(cachedKey)) - Expect(authorizationClient.addCalls).To(Equal(0)) - Expect(serviceAccountClient.createKeyCalls).To(Equal(0)) - Expect(d.projectID).To(Equal("project-123")) - }) - - It("creates key and adds membership", func() { - d := newTestDeployer() - - projectClient := &fakeProjectClient{ - listProjectsResult: []resourcemanager.Project{ - *projectFixture(d.projectName(), "project-123", "container-123", d.managedProjectLabels()), - }, - } - serviceAccountEmail := d.serviceAccountName() + "@sa.stackit.cloud" - serviceAccountClient := &fakeServiceAccountClient{ - listResult: []serviceaccount.ServiceAccount{ - *serviceAccountFixture(serviceAccountEmail, "project-123"), - }, - createKeyResult: createServiceAccountKeyResponseFixture(serviceAccountEmail), - } - authorizationClient := &fakeAuthorizationClient{} - var receivedKey string - - d.projectClient = projectClient - d.serviceAccountClient = serviceAccountClient - d.authorizationClient = authorizationClient - d.serviceEnablementClient = &fakeServiceEnablementClient{getStatusResult: serviceenablement.NewServiceStatus()} - d.skeClientFactory = func(_, serviceAccount, _ string) (skeClient, error) { - receivedKey = serviceAccount - return &fakeSKEClient{}, nil - } - - Expect(d.ensureManagedClusterAccess(context.Background())).To(Succeed()) - Expect(authorizationClient.addCalls).To(Equal(1)) - Expect(serviceAccountClient.createKeyCalls).To(Equal(1)) - Expect(authorizationClient.lastAddedType).To(Equal(projectResourceType)) - Expect(authorizationClient.lastAddedID).To(Equal("project-123")) - - keyBytes, err := os.ReadFile(d.serviceAccountKeyPath) - Expect(err).NotTo(HaveOccurred()) - Expect(receivedKey).To(Equal(string(keyBytes))) - Expect(string(keyBytes)).To(ContainSubstring(`"privateKey":"PRIVATE"`)) - - info, err := os.Stat(d.serviceAccountKeyPath) - Expect(err).NotTo(HaveOccurred()) - Expect(info.Mode().Perm()).To(Equal(os.FileMode(0o600))) - }) -}) - -var _ = Describe("ensureSKEServiceEnabled", func() { - It("skips when already enabled", func() { - d := newTestDeployer() - client := &fakeServiceEnablementClient{getStatusResult: serviceenablement.NewServiceStatus()} - d.serviceEnablementClient = client - - Expect(d.ensureSKEServiceEnabled(context.Background(), "project-123")).To(Succeed()) - Expect(client.enableCalls).To(Equal(0)) - Expect(client.waitCalls).To(Equal(0)) - Expect(client.lastGetStatusServiceID).To(Equal(skeServiceID)) - }) - - It("enables when not found", func() { - d := newTestDeployer() - client := &fakeServiceEnablementClient{ - getStatusErr: &oapierror.GenericOpenAPIError{StatusCode: http.StatusNotFound}, - } - d.serviceEnablementClient = client - - Expect(d.ensureSKEServiceEnabled(context.Background(), "project-123")).To(Succeed()) - Expect(client.enableCalls).To(Equal(1)) - Expect(client.waitCalls).To(Equal(1)) - Expect(client.lastEnableProjectID).To(Equal("project-123")) - Expect(client.lastEnableServiceID).To(Equal(skeServiceID)) - }) - - It("enables when disabled", func() { - d := newTestDeployer() - client := &fakeServiceEnablementClient{ - getStatusResult: disabledServiceStatusFixture(), - } - d.serviceEnablementClient = client - - Expect(d.ensureSKEServiceEnabled(context.Background(), "project-123")).To(Succeed()) - Expect(client.enableCalls).To(Equal(1)) - Expect(client.waitCalls).To(Equal(1)) - }) - - It("fails on get status error", func() { - d := newTestDeployer() - client := &fakeServiceEnablementClient{ - getStatusErr: &oapierror.GenericOpenAPIError{StatusCode: http.StatusForbidden}, - } - d.serviceEnablementClient = client - - err := d.ensureSKEServiceEnabled(context.Background(), "project-123") - Expect(err).To(HaveOccurred()) - Expect(client.enableCalls).To(Equal(0)) - }) -}) - -var _ = Describe("retryWithBackoff", func() { - It("retries until success", func() { - calls := 0 - result, err := retryWithBackoff(context.Background(), wait.Backoff{Duration: 0, Factor: 1, Steps: 3}, func() (string, error) { - calls++ - if calls < 2 { - return "", errors.New("transient") - } - return "done", nil - }) - Expect(err).NotTo(HaveOccurred()) - Expect(result).To(Equal("done")) - Expect(calls).To(Equal(2)) - }) - - It("returns last error when exhausted", func() { - calls := 0 - _, err := retryWithBackoff(context.Background(), wait.Backoff{Duration: 0, Factor: 1, Steps: 3}, func() (int, error) { - calls++ - return 0, errors.New("always fails") - }) - Expect(err).To(HaveOccurred()) - Expect(calls).To(Equal(3)) - }) -}) - -var _ = Describe("Up", func() { - It("uses discovered project and writes kubeconfig", func() { - d := newTestDeployer() - configureValidUpInputs(d) - - cachedKey := `{"credentials":{"privateKey":"cached"}}` - Expect(os.WriteFile(d.serviceAccountKeyPath, []byte(cachedKey), 0o600)).To(Succeed()) - - d.projectClient = &fakeProjectClient{ - listProjectsResult: []resourcemanager.Project{ - *projectFixture(d.projectName(), "project-123", "container-123", d.managedProjectLabels()), - }, - } - serviceAccountEmail := d.serviceAccountName() + "@sa.stackit.cloud" - d.serviceAccountClient = &fakeServiceAccountClient{ - listResult: []serviceaccount.ServiceAccount{ - *serviceAccountFixture(serviceAccountEmail, "project-123"), - }, - } - d.authorizationClient = &fakeAuthorizationClient{ - listMembersResult: []authorization.Member{ - *authorization.NewMember(childProjectRole, serviceAccountEmail), - }, - } - d.serviceEnablementClient = &fakeServiceEnablementClient{getStatusResult: serviceenablement.NewServiceStatus()} - - fakeSKE := &fakeSKEClient{ - providerOptions: providerOptionsFixture(), - createOrUpdateResult: ske.NewClusterWithDefaults(), - waitReadyResult: ske.NewClusterWithDefaults(), - kubeconfigResult: func() *ske.Kubeconfig { - cfg := ske.NewKubeconfig() - cfg.SetKubeconfig("apiVersion: v1\n") - return cfg - }(), - } - d.skeClientFactory = func(_, serviceAccount, _ string) (skeClient, error) { - Expect(serviceAccount).To(Equal(cachedKey)) - return fakeSKE, nil - } - - Expect(d.Up()).To(Succeed()) - Expect(fakeSKE.lastCreateProjectID).To(Equal("project-123")) - Expect(fakeSKE.lastCreateClusterName).To(Equal(d.clusterName())) - - kubeconfigBytes, err := os.ReadFile(d.kubeconfigPath) - Expect(err).NotTo(HaveOccurred()) - Expect(string(kubeconfigBytes)).To(Equal("apiVersion: v1\n")) - }) -}) - -var _ = Describe("Down", func() { - It("succeeds when project is missing", func() { - d := newTestDeployer() - projectClient := &fakeProjectClient{} - d.projectClient = projectClient - - Expect(d.Down()).To(Succeed()) - Expect(projectClient.deleteCalls).To(Equal(0)) - }) - - It("deletes project without touching cluster delete", func() { - d := newTestDeployer() - fakeSKE := &fakeSKEClient{} - d.skeClient = fakeSKE - d.projectClient = &fakeProjectClient{ - listProjectsResult: []resourcemanager.Project{ - *projectFixture(d.projectName(), "project-123", "container-123", d.managedProjectLabels()), - }, - } - - Expect(d.Down()).To(Succeed()) - projectClient := d.projectClient.(*fakeProjectClient) - Expect(projectClient.deleteCalls).To(Equal(1)) - Expect(projectClient.waitDeletedCalls).To(Equal(1)) - Expect(fakeSKE.deleteClusterCalled).To(BeFalse()) - Expect(fakeSKE.waitDeletedCalled).To(BeFalse()) - }) -}) - -var _ = Describe("IsUp", func() { - It("returns false when no project", func() { - d := newTestDeployer() - d.projectClient = &fakeProjectClient{} - - isUp, err := d.IsUp() - Expect(err).NotTo(HaveOccurred()) - Expect(isUp).To(BeFalse()) - }) - - It("returns error when project without cached key", func() { - d := newTestDeployer() - d.projectClient = &fakeProjectClient{ - listProjectsResult: []resourcemanager.Project{ - *projectFixture(d.projectName(), "project-123", "container-123", d.managedProjectLabels()), - }, - } - - _, err := d.IsUp() - Expect(err).To(MatchError(ContainSubstring("child service-account key cache"))) - }) - - It("queries cluster when project with cached key", func() { - d := newTestDeployer() - d.projectClient = &fakeProjectClient{ - listProjectsResult: []resourcemanager.Project{ - *projectFixture(d.projectName(), "project-123", "container-123", d.managedProjectLabels()), - }, - } - cachedKey := `{"credentials":{"privateKey":"cached"}}` - Expect(os.WriteFile(d.serviceAccountKeyPath, []byte(cachedKey), 0o600)).To(Succeed()) - - fakeSKE := &fakeSKEClient{ - getClusterResult: healthyClusterFixture(), - } - d.skeClientFactory = func(_, serviceAccount, _ string) (skeClient, error) { - Expect(serviceAccount).To(Equal(cachedKey)) - return fakeSKE, nil - } - - isUp, err := d.IsUp() - Expect(err).NotTo(HaveOccurred()) - Expect(isUp).To(BeTrue()) - Expect(fakeSKE.lastGetProjectID).To(Equal("project-123")) - }) - - It("returns false when cluster missing", func() { - d := newTestDeployer() - d.projectClient = &fakeProjectClient{ - listProjectsResult: []resourcemanager.Project{ - *projectFixture(d.projectName(), "project-123", "container-123", d.managedProjectLabels()), - }, - } - cachedKey := `{"credentials":{"privateKey":"cached"}}` - Expect(os.WriteFile(d.serviceAccountKeyPath, []byte(cachedKey), 0o600)).To(Succeed()) - - fakeSKE := &fakeSKEClient{ - getClusterErr: &oapierror.GenericOpenAPIError{StatusCode: http.StatusNotFound}, - } - d.skeClientFactory = func(_, _, _ string) (skeClient, error) { - return fakeSKE, nil - } - - isUp, err := d.IsUp() - Expect(err).NotTo(HaveOccurred()) - Expect(isUp).To(BeFalse()) - }) -}) - -func newTestDeployer() *Deployer { - runDir := GinkgoT().TempDir() - return &Deployer{ - options: fakeOptions{runID: "run-123", runDir: runDir}, - region: defaultRegion, - parentContainerID: "parent-123", - projectMemberEmail: "owner@example.com", - kubeconfigPath: filepath.Join(runDir, "kubeconfig"), - serviceAccountKeyPath: filepath.Join(runDir, "service-account-key.json"), - } -} - -func configureValidUpInputs(d *Deployer) { - d.kubernetesVersion = "1.31.0" - d.availabilityZone = defaultAvailabilityZone - d.machineType = "g1.2" - d.nodeImageName = "ubuntu" - d.nodeImageVersion = "v1" - d.nodepoolName = defaultNodepoolName - d.volumeType = "storage" -} - -func setEnvVar(key, value string) { - Expect(os.Setenv(key, value)).To(Succeed()) - DeferCleanup(os.Unsetenv, key) -} - -func providerOptionsFixture() *ske.ProviderOptions { - kubernetesVersion := ske.NewKubernetesVersion() - kubernetesVersion.SetVersion("1.31.0") - - availabilityZone := ske.NewAvailabilityZone() - availabilityZone.SetName(defaultAvailabilityZone) - - machineType := ske.NewMachineType() - machineType.SetName("g1.2") - - imageVersion := ske.NewMachineImageVersion() - imageVersion.SetVersion("v1") - - machineImage := ske.NewMachineImage() - machineImage.SetName("ubuntu") - machineImage.SetVersions([]ske.MachineImageVersion{*imageVersion}) - - volumeType := ske.NewVolumeType() - volumeType.SetName("storage") - - providerOptions := ske.NewProviderOptions() - providerOptions.SetKubernetesVersions([]ske.KubernetesVersion{*kubernetesVersion}) - providerOptions.SetAvailabilityZones([]ske.AvailabilityZone{*availabilityZone}) - providerOptions.SetMachineTypes([]ske.MachineType{*machineType}) - providerOptions.SetMachineImages([]ske.MachineImage{*machineImage}) - providerOptions.SetVolumeTypes([]ske.VolumeType{*volumeType}) - return providerOptions -} - -func healthyClusterFixture() *ske.Cluster { - cluster := ske.NewClusterWithDefaults() - status := ske.NewClusterStatus() - status.SetAggregated(ske.CLUSTERSTATUSSTATE_HEALTHY) - cluster.SetStatus(*status) - return cluster -} - -func disabledServiceStatusFixture() *serviceenablement.ServiceStatus { - status := serviceenablement.NewServiceStatus() - state := serviceenablement.SERVICESTATUSSTATE_DISABLED - status.State = &state - return status -} - -func projectFixture(name, projectID, containerID string, labels map[string]string) *resourcemanager.Project { - project := resourcemanager.NewProjectWithDefaults() - project.SetName(name) - project.SetProjectId(projectID) - project.SetContainerId(containerID) - project.SetLabels(labels) - return project -} - -func projectResponseFixture(name, projectID, containerID string, labels map[string]string) *resourcemanager.GetProjectResponse { - project := resourcemanager.NewGetProjectResponseWithDefaults() - project.SetName(name) - project.SetProjectId(projectID) - project.SetContainerId(containerID) - project.SetLabels(labels) - return project -} - -func serviceAccountFixture(email, projectID string) *serviceaccount.ServiceAccount { - serviceAccountObject := serviceaccount.NewServiceAccountWithDefaults() - serviceAccountObject.SetEmail(email) - serviceAccountObject.SetProjectId(projectID) - serviceAccountObject.SetId("service-account-id") - serviceAccountObject.SetInternal(false) - return serviceAccountObject -} - -func createServiceAccountKeyResponseFixture(email string) *serviceaccount.CreateServiceAccountKeyResponse { - credentials := serviceaccount.NewCreateServiceAccountKeyResponseCredentials( - "https://accounts.stackit.cloud", - email, - "00000000-0000-0000-0000-000000000001", - "00000000-0000-0000-0000-000000000002", - ) - credentials.SetPrivateKey("PRIVATE") - credentials.SetTokenEndpoint("https://accounts.stackit.cloud/oauth/v2/token") - - return serviceaccount.NewCreateServiceAccountKeyResponse( - true, - time.Unix(0, 0).UTC(), - *credentials, - "00000000-0000-0000-0000-000000000003", - serviceaccount.CREATESERVICEACCOUNTKEYRESPONSEKEYALGORITHM_RSA_2048, - serviceaccount.CREATESERVICEACCOUNTKEYRESPONSEKEYORIGIN_GENERATED, - serviceaccount.CREATESERVICEACCOUNTKEYRESPONSEKEYTYPE_USER_MANAGED, - "PUBLIC KEY", - ) -} - -var _ types.Options = fakeOptions{} diff --git a/test/kubetest2/bootstrap_clients.go b/test/kubetest2/clients.go similarity index 77% rename from test/kubetest2/bootstrap_clients.go rename to test/kubetest2/clients.go index d70eea6c..7ca272be 100644 --- a/test/kubetest2/bootstrap_clients.go +++ b/test/kubetest2/clients.go @@ -4,7 +4,6 @@ import ( "context" "fmt" - "github.com/stackitcloud/cloud-provider-stackit/pkg/metrics" sdkconfig "github.com/stackitcloud/stackit-sdk-go/core/config" authorization "github.com/stackitcloud/stackit-sdk-go/services/authorization/v2api" resourcemanager "github.com/stackitcloud/stackit-sdk-go/services/resourcemanager/v0api" @@ -15,16 +14,16 @@ import ( ) const ( - projectLabelScopeKey = "scope" - projectLabelScopeValue = "PUBLIC" - projectLabelManagedKey = "kt2_managed" - projectLabelManagedValue = "true" - projectLabelRunIDKey = "kt2_run_id" - projectOwnerRole = "owner" - projectResourceType = "project" - childProjectRole = "ske.admin" - projectListPageSize float32 = 100 - skeServiceID = "cloud.stackit.ske" + projectLabelScopeKey = "scope" + projectLabelScopeValue = "PUBLIC" + projectLabelManagedKey = "kt2_managed" + projectLabelManagedValue = "true" + projectLabelRunIDKey = "kt2_run_id" + projectOwnerRole = "owner" + projectResourceType = "project" + childProjectRole = "ske.admin" + projectListPageSize = 100 + skeServiceID = "cloud.stackit.ske" ) type projectClient interface { @@ -68,10 +67,9 @@ type sdkServiceEnablementClient struct { api *serviceenablement.APIClient } -func apiClientOptions(serviceAccountKey, endpoint, apiName string) []sdkconfig.ConfigurationOption { +func apiClientOptions(serviceAccountKey, endpoint string) []sdkconfig.ConfigurationOption { opts := []sdkconfig.ConfigurationOption{ sdkconfig.WithServiceAccountKey(serviceAccountKey), - sdkconfig.WithHTTPClient(metrics.NewInstrumentedHTTPClient(apiName)), } if endpoint != "" { opts = append(opts, sdkconfig.WithEndpoint(endpoint)) @@ -87,7 +85,7 @@ func apiEndpointURL(cfg *sdkconfig.Configuration) string { } func newProjectClient(serviceAccountKey, endpoint string) (projectClient, error) { - apiClient, err := resourcemanager.NewAPIClient(apiClientOptions(serviceAccountKey, endpoint, "resourcemanager")...) + apiClient, err := resourcemanager.NewAPIClient(apiClientOptions(serviceAccountKey, endpoint)...) if err != nil { return nil, fmt.Errorf("create Resource Manager client: %w", err) } @@ -95,7 +93,7 @@ func newProjectClient(serviceAccountKey, endpoint string) (projectClient, error) } func newServiceAccountClient(serviceAccountKey, endpoint string) (serviceAccountClient, error) { - apiClient, err := serviceaccount.NewAPIClient(apiClientOptions(serviceAccountKey, endpoint, "serviceaccount")...) + apiClient, err := serviceaccount.NewAPIClient(apiClientOptions(serviceAccountKey, endpoint)...) if err != nil { return nil, fmt.Errorf("create Service Account client: %w", err) } @@ -103,7 +101,7 @@ func newServiceAccountClient(serviceAccountKey, endpoint string) (serviceAccount } func newAuthorizationClient(serviceAccountKey, endpoint string) (authorizationClient, error) { - apiClient, err := authorization.NewAPIClient(apiClientOptions(serviceAccountKey, endpoint, "authorization")...) + apiClient, err := authorization.NewAPIClient(apiClientOptions(serviceAccountKey, endpoint)...) if err != nil { return nil, fmt.Errorf("create Authorization client: %w", err) } @@ -111,7 +109,7 @@ func newAuthorizationClient(serviceAccountKey, endpoint string) (authorizationCl } func newServiceEnablementClient(serviceAccountKey, endpoint string) (serviceEnablementClient, error) { - apiClient, err := serviceenablement.NewAPIClient(apiClientOptions(serviceAccountKey, endpoint, "serviceenablement")...) + apiClient, err := serviceenablement.NewAPIClient(apiClientOptions(serviceAccountKey, endpoint)...) if err != nil { return nil, fmt.Errorf("create Service Enablement client: %w", err) } @@ -119,23 +117,23 @@ func newServiceEnablementClient(serviceAccountKey, endpoint string) (serviceEnab } func (c *sdkProjectClient) ListProjects(ctx context.Context, parentContainerID string) ([]resourcemanager.Project, error) { - projects := make([]resourcemanager.Project, 0) - var offset float32 + projects := make([]resourcemanager.Project, 0, projectListPageSize) + offset := 0 for { resp, err := c.api.DefaultAPI.ListProjects(ctx). ContainerParentId(parentContainerID). - Offset(offset). - Limit(projectListPageSize). + Offset(float32(offset)). + Limit(float32(projectListPageSize)). Execute() if err != nil { return nil, err } items := resp.GetItems() projects = append(projects, items...) - if len(items) < int(projectListPageSize) { + if len(items) < projectListPageSize { return projects, nil } - offset += float32(len(items)) + offset += len(items) } } @@ -210,3 +208,48 @@ func (c *sdkServiceEnablementClient) WaitForServiceEnabled(ctx context.Context, _, err := serviceenablementwait.EnableServiceWaitHandler(ctx, c.api.DefaultAPI, region, projectID, serviceID).WaitWithContext(ctx) return err } + +func (d *Deployer) initializeBootstrapClients() error { + if err := initializeBootstrapClient(d.projectClient != nil, &d.projectClient, func() (projectClient, error) { + return newProjectClient(d.serviceAccount, d.resourceManagerEndpoint) + }); err != nil { + return err + } + if err := initializeBootstrapClient(d.serviceAccountClient != nil, &d.serviceAccountClient, func() (serviceAccountClient, error) { + return newServiceAccountClient(d.serviceAccount, d.serviceAccountEndpoint) + }); err != nil { + return err + } + if err := initializeBootstrapClient(d.authorizationClient != nil, &d.authorizationClient, func() (authorizationClient, error) { + return newAuthorizationClient(d.serviceAccount, d.authorizationEndpoint) + }); err != nil { + return err + } + if err := initializeBootstrapClient(d.serviceEnablementClient != nil, &d.serviceEnablementClient, func() (serviceEnablementClient, error) { + return newServiceEnablementClient(d.serviceAccount, d.serviceEnablementEndpoint) + }); err != nil { + return err + } + return nil +} + +func initializeBootstrapClient[T any](initialized bool, dst *T, build func() (T, error)) error { + if initialized { + return nil + } + client, err := build() + if err != nil { + return err + } + *dst = client + return nil +} + +func (d *Deployer) initializeSKEClient(serviceAccountKey string) error { + client, err := d.skeClientFactory(d.region, serviceAccountKey, d.skeEndpoint) + if err != nil { + return err + } + d.skeClient = client + return nil +} diff --git a/test/kubetest2/deployer.go b/test/kubetest2/deployer.go index 9725dae6..1d92bc2e 100644 --- a/test/kubetest2/deployer.go +++ b/test/kubetest2/deployer.go @@ -17,9 +17,9 @@ const ( defaultNodeCount int64 = 2 defaultNodepoolName = "default" defaultVolumeSizeGiB int64 = 100 - defaultKubeconfigExpiration int64 = 3600 - minKubeconfigExpiration int64 = 600 - maxKubeconfigExpiration int64 = 15552000 + defaultKubeconfigExpiration int64 = 6 * 60 * 60 // 6 hours + minKubeconfigExpiration int64 = 10 * 60 // 10 minutes + maxKubeconfigExpiration int64 = 180 * 24 * 60 * 60 // 180 days ) type Deployer struct { @@ -37,13 +37,12 @@ type Deployer struct { volumeType string kubeconfigExpiresIn int64 - projectID string - serviceAccount string - parentContainerID string - projectMemberEmail string - childServiceAccountEmail string - kubeconfigPath string - serviceAccountKeyPath string + projectID string + serviceAccount string + parentContainerID string + projectMemberEmail string + kubeconfigPath string + serviceAccountKeyPath string resourceManagerEndpoint string serviceAccountEndpoint string diff --git a/test/kubetest2/env_test.go b/test/kubetest2/env_test.go index 267acdae..212d2de7 100644 --- a/test/kubetest2/env_test.go +++ b/test/kubetest2/env_test.go @@ -1,6 +1,8 @@ package kubetest2 import ( + "path/filepath" + . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" ) @@ -22,3 +24,76 @@ var _ = Describe("extractServiceAccountEmail", func() { Entry("blank credentials.iss", `{"credentials":{"iss":" "}}`, "no email in credentials.iss"), ) }) + +var _ = Describe("loadEnvironment", func() { + envVarKeys := []string{ + "STACKIT_SERVICE_ACCOUNT", + "STACKIT_PARENT_CONTAINER_ID", + "STACKIT_RESOURCE_MANAGER_ENDPOINT", + "STACKIT_SERVICE_ACCOUNT_ENDPOINT", + "STACKIT_AUTHORIZATION_ENDPOINT", + "STACKIT_SERVICE_ENABLEMENT_ENDPOINT", + "STACKIT_SKE_ENDPOINT", + } + + DescribeTable("validates required environment variables", + func(env map[string]string, wantErrContains string) { + for _, key := range envVarKeys { + setEnvVar(key, "") + } + for key, value := range env { + setEnvVar(key, value) + } + + runDir := GinkgoT().TempDir() + d := &Deployer{options: fakeOptions{runID: "run-123", runDir: runDir}} + err := d.loadEnvironment() + if wantErrContains != "" { + Expect(err).To(MatchError(ContainSubstring(wantErrContains))) + return + } + Expect(err).NotTo(HaveOccurred()) + Expect(d.parentContainerID).To(Equal("parent-1")) + Expect(d.projectMemberEmail).To(Equal("owner@example.com")) + Expect(d.projectID).To(BeEmpty()) + Expect(d.kubeconfigPath).To(Equal(filepath.Join(runDir, "kubeconfig"))) + Expect(d.serviceAccountKeyPath).To(Equal(filepath.Join(runDir, "service-account-key.json"))) + }, + Entry("missing service account", map[string]string{ + "STACKIT_PARENT_CONTAINER_ID": "parent-1", + }, "STACKIT_SERVICE_ACCOUNT"), + Entry("missing parent container", map[string]string{ + "STACKIT_SERVICE_ACCOUNT": validServiceAccountKey, + }, "STACKIT_PARENT_CONTAINER_ID"), + Entry("invalid service account key", map[string]string{ + "STACKIT_SERVICE_ACCOUNT": "{}", + "STACKIT_PARENT_CONTAINER_ID": "parent-1", + }, "invalid STACKIT_SERVICE_ACCOUNT"), + Entry("project id no longer required", map[string]string{ + "STACKIT_SERVICE_ACCOUNT": validServiceAccountKey, + "STACKIT_PARENT_CONTAINER_ID": "parent-1", + }, ""), + ) + + It("reads optional endpoints", func() { + for _, key := range envVarKeys { + setEnvVar(key, "") + } + setEnvVar("STACKIT_SERVICE_ACCOUNT", validServiceAccountKey) + setEnvVar("STACKIT_PARENT_CONTAINER_ID", "parent-1") + setEnvVar("STACKIT_RESOURCE_MANAGER_ENDPOINT", "https://resource-manager.example.com") + setEnvVar("STACKIT_SERVICE_ACCOUNT_ENDPOINT", "https://service-account.example.com") + setEnvVar("STACKIT_AUTHORIZATION_ENDPOINT", "https://authorization.example.com") + setEnvVar("STACKIT_SKE_ENDPOINT", "https://ske.example.com") + + runDir := GinkgoT().TempDir() + d := &Deployer{options: fakeOptions{runID: "run-123", runDir: runDir}} + Expect(d.loadEnvironment()).To(Succeed()) + + Expect(d.projectMemberEmail).To(Equal("owner@example.com")) + Expect(d.resourceManagerEndpoint).To(Equal("https://resource-manager.example.com")) + Expect(d.serviceAccountEndpoint).To(Equal("https://service-account.example.com")) + Expect(d.authorizationEndpoint).To(Equal("https://authorization.example.com")) + Expect(d.skeEndpoint).To(Equal("https://ske.example.com")) + }) +}) diff --git a/test/kubetest2/lifecycle.go b/test/kubetest2/lifecycle.go index a30f4dd9..8a35723d 100644 --- a/test/kubetest2/lifecycle.go +++ b/test/kubetest2/lifecycle.go @@ -3,13 +3,18 @@ package kubetest2 import ( "context" "fmt" - "os" + "path/filepath" "github.com/stackitcloud/cloud-provider-stackit/pkg/stackit/stackiterrors" "github.com/stackitcloud/stackit-sdk-go/services/ske" "k8s.io/klog/v2" ) +type bootstrapStep struct { + name string + fn func(context.Context) error +} + func (d *Deployer) Init() error { klog.Infof("Initializing STACKIT kubetest2 deployer") @@ -33,37 +38,20 @@ func (d *Deployer) Up() error { klog.Infof("Starting cluster up flow for cluster=%q", d.clusterName()) ctx := context.Background() - if err := d.ensureManagedClusterAccess(ctx); err != nil { - return err - } - if err := d.validateProviderOptions(ctx); err != nil { - return err - } - - clusterName := d.clusterName() - payload := d.clusterPayload() - - klog.Infof("Submitting SKE create/update request for cluster=%q", clusterName) - if _, err := d.skeClient.CreateOrUpdateCluster(ctx, d.projectID, d.region, clusterName, payload); err != nil { - return fmt.Errorf("create or update SKE cluster %q: %w", clusterName, err) - } - klog.Infof("Submitted SKE create/update request for cluster=%q", clusterName) - - if _, err := d.skeClient.WaitForClusterReady(ctx, d.projectID, d.region, clusterName); err != nil { - return fmt.Errorf("wait for SKE cluster %q to become ready: %w", clusterName, err) - } - - klog.Infof("Creating kubeconfig for cluster=%q", clusterName) - kubeconfig, err := d.skeClient.CreateKubeconfig(ctx, d.projectID, d.region, clusterName, d.kubeconfigExpiresIn) - if err != nil { - return fmt.Errorf("create kubeconfig for SKE cluster %q: %w", clusterName, err) - } - klog.Infof("Writing kubeconfig for cluster=%q to %q", clusterName, d.kubeconfigPath) - if err := os.WriteFile(d.kubeconfigPath, []byte(kubeconfig.GetKubeconfig()), 0o600); err != nil { - return fmt.Errorf("write kubeconfig %q: %w", d.kubeconfigPath, err) + for _, step := range []bootstrapStep{ + {"project", d.ensureProject}, + {"service account", d.ensureServiceAccount}, + {"cluster", d.ensureCluster}, + {"csi", d.ensureCSI}, + } { + klog.Infof("Bootstrap step %q starting", step.name) + if err := step.fn(ctx); err != nil { + return fmt.Errorf("bootstrap step %q: %w", step.name, err) + } + klog.Infof("Bootstrap step %q completed", step.name) } - klog.Infof("Cluster up flow completed successfully for cluster=%q", clusterName) + klog.Infof("Cluster up flow completed successfully for cluster=%q", d.clusterName()) return nil } @@ -149,7 +137,7 @@ func (d *Deployer) IsUp() (bool, error) { func (d *Deployer) Kubeconfig() (string, error) { if d.kubeconfigPath == "" { - d.kubeconfigPath = d.options.RunDir() + "/kubeconfig" + d.kubeconfigPath = filepath.Join(d.options.RunDir(), "kubeconfig") } klog.Infof("Returning kubeconfig path %q", d.kubeconfigPath) return d.kubeconfigPath, nil diff --git a/test/kubetest2/lifecycle_test.go b/test/kubetest2/lifecycle_test.go new file mode 100644 index 00000000..599505ea --- /dev/null +++ b/test/kubetest2/lifecycle_test.go @@ -0,0 +1,160 @@ +package kubetest2 + +import ( + "net/http" + "os" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + oapierror "github.com/stackitcloud/stackit-sdk-go/core/oapierror" + authorization "github.com/stackitcloud/stackit-sdk-go/services/authorization/v2api" + resourcemanager "github.com/stackitcloud/stackit-sdk-go/services/resourcemanager/v0api" + serviceaccount "github.com/stackitcloud/stackit-sdk-go/services/serviceaccount/v2api" + serviceenablement "github.com/stackitcloud/stackit-sdk-go/services/serviceenablement/v2api" + "github.com/stackitcloud/stackit-sdk-go/services/ske" +) + +var _ = Describe("Up", func() { + It("uses discovered project and writes kubeconfig", func() { + d := newTestDeployer() + configureValidUpInputs(d) + + cachedKey := `{"credentials":{"privateKey":"cached"}}` + Expect(os.WriteFile(d.serviceAccountKeyPath, []byte(cachedKey), 0o600)).To(Succeed()) + + d.projectClient = &fakeProjectClient{ + listProjectsResult: []resourcemanager.Project{ + *projectFixture(d.projectName(), "project-123", "container-123", d.managedProjectLabels()), + }, + } + serviceAccountEmail := d.serviceAccountName() + "@sa.stackit.cloud" + d.serviceAccountClient = &fakeServiceAccountClient{ + listResult: []serviceaccount.ServiceAccount{ + *serviceAccountFixture(serviceAccountEmail, "project-123"), + }, + } + d.authorizationClient = &fakeAuthorizationClient{ + listMembersResult: []authorization.Member{ + *authorization.NewMember(childProjectRole, serviceAccountEmail), + }, + } + d.serviceEnablementClient = &fakeServiceEnablementClient{getStatusResult: serviceenablement.NewServiceStatus()} + + fakeSKE := &fakeSKEClient{ + providerOptions: providerOptionsFixture(), + createOrUpdateResult: ske.NewClusterWithDefaults(), + waitReadyResult: ske.NewClusterWithDefaults(), + kubeconfigResult: func() *ske.Kubeconfig { + cfg := ske.NewKubeconfig() + cfg.SetKubeconfig("apiVersion: v1\n") + return cfg + }(), + } + d.skeClientFactory = func(_, serviceAccount, _ string) (skeClient, error) { + Expect(serviceAccount).To(Equal(cachedKey)) + return fakeSKE, nil + } + + Expect(d.Up()).To(Succeed()) + Expect(fakeSKE.lastCreateProjectID).To(Equal("project-123")) + Expect(fakeSKE.lastCreateClusterName).To(Equal(d.clusterName())) + + kubeconfigBytes, err := os.ReadFile(d.kubeconfigPath) + Expect(err).NotTo(HaveOccurred()) + Expect(string(kubeconfigBytes)).To(Equal("apiVersion: v1\n")) + }) +}) + +var _ = Describe("Down", func() { + It("succeeds when project is missing", func() { + d := newTestDeployer() + projectClient := &fakeProjectClient{} + d.projectClient = projectClient + + Expect(d.Down()).To(Succeed()) + Expect(projectClient.deleteCalls).To(Equal(0)) + }) + + It("deletes project", func() { + d := newTestDeployer() + d.projectClient = &fakeProjectClient{ + listProjectsResult: []resourcemanager.Project{ + *projectFixture(d.projectName(), "project-123", "container-123", d.managedProjectLabels()), + }, + } + + Expect(d.Down()).To(Succeed()) + projectClient := d.projectClient.(*fakeProjectClient) + Expect(projectClient.deleteCalls).To(Equal(1)) + Expect(projectClient.waitDeletedCalls).To(Equal(1)) + }) +}) + +var _ = Describe("IsUp", func() { + It("returns false when no project", func() { + d := newTestDeployer() + d.projectClient = &fakeProjectClient{} + + isUp, err := d.IsUp() + Expect(err).NotTo(HaveOccurred()) + Expect(isUp).To(BeFalse()) + }) + + It("returns error when project without cached key", func() { + d := newTestDeployer() + d.projectClient = &fakeProjectClient{ + listProjectsResult: []resourcemanager.Project{ + *projectFixture(d.projectName(), "project-123", "container-123", d.managedProjectLabels()), + }, + } + + _, err := d.IsUp() + Expect(err).To(MatchError(ContainSubstring("child service-account key cache"))) + }) + + It("queries cluster when project with cached key", func() { + d := newTestDeployer() + d.projectClient = &fakeProjectClient{ + listProjectsResult: []resourcemanager.Project{ + *projectFixture(d.projectName(), "project-123", "container-123", d.managedProjectLabels()), + }, + } + cachedKey := `{"credentials":{"privateKey":"cached"}}` + Expect(os.WriteFile(d.serviceAccountKeyPath, []byte(cachedKey), 0o600)).To(Succeed()) + + fakeSKE := &fakeSKEClient{ + getClusterResult: healthyClusterFixture(), + } + d.skeClientFactory = func(_, serviceAccount, _ string) (skeClient, error) { + Expect(serviceAccount).To(Equal(cachedKey)) + return fakeSKE, nil + } + + isUp, err := d.IsUp() + Expect(err).NotTo(HaveOccurred()) + Expect(isUp).To(BeTrue()) + Expect(fakeSKE.lastGetProjectID).To(Equal("project-123")) + }) + + It("returns false when cluster missing", func() { + d := newTestDeployer() + d.projectClient = &fakeProjectClient{ + listProjectsResult: []resourcemanager.Project{ + *projectFixture(d.projectName(), "project-123", "container-123", d.managedProjectLabels()), + }, + } + cachedKey := `{"credentials":{"privateKey":"cached"}}` + Expect(os.WriteFile(d.serviceAccountKeyPath, []byte(cachedKey), 0o600)).To(Succeed()) + + fakeSKE := &fakeSKEClient{ + getClusterErr: &oapierror.GenericOpenAPIError{StatusCode: http.StatusNotFound}, + } + d.skeClientFactory = func(_, _, _ string) (skeClient, error) { + return fakeSKE, nil + } + + isUp, err := d.IsUp() + Expect(err).NotTo(HaveOccurred()) + Expect(isUp).To(BeFalse()) + }) +}) diff --git a/test/kubetest2/naming.go b/test/kubetest2/naming.go index 749e76aa..9a8c1c6b 100644 --- a/test/kubetest2/naming.go +++ b/test/kubetest2/naming.go @@ -5,36 +5,23 @@ import ( "encoding/hex" ) -func clusterNameForRun(runID string) string { - sum := sha256.Sum256([]byte(runID)) - return "kt2" + hex.EncodeToString(sum[:4]) -} - func runTokenForRun(runID string) string { sum := sha256.Sum256([]byte(runID)) return hex.EncodeToString(sum[:4]) } -func projectNameForRun(runID string) string { - return "kt2-" + runTokenForRun(runID) -} - -func serviceAccountNameForRun(runID string) string { - return "kt2-" + runTokenForRun(runID) +func (d *Deployer) runToken() string { + return runTokenForRun(d.options.RunID()) } func (d *Deployer) clusterName() string { - return clusterNameForRun(d.options.RunID()) -} - -func (d *Deployer) runToken() string { - return runTokenForRun(d.options.RunID()) + return "kt2" + d.runToken() } func (d *Deployer) projectName() string { - return projectNameForRun(d.options.RunID()) + return "kt2-" + d.runToken() } func (d *Deployer) serviceAccountName() string { - return serviceAccountNameForRun(d.options.RunID()) + return "kt2-" + d.runToken() } diff --git a/test/kubetest2/ske_client.go b/test/kubetest2/ske_client.go index d56060f6..ae9d6a45 100644 --- a/test/kubetest2/ske_client.go +++ b/test/kubetest2/ske_client.go @@ -16,8 +16,6 @@ type skeClient interface { CreateOrUpdateCluster(ctx context.Context, projectID, region, name string, payload ske.CreateOrUpdateClusterPayload) (*ske.Cluster, error) WaitForClusterReady(ctx context.Context, projectID, region, name string) (*ske.Cluster, error) CreateKubeconfig(ctx context.Context, projectID, region, name string, expirationSeconds int64) (*ske.Kubeconfig, error) - DeleteCluster(ctx context.Context, projectID, region, name string) error - WaitForClusterDeleted(ctx context.Context, projectID, region, name string) error } type sdkSKEClient struct { @@ -27,7 +25,7 @@ type sdkSKEClient struct { func newSKEClient(region, serviceAccount, endpoint string) (skeClient, error) { klog.Infof("Creating SKE API client for region=%q with service_account_bytes=%d", region, len(serviceAccount)) - apiClient, err := ske.NewAPIClient(apiClientOptions(serviceAccount, endpoint, "ske")...) + apiClient, err := ske.NewAPIClient(apiClientOptions(serviceAccount, endpoint)...) if err != nil { return nil, fmt.Errorf("create SKE client: %w", err) } @@ -76,15 +74,3 @@ func (c *sdkSKEClient) CreateKubeconfig(ctx context.Context, projectID, region, payload.SetExpirationSeconds(strconv.FormatInt(expirationSeconds, 10)) return c.api.CreateKubeconfig(ctx, projectID, region, name).CreateKubeconfigPayload(*payload).Execute() } - -func (c *sdkSKEClient) DeleteCluster(ctx context.Context, projectID, region, name string) error { - klog.Infof("SKE DeleteCluster: project_id=%q region=%q cluster=%q", projectID, region, name) - _, err := c.api.DeleteCluster(ctx, projectID, region, name).Execute() - return err -} - -func (c *sdkSKEClient) WaitForClusterDeleted(ctx context.Context, projectID, region, name string) error { - klog.Infof("Waiting for SKE cluster deletion: project_id=%q region=%q cluster=%q", projectID, region, name) - _, err := skewait.DeleteClusterWaitHandler(ctx, c.api, projectID, region, name).WaitWithContext(ctx) - return err -} diff --git a/test/kubetest2/suite_test.go b/test/kubetest2/suite_test.go index 52d96754..f16aed5f 100644 --- a/test/kubetest2/suite_test.go +++ b/test/kubetest2/suite_test.go @@ -1,13 +1,381 @@ package kubetest2 import ( + "context" + "os" + "path/filepath" "testing" + "time" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" + authorization "github.com/stackitcloud/stackit-sdk-go/services/authorization/v2api" + resourcemanager "github.com/stackitcloud/stackit-sdk-go/services/resourcemanager/v0api" + serviceaccount "github.com/stackitcloud/stackit-sdk-go/services/serviceaccount/v2api" + serviceenablement "github.com/stackitcloud/stackit-sdk-go/services/serviceenablement/v2api" + "github.com/stackitcloud/stackit-sdk-go/services/ske" + "sigs.k8s.io/kubetest2/pkg/types" ) func TestKubetest2(t *testing.T) { RegisterFailHandler(Fail) RunSpecs(t, "Kubetest2 Suite") } + +const validServiceAccountKey = `{"credentials":{"iss":"owner@example.com"}}` + +type fakeOptions struct { + runID string + runDir string +} + +func (o fakeOptions) HelpRequested() bool { return false } +func (o fakeOptions) ShouldBuild() bool { return false } +func (o fakeOptions) ShouldUp() bool { return false } +func (o fakeOptions) ShouldDown() bool { return false } +func (o fakeOptions) ShouldTest() bool { return false } +func (o fakeOptions) SkipTestJUnitReport() bool { return false } +func (o fakeOptions) RunID() string { return o.runID } +func (o fakeOptions) RunDir() string { return o.runDir } +func (o fakeOptions) RundirInArtifacts() bool { return false } +func (o fakeOptions) PostTestCmd() []string { return nil } +func (o fakeOptions) PreTestCmd() []string { return nil } + +type fakeProjectClient struct { + listProjectsResult []resourcemanager.Project + listProjectsErr error + + createProjectResult *resourcemanager.Project + createProjectErr error + waitActiveResult *resourcemanager.GetProjectResponse + waitActiveErr error + + deleteErr error + waitDeletedErr error + + createCalls int + waitActiveCalls int + deleteCalls int + waitDeletedCalls int + + lastListParentContainerID string + lastCreateParentContainerID string + lastCreateName string + lastCreateOwnerEmail string + lastCreateLabels map[string]string + lastWaitActiveContainerID string + lastDeletedProjectID string + lastWaitDeletedProjectID string +} + +func (c *fakeProjectClient) ListProjects(_ context.Context, parentContainerID string) ([]resourcemanager.Project, error) { + c.lastListParentContainerID = parentContainerID + return c.listProjectsResult, c.listProjectsErr +} + +func (c *fakeProjectClient) CreateProject(_ context.Context, parentContainerID, name, ownerEmail string, labels map[string]string) (*resourcemanager.Project, error) { + c.createCalls++ + c.lastCreateParentContainerID = parentContainerID + c.lastCreateName = name + c.lastCreateOwnerEmail = ownerEmail + c.lastCreateLabels = labels + return c.createProjectResult, c.createProjectErr +} + +func (c *fakeProjectClient) WaitForProjectActive(_ context.Context, containerID string) (*resourcemanager.GetProjectResponse, error) { + c.waitActiveCalls++ + c.lastWaitActiveContainerID = containerID + return c.waitActiveResult, c.waitActiveErr +} + +func (c *fakeProjectClient) DeleteProject(_ context.Context, projectID string) error { + c.deleteCalls++ + c.lastDeletedProjectID = projectID + return c.deleteErr +} + +func (c *fakeProjectClient) WaitForProjectDeleted(_ context.Context, projectID string) error { + c.waitDeletedCalls++ + c.lastWaitDeletedProjectID = projectID + return c.waitDeletedErr +} + +type fakeServiceAccountClient struct { + listResult []serviceaccount.ServiceAccount + listErr error + + createResult *serviceaccount.ServiceAccount + createErr error + + createKeyResult *serviceaccount.CreateServiceAccountKeyResponse + createKeyErr error + + createCalls int + createKeyCalls int + + lastProjectIDForList string + lastProjectIDForCreate string + lastCreatedName string + lastProjectIDForCreateKey string + lastCreateKeyEmail string +} + +func (c *fakeServiceAccountClient) ListServiceAccounts(_ context.Context, projectID string) ([]serviceaccount.ServiceAccount, error) { + c.lastProjectIDForList = projectID + return c.listResult, c.listErr +} + +func (c *fakeServiceAccountClient) CreateServiceAccount(_ context.Context, projectID, name string) (*serviceaccount.ServiceAccount, error) { + c.createCalls++ + c.lastProjectIDForCreate = projectID + c.lastCreatedName = name + return c.createResult, c.createErr +} + +func (c *fakeServiceAccountClient) CreateServiceAccountKey(_ context.Context, projectID, serviceAccountEmail string) (*serviceaccount.CreateServiceAccountKeyResponse, error) { + c.createKeyCalls++ + c.lastProjectIDForCreateKey = projectID + c.lastCreateKeyEmail = serviceAccountEmail + return c.createKeyResult, c.createKeyErr +} + +type fakeAuthorizationClient struct { + listMembersResult []authorization.Member + listMembersErr error + addMembersErr error + + addCalls int + + lastResourceType string + lastResourceID string + lastAddedType string + lastAddedID string + lastAddedMembers []authorization.Member +} + +func (c *fakeAuthorizationClient) ListMembers(_ context.Context, resourceType, resourceID string) ([]authorization.Member, error) { + c.lastResourceType = resourceType + c.lastResourceID = resourceID + return c.listMembersResult, c.listMembersErr +} + +func (c *fakeAuthorizationClient) AddMembers(_ context.Context, resourceID, resourceType string, members []authorization.Member) error { + c.addCalls++ + c.lastAddedID = resourceID + c.lastAddedType = resourceType + c.lastAddedMembers = members + return c.addMembersErr +} + +type fakeServiceEnablementClient struct { + getStatusResult *serviceenablement.ServiceStatus + getStatusErr error + enableErr error + waitErr error + + enableCalls int + waitCalls int + + lastGetStatusRegion string + lastGetStatusProjectID string + lastGetStatusServiceID string + lastEnableRegion string + lastEnableProjectID string + lastEnableServiceID string + lastWaitRegion string + lastWaitProjectID string + lastWaitServiceID string +} + +func (c *fakeServiceEnablementClient) GetServiceStatus(_ context.Context, region, projectID, serviceID string) (*serviceenablement.ServiceStatus, error) { + c.lastGetStatusRegion = region + c.lastGetStatusProjectID = projectID + c.lastGetStatusServiceID = serviceID + return c.getStatusResult, c.getStatusErr +} + +func (c *fakeServiceEnablementClient) EnableService(_ context.Context, region, projectID, serviceID string) error { + c.enableCalls++ + c.lastEnableRegion = region + c.lastEnableProjectID = projectID + c.lastEnableServiceID = serviceID + return c.enableErr +} + +func (c *fakeServiceEnablementClient) WaitForServiceEnabled(_ context.Context, region, projectID, serviceID string) error { + c.waitCalls++ + c.lastWaitRegion = region + c.lastWaitProjectID = projectID + c.lastWaitServiceID = serviceID + return c.waitErr +} + +type fakeSKEClient struct { + providerOptions *ske.ProviderOptions + providerOptionsErr error + getClusterResult *ske.Cluster + getClusterErr error + + createOrUpdateResult *ske.Cluster + createOrUpdateErr error + waitReadyResult *ske.Cluster + waitReadyErr error + kubeconfigResult *ske.Kubeconfig + kubeconfigErr error + + lastGetProjectID string + lastCreateProjectID string + lastCreateRegion string + lastCreateClusterName string + lastKubeconfigProjectID string + lastExpirationSeconds int64 +} + +func (c *fakeSKEClient) GetCluster(_ context.Context, projectID, _, _ string) (*ske.Cluster, error) { + c.lastGetProjectID = projectID + return c.getClusterResult, c.getClusterErr +} + +func (c *fakeSKEClient) ListProviderOptions(_ context.Context, _ string) (*ske.ProviderOptions, error) { + return c.providerOptions, c.providerOptionsErr +} + +func (c *fakeSKEClient) CreateOrUpdateCluster(_ context.Context, projectID, region, name string, _ ske.CreateOrUpdateClusterPayload) (*ske.Cluster, error) { + c.lastCreateProjectID = projectID + c.lastCreateRegion = region + c.lastCreateClusterName = name + return c.createOrUpdateResult, c.createOrUpdateErr +} + +func (c *fakeSKEClient) WaitForClusterReady(_ context.Context, _, _, _ string) (*ske.Cluster, error) { + return c.waitReadyResult, c.waitReadyErr +} + +func (c *fakeSKEClient) CreateKubeconfig(_ context.Context, projectID, _, _ string, expirationSeconds int64) (*ske.Kubeconfig, error) { + c.lastKubeconfigProjectID = projectID + c.lastExpirationSeconds = expirationSeconds + return c.kubeconfigResult, c.kubeconfigErr +} + +func newTestDeployer() *Deployer { + runDir := GinkgoT().TempDir() + return &Deployer{ + options: fakeOptions{runID: "run-123", runDir: runDir}, + region: defaultRegion, + parentContainerID: "parent-123", + projectMemberEmail: "owner@example.com", + kubeconfigPath: filepath.Join(runDir, "kubeconfig"), + serviceAccountKeyPath: filepath.Join(runDir, "service-account-key.json"), + skeClientFactory: newSKEClient, + } +} + +func configureValidUpInputs(d *Deployer) { + d.kubernetesVersion = "1.31.0" + d.availabilityZone = defaultAvailabilityZone + d.machineType = "g1.2" + d.nodeImageName = "ubuntu" + d.nodeImageVersion = "v1" + d.nodepoolName = defaultNodepoolName + d.volumeType = "storage" +} + +func setEnvVar(key, value string) { + Expect(os.Setenv(key, value)).To(Succeed()) + DeferCleanup(os.Unsetenv, key) +} + +func providerOptionsFixture() *ske.ProviderOptions { + kubernetesVersion := ske.NewKubernetesVersion() + kubernetesVersion.SetVersion("1.31.0") + + availabilityZone := ske.NewAvailabilityZone() + availabilityZone.SetName(defaultAvailabilityZone) + + machineType := ske.NewMachineType() + machineType.SetName("g1.2") + + imageVersion := ske.NewMachineImageVersion() + imageVersion.SetVersion("v1") + + machineImage := ske.NewMachineImage() + machineImage.SetName("ubuntu") + machineImage.SetVersions([]ske.MachineImageVersion{*imageVersion}) + + volumeType := ske.NewVolumeType() + volumeType.SetName("storage") + + providerOptions := ske.NewProviderOptions() + providerOptions.SetKubernetesVersions([]ske.KubernetesVersion{*kubernetesVersion}) + providerOptions.SetAvailabilityZones([]ske.AvailabilityZone{*availabilityZone}) + providerOptions.SetMachineTypes([]ske.MachineType{*machineType}) + providerOptions.SetMachineImages([]ske.MachineImage{*machineImage}) + providerOptions.SetVolumeTypes([]ske.VolumeType{*volumeType}) + return providerOptions +} + +func healthyClusterFixture() *ske.Cluster { + cluster := ske.NewClusterWithDefaults() + status := ske.NewClusterStatus() + status.SetAggregated(ske.CLUSTERSTATUSSTATE_HEALTHY) + cluster.SetStatus(*status) + return cluster +} + +func disabledServiceStatusFixture() *serviceenablement.ServiceStatus { + status := serviceenablement.NewServiceStatus() + state := serviceenablement.SERVICESTATUSSTATE_DISABLED + status.State = &state + return status +} + +func projectFixture(name, projectID, containerID string, labels map[string]string) *resourcemanager.Project { + project := resourcemanager.NewProjectWithDefaults() + project.SetName(name) + project.SetProjectId(projectID) + project.SetContainerId(containerID) + project.SetLabels(labels) + return project +} + +func projectResponseFixture(name, projectID, containerID string, labels map[string]string) *resourcemanager.GetProjectResponse { + project := resourcemanager.NewGetProjectResponseWithDefaults() + project.SetName(name) + project.SetProjectId(projectID) + project.SetContainerId(containerID) + project.SetLabels(labels) + return project +} + +func serviceAccountFixture(email, projectID string) *serviceaccount.ServiceAccount { + serviceAccountObject := serviceaccount.NewServiceAccountWithDefaults() + serviceAccountObject.SetEmail(email) + serviceAccountObject.SetProjectId(projectID) + serviceAccountObject.SetId("service-account-id") + serviceAccountObject.SetInternal(false) + return serviceAccountObject +} + +func createServiceAccountKeyResponseFixture(email string) *serviceaccount.CreateServiceAccountKeyResponse { + credentials := serviceaccount.NewCreateServiceAccountKeyResponseCredentials( + "https://accounts.stackit.cloud", + email, + "00000000-0000-0000-0000-000000000001", + "00000000-0000-0000-0000-000000000002", + ) + credentials.SetPrivateKey("PRIVATE") + credentials.SetTokenEndpoint("https://accounts.stackit.cloud/oauth/v2/token") + + return serviceaccount.NewCreateServiceAccountKeyResponse( + true, + time.Unix(0, 0).UTC(), + *credentials, + "00000000-0000-0000-0000-000000000003", + serviceaccount.CREATESERVICEACCOUNTKEYRESPONSEKEYALGORITHM_RSA_2048, + serviceaccount.CREATESERVICEACCOUNTKEYRESPONSEKEYORIGIN_GENERATED, + serviceaccount.CREATESERVICEACCOUNTKEYRESPONSEKEYTYPE_USER_MANAGED, + "PUBLIC KEY", + ) +} + +var _ types.Options = fakeOptions{} From 5cde9134972e99eba55ba9448ecb02a4a3328423 Mon Sep 17 00:00:00 2001 From: Felix Breuer Date: Mon, 24 Aug 2026 12:15:22 +0200 Subject: [PATCH 09/11] deploy csi and run parallel tests Signed-off-by: Felix Breuer --- .gitignore | 1 + Makefile | 2 +- docs/testing.md | 2 +- go.mod | 6 + go.sum | 15 + test/kubetest2/bootstrap_csi.go | 538 +++++++++++++++++- test/kubetest2/bootstrap_csi_test.go | 235 ++++++++ test/kubetest2/bootstrap_serviceaccount.go | 81 +-- .../bootstrap_serviceaccount_test.go | 53 +- test/kubetest2/clients.go | 3 +- test/kubetest2/deployer.go | 6 + test/kubetest2/env.go | 4 +- test/kubetest2/env_test.go | 3 + test/kubetest2/flags.go | 8 +- test/kubetest2/flags_test.go | 2 + test/kubetest2/lifecycle_test.go | 3 +- test/kubetest2/suite_test.go | 15 + test/kustomize/cloud-config.yaml | 2 +- test/kustomize/storageclass.yaml | 2 + 19 files changed, 932 insertions(+), 49 deletions(-) create mode 100644 test/kubetest2/bootstrap_csi_test.go diff --git a/.gitignore b/.gitignore index 63550e4f..b85d1921 100644 --- a/.gitignore +++ b/.gitignore @@ -25,4 +25,5 @@ test/e2e/kubeconfig* # kubetest2 _artifacts +_artifacts-parallel _rundir diff --git a/Makefile b/Makefile index 4b1bc2d9..b09ff333 100644 --- a/Makefile +++ b/Makefile @@ -119,7 +119,7 @@ verify-e2e-csi: $(KUBERNETES_TEST) -focus=$(FOCUS) \ -skip=$(SKIP) \ $(KUBERNETES_TEST) -- \ - -storage.testdriver=$(PWD)/test/e2e/csi-plugin/block-storage.yaml + -storage.testdriver=$(PWD)/test/csi-plugin/block-storage.yaml verify-image-stackit-csi-plugin: LOCAL = true verify-image-stackit-csi-plugin: APKO_EXTRA_PACKAGES = busybox diff --git a/docs/testing.md b/docs/testing.md index 97000ba2..550e7ade 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -236,7 +236,7 @@ You can apply this manifest using `kubectl apply -f ` or ensuring The test configuration file allows for granular control over which specific tests are executed. This is useful for debugging or targeting a subset of features. -- **Configuration File:** `test/e2e/csi/block-storage.yaml` +- **Configuration File:** `test/csi-plugin/block-storage.yaml` - **Action:** Modify this YAML file to include or exclude specific test cases, adjust parameters, or change timeout settings for the E2E runs. This allows for focused testing without running the entire suite. ### Full Example diff --git a/go.mod b/go.mod index 52e5256b..29b29be8 100644 --- a/go.mod +++ b/go.mod @@ -37,6 +37,8 @@ require ( k8s.io/mount-utils v0.36.3 k8s.io/utils v0.0.0-20260707023825-cf1189d6abe3 sigs.k8s.io/kubetest2 v0.0.0-20260510045745-558f16b589d1 + sigs.k8s.io/kustomize/api v0.19.0 + sigs.k8s.io/kustomize/kyaml v0.19.0 ) replace k8s.io/cloud-provider => github.com/stackitcloud/cloud-provider v0.36.0-ske-2 @@ -58,6 +60,7 @@ require ( github.com/felixge/httpsnoop v1.0.4 // indirect github.com/fsnotify/fsnotify v1.9.0 // indirect github.com/fxamacker/cbor/v2 v2.9.0 // indirect + github.com/go-errors/errors v1.4.2 // indirect github.com/go-logr/logr v1.4.3 // indirect github.com/go-logr/stdr v1.2.2 // indirect github.com/go-openapi/jsonpointer v0.22.1 // indirect @@ -82,6 +85,7 @@ require ( github.com/google/gnostic-models v0.7.0 // indirect github.com/google/go-cmp v0.7.0 // indirect github.com/google/pprof v0.0.0-20260402051712-545e8a4df936 // indirect + github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 // indirect github.com/grpc-ecosystem/go-grpc-middleware/providers/prometheus v1.1.0 // indirect github.com/grpc-ecosystem/go-grpc-middleware/v2 v2.3.3 // indirect github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.7 // indirect @@ -93,6 +97,7 @@ require ( github.com/moby/term v0.5.2 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee // indirect + github.com/monochromegane/go-gitignore v0.0.0-20200626010858-205db1a8cc00 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect github.com/prometheus/common v0.70.1 // indirect @@ -100,6 +105,7 @@ require ( github.com/stoewer/go-strcase v1.3.1 // indirect github.com/stretchr/objx v0.5.3 // indirect github.com/x448/float16 v0.8.4 // indirect + github.com/xlab/treeprint v1.2.0 // indirect go.etcd.io/etcd/api/v3 v3.6.8 // indirect go.etcd.io/etcd/client/pkg/v3 v3.6.8 // indirect go.etcd.io/etcd/client/v3 v3.6.8 // indirect diff --git a/go.sum b/go.sum index 56f24f10..fc8fd7b9 100644 --- a/go.sum +++ b/go.sum @@ -45,6 +45,8 @@ github.com/gkampitakis/go-diff v1.3.2 h1:Qyn0J9XJSDTgnsgHRdz9Zp24RaJeKMUHg2+PDZZ github.com/gkampitakis/go-diff v1.3.2/go.mod h1:LLgOrpqleQe26cte8s36HTWcTmMEur6OPYerdAAS9tk= github.com/gkampitakis/go-snaps v0.5.15 h1:amyJrvM1D33cPHwVrjo9jQxX8g/7E2wYdZ+01KS3zGE= github.com/gkampitakis/go-snaps v0.5.15/go.mod h1:HNpx/9GoKisdhw9AFOBT1N7DBs9DiHo/hGheFGBZ+mc= +github.com/go-errors/errors v1.4.2 h1:J6MZopCL4uSllY1OfXM374weqZFFItUbrImctkmUxIA= +github.com/go-errors/errors v1.4.2/go.mod h1:sIVyrIiJhuEF+Pj9Ebtd6P/rEYROXFi3BopGUQ5a5Og= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI= github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= @@ -105,6 +107,8 @@ github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/pprof v0.0.0-20260402051712-545e8a4df936 h1:EwtI+Al+DeppwYX2oXJCETMO23COyaKGP6fHVpkpWpg= github.com/google/pprof v0.0.0-20260402051712-545e8a4df936/go.mod h1:MxpfABSjhmINe3F1It9d+8exIHFvUqtLIRCdOGNXqiI= +github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 h1:El6M4kTTCOh6aBiKaUGG7oYTSPP8MxqL4YI3kZKwcP4= +github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510/go.mod h1:pupxD2MaaD3pAXIBCelhxNneeOaAeabZDe5s4K6zSpQ= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/gorilla/websocket v1.5.4-0.20250319132907-e064f32e3674 h1:JeSE6pjso5THxAzdVpqr6/geYxZytqFMBCOtn/ujyeo= @@ -153,6 +157,8 @@ github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJ github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee h1:W5t00kpgFdJifH4BDsTlE89Zl93FEloxaWZfGcifgq8= github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/monochromegane/go-gitignore v0.0.0-20200626010858-205db1a8cc00 h1:n6/2gBQ3RWajuToeY6ZtZTIKv2v7ThUy5KKusIT0yc0= +github.com/monochromegane/go-gitignore v0.0.0-20200626010858-205db1a8cc00/go.mod h1:Pm3mSP3c5uWn86xMLZ5Sa7JB9GsEZySvHYXCTK4E9q4= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/onsi/ginkgo/v2 v2.32.1 h1:6tlvcDm/3sE8lGJbZ4+d4mO3RLy24/tQWOFzVSQNIfw= @@ -173,6 +179,8 @@ github.com/prometheus/procfs v0.21.1/go.mod h1:aB55Cww9pdSJVHk0hUf0inxWyyjPogFIj github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ= github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3 h1:n661drycOFuPLCN3Uc8sB6B/s6Z4t2xvBgU1htSHuq8= +github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3/go.mod h1:A0bzQcvG0E7Rwjx0REVgAGH58e96+X0MeOfepqsbeW4= github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= github.com/soheilhy/cmux v0.1.5 h1:jjzc5WVemNEDTLwv9tlmemhC73tI08BNOIGwBOo10Js= @@ -208,6 +216,7 @@ github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpE github.com/stretchr/objx v0.5.3 h1:jmXUvGomnU1o3W/V5h2VEradbpJDwGrzugQQvL0POH4= github.com/stretchr/objx v0.5.3/go.mod h1:rDQraq+vQZU7Fde9LOZLr8Tax6zZvy4kuNKF+QYS+U0= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= @@ -227,6 +236,8 @@ github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM= github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg= github.com/xiang90/probing v0.0.0-20221125231312-a49e3df8f510 h1:S2dVYn90KE98chqDkyE9Z4N61UnQd+KOfgp5Iu53llk= github.com/xiang90/probing v0.0.0-20221125231312-a49e3df8f510/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= +github.com/xlab/treeprint v1.2.0 h1:HzHnuAF1plUN2zGlAFHbSQP2qJ0ZAD3XF5XD7OesXRQ= +github.com/xlab/treeprint v1.2.0/go.mod h1:gj5Gd3gPdKtR1ikdDK6fnFLdmIS0X30kTTuNd/WEJu0= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= go.etcd.io/bbolt v1.4.3 h1:dEadXpI6G79deX5prL3QRNP6JB8UxVkqo4UPnHaNXJo= @@ -381,6 +392,10 @@ sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730 h1:IpInykpT6ceI+QxKBbEflcR5E sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730/go.mod h1:mdzfpAEoE6DHQEN0uh9ZbOCuHbLK5wOm7dK4ctXE9Tg= sigs.k8s.io/kubetest2 v0.0.0-20260510045745-558f16b589d1 h1:ttox7lK6ZgN/goexDpOxStOl1Q6j4CwprvK+rkS0c1M= sigs.k8s.io/kubetest2 v0.0.0-20260510045745-558f16b589d1/go.mod h1:1IMaKFT+Htrbn6ixEZq6GTCk13CP/S0vwujFC5qs55U= +sigs.k8s.io/kustomize/api v0.19.0 h1:F+2HB2mU1MSiR9Hp1NEgoU2q9ItNOaBJl0I4Dlus5SQ= +sigs.k8s.io/kustomize/api v0.19.0/go.mod h1:/BbwnivGVcBh1r+8m3tH1VNxJmHSk1PzP5fkP6lbL1o= +sigs.k8s.io/kustomize/kyaml v0.19.0 h1:RFge5qsO1uHhwJsu3ipV7RNolC7Uozc0jUBC/61XSlA= +sigs.k8s.io/kustomize/kyaml v0.19.0/go.mod h1:FeKD5jEOH+FbZPpqUghBP8mrLjJ3+zD3/rf9NNu1cwY= sigs.k8s.io/randfill v1.0.0 h1:JfjMILfT8A6RbawdsK2JXGBR5AQVfd+9TbzrlneTyrU= sigs.k8s.io/randfill v1.0.0/go.mod h1:XeLlZ/jmk4i1HRopwe7/aU3H5n1zNUcX6TM94b3QxOY= sigs.k8s.io/structured-merge-diff/v6 v6.3.3 h1:u08YRbVUi59ri4YD6cg0UqNM4Dimn0sIl+wldcx5PYw= diff --git a/test/kubetest2/bootstrap_csi.go b/test/kubetest2/bootstrap_csi.go index 728257ac..82da08d3 100644 --- a/test/kubetest2/bootstrap_csi.go +++ b/test/kubetest2/bootstrap_csi.go @@ -1,15 +1,545 @@ package kubetest2 import ( + "bytes" "context" + "encoding/json" + "fmt" + "io" + "io/fs" + "os" + "path/filepath" + "strings" + "time" + "gopkg.in/yaml.v3" + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/wait" + k8syaml "k8s.io/apimachinery/pkg/util/yaml" + "k8s.io/client-go/discovery" + memory "k8s.io/client-go/discovery/cached/memory" + "k8s.io/client-go/dynamic" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/restmapper" + "k8s.io/client-go/tools/clientcmd" "k8s.io/klog/v2" + "sigs.k8s.io/kustomize/api/krusty" + "sigs.k8s.io/kustomize/kyaml/filesys" +) + +type csiApplier func(ctx context.Context, kubeconfigPath, stagedKustomizeDir string) error + +const ( + csiFieldManager = "kubetest2-stackit" + csiPollInterval = 2 * time.Second + csiRolloutTimeout = 5 * time.Minute + + csiManifestsPath = "test/kustomize" + baseManifestsPath = "deploy/csi-plugin" + csiTestDriverTemplatePath = "test/csi-plugin/block-storage.yaml" + csiTestDriverFileName = "csi-testdriver.yaml" + + kubetest2CSIStorageClassName = "premium-perf4-stackit-kubetest2" + kubetest2CSIDriverName = "kubetest2.csi.stackit.cloud" + + csiConfigMapName = "stackit-cloud-config" + csiConfigMapKey = "cloud.yaml" + csiConfigNamespace = "kube-system" ) // ensureCSI deploys the STACKIT CSI driver into the freshly provisioned -// cluster as the last bootstrap step. Not yet implemented; will install the -// CSI Helm chart here. -func (d *Deployer) ensureCSI(_ context.Context) error { - klog.Infof("CSI deployment not yet implemented, skipping") +// cluster as the last bootstrap step. It stages the Kustomize overlay and base +// manifests into the run directory, applies the necessary image, project, region, +// and credential substitutions, and applies the manifests to the cluster using +// native Go Kubernetes clients. +func (d *Deployer) ensureCSI(ctx context.Context) error { + klog.Infof("Starting CSI deployment into cluster=%q", d.clusterName()) + + serviceAccountKey, ok, err := d.readCachedChildServiceAccountKey() + if err != nil { + return fmt.Errorf("read cached child service-account key %q: %w", d.serviceAccountKeyPath, err) + } + if !ok || strings.TrimSpace(serviceAccountKey) == "" { + return fmt.Errorf("cached child service-account key is missing or empty at %q", d.serviceAccountKeyPath) + } + + stagedKustomizeDir, err := d.stageCSIManifests(serviceAccountKey) + if err != nil { + return fmt.Errorf("stage CSI manifests: %w", err) + } + if _, err := d.stageCSITestDriver(); err != nil { + return fmt.Errorf("stage CSI testdriver: %w", err) + } + + applier := d.csiApplier + if applier == nil { + applier = applyCSIManifestsNative + } + + if err := applier(ctx, d.kubeconfigPath, stagedKustomizeDir); err != nil { + return fmt.Errorf("apply CSI manifests: %w", err) + } + + klog.Infof("CSI deployment completed successfully for cluster=%q", d.clusterName()) + return nil +} + +func applyCSIManifestsNative(ctx context.Context, kubeconfigPath, stagedKustomizeDir string) error { + objects, err := renderKustomize(stagedKustomizeDir) + if err != nil { + return fmt.Errorf("render Kustomize overlay %q: %w", stagedKustomizeDir, err) + } + + restConfig, err := clientcmd.BuildConfigFromFlags("", kubeconfigPath) + if err != nil { + return fmt.Errorf("build rest config from %q: %w", kubeconfigPath, err) + } + + dynClient, err := dynamic.NewForConfig(restConfig) + if err != nil { + return fmt.Errorf("create dynamic client: %w", err) + } + + discoClient, err := discovery.NewDiscoveryClientForConfig(restConfig) + if err != nil { + return fmt.Errorf("create discovery client: %w", err) + } + + cachedDisco := memory.NewMemCacheClient(discoClient) + mapper := restmapper.NewDeferredDiscoveryRESTMapper(cachedDisco) + + klog.Infof("Applying %d rendered Kubernetes objects via server-side apply", len(objects)) + if err := applyUnstructuredObjects(ctx, dynClient, mapper, objects); err != nil { + return fmt.Errorf("apply Kubernetes objects: %w", err) + } + + clientset, err := kubernetes.NewForConfig(restConfig) + if err != nil { + return fmt.Errorf("create kubernetes clientset: %w", err) + } + + klog.Infof("Waiting for CSI controller deployment rollout") + if err := waitForDeploymentReady(ctx, clientset, "kube-system", "csi-stackit-controllerplugin", csiRolloutTimeout); err != nil { + return fmt.Errorf("wait for CSI controller deployment rollout: %w", err) + } + + klog.Infof("Waiting for CSI node daemonset rollout") + if err := waitForDaemonSetReady(ctx, clientset, "kube-system", "csi-stackit-nodeplugin", csiRolloutTimeout); err != nil { + return fmt.Errorf("wait for CSI node daemonset rollout: %w", err) + } + + return nil +} + +func renderKustomize(dir string) ([]*unstructured.Unstructured, error) { + k := krusty.MakeKustomizer(krusty.MakeDefaultOptions()) + fSys := filesys.MakeFsOnDisk() + + resMap, err := k.Run(fSys, dir) + if err != nil { + return nil, fmt.Errorf("kustomize run: %w", err) + } + + yamlBytes, err := resMap.AsYaml() + if err != nil { + return nil, fmt.Errorf("kustomize as yaml: %w", err) + } + + return decodeYAMLToUnstructured(yamlBytes) +} + +func decodeYAMLToUnstructured(yamlBytes []byte) ([]*unstructured.Unstructured, error) { + decoder := k8syaml.NewYAMLOrJSONDecoder(bytes.NewReader(yamlBytes), 4096) + var objects []*unstructured.Unstructured + + for { + var rawObj map[string]interface{} + err := decoder.Decode(&rawObj) + if err != nil { + if err == io.EOF { + break + } + return nil, fmt.Errorf("decode YAML object: %w", err) + } + if len(rawObj) == 0 { + continue + } + + u := &unstructured.Unstructured{Object: rawObj} + if u.GetKind() == "" || u.GetName() == "" { + continue + } + objects = append(objects, u) + } + + return objects, nil +} + +func applyUnstructuredObjects(ctx context.Context, dynClient dynamic.Interface, mapper meta.RESTMapper, objects []*unstructured.Unstructured) error { + for _, obj := range objects { + gvk := obj.GroupVersionKind() + mapping, err := mapper.RESTMapping(gvk.GroupKind(), gvk.Version) + if err != nil { + return fmt.Errorf("get REST mapping for %s: %w", gvk.String(), err) + } + + var ri dynamic.ResourceInterface + if mapping.Scope.Name() == meta.RESTScopeNameNamespace { + ns := obj.GetNamespace() + if ns == "" { + ns = metav1.NamespaceDefault + } + ri = dynClient.Resource(mapping.Resource).Namespace(ns) + } else { + ri = dynClient.Resource(mapping.Resource) + } + + objJSON, err := json.Marshal(obj) + if err != nil { + return fmt.Errorf("marshal object %s/%s to JSON: %w", gvk.Kind, obj.GetName(), err) + } + + force := true + applyOpts := metav1.PatchOptions{ + FieldManager: csiFieldManager, + Force: &force, + } + + klog.Infof("Server-side applying %s %s/%s", gvk.Kind, obj.GetNamespace(), obj.GetName()) + if _, err := ri.Patch(ctx, obj.GetName(), types.ApplyPatchType, objJSON, applyOpts); err != nil { + return fmt.Errorf("server-side apply %s %s/%s: %w", gvk.Kind, obj.GetNamespace(), obj.GetName(), err) + } + } + return nil +} + +func waitForDeploymentReady(ctx context.Context, clientset kubernetes.Interface, namespace, name string, timeout time.Duration) error { + return wait.PollUntilContextTimeout(ctx, csiPollInterval, timeout, true, func(ctx context.Context) (bool, error) { + deploy, err := clientset.AppsV1().Deployments(namespace).Get(ctx, name, metav1.GetOptions{}) + if err != nil { + if apierrors.IsNotFound(err) { + return false, nil + } + return false, err + } + + specReplicas := int32(1) + if deploy.Spec.Replicas != nil { + specReplicas = *deploy.Spec.Replicas + } + + if deploy.Status.ObservedGeneration >= deploy.Generation && + deploy.Status.UpdatedReplicas == specReplicas && + deploy.Status.AvailableReplicas == specReplicas && + deploy.Status.UnavailableReplicas == 0 { + return true, nil + } + return false, nil + }) +} + +func waitForDaemonSetReady(ctx context.Context, clientset kubernetes.Interface, namespace, name string, timeout time.Duration) error { + return wait.PollUntilContextTimeout(ctx, csiPollInterval, timeout, true, func(ctx context.Context) (bool, error) { + ds, err := clientset.AppsV1().DaemonSets(namespace).Get(ctx, name, metav1.GetOptions{}) + if err != nil { + if apierrors.IsNotFound(err) { + return false, nil + } + return false, err + } + + if ds.Status.ObservedGeneration >= ds.Generation && + ds.Status.DesiredNumberScheduled > 0 && + ds.Status.UpdatedNumberScheduled == ds.Status.DesiredNumberScheduled && + ds.Status.NumberAvailable == ds.Status.DesiredNumberScheduled && + ds.Status.NumberUnavailable == 0 { + return true, nil + } + return false, nil + }) +} + +func resolveManifestPath(relPath string) (string, error) { + candidates := []string{ + relPath, + filepath.Join("..", "..", relPath), + } + + for _, candidate := range candidates { + if _, err := os.Stat(candidate); err == nil { + return filepath.Abs(candidate) + } + } + + return "", fmt.Errorf("manifest path %q not found (tried %v)", relPath, candidates) +} + +func (d *Deployer) stageCSIManifests(serviceAccountKey string) (string, error) { + stagingRoot := filepath.Join(d.options.RunDir(), "manifests") + + stagedBaseDir := filepath.Join(stagingRoot, "deploy", "csi-plugin") + stagedKustomizeDir := filepath.Join(stagingRoot, "test", "kustomize") + + baseSrc, err := resolveManifestPath(baseManifestsPath) + if err != nil { + return "", fmt.Errorf("resolve base manifests path: %w", err) + } + + csiSrc, err := resolveManifestPath(csiManifestsPath) + if err != nil { + return "", fmt.Errorf("resolve CSI overlay manifests path: %w", err) + } + + if err := os.RemoveAll(stagingRoot); err != nil { + return "", fmt.Errorf("clean existing manifests staging dir %q: %w", stagingRoot, err) + } + + if err := copyDir(baseSrc, stagedBaseDir); err != nil { + return "", fmt.Errorf("copy base CSI manifests from %q to %q: %w", baseSrc, stagedBaseDir, err) + } + + if err := copyDir(csiSrc, stagedKustomizeDir); err != nil { + return "", fmt.Errorf("copy CSI overlay manifests from %q to %q: %w", csiSrc, stagedKustomizeDir, err) + } + + // 1. Replace image and tag in kustomization.yaml + kustomizationPath := filepath.Join(stagedKustomizeDir, "kustomization.yaml") + if err := replaceInFile(kustomizationPath, map[string]string{ + "REPLACE_WITH_IMAGE_NAME": d.csiImageName, + "REPLACE_WITH_TAG": d.csiImageTag, + }); err != nil { + return "", fmt.Errorf("substitute values in %q: %w", kustomizationPath, err) + } + + cloudConfigPath := filepath.Join(stagedKustomizeDir, "cloud-config.yaml") + cloudConfigContent, err := renderCSICloudConfigManifest(d.projectID, d.region, d.iaasEndpoint) + if err != nil { + return "", fmt.Errorf("render CSI cloud config manifest: %w", err) + } + if err := os.WriteFile(cloudConfigPath, cloudConfigContent, 0o600); err != nil { + return "", fmt.Errorf("write CSI cloud config manifest %q: %w", cloudConfigPath, err) + } + + cloudSecretPath := filepath.Join(stagedKustomizeDir, "cloud-secret.yaml") + if err := replaceInFile(cloudSecretPath, map[string]string{ + "REPLACE_WITH_SERVICEACCOUNT_JSON": serviceAccountKey, + }); err != nil { + return "", fmt.Errorf("substitute values in %q: %w", cloudSecretPath, err) + } + + return stagedKustomizeDir, nil +} + +func (d *Deployer) stageCSITestDriver() (string, error) { + templatePath, err := resolveManifestPath(csiTestDriverTemplatePath) + if err != nil { + return "", fmt.Errorf("resolve CSI testdriver template path: %w", err) + } + + content, err := renderCSITestDriverConfig(templatePath) + if err != nil { + return "", err + } + + outputPath := filepath.Join(d.options.RunDir(), csiTestDriverFileName) + if err := os.WriteFile(outputPath, content, 0o600); err != nil { + return "", fmt.Errorf("write CSI testdriver config %q: %w", outputPath, err) + } + + return outputPath, nil +} + +func renderCSITestDriverConfig(templatePath string) ([]byte, error) { + content, err := os.ReadFile(templatePath) + if err != nil { + return nil, fmt.Errorf("read CSI testdriver template %q: %w", templatePath, err) + } + + var config csiTestDriverConfig + if err := yaml.Unmarshal(content, &config); err != nil { + return nil, fmt.Errorf("decode CSI testdriver template %q: %w", templatePath, err) + } + + config.StorageClass.FromExistingClassName = kubetest2CSIStorageClassName + config.DriverInfo.Name = kubetest2CSIDriverName + + rendered, err := yaml.Marshal(&config) + if err != nil { + return nil, fmt.Errorf("encode CSI testdriver config: %w", err) + } + + return rendered, nil +} + +type csiTestDriverConfig struct { + StorageClass csiTestClassConfig `yaml:"StorageClass"` + SnapshotClass csiTestClassConfig `yaml:"SnapshotClass"` + DriverInfo csiTestDriverInfo `yaml:"DriverInfo"` +} + +type csiTestClassConfig struct { + FromExistingClassName string `yaml:"FromExistingClassName"` +} + +type csiTestDriverInfo struct { + Name string `yaml:"Name"` + SupportedFsType map[string]map[string]any `yaml:"SupportedFsType,omitempty"` + Capabilities map[string]bool `yaml:"Capabilities,omitempty"` +} + +type csiCloudConfigManifest struct { + APIVersion string `yaml:"apiVersion"` + Kind string `yaml:"kind"` + Metadata csiCloudConfigMetadata `yaml:"metadata"` + Data map[string]string `yaml:"data"` +} + +type csiCloudConfigMetadata struct { + Name string `yaml:"name"` + Namespace string `yaml:"namespace"` +} + +type csiCloudConfig struct { + Global csiCloudConfigGlobal `yaml:"global"` + BlockStorage csiCloudConfigBlockStorage `yaml:"blockStorage"` +} + +type csiCloudConfigGlobal struct { + ProjectID string `yaml:"projectId"` + Region string `yaml:"region"` + APIEndpoints *csiCloudConfigAPIEndpoints `yaml:"apiEndpoints,omitempty"` +} + +type csiCloudConfigAPIEndpoints struct { + IaaSAPI string `yaml:"iaasApi"` +} + +type csiCloudConfigBlockStorage struct { + RescanOnResize bool `yaml:"rescanOnResize"` +} + +func renderCSICloudConfigManifest(projectID, region, iaasEndpoint string) ([]byte, error) { + cloudConfig := csiCloudConfig{ + Global: csiCloudConfigGlobal{ + ProjectID: projectID, + Region: region, + }, + BlockStorage: csiCloudConfigBlockStorage{ + RescanOnResize: true, + }, + } + if iaasEndpoint != "" { + cloudConfig.Global.APIEndpoints = &csiCloudConfigAPIEndpoints{ + IaaSAPI: iaasEndpoint, + } + } + + cloudConfigBytes, err := yaml.Marshal(&cloudConfig) + if err != nil { + return nil, fmt.Errorf("encode CSI cloud config: %w", err) + } + + manifest := csiCloudConfigManifest{ + APIVersion: "v1", + Kind: "ConfigMap", + Metadata: csiCloudConfigMetadata{ + Name: csiConfigMapName, + Namespace: csiConfigNamespace, + }, + Data: map[string]string{ + csiConfigMapKey: string(cloudConfigBytes), + }, + } + + rendered, err := yaml.Marshal(&manifest) + if err != nil { + return nil, fmt.Errorf("encode CSI cloud config manifest: %w", err) + } + + return rendered, nil +} + +func replaceInFile(path string, replacements map[string]string) error { + contentBytes, err := os.ReadFile(path) + if err != nil { + return fmt.Errorf("read file %q: %w", path, err) + } + + content := string(contentBytes) + for oldVal, newVal := range replacements { + content = strings.ReplaceAll(content, oldVal, newVal) + } + + if err := os.WriteFile(path, []byte(content), 0o600); err != nil { + return fmt.Errorf("write file %q: %w", path, err) + } + return nil +} + +func copyDir(src, dst string) error { + srcInfo, err := os.Stat(src) + if err != nil { + return err + } + + if err := os.MkdirAll(dst, srcInfo.Mode()); err != nil { + return err + } + + return filepath.WalkDir(src, func(path string, d fs.DirEntry, walkErr error) error { + if walkErr != nil { + return walkErr + } + + relPath, err := filepath.Rel(src, path) + if err != nil { + return err + } + if relPath == "." { + return nil + } + + targetPath := filepath.Join(dst, relPath) + if d.IsDir() { + dirInfo, err := d.Info() + if err != nil { + return err + } + return os.MkdirAll(targetPath, dirInfo.Mode()) + } + + return copyFile(path, targetPath) + }) +} + +func copyFile(src, dst string) error { + srcFile, err := os.Open(src) + if err != nil { + return err + } + defer srcFile.Close() + + srcInfo, err := srcFile.Stat() + if err != nil { + return err + } + + if err := os.MkdirAll(filepath.Dir(dst), 0o755); err != nil { + return err + } + + dstFile, err := os.OpenFile(dst, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, srcInfo.Mode()) + if err != nil { + return err + } + defer dstFile.Close() + + if _, err := io.Copy(dstFile, srcFile); err != nil { + return err + } return nil } diff --git a/test/kubetest2/bootstrap_csi_test.go b/test/kubetest2/bootstrap_csi_test.go new file mode 100644 index 00000000..82d9b82e --- /dev/null +++ b/test/kubetest2/bootstrap_csi_test.go @@ -0,0 +1,235 @@ +package kubetest2 + +import ( + "context" + "os" + "path/filepath" + "strings" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + appsv1 "k8s.io/api/apps/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes/fake" +) + +var _ = Describe("ensureCSI", func() { + It("stages manifests with rendered cloud config and calls csiApplier", func() { + d := newTestDeployer() + configureValidUpInputs(d) + d.projectID = "project-123" + d.iaasEndpoint = "https://iaas.api.qa.stackit.cloud" + + serviceAccountKey := `{"credentials":{"privateKey":"dummy-private-key"}}` + Expect(os.WriteFile(d.serviceAccountKeyPath, []byte(serviceAccountKey), 0o600)).To(Succeed()) + + var recordedKubeconfig string + var recordedStagedDir string + d.csiApplier = func(_ context.Context, kubeconfigPath, stagedKustomizeDir string) error { + recordedKubeconfig = kubeconfigPath + recordedStagedDir = stagedKustomizeDir + return nil + } + + Expect(d.ensureCSI(context.Background())).To(Succeed()) + + Expect(recordedKubeconfig).To(Equal(d.kubeconfigPath)) + Expect(recordedStagedDir).To(Equal(filepath.Join(d.options.RunDir(), "manifests", "test", "kustomize"))) + + // Verify substitutions in staged files + stagedKustomizeDir := filepath.Join(d.options.RunDir(), "manifests", "test", "kustomize") + + kustomizationBytes, err := os.ReadFile(filepath.Join(stagedKustomizeDir, "kustomization.yaml")) + Expect(err).NotTo(HaveOccurred()) + Expect(string(kustomizationBytes)).To(ContainSubstring("newName: " + d.csiImageName)) + Expect(string(kustomizationBytes)).To(ContainSubstring("newTag: " + d.csiImageTag)) + Expect(string(kustomizationBytes)).NotTo(ContainSubstring("REPLACE_WITH_IMAGE_NAME")) + Expect(string(kustomizationBytes)).NotTo(ContainSubstring("REPLACE_WITH_TAG")) + + cloudConfigBytes, err := os.ReadFile(filepath.Join(stagedKustomizeDir, "cloud-config.yaml")) + Expect(err).NotTo(HaveOccurred()) + Expect(string(cloudConfigBytes)).To(ContainSubstring("projectId: project-123")) + Expect(string(cloudConfigBytes)).To(ContainSubstring("region: " + d.region)) + Expect(string(cloudConfigBytes)).To(ContainSubstring("apiEndpoints:")) + Expect(string(cloudConfigBytes)).To(ContainSubstring("iaasApi: " + d.iaasEndpoint)) + Expect(string(cloudConfigBytes)).To(ContainSubstring("name: " + csiConfigMapName)) + Expect(string(cloudConfigBytes)).To(ContainSubstring("namespace: " + csiConfigNamespace)) + Expect(string(cloudConfigBytes)).NotTo(ContainSubstring("REPLACE_WITH_PROJECTID")) + Expect(string(cloudConfigBytes)).NotTo(ContainSubstring("REPLACE_WITH_OPTIONAL_IAAS_API")) + + cloudSecretBytes, err := os.ReadFile(filepath.Join(stagedKustomizeDir, "cloud-secret.yaml")) + Expect(err).NotTo(HaveOccurred()) + Expect(string(cloudSecretBytes)).To(ContainSubstring(serviceAccountKey)) + Expect(string(cloudSecretBytes)).NotTo(ContainSubstring("REPLACE_WITH_SERVICEACCOUNT_JSON")) + + testDriverBytes, err := os.ReadFile(filepath.Join(d.options.RunDir(), csiTestDriverFileName)) + Expect(err).NotTo(HaveOccurred()) + Expect(string(testDriverBytes)).To(ContainSubstring("FromExistingClassName: " + kubetest2CSIStorageClassName)) + Expect(string(testDriverBytes)).To(ContainSubstring("Name: " + kubetest2CSIDriverName)) + Expect(string(testDriverBytes)).NotTo(ContainSubstring("FromExistingClassName: premium-perf4-stackit\n")) + Expect(string(testDriverBytes)).NotTo(ContainSubstring("Name: block-storage.csi.stackit.cloud")) + + // Verify that in-process renderKustomize parses the staged overlay correctly into unstructured objects + objects, err := renderKustomize(stagedKustomizeDir) + Expect(err).NotTo(HaveOccurred()) + Expect(objects).NotTo(BeEmpty()) + + var foundControllerPlugin, foundNodePlugin bool + for _, obj := range objects { + if obj.GetKind() == "Deployment" && obj.GetName() == "csi-stackit-controllerplugin" { + foundControllerPlugin = true + } + if obj.GetKind() == "DaemonSet" && obj.GetName() == "csi-stackit-nodeplugin" { + foundNodePlugin = true + } + } + Expect(foundControllerPlugin).To(BeTrue(), "expected csi-stackit-controllerplugin in rendered objects") + Expect(foundNodePlugin).To(BeTrue(), "expected csi-stackit-nodeplugin in rendered objects") + }) + + It("does not write iaas endpoint override when not set", func() { + d := newTestDeployer() + configureValidUpInputs(d) + d.projectID = "project-123" + + serviceAccountKey := `{"credentials":{"privateKey":"dummy-private-key"}}` + Expect(os.WriteFile(d.serviceAccountKeyPath, []byte(serviceAccountKey), 0o600)).To(Succeed()) + + d.csiApplier = func(_ context.Context, _, _ string) error { + return nil + } + + Expect(d.ensureCSI(context.Background())).To(Succeed()) + + cloudConfigBytes, err := os.ReadFile(filepath.Join(d.options.RunDir(), "manifests", "test", "kustomize", "cloud-config.yaml")) + Expect(err).NotTo(HaveOccurred()) + Expect(string(cloudConfigBytes)).NotTo(ContainSubstring("apiEndpoints:")) + Expect(string(cloudConfigBytes)).NotTo(ContainSubstring("iaasApi:")) + cloudSecretBytes, err := os.ReadFile(filepath.Join(d.options.RunDir(), "manifests", "test", "kustomize", "cloud-secret.yaml")) + Expect(err).NotTo(HaveOccurred()) + Expect(string(cloudSecretBytes)).To(ContainSubstring(serviceAccountKey)) + }) + + It("fails when cached service account key is missing", func() { + d := newTestDeployer() + configureValidUpInputs(d) + d.projectID = "project-123" + + _ = os.Remove(d.serviceAccountKeyPath) + + err := d.ensureCSI(context.Background()) + Expect(err).To(MatchError(ContainSubstring("cached child service-account key is missing or empty"))) + }) + + It("fails when csiApplier returns an error", func() { + d := newTestDeployer() + configureValidUpInputs(d) + d.projectID = "project-123" + + serviceAccountKey := `{"credentials":{"privateKey":"dummy"}}` + Expect(os.WriteFile(d.serviceAccountKeyPath, []byte(serviceAccountKey), 0o600)).To(Succeed()) + + d.csiApplier = func(_ context.Context, _, _ string) error { + return os.ErrInvalid + } + + err := d.ensureCSI(context.Background()) + Expect(err).To(MatchError(ContainSubstring("apply CSI manifests"))) + }) +}) + +var _ = Describe("renderCSICloudConfigManifest", func() { + It("renders the optional iaas endpoint only when configured", func() { + rendered, err := renderCSICloudConfigManifest("project-123", "eu01", "https://iaas.example.com") + Expect(err).NotTo(HaveOccurred()) + + text := string(rendered) + Expect(text).To(ContainSubstring("name: " + csiConfigMapName)) + Expect(text).To(ContainSubstring("namespace: " + csiConfigNamespace)) + Expect(text).To(ContainSubstring("projectId: project-123")) + Expect(text).To(ContainSubstring("region: eu01")) + Expect(text).To(ContainSubstring("apiEndpoints:")) + Expect(text).To(ContainSubstring("iaasApi: https://iaas.example.com")) + Expect(text).To(ContainSubstring("rescanOnResize: true")) + }) + + It("omits apiEndpoints when no iaas endpoint override is configured", func() { + rendered, err := renderCSICloudConfigManifest("project-123", "eu01", "") + Expect(err).NotTo(HaveOccurred()) + + text := string(rendered) + Expect(text).To(ContainSubstring("projectId: project-123")) + Expect(text).To(ContainSubstring("region: eu01")) + Expect(text).NotTo(ContainSubstring("apiEndpoints:")) + Expect(text).NotTo(ContainSubstring("iaasApi:")) + }) +}) + +var _ = Describe("renderCSITestDriverConfig", func() { + It("rewrites the checked-in testdriver for the kubetest2 CSI overlay", func() { + templatePath, err := resolveManifestPath(csiTestDriverTemplatePath) + Expect(err).NotTo(HaveOccurred()) + + rendered, err := renderCSITestDriverConfig(templatePath) + Expect(err).NotTo(HaveOccurred()) + + text := string(rendered) + Expect(text).To(ContainSubstring("FromExistingClassName: " + kubetest2CSIStorageClassName)) + Expect(text).To(ContainSubstring("Name: " + kubetest2CSIDriverName)) + Expect(text).To(ContainSubstring("SnapshotClass:")) + Expect(text).To(ContainSubstring("SupportedFsType:")) + Expect(text).To(ContainSubstring("controllerExpansion: true")) + Expect(strings.Count(text, kubetest2CSIStorageClassName)).To(Equal(1)) + }) +}) + +var _ = Describe("waitForDeploymentReady", func() { + It("succeeds when deployment is ready", func() { + replicas := int32(1) + deploy := &appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{ + Name: "csi-stackit-controllerplugin", + Namespace: "kube-system", + Generation: 1, + }, + Spec: appsv1.DeploymentSpec{ + Replicas: &replicas, + }, + Status: appsv1.DeploymentStatus{ + ObservedGeneration: 1, + UpdatedReplicas: 1, + AvailableReplicas: 1, + UnavailableReplicas: 0, + }, + } + + fakeClientset := fake.NewClientset(deploy) + err := waitForDeploymentReady(context.Background(), fakeClientset, "kube-system", "csi-stackit-controllerplugin", timeoutDuration) + Expect(err).NotTo(HaveOccurred()) + }) +}) + +var _ = Describe("waitForDaemonSetReady", func() { + It("succeeds when daemonset is ready", func() { + ds := &appsv1.DaemonSet{ + ObjectMeta: metav1.ObjectMeta{ + Name: "csi-stackit-nodeplugin", + Namespace: "kube-system", + Generation: 1, + }, + Status: appsv1.DaemonSetStatus{ + ObservedGeneration: 1, + DesiredNumberScheduled: 2, + UpdatedNumberScheduled: 2, + NumberAvailable: 2, + NumberUnavailable: 0, + }, + } + + fakeClientset := fake.NewClientset(ds) + err := waitForDaemonSetReady(context.Background(), fakeClientset, "kube-system", "csi-stackit-nodeplugin", timeoutDuration) + Expect(err).NotTo(HaveOccurred()) + }) +}) + +const timeoutDuration = 2 * csiPollInterval diff --git a/test/kubetest2/bootstrap_serviceaccount.go b/test/kubetest2/bootstrap_serviceaccount.go index 36683e23..2d197ab0 100644 --- a/test/kubetest2/bootstrap_serviceaccount.go +++ b/test/kubetest2/bootstrap_serviceaccount.go @@ -46,16 +46,22 @@ type serviceAccountKeyCredentialsFile struct { TokenEndpoint string `json:"tokenEndpoint"` } +var childProjectRoles = []string{ + childProjectSKERole, + childProjectStorageRole, +} + // ensureServiceAccount idempotently resolves (or creates) the managed child -// service account, grants it the SKE admin role, caches a service-account key -// and initializes the SKE client from it. Depends on d.projectID being set. +// service account, grants it the required project roles, caches a +// service-account key and initializes the SKE client from it. Depends on +// d.projectID being set. func (d *Deployer) ensureServiceAccount(ctx context.Context) error { childServiceAccount, err := d.resolveManagedServiceAccount(ctx, d.projectID) if err != nil { return err } - if err := d.ensureProjectServiceAccountRole(ctx, d.projectID, childServiceAccount.Email); err != nil { + if err := d.ensureProjectServiceAccountRoles(ctx, d.projectID, childServiceAccount.Email); err != nil { return err } @@ -72,61 +78,65 @@ func (d *Deployer) resolveManagedServiceAccount(ctx context.Context, projectID s return nil, fmt.Errorf("list service accounts in STACKIT project %q: %w", projectID, err) } - matches := make([]managedServiceAccount, 0, 1) for _, serviceAccount := range serviceAccounts { if !d.matchesManagedServiceAccountEmail(serviceAccount.GetEmail()) { continue } - matches = append(matches, managedServiceAccount{ + klog.Infof("Reusing managed service account=%q in project_id=%q", serviceAccount.GetEmail(), projectID) + return &managedServiceAccount{ Email: serviceAccount.GetEmail(), ProjectID: serviceAccount.GetProjectId(), - }) + }, nil } - switch len(matches) { - case 0: - klog.Infof("Creating managed service account=%q in project_id=%q", d.serviceAccountName(), projectID) - createdServiceAccount, err := d.serviceAccountClient.CreateServiceAccount(ctx, projectID, d.serviceAccountName()) - if err != nil { - return nil, fmt.Errorf("create service account %q in STACKIT project %q: %w", d.serviceAccountName(), projectID, err) - } - return &managedServiceAccount{ - Email: createdServiceAccount.GetEmail(), - ProjectID: createdServiceAccount.GetProjectId(), - }, nil - case 1: - klog.Infof("Reusing managed service account=%q in project_id=%q", matches[0].Email, projectID) - return &matches[0], nil - default: - return nil, fmt.Errorf( - "found %d managed service accounts for run token %q in project %q", - len(matches), - d.runToken(), - projectID, - ) + klog.Infof("Creating managed service account=%q in project_id=%q", d.serviceAccountName(), projectID) + createdServiceAccount, err := d.serviceAccountClient.CreateServiceAccount(ctx, projectID, d.serviceAccountName()) + if err != nil { + return nil, fmt.Errorf("create service account %q in STACKIT project %q: %w", d.serviceAccountName(), projectID, err) } + return &managedServiceAccount{ + Email: createdServiceAccount.GetEmail(), + ProjectID: createdServiceAccount.GetProjectId(), + }, nil } -func (d *Deployer) ensureProjectServiceAccountRole(ctx context.Context, projectID, serviceAccountEmail string) error { +func (d *Deployer) ensureProjectServiceAccountRoles(ctx context.Context, projectID, serviceAccountEmail string) error { members, err := d.authorizationClient.ListMembers(ctx, projectResourceType, projectID) if err != nil { return fmt.Errorf("list members for STACKIT project %q: %w", projectID, err) } + + assignedRoles := make(map[string]struct{}, len(childProjectRoles)) for _, member := range members { - if member.GetSubject() == serviceAccountEmail && member.GetRole() == childProjectRole { - klog.Infof("Managed service account=%q already has role=%q in project_id=%q", serviceAccountEmail, childProjectRole, projectID) - return nil + if member.GetSubject() != serviceAccountEmail { + continue } + assignedRoles[member.GetRole()] = struct{}{} + } + + missingMembers := make([]authorization.Member, 0, len(childProjectRoles)) + missingRoles := make([]string, 0, len(childProjectRoles)) + for _, role := range childProjectRoles { + if _, ok := assignedRoles[role]; ok { + klog.Infof("Managed service account=%q already has role=%q in project_id=%q", serviceAccountEmail, role, projectID) + continue + } + missingRoles = append(missingRoles, role) + missingMembers = append(missingMembers, *authorization.NewMember(role, serviceAccountEmail)) + } + + if len(missingMembers) == 0 { + return nil } - klog.Infof("Adding role=%q for managed service account=%q in project_id=%q", childProjectRole, serviceAccountEmail, projectID) + klog.Infof("Adding roles=%q for managed service account=%q in project_id=%q", strings.Join(missingRoles, ","), serviceAccountEmail, projectID) if err := d.authorizationClient.AddMembers( ctx, projectID, projectResourceType, - []authorization.Member{*authorization.NewMember(childProjectRole, serviceAccountEmail)}, + missingMembers, ); err != nil { - return fmt.Errorf("add role %q for service account %q in STACKIT project %q: %w", childProjectRole, serviceAccountEmail, projectID, err) + return fmt.Errorf("add roles %q for service account %q in STACKIT project %q: %w", strings.Join(missingRoles, ","), serviceAccountEmail, projectID, err) } return nil } @@ -210,7 +220,8 @@ func (d *Deployer) matchesManagedServiceAccountEmail(email string) bool { if !found { return false } - return localPart == d.serviceAccountName() + name := d.serviceAccountName() + return localPart == name || strings.HasPrefix(localPart, name+"-") } func serviceAccountKeyJSON(createdKey *serviceaccount.CreateServiceAccountKeyResponse) (string, error) { diff --git a/test/kubetest2/bootstrap_serviceaccount_test.go b/test/kubetest2/bootstrap_serviceaccount_test.go index 99c23d25..ce1ed2b8 100644 --- a/test/kubetest2/bootstrap_serviceaccount_test.go +++ b/test/kubetest2/bootstrap_serviceaccount_test.go @@ -18,6 +18,16 @@ var _ = Describe("matchesManagedServiceAccountEmail", func() { Expect(d.matchesManagedServiceAccountEmail(d.serviceAccountName() + "@sa.stackit.cloud")).To(BeTrue()) }) + It("matches emails with generated suffixes", func() { + d := newTestDeployer() + Expect(d.matchesManagedServiceAccountEmail(d.serviceAccountName() + "-aBc2defg@sa.stackit.cloud")).To(BeTrue()) + }) + + It("rejects longer prefixes without the generated suffix separator", func() { + d := newTestDeployer() + Expect(d.matchesManagedServiceAccountEmail(d.serviceAccountName() + "extra@sa.stackit.cloud")).To(BeFalse()) + }) + DescribeTable("rejects non-matching emails", func(email string) { d := newTestDeployer() @@ -36,7 +46,7 @@ var _ = Describe("ensureServiceAccount", func() { cachedKey := `{"credentials":{"privateKey":"cached"}}` Expect(os.WriteFile(d.serviceAccountKeyPath, []byte(cachedKey), 0o600)).To(Succeed()) - serviceAccountEmail := d.serviceAccountName() + "@sa.stackit.cloud" + serviceAccountEmail := d.serviceAccountName() + "-aBc2defg@sa.stackit.cloud" serviceAccountClient := &fakeServiceAccountClient{ listResult: []serviceaccount.ServiceAccount{ *serviceAccountFixture(serviceAccountEmail, "project-123"), @@ -44,7 +54,8 @@ var _ = Describe("ensureServiceAccount", func() { } authorizationClient := &fakeAuthorizationClient{ listMembersResult: []authorization.Member{ - *authorization.NewMember(childProjectRole, serviceAccountEmail), + *authorization.NewMember(childProjectSKERole, serviceAccountEmail), + *authorization.NewMember(childProjectStorageRole, serviceAccountEmail), }, } fakeSKE := &fakeSKEClient{} @@ -67,7 +78,7 @@ var _ = Describe("ensureServiceAccount", func() { d := newTestDeployer() d.projectID = "project-123" - serviceAccountEmail := d.serviceAccountName() + "@sa.stackit.cloud" + serviceAccountEmail := d.serviceAccountName() + "-aBc2defg@sa.stackit.cloud" serviceAccountClient := &fakeServiceAccountClient{ listResult: []serviceaccount.ServiceAccount{ *serviceAccountFixture(serviceAccountEmail, "project-123"), @@ -89,6 +100,10 @@ var _ = Describe("ensureServiceAccount", func() { Expect(serviceAccountClient.createKeyCalls).To(Equal(1)) Expect(authorizationClient.lastAddedType).To(Equal(projectResourceType)) Expect(authorizationClient.lastAddedID).To(Equal("project-123")) + Expect(authorizationClient.lastAddedMembers).To(ConsistOf( + *authorization.NewMember(childProjectSKERole, serviceAccountEmail), + *authorization.NewMember(childProjectStorageRole, serviceAccountEmail), + )) keyBytes, err := os.ReadFile(d.serviceAccountKeyPath) Expect(err).NotTo(HaveOccurred()) @@ -99,6 +114,38 @@ var _ = Describe("ensureServiceAccount", func() { Expect(err).NotTo(HaveOccurred()) Expect(info.Mode().Perm()).To(Equal(os.FileMode(0o600))) }) + + It("creates the managed service account when none exists", func() { + d := newTestDeployer() + d.projectID = "project-123" + + serviceAccountEmail := d.serviceAccountName() + "-aBc2defg@sa.stackit.cloud" + serviceAccountClient := &fakeServiceAccountClient{ + createResult: serviceAccountFixture(serviceAccountEmail, "project-123"), + createKeyResult: createServiceAccountKeyResponseFixture(serviceAccountEmail), + } + authorizationClient := &fakeAuthorizationClient{} + var receivedKey string + + d.serviceAccountClient = serviceAccountClient + d.authorizationClient = authorizationClient + d.skeClientFactory = func(_, serviceAccount, _ string) (skeClient, error) { + receivedKey = serviceAccount + return &fakeSKEClient{}, nil + } + + Expect(d.ensureServiceAccount(context.Background())).To(Succeed()) + Expect(serviceAccountClient.createCalls).To(Equal(1)) + Expect(serviceAccountClient.listCalls).To(Equal(1)) + Expect(serviceAccountClient.createKeyCalls).To(Equal(1)) + Expect(authorizationClient.addCalls).To(Equal(1)) + Expect(authorizationClient.lastAddedMembers).To(ConsistOf( + *authorization.NewMember(childProjectSKERole, serviceAccountEmail), + *authorization.NewMember(childProjectStorageRole, serviceAccountEmail), + )) + Expect(serviceAccountClient.lastCreatedName).To(Equal(d.serviceAccountName())) + Expect(receivedKey).To(ContainSubstring(`"privateKey":"PRIVATE"`)) + }) }) var _ = Describe("retryWithBackoff", func() { diff --git a/test/kubetest2/clients.go b/test/kubetest2/clients.go index 7ca272be..25a173d7 100644 --- a/test/kubetest2/clients.go +++ b/test/kubetest2/clients.go @@ -21,7 +21,8 @@ const ( projectLabelRunIDKey = "kt2_run_id" projectOwnerRole = "owner" projectResourceType = "project" - childProjectRole = "ske.admin" + childProjectSKERole = "ske.admin" + childProjectStorageRole = "blockstorage.admin" projectListPageSize = 100 skeServiceID = "cloud.stackit.ske" ) diff --git a/test/kubetest2/deployer.go b/test/kubetest2/deployer.go index 1d92bc2e..434da5a7 100644 --- a/test/kubetest2/deployer.go +++ b/test/kubetest2/deployer.go @@ -37,6 +37,9 @@ type Deployer struct { volumeType string kubeconfigExpiresIn int64 + csiImageName string + csiImageTag string + projectID string serviceAccount string parentContainerID string @@ -49,6 +52,7 @@ type Deployer struct { authorizationEndpoint string serviceEnablementEndpoint string skeEndpoint string + iaasEndpoint string projectClient projectClient serviceAccountClient serviceAccountClient @@ -56,6 +60,7 @@ type Deployer struct { serviceEnablementClient serviceEnablementClient skeClient skeClient skeClientFactory func(region, serviceAccount, endpoint string) (skeClient, error) + csiApplier csiApplier } var _ types.NewDeployer = New @@ -75,6 +80,7 @@ func New(opts types.Options) (types.Deployer, *pflag.FlagSet) { volumeSizeGiB: defaultVolumeSizeGiB, kubeconfigExpiresIn: defaultKubeconfigExpiration, skeClientFactory: newSKEClient, + csiApplier: applyCSIManifestsNative, } fs := pflag.NewFlagSet(Name, pflag.ContinueOnError) diff --git a/test/kubetest2/env.go b/test/kubetest2/env.go index 51bfbcfb..0709de98 100644 --- a/test/kubetest2/env.go +++ b/test/kubetest2/env.go @@ -36,6 +36,7 @@ func (d *Deployer) loadEnvironment() error { d.authorizationEndpoint = strings.TrimSpace(os.Getenv("STACKIT_AUTHORIZATION_ENDPOINT")) d.serviceEnablementEndpoint = strings.TrimSpace(os.Getenv("STACKIT_SERVICE_ENABLEMENT_ENDPOINT")) d.skeEndpoint = strings.TrimSpace(os.Getenv("STACKIT_SKE_ENDPOINT")) + d.iaasEndpoint = strings.TrimSpace(os.Getenv("STACKIT_IAAS_ENDPOINT")) d.kubeconfigPath = filepath.Join(d.options.RunDir(), "kubeconfig") d.serviceAccountKeyPath = filepath.Join(d.options.RunDir(), "service-account-key.json") @@ -49,12 +50,13 @@ func (d *Deployer) loadEnvironment() error { ) klog.Infof( - "STACKIT API endpoint overrides: resource_manager=%q service_account=%q authorization=%q service_enablement=%q ske=%q", + "STACKIT API endpoint overrides: resource_manager=%q service_account=%q authorization=%q service_enablement=%q ske=%q iaas=%q", d.resourceManagerEndpoint, d.serviceAccountEndpoint, d.authorizationEndpoint, d.serviceEnablementEndpoint, d.skeEndpoint, + d.iaasEndpoint, ) return nil diff --git a/test/kubetest2/env_test.go b/test/kubetest2/env_test.go index 212d2de7..2ec42684 100644 --- a/test/kubetest2/env_test.go +++ b/test/kubetest2/env_test.go @@ -34,6 +34,7 @@ var _ = Describe("loadEnvironment", func() { "STACKIT_AUTHORIZATION_ENDPOINT", "STACKIT_SERVICE_ENABLEMENT_ENDPOINT", "STACKIT_SKE_ENDPOINT", + "STACKIT_IAAS_ENDPOINT", } DescribeTable("validates required environment variables", @@ -85,6 +86,7 @@ var _ = Describe("loadEnvironment", func() { setEnvVar("STACKIT_SERVICE_ACCOUNT_ENDPOINT", "https://service-account.example.com") setEnvVar("STACKIT_AUTHORIZATION_ENDPOINT", "https://authorization.example.com") setEnvVar("STACKIT_SKE_ENDPOINT", "https://ske.example.com") + setEnvVar("STACKIT_IAAS_ENDPOINT", "https://iaas.example.com") runDir := GinkgoT().TempDir() d := &Deployer{options: fakeOptions{runID: "run-123", runDir: runDir}} @@ -95,5 +97,6 @@ var _ = Describe("loadEnvironment", func() { Expect(d.serviceAccountEndpoint).To(Equal("https://service-account.example.com")) Expect(d.authorizationEndpoint).To(Equal("https://authorization.example.com")) Expect(d.skeEndpoint).To(Equal("https://ske.example.com")) + Expect(d.iaasEndpoint).To(Equal("https://iaas.example.com")) }) }) diff --git a/test/kubetest2/flags.go b/test/kubetest2/flags.go index 530610d9..bc40c812 100644 --- a/test/kubetest2/flags.go +++ b/test/kubetest2/flags.go @@ -21,11 +21,13 @@ func bindFlags(fs *pflag.FlagSet, d *Deployer) { fs.Int64Var(&d.volumeSizeGiB, "volume-size", defaultVolumeSizeGiB, "Root volume size in GiB for the SKE nodepool") fs.StringVar(&d.volumeType, "volume-type", "", "Root volume type for the SKE nodepool") fs.Int64Var(&d.kubeconfigExpiresIn, "kubeconfig-expiration-seconds", defaultKubeconfigExpiration, "Admin kubeconfig expiration in seconds") + fs.StringVar(&d.csiImageName, "csi-image-name", "", "Image name (repository) for the STACKIT CSI plugin") + fs.StringVar(&d.csiImageTag, "csi-image-tag", "", "Image tag for the STACKIT CSI plugin") } func (d *Deployer) validate() error { klog.Infof( - "Validating deployer configuration: run_id=%q region=%q kubernetes_version=%q availability_zone=%q machine_type=%q node_image_name=%q node_image_version=%q node_count=%d nodepool_name=%q volume_size=%d volume_type=%q kubeconfig_expiration_seconds=%d", + "Validating deployer configuration: run_id=%q region=%q kubernetes_version=%q availability_zone=%q machine_type=%q node_image_name=%q node_image_version=%q node_count=%d nodepool_name=%q volume_size=%d volume_type=%q kubeconfig_expiration_seconds=%d csi_image_name=%q csi_image_tag=%q", d.options.RunID(), d.region, d.kubernetesVersion, @@ -38,6 +40,8 @@ func (d *Deployer) validate() error { d.volumeSizeGiB, d.volumeType, d.kubeconfigExpiresIn, + d.csiImageName, + d.csiImageTag, ) requiredFlags := map[string]string{ @@ -47,6 +51,8 @@ func (d *Deployer) validate() error { "--machine-type": d.machineType, "--node-image-name": d.nodeImageName, "--node-image-version": d.nodeImageVersion, + "--csi-image-name": d.csiImageName, + "--csi-image-tag": d.csiImageTag, } for flagName, value := range requiredFlags { diff --git a/test/kubetest2/flags_test.go b/test/kubetest2/flags_test.go index b0e9f083..ee66676b 100644 --- a/test/kubetest2/flags_test.go +++ b/test/kubetest2/flags_test.go @@ -37,6 +37,8 @@ var _ = Describe("validate", func() { Entry("zero volume size", func(d *Deployer) { d.volumeSizeGiB = 0 }, "--volume-size must be greater than 0"), Entry("kubeconfig expiration too small", func(d *Deployer) { d.kubeconfigExpiresIn = minKubeconfigExpiration - 1 }, "--kubeconfig-expiration-seconds must be between"), Entry("kubeconfig expiration too large", func(d *Deployer) { d.kubeconfigExpiresIn = maxKubeconfigExpiration + 1 }, "--kubeconfig-expiration-seconds must be between"), + Entry("missing csi image name", func(d *Deployer) { d.csiImageName = "" }, "--csi-image-name is required"), + Entry("missing csi image tag", func(d *Deployer) { d.csiImageTag = "" }, "--csi-image-tag is required"), Entry("empty run id", func(d *Deployer) { d.options = fakeOptions{runID: "", runDir: d.options.RunDir()} }, "run-id must not be empty"), ) }) diff --git a/test/kubetest2/lifecycle_test.go b/test/kubetest2/lifecycle_test.go index 599505ea..7438f147 100644 --- a/test/kubetest2/lifecycle_test.go +++ b/test/kubetest2/lifecycle_test.go @@ -35,7 +35,8 @@ var _ = Describe("Up", func() { } d.authorizationClient = &fakeAuthorizationClient{ listMembersResult: []authorization.Member{ - *authorization.NewMember(childProjectRole, serviceAccountEmail), + *authorization.NewMember(childProjectSKERole, serviceAccountEmail), + *authorization.NewMember(childProjectStorageRole, serviceAccountEmail), }, } d.serviceEnablementClient = &fakeServiceEnablementClient{getStatusResult: serviceenablement.NewServiceStatus()} diff --git a/test/kubetest2/suite_test.go b/test/kubetest2/suite_test.go index f16aed5f..3bef6c0b 100644 --- a/test/kubetest2/suite_test.go +++ b/test/kubetest2/suite_test.go @@ -102,6 +102,7 @@ func (c *fakeProjectClient) WaitForProjectDeleted(_ context.Context, projectID s type fakeServiceAccountClient struct { listResult []serviceaccount.ServiceAccount + listResults [][]serviceaccount.ServiceAccount listErr error createResult *serviceaccount.ServiceAccount @@ -118,10 +119,19 @@ type fakeServiceAccountClient struct { lastCreatedName string lastProjectIDForCreateKey string lastCreateKeyEmail string + listCalls int } func (c *fakeServiceAccountClient) ListServiceAccounts(_ context.Context, projectID string) ([]serviceaccount.ServiceAccount, error) { c.lastProjectIDForList = projectID + c.listCalls++ + if len(c.listResults) > 0 { + result := c.listResults[0] + if len(c.listResults) > 1 { + c.listResults = c.listResults[1:] + } + return result, c.listErr + } return c.listResult, c.listErr } @@ -267,6 +277,9 @@ func newTestDeployer() *Deployer { kubeconfigPath: filepath.Join(runDir, "kubeconfig"), serviceAccountKeyPath: filepath.Join(runDir, "service-account-key.json"), skeClientFactory: newSKEClient, + csiApplier: func(_ context.Context, _, _ string) error { + return nil + }, } } @@ -278,6 +291,8 @@ func configureValidUpInputs(d *Deployer) { d.nodeImageVersion = "v1" d.nodepoolName = defaultNodepoolName d.volumeType = "storage" + d.csiImageName = "ghcr.io/stackitcloud/cloud-provider-stackit/stackit-csi-plugin" + d.csiImageTag = "v1.0.0" } func setEnvVar(key, value string) { diff --git a/test/kustomize/cloud-config.yaml b/test/kustomize/cloud-config.yaml index a96763e0..dd74e91c 100644 --- a/test/kustomize/cloud-config.yaml +++ b/test/kustomize/cloud-config.yaml @@ -6,7 +6,7 @@ metadata: data: cloud.yaml: |- global: - projectId: REPLACE_WITH_PROJECTID + projectId: stackit-project-id region: eu01 blockStorage: rescanOnResize: true diff --git a/test/kustomize/storageclass.yaml b/test/kustomize/storageclass.yaml index 2bba6f99..0b454e66 100644 --- a/test/kustomize/storageclass.yaml +++ b/test/kustomize/storageclass.yaml @@ -3,5 +3,7 @@ kind: StorageClass metadata: name: premium-perf4-stackit-kubetest2 provisioner: kubetest2.csi.stackit.cloud +allowVolumeExpansion: true +volumeBindingMode: WaitForFirstConsumer parameters: type: "storage_premium_perf4" From ff65866fcb03ee97776e49b2d160432e47f023b6 Mon Sep 17 00:00:00 2001 From: Robert Kaussow Date: Mon, 24 Aug 2026 15:23:36 +0200 Subject: [PATCH 10/11] support workload identity --- test/kubetest2/clients.go | 5 +++-- test/kubetest2/env.go | 22 ++++++++++++-------- test/kubetest2/env_test.go | 42 +++++++++++++++++++++++++++++++++----- 3 files changed, 54 insertions(+), 15 deletions(-) diff --git a/test/kubetest2/clients.go b/test/kubetest2/clients.go index 25a173d7..b749bdd9 100644 --- a/test/kubetest2/clients.go +++ b/test/kubetest2/clients.go @@ -69,8 +69,9 @@ type sdkServiceEnablementClient struct { } func apiClientOptions(serviceAccountKey, endpoint string) []sdkconfig.ConfigurationOption { - opts := []sdkconfig.ConfigurationOption{ - sdkconfig.WithServiceAccountKey(serviceAccountKey), + var opts []sdkconfig.ConfigurationOption + if serviceAccountKey != "" { + opts = append(opts, sdkconfig.WithServiceAccountKey(serviceAccountKey)) } if endpoint != "" { opts = append(opts, sdkconfig.WithEndpoint(endpoint)) diff --git a/test/kubetest2/env.go b/test/kubetest2/env.go index 0709de98..4de69cea 100644 --- a/test/kubetest2/env.go +++ b/test/kubetest2/env.go @@ -13,19 +13,25 @@ import ( func (d *Deployer) loadEnvironment() error { klog.Infof("Loading STACKIT environment variables") - serviceAccount, ok := os.LookupEnv("STACKIT_SERVICE_ACCOUNT") - if !ok || strings.TrimSpace(serviceAccount) == "" { - return incorrectUsagef("STACKIT_SERVICE_ACCOUNT environment variable is required") - } - parentContainerID, ok := os.LookupEnv("STACKIT_PARENT_CONTAINER_ID") if !ok || strings.TrimSpace(parentContainerID) == "" { return incorrectUsagef("STACKIT_PARENT_CONTAINER_ID environment variable is required") } - projectMemberEmail, err := extractServiceAccountEmail(serviceAccount) - if err != nil { - return incorrectUsagef("invalid STACKIT_SERVICE_ACCOUNT: %v", err) + serviceAccount := strings.TrimSpace(os.Getenv("STACKIT_SERVICE_ACCOUNT")) + serviceAccountEmail := strings.TrimSpace(os.Getenv("STACKIT_SERVICE_ACCOUNT_EMAIL")) + + var projectMemberEmail string + if serviceAccount != "" { + var err error + projectMemberEmail, err = extractServiceAccountEmail(serviceAccount) + if err != nil { + return incorrectUsagef("invalid STACKIT_SERVICE_ACCOUNT: %v", err) + } + } else if serviceAccountEmail != "" { + projectMemberEmail = serviceAccountEmail + } else { + return incorrectUsagef("either STACKIT_SERVICE_ACCOUNT or STACKIT_SERVICE_ACCOUNT_EMAIL must be provided") } d.serviceAccount = serviceAccount diff --git a/test/kubetest2/env_test.go b/test/kubetest2/env_test.go index 2ec42684..0de66940 100644 --- a/test/kubetest2/env_test.go +++ b/test/kubetest2/env_test.go @@ -28,6 +28,7 @@ var _ = Describe("extractServiceAccountEmail", func() { var _ = Describe("loadEnvironment", func() { envVarKeys := []string{ "STACKIT_SERVICE_ACCOUNT", + "STACKIT_SERVICE_ACCOUNT_EMAIL", "STACKIT_PARENT_CONTAINER_ID", "STACKIT_RESOURCE_MANAGER_ENDPOINT", "STACKIT_SERVICE_ACCOUNT_ENDPOINT", @@ -60,23 +61,30 @@ var _ = Describe("loadEnvironment", func() { Expect(d.kubeconfigPath).To(Equal(filepath.Join(runDir, "kubeconfig"))) Expect(d.serviceAccountKeyPath).To(Equal(filepath.Join(runDir, "service-account-key.json"))) }, - Entry("missing service account", map[string]string{ + Entry("missing all auth", map[string]string{ "STACKIT_PARENT_CONTAINER_ID": "parent-1", - }, "STACKIT_SERVICE_ACCOUNT"), - Entry("missing parent container", map[string]string{ + }, "either STACKIT_SERVICE_ACCOUNT or STACKIT_SERVICE_ACCOUNT_EMAIL"), + Entry("missing parent container with service account", map[string]string{ "STACKIT_SERVICE_ACCOUNT": validServiceAccountKey, }, "STACKIT_PARENT_CONTAINER_ID"), + Entry("missing parent container with workload identity", map[string]string{ + "STACKIT_SERVICE_ACCOUNT_EMAIL": "owner@example.com", + }, "STACKIT_PARENT_CONTAINER_ID"), Entry("invalid service account key", map[string]string{ "STACKIT_SERVICE_ACCOUNT": "{}", "STACKIT_PARENT_CONTAINER_ID": "parent-1", }, "invalid STACKIT_SERVICE_ACCOUNT"), - Entry("project id no longer required", map[string]string{ + Entry("valid with service account key", map[string]string{ "STACKIT_SERVICE_ACCOUNT": validServiceAccountKey, "STACKIT_PARENT_CONTAINER_ID": "parent-1", }, ""), + Entry("valid with workload identity", map[string]string{ + "STACKIT_SERVICE_ACCOUNT_EMAIL": "owner@example.com", + "STACKIT_PARENT_CONTAINER_ID": "parent-1", + }, ""), ) - It("reads optional endpoints", func() { + It("reads optional endpoints with service account", func() { for _, key := range envVarKeys { setEnvVar(key, "") } @@ -99,4 +107,28 @@ var _ = Describe("loadEnvironment", func() { Expect(d.skeEndpoint).To(Equal("https://ske.example.com")) Expect(d.iaasEndpoint).To(Equal("https://iaas.example.com")) }) + + It("reads optional endpoints with workload identity", func() { + for _, key := range envVarKeys { + setEnvVar(key, "") + } + setEnvVar("STACKIT_SERVICE_ACCOUNT_EMAIL", "owner@example.com") + setEnvVar("STACKIT_PARENT_CONTAINER_ID", "parent-1") + setEnvVar("STACKIT_RESOURCE_MANAGER_ENDPOINT", "https://resource-manager.example.com") + setEnvVar("STACKIT_SERVICE_ACCOUNT_ENDPOINT", "https://service-account.example.com") + setEnvVar("STACKIT_AUTHORIZATION_ENDPOINT", "https://authorization.example.com") + setEnvVar("STACKIT_SKE_ENDPOINT", "https://ske.example.com") + setEnvVar("STACKIT_IAAS_ENDPOINT", "https://iaas.example.com") + + runDir := GinkgoT().TempDir() + d := &Deployer{options: fakeOptions{runID: "run-123", runDir: runDir}} + Expect(d.loadEnvironment()).To(Succeed()) + + Expect(d.projectMemberEmail).To(Equal("owner@example.com")) + Expect(d.resourceManagerEndpoint).To(Equal("https://resource-manager.example.com")) + Expect(d.serviceAccountEndpoint).To(Equal("https://service-account.example.com")) + Expect(d.authorizationEndpoint).To(Equal("https://authorization.example.com")) + Expect(d.skeEndpoint).To(Equal("https://ske.example.com")) + Expect(d.iaasEndpoint).To(Equal("https://iaas.example.com")) + }) }) From db8aa5c49058321dd93269cf4196231d54ae9cdf Mon Sep 17 00:00:00 2001 From: Robert Kaussow Date: Mon, 24 Aug 2026 16:18:06 +0200 Subject: [PATCH 11/11] add make targte for kubetest e2e --- Makefile | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/Makefile b/Makefile index b09ff333..a31d9908 100644 --- a/Makefile +++ b/Makefile @@ -10,6 +10,9 @@ REPO ?= stackitcloud/cloud-provider-stackit PLATFORMS ?= amd64 arm64 IS_DEV ?= true +E2E_RUN_ID ?= $(or $(PROW_JOB_ID),local-test-e2e) +E2E_PARALLEL_NODES ?= 4 + .PHONY: all all: verify @@ -48,6 +51,8 @@ image-%: $(APKO) $(KO) .PHONY: image-stackit-csi-plugin-test image-stackit-csi-plugin-test: export KO_CONFIG_PATH = test/e2e/.ko-kubetest2.yaml +image-stackit-csi-plugin-test: export IS_DEV = true +image-stackit-csi-plugin-test: export VERSION = $(VERSION)-kubetest2 image-stackit-csi-plugin-test: image-stackit-csi-plugin .PHONY: clean-tools-bin @@ -127,6 +132,33 @@ verify-image-stackit-csi-plugin: image-stackit-csi-plugin @echo "verifying binaries in image" @docker run -v ./tools/csi-deps-check.sh:/tools/csi-deps-check.sh --entrypoint=/tools/csi-deps-check.sh $(REGISTRY)/$(REPO)/stackit-csi-plugin-dev:$(VERSION) +.PHONY: test-e2e +test-e2e: image-stackit-csi-plugin-test + go run ./test \ + --up \ + --test=ginkgo \ + --run-id "$(E2E_RUN_ID)" \ + --rundir "_rundir" \ + --region "eu01" \ + --kubernetes-version "$(E2E_K8S_VERSION)" \ + --availability-zone "eu01-1" \ + --machine-type "$(E2E_MACHINE_TYPE)" \ + --node-image-name "$(E2E_IMAGE_NAME)" \ + --node-image-version "$(E2E_IMAGE_VERSION)" \ + --nodepool-name "default" \ + --volume-size 100 \ + --kubeconfig-expiration-seconds 3600 \ + --csi-image-name "$(REGISTRY)/$(REPO)/stackit-csi-plugin-dev" \ + --csi-image-tag "$(VERSION)-kubetest2" \ + "$@" \ + -- \ + --test-package-version="v$(E2E_K8S_VERSION)" \ + --focus-regex="External.Storage" \ + --skip-regex="\[Feature:|\[Disruptive\]|\[Serial\]" \ + --ginkgo-args="-v" \ + --parallel="$(E2E_PARALLEL_NODES)" \ + --test-args="--storage.testdriver=$(PWD)/_rundir/$(E2E_RUN_ID)/csi-testdriver.yaml" + # generate mock types for the following services (space-separated list) .PHONY: mocks mocks: $(MOCKGEN)