diff --git a/api/v1alpha/workload_types.go b/api/v1alpha/workload_types.go index 5444e3c0..12cab489 100644 --- a/api/v1alpha/workload_types.go +++ b/api/v1alpha/workload_types.go @@ -229,7 +229,13 @@ type HorizontalScaleSettings struct { // Controls how instances are managed during scale up and down, as well as // during maintenance events. // + // OrderedReady creates and deletes instances one at a time, waiting for + // each to become ready before moving to the next. Parallel creates and + // deletes instances all at once without waiting on readiness. Both replace + // instances on a template change one at a time. + // // +kubebuilder:validation:Required + // +kubebuilder:validation:Enum=OrderedReady;Parallel // +kubebuilder:default=OrderedReady InstanceManagementPolicy InstanceManagementPolicyType `json:"instanceManagementPolicy,omitempty"` } @@ -238,7 +244,7 @@ type InstanceManagementPolicyType string const ( OrderedReadyInstanceManagementPolicyType InstanceManagementPolicyType = "OrderedReady" - // ParallelInstanceManagementPolicyType InstanceManagementPolicyType = "Parallel" + ParallelInstanceManagementPolicyType InstanceManagementPolicyType = "Parallel" ) type MetricSpec struct { diff --git a/config/base/crd/bases/compute.datumapis.com_workloaddeployments.yaml b/config/base/crd/bases/compute.datumapis.com_workloaddeployments.yaml index 73ade506..f0ae078a 100644 --- a/config/base/crd/bases/compute.datumapis.com_workloaddeployments.yaml +++ b/config/base/crd/bases/compute.datumapis.com_workloaddeployments.yaml @@ -106,6 +106,14 @@ spec: description: |- Controls how instances are managed during scale up and down, as well as during maintenance events. + + OrderedReady creates and deletes instances one at a time, waiting for + each to become ready before moving to the next. Parallel creates and + deletes instances all at once without waiting on readiness. Both replace + instances on a template change one at a time. + enum: + - OrderedReady + - Parallel type: string maxReplicas: description: The maximum number of replicas. diff --git a/config/base/crd/bases/compute.datumapis.com_workloads.yaml b/config/base/crd/bases/compute.datumapis.com_workloads.yaml index d64fbb4f..a7a8ae24 100644 --- a/config/base/crd/bases/compute.datumapis.com_workloads.yaml +++ b/config/base/crd/bases/compute.datumapis.com_workloads.yaml @@ -163,6 +163,14 @@ spec: description: |- Controls how instances are managed during scale up and down, as well as during maintenance events. + + OrderedReady creates and deletes instances one at a time, waiting for + each to become ready before moving to the next. Parallel creates and + deletes instances all at once without waiting on readiness. Both replace + instances on a template change one at a time. + enum: + - OrderedReady + - Parallel type: string maxReplicas: description: The maximum number of replicas. diff --git a/docs/api/workloaddeployments.md b/docs/api/workloaddeployments.md index 365579e9..8f61c6cc 100644 --- a/docs/api/workloaddeployments.md +++ b/docs/api/workloaddeployments.md @@ -192,11 +192,17 @@ Scale settings such as minimum and maximum replica counts. instanceManagementPolicy - string + enum Controls how instances are managed during scale up and down, as well as -during maintenance events.
+during maintenance events. + +OrderedReady creates and deletes instances one at a time, waiting for +each to become ready before moving to the next. Parallel creates and +deletes instances all at once without waiting on readiness. Both replace +instances on a template change one at a time.

+ Enum: OrderedReady, Parallel
Default: OrderedReady
true diff --git a/docs/api/workloads.md b/docs/api/workloads.md index 6ea07fbf..0c4a046b 100644 --- a/docs/api/workloads.md +++ b/docs/api/workloads.md @@ -188,11 +188,17 @@ Scale settings such as minimum and maximum replica counts. instanceManagementPolicy - string + enum Controls how instances are managed during scale up and down, as well as -during maintenance events.
+during maintenance events. + +OrderedReady creates and deletes instances one at a time, waiting for +each to become ready before moving to the next. Parallel creates and +deletes instances all at once without waiting on readiness. Both replace +instances on a template change one at a time.

+ Enum: OrderedReady, Parallel
Default: OrderedReady
true diff --git a/internal/controller/instancecontrol/stateful/stateful_control.go b/internal/controller/instancecontrol/stateful/stateful_control.go index f61f7421..b446709a 100644 --- a/internal/controller/instancecontrol/stateful/stateful_control.go +++ b/internal/controller/instancecontrol/stateful/stateful_control.go @@ -208,10 +208,25 @@ func (c *statefulControl) GetActions( slices.SortFunc(recreateActions, descendingOrdinal) slices.SortFunc(deleteActions, descendingOrdinal) - actions := make([]instancecontrol.Action, 0, len(createActions)+len(waitActions)+len(recreateActions)+len(deleteActions)+len(patchLabelActions)+len(patchTemplateHashActions)) + actions, err := orderRolloutActions(deployment.Spec.ScaleSettings.InstanceManagementPolicy, createActions, waitActions, recreateActions, deleteActions) + if err != nil { + return nil, err + } + + actions = append(actions, patchLabelActions...) + actions = append(actions, patchTemplateHashActions...) - switch deployment.Spec.ScaleSettings.InstanceManagementPolicy { - case v1alpha.OrderedReadyInstanceManagementPolicyType: + return actions, nil +} + +func orderRolloutActions( + policy v1alpha.InstanceManagementPolicyType, + createActions, waitActions, recreateActions, deleteActions []instancecontrol.Action, +) ([]instancecontrol.Action, error) { + actions := make([]instancecontrol.Action, 0, len(createActions)+len(waitActions)+len(recreateActions)+len(deleteActions)) + + switch policy { + case v1alpha.OrderedReadyInstanceManagementPolicyType, "": // Add create and wait actions, and sort by ordinal. This allows us to wait // for instances to be processed in the correct order. @@ -233,10 +248,24 @@ func (c *statefulControl) GetActions( } } - } + case v1alpha.ParallelInstanceManagementPolicyType: + slices.SortFunc(createActions, ascendingOrdinal) + slices.SortFunc(waitActions, ascendingOrdinal) - actions = append(actions, patchLabelActions...) - actions = append(actions, patchTemplateHashActions...) + actions = append(actions, createActions...) + actions = append(actions, deleteActions...) + actions = append(actions, waitActions...) + + for i := range recreateActions { + if i > 0 || len(createActions) > 0 || len(waitActions) > 0 { + recreateActions[i].SkipExecution() + } + } + actions = append(actions, recreateActions...) + + default: + return nil, fmt.Errorf("unsupported instance management policy %q", policy) + } return actions, nil } diff --git a/internal/controller/instancecontrol/stateful/stateful_control_test.go b/internal/controller/instancecontrol/stateful/stateful_control_test.go index 98106543..09586f45 100644 --- a/internal/controller/instancecontrol/stateful/stateful_control_test.go +++ b/internal/controller/instancecontrol/stateful/stateful_control_test.go @@ -20,6 +20,8 @@ import ( "go.datum.net/compute/internal/controller/instancecontrol" ) +const testImageUpdate = "test-image-update" + var ( scheme = runtime.NewScheme() ) @@ -101,7 +103,7 @@ func TestUpdateWithAllReadyInstances(t *testing.T) { currentInstances = append(currentInstances, *getInstanceForDeployment(deployment, 0)) currentInstances = append(currentInstances, *getInstanceForDeployment(deployment, 1)) - deployment.Spec.Template.Spec.Runtime.Sandbox.Containers[0].Image = "test-image-update" + deployment.Spec.Template.Spec.Runtime.Sandbox.Containers[0].Image = testImageUpdate actions, err := control.GetActions(ctx, scheme, deployment, deployment.Spec.ScaleSettings.MinReplicas, currentInstances) @@ -814,3 +816,118 @@ func getInstanceTemplate(name string, ordinal int) *v1alpha.Instance { return instance } + +func getParallelWorkloadDeployment(minReplicas int32) *v1alpha.WorkloadDeployment { + deployment := getWorkloadDeployment("test-deploy", minReplicas) + deployment.Spec.ScaleSettings.InstanceManagementPolicy = v1alpha.ParallelInstanceManagementPolicyType + return deployment +} + +func TestParallel_FreshDeploymentCreatesAllInstances(t *testing.T) { + control := NewWithOptions(Options{}) + deployment := getParallelWorkloadDeployment(3) + + actions, err := control.GetActions(context.Background(), scheme, deployment, 3, nil) + + require.NoError(t, err) + require.Len(t, actions, 3) + for i, action := range actions { + assert.Equal(t, fmt.Sprintf("test-deploy-%d", i), action.Object.GetName()) + assert.Equal(t, instancecontrol.ActionTypeCreate, action.ActionType()) + assert.False(t, action.IsSkipped()) + } +} + +func TestParallel_ScaleUpDoesNotWaitOnNotReadyInstance(t *testing.T) { + control := NewWithOptions(Options{}) + deployment := getParallelWorkloadDeployment(3) + + notReady := getInstanceForDeployment(deployment, 0) + apimeta.SetStatusCondition(¬Ready.Status.Conditions, metav1.Condition{ + Type: v1alpha.InstanceReady, + Status: metav1.ConditionFalse, + }) + + actions, err := control.GetActions(context.Background(), scheme, deployment, 3, []v1alpha.Instance{*notReady}) + + require.NoError(t, err) + created := map[string]bool{} + for _, action := range actions { + if action.ActionType() == instancecontrol.ActionTypeCreate { + assert.False(t, action.IsSkipped(), action.Object.GetName()) + created[action.Object.GetName()] = true + } + } + assert.Equal(t, map[string]bool{"test-deploy-1": true, "test-deploy-2": true}, created) +} + +func TestParallel_ScaleDownDeletesAllExtraInstances(t *testing.T) { + control := NewWithOptions(Options{}) + deployment := getParallelWorkloadDeployment(1) + + current := []v1alpha.Instance{ + *getInstanceForDeployment(deployment, 0), + *getInstanceForDeployment(deployment, 1), + *getInstanceForDeployment(deployment, 2), + } + + actions, err := control.GetActions(context.Background(), scheme, deployment, 1, current) + + require.NoError(t, err) + require.Len(t, actions, 2) + for _, action := range actions { + assert.Equal(t, instancecontrol.ActionTypeDelete, action.ActionType()) + assert.False(t, action.IsSkipped(), action.Object.GetName()) + } +} + +func TestParallel_TemplateChangeReplacesOneInstanceAtATime(t *testing.T) { + control := NewWithOptions(Options{}) + deployment := getParallelWorkloadDeployment(2) + + current := []v1alpha.Instance{ + *getInstanceForDeployment(deployment, 0), + *getInstanceForDeployment(deployment, 1), + } + deployment.Spec.Template.Spec.Runtime.Sandbox.Containers[0].Image = testImageUpdate + + actions, err := control.GetActions(context.Background(), scheme, deployment, 2, current) + + require.NoError(t, err) + require.Len(t, actions, 2) + assert.Equal(t, "test-deploy-1", actions[0].Object.GetName()) + assert.Equal(t, instancecontrol.ActionTypeDelete, actions[0].ActionType()) + assert.False(t, actions[0].IsSkipped()) + assert.Equal(t, "test-deploy-0", actions[1].Object.GetName()) + assert.True(t, actions[1].IsSkipped()) +} + +func TestParallel_TemplateChangeHeldWhileInstanceIsReplaced(t *testing.T) { + control := NewWithOptions(Options{}) + deployment := getParallelWorkloadDeployment(2) + + current := []v1alpha.Instance{*getInstanceForDeployment(deployment, 0)} + deployment.Spec.Template.Spec.Runtime.Sandbox.Containers[0].Image = testImageUpdate + + actions, err := control.GetActions(context.Background(), scheme, deployment, 2, current) + + require.NoError(t, err) + for _, action := range actions { + switch action.ActionType() { + case instancecontrol.ActionTypeCreate: + assert.False(t, action.IsSkipped(), action.Object.GetName()) + case instancecontrol.ActionTypeDelete: + assert.True(t, action.IsSkipped(), action.Object.GetName()) + } + } +} + +func TestUnsupportedInstanceManagementPolicyReturnsError(t *testing.T) { + control := NewWithOptions(Options{}) + deployment := getWorkloadDeployment("test-deploy", 1) + deployment.Spec.ScaleSettings.InstanceManagementPolicy = "Bogus" + + _, err := control.GetActions(context.Background(), scheme, deployment, 1, nil) + + assert.ErrorContains(t, err, "Bogus") +}