fix: create instances for the Parallel scale policy

api/v1alpha/workload_types.go

@@ -229,7 +229,13 @@ type HorizontalScaleSettings struct {
 	// Controls how instances are managed during scale up and down, as well as
 	// during maintenance events.
 	//
+	// OrderedReady creates and deletes instances one at a time, waiting for
+	// each to become ready before moving to the next. Parallel creates and
+	// deletes instances all at once without waiting on readiness. Both replace
+	// instances on a template change one at a time.
+	//
 	// +kubebuilder:validation:Required
+	// +kubebuilder:validation:Enum=OrderedReady;Parallel
 	// +kubebuilder:default=OrderedReady
 	InstanceManagementPolicy InstanceManagementPolicyType `json:"instanceManagementPolicy,omitempty"`
 }
@@ -238,7 +244,7 @@ type InstanceManagementPolicyType string
 
 const (
 	OrderedReadyInstanceManagementPolicyType InstanceManagementPolicyType = "OrderedReady"
-	// ParallelInstanceManagementPolicyType     InstanceManagementPolicyType = "Parallel"
+	ParallelInstanceManagementPolicyType     InstanceManagementPolicyType = "Parallel"
 )
 
 type MetricSpec struct {

config/base/crd/bases/compute.datumapis.com_workloaddeployments.yaml

@@ -106,6 +106,14 @@ spec:
                     description: |-
                       Controls how instances are managed during scale up and down, as well as
                       during maintenance events.
+
+                      OrderedReady creates and deletes instances one at a time, waiting for
+                      each to become ready before moving to the next. Parallel creates and
+                      deletes instances all at once without waiting on readiness. Both replace
+                      instances on a template change one at a time.
+                    enum:
+                    - OrderedReady
+                    - Parallel
                     type: string
                   maxReplicas:
                     description: The maximum number of replicas.

config/base/crd/bases/compute.datumapis.com_workloads.yaml

@@ -163,6 +163,14 @@ spec:
                           description: |-
                             Controls how instances are managed during scale up and down, as well as
                             during maintenance events.
+
+                            OrderedReady creates and deletes instances one at a time, waiting for
+                            each to become ready before moving to the next. Parallel creates and
+                            deletes instances all at once without waiting on readiness. Both replace
+                            instances on a template change one at a time.
+                          enum:
+                          - OrderedReady
+                          - Parallel
                           type: string
                         maxReplicas:
                           description: The maximum number of replicas.

docs/api/workloaddeployments.md

@@ -192,11 +192,17 @@ Scale settings such as minimum and maximum replica counts.
     </thead>
     <tbody><tr>
         <td><b>instanceManagementPolicy</b></td>
-        <td>string</td>
+        <td>enum</td>
         <td>
           Controls how instances are managed during scale up and down, as well as
-during maintenance events.<br/>
+during maintenance events.
+
+OrderedReady creates and deletes instances one at a time, waiting for
+each to become ready before moving to the next. Parallel creates and
+deletes instances all at once without waiting on readiness. Both replace
+instances on a template change one at a time.<br/>
           <br/>
+            <i>Enum</i>: OrderedReady, Parallel<br/>
             <i>Default</i>: OrderedReady<br/>
         </td>
         <td>true</td>

docs/api/workloads.md

@@ -188,11 +188,17 @@ Scale settings such as minimum and maximum replica counts.
     </thead>
     <tbody><tr>
         <td><b>instanceManagementPolicy</b></td>
-        <td>string</td>
+        <td>enum</td>
         <td>
           Controls how instances are managed during scale up and down, as well as
-during maintenance events.<br/>
+during maintenance events.
+
+OrderedReady creates and deletes instances one at a time, waiting for
+each to become ready before moving to the next. Parallel creates and
+deletes instances all at once without waiting on readiness. Both replace
+instances on a template change one at a time.<br/>
           <br/>
+            <i>Enum</i>: OrderedReady, Parallel<br/>
             <i>Default</i>: OrderedReady<br/>
         </td>
         <td>true</td>

internal/controller/instancecontrol/stateful/stateful_control.go

@@ -208,10 +208,25 @@ func (c *statefulControl) GetActions(
 	slices.SortFunc(recreateActions, descendingOrdinal)
 	slices.SortFunc(deleteActions, descendingOrdinal)
 
-	actions := make([]instancecontrol.Action, 0, len(createActions)+len(waitActions)+len(recreateActions)+len(deleteActions)+len(patchLabelActions)+len(patchTemplateHashActions))
+	actions, err := orderRolloutActions(deployment.Spec.ScaleSettings.InstanceManagementPolicy, createActions, waitActions, recreateActions, deleteActions)
+	if err != nil {
+		return nil, err
+	}
+
+	actions = append(actions, patchLabelActions...)
+	actions = append(actions, patchTemplateHashActions...)
 
-	switch deployment.Spec.ScaleSettings.InstanceManagementPolicy {
-	case v1alpha.OrderedReadyInstanceManagementPolicyType:
+	return actions, nil
+}
+
+func orderRolloutActions(
+	policy v1alpha.InstanceManagementPolicyType,
+	createActions, waitActions, recreateActions, deleteActions []instancecontrol.Action,
+) ([]instancecontrol.Action, error) {
+	actions := make([]instancecontrol.Action, 0, len(createActions)+len(waitActions)+len(recreateActions)+len(deleteActions))
+
+	switch policy {
+	case v1alpha.OrderedReadyInstanceManagementPolicyType, "":
 
 		// Add create and wait actions, and sort by ordinal. This allows us to wait
 		// for instances to be processed in the correct order.
@@ -233,10 +248,24 @@ func (c *statefulControl) GetActions(
 			}
 		}
 
-	}
+	case v1alpha.ParallelInstanceManagementPolicyType:
+		slices.SortFunc(createActions, ascendingOrdinal)
+		slices.SortFunc(waitActions, ascendingOrdinal)
 
-	actions = append(actions, patchLabelActions...)
-	actions = append(actions, patchTemplateHashActions...)
+		actions = append(actions, createActions...)
+		actions = append(actions, deleteActions...)
+		actions = append(actions, waitActions...)
+
+		for i := range recreateActions {
+			if i > 0 || len(createActions) > 0 || len(waitActions) > 0 {
+				recreateActions[i].SkipExecution()
+			}
+		}
+		actions = append(actions, recreateActions...)
+
+	default:
+		return nil, fmt.Errorf("unsupported instance management policy %q", policy)
+	}
 
 	return actions, nil
 }

internal/controller/instancecontrol/stateful/stateful_control_test.go

@@ -20,6 +20,8 @@ import (
 	"go.datum.net/compute/internal/controller/instancecontrol"
 )
 
+const testImageUpdate = "test-image-update"
+
 var (
 	scheme = runtime.NewScheme()
 )
@@ -101,7 +103,7 @@ func TestUpdateWithAllReadyInstances(t *testing.T) {
 	currentInstances = append(currentInstances, *getInstanceForDeployment(deployment, 0))
 	currentInstances = append(currentInstances, *getInstanceForDeployment(deployment, 1))
 
-	deployment.Spec.Template.Spec.Runtime.Sandbox.Containers[0].Image = "test-image-update"
+	deployment.Spec.Template.Spec.Runtime.Sandbox.Containers[0].Image = testImageUpdate
 
 	actions, err := control.GetActions(ctx, scheme, deployment, deployment.Spec.ScaleSettings.MinReplicas, currentInstances)
 
@@ -814,3 +816,118 @@ func getInstanceTemplate(name string, ordinal int) *v1alpha.Instance {
 
 	return instance
 }
+
+func getParallelWorkloadDeployment(minReplicas int32) *v1alpha.WorkloadDeployment {
+	deployment := getWorkloadDeployment("test-deploy", minReplicas)
+	deployment.Spec.ScaleSettings.InstanceManagementPolicy = v1alpha.ParallelInstanceManagementPolicyType
+	return deployment
+}
+
+func TestParallel_FreshDeploymentCreatesAllInstances(t *testing.T) {
+	control := NewWithOptions(Options{})
+	deployment := getParallelWorkloadDeployment(3)
+
+	actions, err := control.GetActions(context.Background(), scheme, deployment, 3, nil)
+
+	require.NoError(t, err)
+	require.Len(t, actions, 3)
+	for i, action := range actions {
+		assert.Equal(t, fmt.Sprintf("test-deploy-%d", i), action.Object.GetName())
+		assert.Equal(t, instancecontrol.ActionTypeCreate, action.ActionType())
+		assert.False(t, action.IsSkipped())
+	}
+}
+
+func TestParallel_ScaleUpDoesNotWaitOnNotReadyInstance(t *testing.T) {
+	control := NewWithOptions(Options{})
+	deployment := getParallelWorkloadDeployment(3)
+
+	notReady := getInstanceForDeployment(deployment, 0)
+	apimeta.SetStatusCondition(&notReady.Status.Conditions, metav1.Condition{
+		Type:   v1alpha.InstanceReady,
+		Status: metav1.ConditionFalse,
+	})
+
+	actions, err := control.GetActions(context.Background(), scheme, deployment, 3, []v1alpha.Instance{*notReady})
+
+	require.NoError(t, err)
+	created := map[string]bool{}
+	for _, action := range actions {
+		if action.ActionType() == instancecontrol.ActionTypeCreate {
+			assert.False(t, action.IsSkipped(), action.Object.GetName())
+			created[action.Object.GetName()] = true
+		}
+	}
+	assert.Equal(t, map[string]bool{"test-deploy-1": true, "test-deploy-2": true}, created)
+}
+
+func TestParallel_ScaleDownDeletesAllExtraInstances(t *testing.T) {
+	control := NewWithOptions(Options{})
+	deployment := getParallelWorkloadDeployment(1)
+
+	current := []v1alpha.Instance{
+		*getInstanceForDeployment(deployment, 0),
+		*getInstanceForDeployment(deployment, 1),
+		*getInstanceForDeployment(deployment, 2),
+	}
+
+	actions, err := control.GetActions(context.Background(), scheme, deployment, 1, current)
+
+	require.NoError(t, err)
+	require.Len(t, actions, 2)
+	for _, action := range actions {
+		assert.Equal(t, instancecontrol.ActionTypeDelete, action.ActionType())
+		assert.False(t, action.IsSkipped(), action.Object.GetName())
+	}
+}
+
+func TestParallel_TemplateChangeReplacesOneInstanceAtATime(t *testing.T) {
+	control := NewWithOptions(Options{})
+	deployment := getParallelWorkloadDeployment(2)
+
+	current := []v1alpha.Instance{
+		*getInstanceForDeployment(deployment, 0),
+		*getInstanceForDeployment(deployment, 1),
+	}
+	deployment.Spec.Template.Spec.Runtime.Sandbox.Containers[0].Image = testImageUpdate
+
+	actions, err := control.GetActions(context.Background(), scheme, deployment, 2, current)
+
+	require.NoError(t, err)
+	require.Len(t, actions, 2)
+	assert.Equal(t, "test-deploy-1", actions[0].Object.GetName())
+	assert.Equal(t, instancecontrol.ActionTypeDelete, actions[0].ActionType())
+	assert.False(t, actions[0].IsSkipped())
+	assert.Equal(t, "test-deploy-0", actions[1].Object.GetName())
+	assert.True(t, actions[1].IsSkipped())
+}
+
+func TestParallel_TemplateChangeHeldWhileInstanceIsReplaced(t *testing.T) {
+	control := NewWithOptions(Options{})
+	deployment := getParallelWorkloadDeployment(2)
+
+	current := []v1alpha.Instance{*getInstanceForDeployment(deployment, 0)}
+	deployment.Spec.Template.Spec.Runtime.Sandbox.Containers[0].Image = testImageUpdate
+
+	actions, err := control.GetActions(context.Background(), scheme, deployment, 2, current)
+
+	require.NoError(t, err)
+	for _, action := range actions {
+		switch action.ActionType() {
+		case instancecontrol.ActionTypeCreate:
+			assert.False(t, action.IsSkipped(), action.Object.GetName())
+		case instancecontrol.ActionTypeDelete:
+			assert.True(t, action.IsSkipped(), action.Object.GetName())
+		}
+	}
+}
+
+func TestUnsupportedInstanceManagementPolicyReturnsError(t *testing.T) {
+	control := NewWithOptions(Options{})
+	deployment := getWorkloadDeployment("test-deploy", 1)
+	deployment.Spec.ScaleSettings.InstanceManagementPolicy = "Bogus"
+
+	_, err := control.GetActions(context.Background(), scheme, deployment, 1, nil)
+
+	assert.ErrorContains(t, err, "Bogus")
+}