✨ ClusterObjectDeployment: Progressing reasons for impeded head revision rollouts

api/v1/clusterobjectdeployment_types.go

@@ -33,6 +33,9 @@ const (
 	ClusterObjectDeploymentReasonNewClusterObjectSetProgressing = "NewClusterObjectSetProgressing"
 	ClusterObjectDeploymentReasonProgressDeadlineExceeded       = ReasonProgressDeadlineExceeded
 	ClusterObjectDeploymentReasonInvalidSpecification           = "InvalidSpecification"
+	ClusterObjectDeploymentReasonRevisionPreconditionsNotMet    = "RevisionPreconditionsNotMet"
+	ClusterObjectDeploymentReasonRevisionObjectCollision        = "RevisionObjectCollision"
+	ClusterObjectDeploymentReasonRevisionInvalidSpecification   = "RevisionInvalidSpecification"
 
 	// Available condition reasons.
 	ClusterObjectDeploymentReasonAvailable        = "Available"
@@ -150,6 +153,17 @@ type ClusterObjectDeploymentStatus struct {
 	// contains labels or annotations that are not valid for the resulting
 	// ClusterObjectSet; no new revision is created until the metadata is corrected.
 	//
+	// While a rollout is incomplete, Progressing also reflects why the latest (head
+	// active) ClusterObjectSet revision is not yet Ready. It is True with reason
+	// RevisionPreconditionsNotMet when the revision's preconditions are not yet met
+	// (for example RBAC or a dependency CRD that has not propagated); the controller
+	// keeps retrying. It is False with reason RevisionObjectCollision when the
+	// revision's objects are owned by another controller or already exist, which the
+	// user must resolve. It is False with reason RevisionInvalidSpecification when the
+	// revision's own objects are invalid; this is distinct from InvalidSpecification
+	// above, which refers to the deployment's spec.template.metadata. Inspect the
+	// named ClusterObjectSet revision for the specific cause.
+	//
 	// Available is True with reason Available when at least one active (non-archived)
 	// ClusterObjectSet revision is Ready, so the deployment remains available across
 	// rollouts while a previous revision is still Ready. It is False with reason

docs/api-reference/olmv1-api-reference.md

@@ -458,7 +458,7 @@ _Appears in:_
 
 | Field | Description | Default | Validation |
 | --- | --- | --- | --- |
-| `conditions` _[Condition](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.31/#condition-v1-meta) array_ | conditions describes rollout progress and availability for the observed generation.<br />Progressing is True with reason NewClusterObjectSetProgressing while the desired<br />rollout is incomplete, or False with reason NewClusterObjectSetProgressed after<br />completion. Completion remains recorded even if the completed revision later<br />becomes unhealthy. Progressing is False with reason ProgressDeadlineExceeded when<br />the latest revision does not roll out within spec.template.spec.progressDeadlineMinutes.<br />Progressing is False with reason InvalidSpecification when spec.template.metadata<br />contains labels or annotations that are not valid for the resulting<br />ClusterObjectSet; no new revision is created until the metadata is corrected.<br />Available is True with reason Available when at least one active (non-archived)<br />ClusterObjectSet revision is Ready, so the deployment remains available across<br />rollouts while a previous revision is still Ready. It is False with reason<br />Unavailable when active revisions exist but none are Ready yet, or with reason<br />NoActiveRevision when no active revision exists. |  | Optional: \{\} <br /> |
+| `conditions` _[Condition](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.31/#condition-v1-meta) array_ | conditions describes rollout progress and availability for the observed generation.<br />Progressing is True with reason NewClusterObjectSetProgressing while the desired<br />rollout is incomplete, or False with reason NewClusterObjectSetProgressed after<br />completion. Completion remains recorded even if the completed revision later<br />becomes unhealthy. Progressing is False with reason ProgressDeadlineExceeded when<br />the latest revision does not roll out within spec.template.spec.progressDeadlineMinutes.<br />Progressing is False with reason InvalidSpecification when spec.template.metadata<br />contains labels or annotations that are not valid for the resulting<br />ClusterObjectSet; no new revision is created until the metadata is corrected.<br />While a rollout is incomplete, Progressing also reflects why the latest (head<br />active) ClusterObjectSet revision is not yet Ready. It is True with reason<br />RevisionPreconditionsNotMet when the revision's preconditions are not yet met<br />(for example RBAC or a dependency CRD that has not propagated); the controller<br />keeps retrying. It is False with reason RevisionObjectCollision when the<br />revision's objects are owned by another controller or already exist, which the<br />user must resolve. It is False with reason RevisionInvalidSpecification when the<br />revision's own objects are invalid; this is distinct from InvalidSpecification<br />above, which refers to the deployment's spec.template.metadata. Inspect the<br />named ClusterObjectSet revision for the specific cause.<br />Available is True with reason Available when at least one active (non-archived)<br />ClusterObjectSet revision is Ready, so the deployment remains available across<br />rollouts while a previous revision is still Ready. It is False with reason<br />Unavailable when active revisions exist but none are Ready yet, or with reason<br />NoActiveRevision when no active revision exists. |  | Optional: \{\} <br /> |
 
 
 #### ClusterObjectDeploymentTemplate

docs/draft/concepts/clusterobjectsets.md

@@ -264,6 +264,33 @@ recorded, the COD reports `Progressing=False` with reason
 timing. The COD can remain `Available=True` if an older active revision is still
 ready.
 
+### ClusterObjectDeployment Progressing
+
+While a rollout is incomplete, a ClusterObjectDeployment (COD) reflects *why* its
+head active revision is not yet `Ready` on its own `Progressing` condition, so you
+can tell a rollout that is merely in progress from one that is stalled on
+something you must fix. Each reason names the revision; inspect that
+ClusterObjectSet for the specific cause.
+
+| Status | Reason | Meaning | What to do |
+| --- | --- | --- | --- |
+| True | `NewClusterObjectSetProgressing` | The head revision is rolling out (`Pending`/`ObjectsNotReady`) | Wait |
+| True | `RevisionPreconditionsNotMet` | The head revision's precondition is not yet met (e.g. a mutable/invalid referenced Secret, or RBAC/dependency CRDs not yet propagated); it keeps retrying | Satisfy the precondition (grant RBAC, install the dependency, fix the Secret); it recovers automatically |
+| False | `RevisionObjectCollision` | The head revision's objects are owned by another controller or already exist; it will not roll out without intervention | Resolve ownership of the conflicting objects |
+| False | `RevisionInvalidSpecification` | The head revision's own objects are invalid | Correct the template so the rendered objects are valid |
+| False | `ProgressDeadlineExceeded` | A still-retrying revision exceeded its deadline (see [Progress deadline](#progress-deadline)) | See above |
+| False | `InvalidSpecification` | The deployment's own `spec.template.metadata` is invalid, so no revision is created | Fix the template labels/annotations |
+| False | `NewClusterObjectSetProgressed` | The head revision completed its rollout | None |
+
+`RevisionObjectCollision` and `RevisionInvalidSpecification` are *hard stalls*:
+they will not resolve on their own, so they report `Progressing=False` and take
+precedence over `ProgressDeadlineExceeded` (the actionable cause is more useful
+than "the clock ran out"). `RevisionPreconditionsNotMet` keeps retrying, so it
+stays `Progressing=True` and the progress deadline still governs it. Note that
+`RevisionInvalidSpecification` (the revision's rendered objects) is distinct from
+the deployment's own `InvalidSpecification` (invalid `spec.template.metadata`,
+reported before any revision is created).
+
 ### Completion (`status.completedAt`)
 
 The first time a revision becomes ready, the controller stamps `status.completedAt` with the current time. This timestamp is set once and never changes, marking that this revision successfully rolled out at least once — it persists even if the revision later becomes not ready. The ClusterExtension controller uses it to determine the installed revision.

helm/olmv1/base/object-controller/crd/experimental/olm.operatorframework.io_clusterobjectdeployments.yaml

@@ -551,6 +551,17 @@ spec:
                   contains labels or annotations that are not valid for the resulting
                   ClusterObjectSet; no new revision is created until the metadata is corrected.
 
+                  While a rollout is incomplete, Progressing also reflects why the latest (head
+                  active) ClusterObjectSet revision is not yet Ready. It is True with reason
+                  RevisionPreconditionsNotMet when the revision's preconditions are not yet met
+                  (for example RBAC or a dependency CRD that has not propagated); the controller
+                  keeps retrying. It is False with reason RevisionObjectCollision when the
+                  revision's objects are owned by another controller or already exist, which the
+                  user must resolve. It is False with reason RevisionInvalidSpecification when the
+                  revision's own objects are invalid; this is distinct from InvalidSpecification
+                  above, which refers to the deployment's spec.template.metadata. Inspect the
+                  named ClusterObjectSet revision for the specific cause.
+
                   Available is True with reason Available when at least one active (non-archived)
                   ClusterObjectSet revision is Ready, so the deployment remains available across
                   rollouts while a previous revision is still Ready. It is False with reason

internal/object-controller/controllers/clusterobjectdeployment_controller.go

@@ -142,19 +142,58 @@ func (c *ClusterObjectDeploymentReconciler) reconcileRollout(ctx context.Context
 		return ctrl.Result{}, nil
 	}
 
+	// Reflect why the head revision is not yet Ready. Hard stalls will not resolve
+	// without user action, so they report Progressing=False and short-circuit the
+	// deadline (a more actionable reason than ProgressDeadlineExceeded). The COD
+	// surfaces its own generic, revision-naming message and does not echo the
+	// revision's Ready message; the user inspects the named ClusterObjectSet for the
+	// specific cause.
+	readyReason := headRevisionReadyReason(latest)
+	switch readyReason {
+	case ocv1.ClusterObjectSetReasonInvalidSpecification:
+		setProgressing(cod, metav1.ConditionFalse, ocv1.ClusterObjectDeploymentReasonRevisionInvalidSpecification,
+			fmt.Sprintf("latest revision %q has an invalid specification; inspect the ClusterObjectSet for details", latest.Name))
+		return ctrl.Result{}, nil
+	case ocv1.ClusterObjectSetReasonObjectCollision:
+		setProgressing(cod, metav1.ConditionFalse, ocv1.ClusterObjectDeploymentReasonRevisionObjectCollision,
+			fmt.Sprintf("latest revision %q cannot take ownership of one or more objects owned by another controller or already present; inspect the ClusterObjectSet for details", latest.Name))
+		return ctrl.Result{}, nil
+	}
+
 	remaining, hasDeadline := durationUntilDeploymentDeadline(c.Clock, cod, latest)
 	if hasDeadline && remaining <= 0 {
 		setProgressing(cod, metav1.ConditionFalse, ocv1.ClusterObjectDeploymentReasonProgressDeadlineExceeded, "latest revision exceeded its progress deadline")
 		return ctrl.Result{}, nil
 	}
 
+	// PreconditionsNotMet is a still-retrying state: the controller keeps retrying,
+	// so Progressing stays True and the deadline above governs it.
+	if readyReason == ocv1.ClusterObjectSetReasonPreconditionsNotMet {
+		setProgressing(cod, metav1.ConditionTrue, ocv1.ClusterObjectDeploymentReasonRevisionPreconditionsNotMet,
+			fmt.Sprintf("latest revision %q is waiting for preconditions to be met; inspect the ClusterObjectSet for details", latest.Name))
+		if hasDeadline {
+			return ctrl.Result{RequeueAfter: remaining}, nil
+		}
+		return ctrl.Result{}, nil
+	}
+
 	setProgressing(cod, metav1.ConditionTrue, ocv1.ClusterObjectDeploymentReasonNewClusterObjectSetProgressing, "latest revision is progressing")
 	if hasDeadline {
 		return ctrl.Result{RequeueAfter: remaining}, nil
 	}
 	return ctrl.Result{}, nil
 }
 
+// headRevisionReadyReason returns the Ready condition reason of the head active
+// revision, or "" when the revision has no Ready condition yet.
+func headRevisionReadyReason(cos *ocv1.ClusterObjectSet) string {
+	cond := apimeta.FindStatusCondition(cos.Status.Conditions, ocv1.ClusterObjectSetTypeReady)
+	if cond == nil {
+		return ""
+	}
+	return cond.Reason
+}
+
 // listRevisions returns all ClusterObjectSets in the deployment's group, sorted
 // ascending by revision number.
 func (c *ClusterObjectDeploymentReconciler) listRevisions(ctx context.Context, group string) ([]ocv1.ClusterObjectSet, error) {

internal/object-controller/controllers/clusterobjectdeployment_controller_test.go

@@ -122,6 +122,26 @@ func expireRevisionDeadline(t *testing.T, c client.Client, name string) {
 	require.NoError(t, c.Update(t.Context(), cos))
 }
 
+// revisionReadyProbeMessage is a distinctive message written onto a revision's
+// Ready condition so tests can assert the COD surfaces its own generic message
+// rather than echoing the revision's message.
+const revisionReadyProbeMessage = "COS-ONLY-REVISION-READY-MESSAGE"
+
+// setRevisionReadyReason sets the named revision's Ready condition to False with
+// the given reason and a distinctive message (revisionReadyProbeMessage).
+func setRevisionReadyReason(t *testing.T, c client.Client, name, reason string) {
+	t.Helper()
+	cos, ok := getCOS(t, c, name)
+	require.True(t, ok)
+	apimeta.SetStatusCondition(&cos.Status.Conditions, metav1.Condition{
+		Type:    ocv1.ClusterObjectSetTypeReady,
+		Status:  metav1.ConditionFalse,
+		Reason:  reason,
+		Message: revisionReadyProbeMessage,
+	})
+	require.NoError(t, c.Status().Update(t.Context(), cos))
+}
+
 func Test_ClusterObjectDeploymentReconciler_Available(t *testing.T) {
 	t.Run("no active revision before the first revision exists", func(t *testing.T) {
 		c := newCODClient(t, codFixture())
@@ -381,10 +401,15 @@ func Test_TemplateHash_MetadataParticipates(t *testing.T) {
 
 func Test_ClusterObjectDeploymentReconciler_NoNewRevisionWhenTemplateUnchanged(t *testing.T) {
 	for _, tc := range []struct {
-		name       string
-		prep       func(t *testing.T, c client.Client, cosName string)
+		name string
+		prep func(t *testing.T, c client.Client, cosName string)
+
 		wantStatus metav1.ConditionStatus
 		wantReason string
+		// wantMsgContains, when set, must be a substring of the COD Progressing
+		// message; wantMsgExcludes, when set, must not be.
+		wantMsgContains string
+		wantMsgExcludes string
 	}{
 		{
 			name: "completed latest revision reports Progressed",
@@ -409,6 +434,62 @@ func Test_ClusterObjectDeploymentReconciler_NoNewRevisionWhenTemplateUnchanged(t
 			wantStatus: metav1.ConditionFalse,
 			wantReason: ocv1.ClusterObjectDeploymentReasonProgressDeadlineExceeded,
 		},
+		{
+			name: "head revision ObjectCollision reports RevisionObjectCollision and is not progressing",
+			prep: func(t *testing.T, c client.Client, cosName string) {
+				setRevisionReadyReason(t, c, cosName, ocv1.ClusterObjectSetReasonObjectCollision)
+			},
+			wantStatus:      metav1.ConditionFalse,
+			wantReason:      ocv1.ClusterObjectDeploymentReasonRevisionObjectCollision,
+			wantMsgContains: clusterObjectDeploymentName + "-1",
+			wantMsgExcludes: revisionReadyProbeMessage,
+		},
+		{
+			name: "head revision InvalidSpecification reports RevisionInvalidSpecification and is not progressing",
+			prep: func(t *testing.T, c client.Client, cosName string) {
+				setRevisionReadyReason(t, c, cosName, ocv1.ClusterObjectSetReasonInvalidSpecification)
+			},
+			wantStatus:      metav1.ConditionFalse,
+			wantReason:      ocv1.ClusterObjectDeploymentReasonRevisionInvalidSpecification,
+			wantMsgContains: clusterObjectDeploymentName + "-1",
+			wantMsgExcludes: revisionReadyProbeMessage,
+		},
+		{
+			name: "head revision PreconditionsNotMet reports RevisionPreconditionsNotMet and keeps progressing",
+			prep: func(t *testing.T, c client.Client, cosName string) {
+				setRevisionReadyReason(t, c, cosName, ocv1.ClusterObjectSetReasonPreconditionsNotMet)
+			},
+			wantStatus:      metav1.ConditionTrue,
+			wantReason:      ocv1.ClusterObjectDeploymentReasonRevisionPreconditionsNotMet,
+			wantMsgContains: clusterObjectDeploymentName + "-1",
+			wantMsgExcludes: revisionReadyProbeMessage,
+		},
+		{
+			name:       "head revision with no Ready condition falls through to NewClusterObjectSetProgressing",
+			prep:       nil,
+			wantStatus: metav1.ConditionTrue,
+			wantReason: ocv1.ClusterObjectDeploymentReasonNewClusterObjectSetProgressing,
+		},
+		{
+			name: "head revision with unrecognized Ready reason falls through to NewClusterObjectSetProgressing",
+			prep: func(t *testing.T, c client.Client, cosName string) {
+				setRevisionReadyReason(t, c, cosName, ocv1.ClusterObjectSetReasonObjectsNotReady)
+			},
+			wantStatus: metav1.ConditionTrue,
+			wantReason: ocv1.ClusterObjectDeploymentReasonNewClusterObjectSetProgressing,
+		},
+		{
+			name: "completed revision that also reports ObjectCollision still reports Progressed",
+			prep: func(t *testing.T, c client.Client, cosName string) {
+				setRevisionReadyReason(t, c, cosName, ocv1.ClusterObjectSetReasonObjectCollision)
+				cos, ok := getCOS(t, c, cosName)
+				require.True(t, ok)
+				cos.Status.CompletedAt = metav1.Now()
+				require.NoError(t, c.Status().Update(t.Context(), cos))
+			},
+			wantStatus: metav1.ConditionFalse,
+			wantReason: ocv1.ClusterObjectDeploymentReasonNewClusterObjectSetProgressed,
+		},
 	} {
 		t.Run(tc.name, func(t *testing.T) {
 			c := newCODClient(t, codFixture())
@@ -432,6 +513,12 @@ func Test_ClusterObjectDeploymentReconciler_NoNewRevisionWhenTemplateUnchanged(t
 			require.NotNil(t, cond)
 			require.Equal(t, tc.wantStatus, cond.Status)
 			require.Equal(t, tc.wantReason, cond.Reason)
+			if tc.wantMsgContains != "" {
+				require.Contains(t, cond.Message, tc.wantMsgContains)
+			}
+			if tc.wantMsgExcludes != "" {
+				require.NotContains(t, cond.Message, tc.wantMsgExcludes)
+			}
 		})
 	}
 }