From 835dec893f272649ce6f85f3fba12ab4fee4b048 Mon Sep 17 00:00:00 2001 From: Fabricio Aguiar Date: Thu, 22 Oct 2026 16:20:52 +0100 Subject: [PATCH 1/3] test(object-controller): define revision deadline semantics Document one deadline for the entire revision, starting at COS creation. Time spent waiting to create the COS is outside the MVP budget. Cover a deployment older than its deadline, five phases sharing one budget, late completion, later health regressions, and a fresh budget for the next revision without depending on phase completion timestamps. Signed-off-by: Fabricio Aguiar rh-pre-commit.version: 2.3.2 rh-pre-commit.check-secrets: ENABLED --- api/v1/clusterobjectdeployment_types.go | 11 +- docs/draft/concepts/clusterobjectsets.md | 20 ++ .../clusterobjectdeployment_deadline_test.go | 193 ++++++++++++++++++ 3 files changed, 220 insertions(+), 4 deletions(-) create mode 100644 internal/object-controller/controllers/clusterobjectdeployment_deadline_test.go diff --git a/api/v1/clusterobjectdeployment_types.go b/api/v1/clusterobjectdeployment_types.go index 8d242ba50a..b7ed616ea3 100644 --- a/api/v1/clusterobjectdeployment_types.go +++ b/api/v1/clusterobjectdeployment_types.go @@ -109,10 +109,13 @@ type ClusterObjectDeploymentTemplateSpec struct { // +listMapKey=name Phases []ClusterObjectSetPhase `json:"phases"` //nolint:kubeapilinter // These are ordered groups of objects, not an enum describing lifecycle state. - // progressDeadlineMinutes is an optional field that defines the maximum period - // of time in minutes after which an installation should be considered failed and - // require manual intervention. This functionality is disabled when no value - // is provided. The minimum period is 10 minutes, and the maximum is 720 minutes (12 hours). + // progressDeadlineMinutes is the maximum number of minutes a revision may take + // to roll out, measured from the ClusterObjectSet's creation timestamp. Time + // spent waiting for the ClusterObjectSet to be created is not counted. The + // deadline covers all phases together; completing a phase does not reset it. + // Once a revision has completed, the deadline no longer applies, even if it + // later becomes unhealthy. When omitted, no deadline is enforced. The minimum + // period is 10 minutes, and the maximum is 720 minutes (12 hours). // // The value is passed through to each ClusterObjectSet revision created from this template. // diff --git a/docs/draft/concepts/clusterobjectsets.md b/docs/draft/concepts/clusterobjectsets.md index d568561f8c..fbc0aa97cf 100644 --- a/docs/draft/concepts/clusterobjectsets.md +++ b/docs/draft/concepts/clusterobjectsets.md @@ -233,6 +233,26 @@ Indicates whether all objects in the revision have been successfully rolled out On a revision's first reconciliation the `Ready` condition is initialized to `Unknown`/`Pending`, so it is always present. A transient internal error (e.g. an API read failure) leaves the `Ready` condition unchanged and is retried with backoff; it is not reported as a distinct reason (while the revision is still `Pending`, it stays `Unknown`/`Pending`). +### Progress deadline + +`spec.progressDeadlineMinutes` sets one time budget for the entire revision, +measured from the ClusterObjectSet's creation timestamp. Completing a phase does +not restart the timer. For example, a five-phase revision with a ten-minute +deadline reports `Ready=False` with reason `ProgressDeadlineExceeded` if it is +still incomplete ten minutes after creation, even if four phases have completed. + +ClusterObjectDeployments configure this budget through +`spec.template.spec.progressDeadlineMinutes` and pass it to each new revision. +Time spent waiting for the deployment controller to create a ClusterObjectSet +is not counted. Each revision receives its own budget. The deployment reports +`Progressing=False` with reason `ProgressDeadlineExceeded` when its latest +revision exceeds the deadline. It can remain `Available=True` if an older active +revision is still ready. + +Omitting the field disables the deadline. Reconciliation continues after a +timeout, so a revision can recover when all its objects become ready. Once a +revision has completed, later health regressions do not restart deadline timing. + ### Completion (`status.completedAt`) The first time a revision becomes ready, the controller stamps `status.completedAt` with the current time. This timestamp is set once and never changes, marking that this revision successfully rolled out at least once — it persists even if the revision later becomes not ready. The ClusterExtension controller uses it to determine the installed revision. diff --git a/internal/object-controller/controllers/clusterobjectdeployment_deadline_test.go b/internal/object-controller/controllers/clusterobjectdeployment_deadline_test.go new file mode 100644 index 0000000000..cd18940bff --- /dev/null +++ b/internal/object-controller/controllers/clusterobjectdeployment_deadline_test.go @@ -0,0 +1,193 @@ +//go:build !standard + +package controllers_test + +import ( + "context" + "fmt" + "testing" + "time" + + "github.com/stretchr/testify/require" + "go.uber.org/mock/gomock" + corev1 "k8s.io/api/core/v1" + apimeta "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + clocktesting "k8s.io/utils/clock/testing" + "pkg.package-operator.run/boxcutter" + "pkg.package-operator.run/boxcutter/machinery" + machinerytypes "pkg.package-operator.run/boxcutter/machinery/types" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + "sigs.k8s.io/controller-runtime/pkg/client/interceptor" + + ocv1 "github.com/operator-framework/operator-controller/api/v1" + "github.com/operator-framework/operator-controller/internal/object-controller/controllers" + "github.com/operator-framework/operator-controller/internal/testutil" +) + +func Test_ClusterObjectDeploymentReconciler_DeadlineCoversEntireRevision(t *testing.T) { + creation := time.Date(2026, 10, 22, 12, 0, 0, 0, time.UTC) + clk := clocktesting.NewFakeClock(creation) + scheme := newScheme(t) + require.NoError(t, corev1.AddToScheme(scheme)) + mockCtrl := gomock.NewController(t) + + cod := codFixture() + cod.UID = types.UID(cod.Name) + // Time before the COS exists is outside the revision's budget, even when + // creating it took longer than the configured deadline. + cod.CreationTimestamp = metav1.NewTime(creation.Add(-30 * time.Minute)) + cod.Spec.Template.Spec.ProgressDeadlineMinutes = 10 + phaseTemplate := cod.Spec.Template.Spec.Phases[0] + cod.Spec.Template.Spec.Phases = nil + for i := range 5 { + phase := phaseTemplate.DeepCopy() + phase.Name = fmt.Sprintf("phase-%d", i+1) + phase.Objects[0].Object.SetName(phase.Name) + cod.Spec.Template.Spec.Phases = append(cod.Spec.Template.Spec.Phases, *phase) + } + + c := testutil.WithIndexes(t, fake.NewClientBuilder().WithScheme(scheme), controllers.SetupIndexes). + WithStatusSubresource(&ocv1.ClusterObjectDeployment{}, &ocv1.ClusterObjectSet{}). + WithObjects(cod). + WithInterceptorFuncs(interceptor.Funcs{ + Create: func(ctx context.Context, cl client.WithWatch, obj client.Object, opts ...client.CreateOption) error { + // Supply the creation metadata normally assigned by the API server. + obj.SetCreationTimestamp(metav1.NewTime(clk.Now())) + obj.SetUID(types.UID(obj.GetName())) + return cl.Create(ctx, obj, opts...) + }, + }).Build() + _, err := reconcileCOD(t, c) + require.NoError(t, err) + cosName := cod.Name + "-1" + cos, found := getCOS(t, c, cosName) + require.True(t, found) + require.True(t, cos.CreationTimestamp.Time.Equal(creation)) + require.Equal(t, int32(10), cos.Spec.ProgressDeadlineMinutes) + require.Len(t, cos.Spec.Phases, 5) + require.Equal(t, metav1.ConditionTrue, progressing(t, c).Status) + // Start after finalizer initialization; the fake client's metadata patch + // otherwise drops the first reconcile's in-memory phase digests. + cos.Finalizers = []string{"olm.operatorframework.io/teardown"} + require.NoError(t, c.Update(t.Context(), cos)) + + completedPhases := 0 + engine := newMockRevisionEngineWithReconcile(mockCtrl, + func(context.Context, machinerytypes.Revision, ...machinerytypes.RevisionReconcileOption) (machinery.RevisionResult, error) { + phases := make([]machinery.PhaseResult, 0, len(cos.Spec.Phases)) + for i, phase := range cos.Spec.Phases { + probeStatus := machinerytypes.ProbeStatusFalse + if i < completedPhases { + probeStatus = machinerytypes.ProbeStatusTrue + } + phases = append(phases, newMockPhaseResult(mockCtrl, phaseResultConfig{ + name: phase.Name, + isComplete: i < completedPhases, + objects: []machinery.ObjectResult{newMockObjectResult(mockCtrl, objectResultConfig{ + action: machinery.ActionCreated, + object: phase.Objects[0].Object.DeepCopy(), + complete: i < completedPhases, + paused: i > completedPhases, + probes: machinerytypes.ProbeResultContainer{ + boxcutter.ProgressProbeType: {Status: probeStatus}, + }, + })}, + })) + } + return newMockRevisionResult(mockCtrl, revisionResultConfig{ + phases: phases, + isComplete: completedPhases == len(cos.Spec.Phases), + }), nil + }, nil, + ) + cache := newMockTrackingCache(mockCtrl, c, nil) + factory := newMockRevisionEngineFactoryWithEngine(mockCtrl, engine, nil) + reconcileRevision := func() ctrl.Result { + t.Helper() + // Reconstruct the reconciler each time so timing also survives restarts. + r := &controllers.ClusterObjectSetReconciler{ + Client: c, Clock: clk, TrackingCache: cache, RevisionEngineFactory: factory, + } + result, err := r.Reconcile(t.Context(), ctrl.Request{NamespacedName: types.NamespacedName{Name: cosName}}) + require.NoError(t, err) + _, err = reconcileCOD(t, c) + require.NoError(t, err) + return result + } + + // Four phase completions consume eight minutes of the same ten-minute budget. + for completedPhases = 0; completedPhases < 5; completedPhases++ { + elapsed := time.Duration(completedPhases*2) * time.Minute + clk.SetTime(creation.Add(elapsed)) + result := reconcileRevision() + cos, found = getCOS(t, c, cosName) + require.True(t, found) + require.Equal(t, 10*time.Minute-elapsed, result.RequeueAfter, "revision conditions: %+v", cos.Status.Conditions) + require.EqualValues(t, completedPhases, cos.Status.ObjectCounts.Ready) + require.True(t, cos.Status.CompletedAt.IsZero()) + condition := progressing(t, c) + require.NotNil(t, condition) + require.Equal(t, metav1.ConditionTrue, condition.Status) + require.Equal(t, ocv1.ClusterObjectDeploymentReasonNewClusterObjectSetProgressing, condition.Reason) + } + + // The last phase remains incomplete when the revision reaches its deadline. + completedPhases = 4 + clk.SetTime(creation.Add(10 * time.Minute)) + require.Zero(t, reconcileRevision().RequeueAfter) + cos, found = getCOS(t, c, cosName) + require.True(t, found) + ready := apimeta.FindStatusCondition(cos.Status.Conditions, ocv1.ClusterObjectSetTypeReady) + require.NotNil(t, ready) + require.Equal(t, metav1.ConditionFalse, ready.Status) + require.Equal(t, ocv1.ReasonProgressDeadlineExceeded, ready.Reason) + condition := progressing(t, c) + require.NotNil(t, condition) + require.Equal(t, metav1.ConditionFalse, condition.Status) + require.Equal(t, ocv1.ClusterObjectDeploymentReasonProgressDeadlineExceeded, condition.Reason) + + // Completing the revision still recovers after timeout. + completedPhases = 5 + clk.Step(time.Minute) + require.Zero(t, reconcileRevision().RequeueAfter) + cos, found = getCOS(t, c, cosName) + require.True(t, found) + require.True(t, cos.Status.CompletedAt.Time.Equal(clk.Now())) + condition = progressing(t, c) + require.NotNil(t, condition) + require.Equal(t, ocv1.ClusterObjectDeploymentReasonNewClusterObjectSetProgressed, condition.Reason) + + // Later health regressions do not make the completed revision time out again. + completedPhases = 4 + clk.Step(30 * time.Minute) + require.Zero(t, reconcileRevision().RequeueAfter) + cos, found = getCOS(t, c, cosName) + require.True(t, found) + ready = apimeta.FindStatusCondition(cos.Status.Conditions, ocv1.ClusterObjectSetTypeReady) + require.NotNil(t, ready) + require.Equal(t, ocv1.ClusterObjectSetReasonObjectsNotReady, ready.Reason) + condition = progressing(t, c) + require.NotNil(t, condition) + require.Equal(t, ocv1.ClusterObjectDeploymentReasonNewClusterObjectSetProgressed, condition.Reason) + + // A later revision receives a fresh budget despite the deployment's age and + // the previous revision's earlier timeout. + require.NoError(t, c.Get(t.Context(), client.ObjectKeyFromObject(cod), cod)) + cod.Spec.Template.Spec.Phases[0].Objects[0].Object.Object["data"] = map[string]interface{}{"version": "second"} + require.NoError(t, c.Update(t.Context(), cod)) + _, err = reconcileCOD(t, c) + require.NoError(t, err) + cosName = cod.Name + "-2" + cos, found = getCOS(t, c, cosName) + require.True(t, found) + require.True(t, cos.CreationTimestamp.Time.Equal(clk.Now())) + cos.Finalizers = []string{"olm.operatorframework.io/teardown"} + require.NoError(t, c.Update(t.Context(), cos)) + completedPhases = 0 + require.Equal(t, 10*time.Minute, reconcileRevision().RequeueAfter) + require.Equal(t, metav1.ConditionTrue, progressing(t, c).Status) +} From 5eb370b15fb0f58056e892405680144f210abd4c Mon Sep 17 00:00:00 2001 From: Fabricio Aguiar Date: Thu, 22 Oct 2026 17:05:20 +0100 Subject: [PATCH 2/3] fix(object-controller): enforce COD revision progress deadlines Calculate the deployment deadline from the desired COS creation timestamp through its first successful completion. Enforce the deadline even when COS readiness remains Pending, PreconditionsNotMet, or another non-deadline reason. Schedule the COD at the deadline and cap retry backoff so a quiet or failing revision cannot leave the deployment progressing indefinitely. Reuse the revision completion latch to allow late recovery without timing completed revisions again. Keep time waiting for COS creation outside the MVP budget. Cover fixed budgets across phases, blocked revisions, retry scheduling, creation retries, late completion, and availability from an older revision. Signed-off-by: Fabricio Aguiar rh-pre-commit.version: 2.3.2 rh-pre-commit.check-secrets: ENABLED --- api/v1/clusterobjectdeployment_types.go | 4 +- docs/draft/concepts/clusterobjectsets.md | 41 +-- .../clusterobjectdeployment_controller.go | 138 +++++++--- ...clusterobjectdeployment_controller_test.go | 23 +- .../clusterobjectdeployment_deadline_test.go | 255 ++++++++++++++++-- .../controllers/progress_deadline.go | 28 +- .../controllers/progress_deadline_test.go | 230 ++++++++++++++++ .../features/cod_progression_timeout.feature | 9 +- 8 files changed, 625 insertions(+), 103 deletions(-) diff --git a/api/v1/clusterobjectdeployment_types.go b/api/v1/clusterobjectdeployment_types.go index b7ed616ea3..86d02882bf 100644 --- a/api/v1/clusterobjectdeployment_types.go +++ b/api/v1/clusterobjectdeployment_types.go @@ -117,8 +117,6 @@ type ClusterObjectDeploymentTemplateSpec struct { // later becomes unhealthy. When omitted, no deadline is enforced. The minimum // period is 10 minutes, and the maximum is 720 minutes (12 hours). // - // The value is passed through to each ClusterObjectSet revision created from this template. - // // +kubebuilder:validation:Minimum:=10 // +kubebuilder:validation:Maximum:=720 // +optional @@ -147,7 +145,7 @@ type ClusterObjectDeploymentStatus struct { // rollout is incomplete, or False with reason NewClusterObjectSetProgressed after // completion. Completion remains recorded even if the completed revision later // becomes unhealthy. Progressing is False with reason ProgressDeadlineExceeded when - // the latest revision does not roll out within its spec.progressDeadlineMinutes. + // the latest revision does not roll out within spec.template.spec.progressDeadlineMinutes. // Progressing is False with reason InvalidSpecification when spec.template.metadata // contains labels or annotations that are not valid for the resulting // ClusterObjectSet; no new revision is created until the metadata is corrected. diff --git a/docs/draft/concepts/clusterobjectsets.md b/docs/draft/concepts/clusterobjectsets.md index fbc0aa97cf..f7f328d671 100644 --- a/docs/draft/concepts/clusterobjectsets.md +++ b/docs/draft/concepts/clusterobjectsets.md @@ -235,23 +235,34 @@ On a revision's first reconciliation the `Ready` condition is initialized to `Un ### Progress deadline -`spec.progressDeadlineMinutes` sets one time budget for the entire revision, -measured from the ClusterObjectSet's creation timestamp. Completing a phase does -not restart the timer. For example, a five-phase revision with a ten-minute -deadline reports `Ready=False` with reason `ProgressDeadlineExceeded` if it is -still incomplete ten minutes after creation, even if four phases have completed. - -ClusterObjectDeployments configure this budget through -`spec.template.spec.progressDeadlineMinutes` and pass it to each new revision. -Time spent waiting for the deployment controller to create a ClusterObjectSet -is not counted. Each revision receives its own budget. The deployment reports -`Progressing=False` with reason `ProgressDeadlineExceeded` when its latest -revision exceeds the deadline. It can remain `Available=True` if an older active -revision is still ready. +`spec.progressDeadlineMinutes` sets one time budget for the entire +ClusterObjectSet (COS) revision, measured from its creation timestamp until its +first successful rollout, recorded in `status.completedAt`. Completing a phase +does not reset the timer. + +For a ClusterObjectDeployment (COD), this is the time allowed for the deployment to +progress to its desired new revision. Configure it through +`spec.template.spec.progressDeadlineMinutes`. The COD enforces this budget without +setting `spec.progressDeadlineMinutes` on the ClusterObjectSets it creates. Time +spent waiting for the ClusterObjectSet to be created is not counted. Each new +revision receives its own budget. + +If the desired revision is still incomplete when that budget expires, the COD +reports `Progressing=False` with reason `ProgressDeadlineExceeded`. A five-phase +revision with a ten-minute deadline has ten minutes total, even if four phases +have completed. The COD schedules a deadline check even if the COS emits no +further events. + +The COS's `Ready` condition continues to report readiness failures, such as +`ObjectsNotReady`, `PreconditionsNotMet`, or `InvalidSpecification`. The COD checks +elapsed time independently of the COS's readiness condition. Omitting the field disables the deadline. Reconciliation continues after a -timeout, so a revision can recover when all its objects become ready. Once a -revision has completed, later health regressions do not restart deadline timing. +timeout, so a revision can still complete later. Once `status.completedAt` is +recorded, the COD reports `Progressing=False` with reason +`NewClusterObjectSetProgressed`. Later health regressions do not restart deadline +timing. The COD can remain `Available=True` if an older active revision is still +ready. ### Completion (`status.completedAt`) diff --git a/internal/object-controller/controllers/clusterobjectdeployment_controller.go b/internal/object-controller/controllers/clusterobjectdeployment_controller.go index b5da2ac791..e5bb4302ad 100644 --- a/internal/object-controller/controllers/clusterobjectdeployment_controller.go +++ b/internal/object-controller/controllers/clusterobjectdeployment_controller.go @@ -9,6 +9,7 @@ import ( "fmt" "maps" "slices" + "time" "unicode/utf8" "k8s.io/apimachinery/pkg/api/equality" @@ -20,9 +21,12 @@ import ( "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/validation/field" + "k8s.io/client-go/util/workqueue" + "k8s.io/utils/clock" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/builder" "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller" "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" "sigs.k8s.io/controller-runtime/pkg/handler" "sigs.k8s.io/controller-runtime/pkg/log" @@ -43,6 +47,7 @@ const defaultRevisionHistoryLimit = 5 type ClusterObjectDeploymentReconciler struct { Client client.Client Scheme *runtime.Scheme + Clock clock.Clock } //+kubebuilder:rbac:groups=olm.operatorframework.io,resources=clusterobjectdeployments,verbs=get;list;watch;update;patch;create;delete @@ -53,6 +58,9 @@ type ClusterObjectDeploymentReconciler struct { func (c *ClusterObjectDeploymentReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { l := log.FromContext(ctx).WithName("cluster-object-deployment") ctx = log.IntoContext(ctx, l) + if c.Clock == nil { + c.Clock = clock.RealClock{} + } existing := &ocv1.ClusterObjectDeployment{} if err := c.Client.Get(ctx, req.NamespacedName, existing); err != nil { @@ -68,7 +76,7 @@ func (c *ClusterObjectDeploymentReconciler) Reconcile(ctx context.Context, req c defer l.Info("reconcile ending") reconciled := existing.DeepCopy() - reconcileErr := c.reconcile(ctx, reconciled) + result, reconcileErr := c.reconcile(ctx, reconciled) if !equality.Semantic.DeepEqual(existing.Status, reconciled.Status) { if err := c.Client.Status().Update(ctx, reconciled); err != nil { @@ -76,16 +84,16 @@ func (c *ClusterObjectDeploymentReconciler) Reconcile(ctx context.Context, req c } } - return ctrl.Result{}, reconcileErr + return result, reconcileErr } // reconcile garbage-collects excess archived revisions and reconciles the -// deployment's rollout (see reconcileRollout). Requeues are driven by the -// controller's watch on owned ClusterObjectSets. -func (c *ClusterObjectDeploymentReconciler) reconcile(ctx context.Context, cod *ocv1.ClusterObjectDeployment) error { +// deployment's rollout (see reconcileRollout). Child events and the revision's +// fixed deadline drive subsequent reconciliations. +func (c *ClusterObjectDeploymentReconciler) reconcile(ctx context.Context, cod *ocv1.ClusterObjectDeployment) (ctrl.Result, error) { revisions, err := c.listRevisions(ctx, cod.Name) if err != nil { - return err + return ctrl.Result{}, err } // Garbage collection is best-effort housekeeping and must not block rollout or @@ -96,13 +104,14 @@ func (c *ClusterObjectDeploymentReconciler) reconcile(ctx context.Context, cod * // rollout outcome so the failure is still surfaced and retried. gcErr := c.garbageCollectArchivedRevisions(ctx, cod, revisions) - return errors.Join(gcErr, c.reconcileRollout(ctx, cod, revisions)) + result, rolloutErr := c.reconcileRollout(ctx, cod, revisions) + return result, errors.Join(gcErr, rolloutErr) } // reconcileRollout creates a new ClusterObjectSet revision when the template has // changed (or no active revision exists) and reflects the latest revision's // rollout progress in the deployment's Available and Progressing conditions. -func (c *ClusterObjectDeploymentReconciler) reconcileRollout(ctx context.Context, cod *ocv1.ClusterObjectDeployment, revisions []ocv1.ClusterObjectSet) error { +func (c *ClusterObjectDeploymentReconciler) reconcileRollout(ctx context.Context, cod *ocv1.ClusterObjectDeployment, revisions []ocv1.ClusterObjectSet) (ctrl.Result, error) { hash := templateHash(cod.Spec.Template) setAvailable(cod, revisions) @@ -113,35 +122,37 @@ func (c *ClusterObjectDeploymentReconciler) reconcileRollout(ctx context.Context // in logs, so validate here and surface it on the Progressing condition instead. if msg := validateTemplateMetadata(cod.Spec.Template.Metadata); msg != "" { setProgressing(cod, metav1.ConditionFalse, ocv1.ClusterObjectDeploymentReasonInvalidSpecification, msg) - return nil + return ctrl.Result{}, nil } latest := latestActiveRevision(revisions) if latest == nil || latest.Labels[labels.TemplateHashKey] != hash { - if err := c.createRevision(ctx, cod, revisions, hash); err != nil { + var err error + latest, err = c.createRevision(ctx, cod, revisions, hash) + if err != nil { setProgressing(cod, metav1.ConditionTrue, ocv1.ClusterObjectDeploymentReasonNewClusterObjectSetProgressing, "waiting for new revision to be created") - return err + return ctrl.Result{}, err } - setProgressing(cod, metav1.ConditionTrue, ocv1.ClusterObjectDeploymentReasonNewClusterObjectSetProgressing, "waiting for new revision to be created") - return nil } - // A revision that has exceeded its progress deadline reports Ready=False with - // reason ProgressDeadlineExceeded and never records completedAt. Surface that - // terminal rollout state on the deployment's Progressing condition so it is - // actionable, rather than reporting an indefinitely-progressing rollout. - if ready := apimeta.FindStatusCondition(latest.Status.Conditions, ocv1.ClusterObjectSetTypeReady); ready != nil && - ready.Status == metav1.ConditionFalse && ready.Reason == ocv1.ReasonProgressDeadlineExceeded { + // Completion is historical: later health changes cannot restart the deadline + // or turn an already completed rollout into a failed rollout. + if !latest.Status.CompletedAt.IsZero() { + setProgressing(cod, metav1.ConditionFalse, ocv1.ClusterObjectDeploymentReasonNewClusterObjectSetProgressed, "latest revision has progressed") + return ctrl.Result{}, nil + } + + remaining, hasDeadline := durationUntilDeploymentDeadline(c.Clock, cod, latest) + if hasDeadline && remaining <= 0 { setProgressing(cod, metav1.ConditionFalse, ocv1.ClusterObjectDeploymentReasonProgressDeadlineExceeded, "latest revision exceeded its progress deadline") - return nil + return ctrl.Result{}, nil } - if latest.Status.CompletedAt.IsZero() { - setProgressing(cod, metav1.ConditionTrue, ocv1.ClusterObjectDeploymentReasonNewClusterObjectSetProgressing, "latest revision is progressing") - } else { - setProgressing(cod, metav1.ConditionFalse, ocv1.ClusterObjectDeploymentReasonNewClusterObjectSetProgressed, "latest revision has progressed") + setProgressing(cod, metav1.ConditionTrue, ocv1.ClusterObjectDeploymentReasonNewClusterObjectSetProgressing, "latest revision is progressing") + if hasDeadline { + return ctrl.Result{RequeueAfter: remaining}, nil } - return nil + return ctrl.Result{}, nil } // listRevisions returns all ClusterObjectSets in the deployment's group, sorted @@ -229,19 +240,64 @@ func archivedTeardownComplete(rev *ocv1.ClusterObjectSet) bool { // the deployment. Any create error, including AlreadyExists, is returned. // // TODO: revisit AlreadyExists handling under the adoption use-case. -func (c *ClusterObjectDeploymentReconciler) createRevision(ctx context.Context, cod *ocv1.ClusterObjectDeployment, revisions []ocv1.ClusterObjectSet, hash string) error { +func (c *ClusterObjectDeploymentReconciler) createRevision(ctx context.Context, cod *ocv1.ClusterObjectDeployment, revisions []ocv1.ClusterObjectSet, hash string) (*ocv1.ClusterObjectSet, error) { revision, err := nextRevisionNumber(revisions) if err != nil { - return fmt.Errorf("allocating revision for group %q: %w", cod.Name, err) + return nil, fmt.Errorf("allocating revision for group %q: %w", cod.Name, err) } cos := buildClusterObjectSet(cod, revision, hash) if err := controllerutil.SetControllerReference(cod, cos, c.Scheme); err != nil { - return fmt.Errorf("setting controller reference: %w", err) + return nil, fmt.Errorf("setting controller reference: %w", err) } if err := c.Client.Create(ctx, cos); err != nil { - return fmt.Errorf("creating revision %d: %w", revision, err) + return nil, fmt.Errorf("creating revision %d: %w", revision, err) + } + return cos, nil +} + +// durationUntilDeploymentDeadline excludes time before the revision has its +// server-assigned creation timestamp. The deployment supplies the budget, while +// the revision's completion latch ends it independently of current readiness. +func durationUntilDeploymentDeadline(clk clock.Clock, cod *ocv1.ClusterObjectDeployment, cos *ocv1.ClusterObjectSet) (time.Duration, bool) { + if cos == nil || cos.CreationTimestamp.IsZero() || !cos.Status.CompletedAt.IsZero() || + cos.Spec.LifecycleState == ocv1.ClusterObjectSetLifecycleStateArchived || !cos.DeletionTimestamp.IsZero() { + return 0, false + } + pd := cod.Spec.Template.Spec.ProgressDeadlineMinutes + if pd <= 0 { + return 0, false + } + deadline := cos.CreationTimestamp.Add(time.Duration(pd) * time.Minute) + return deadline.Sub(clk.Now()), true +} + +// newCODDeadlineAwareRateLimiter bounds error backoff as well as successful +// reconciles: controller-runtime ignores RequeueAfter when an error is returned. +func newCODDeadlineAwareRateLimiter(delegate workqueue.TypedRateLimiter[ctrl.Request], reader client.Reader, clk clock.Clock) *deadlineAwareRateLimiter { + return &deadlineAwareRateLimiter{ + delegate: delegate, + remaining: func(req ctrl.Request) (time.Duration, bool, error) { + ctx := context.Background() + cod := &ocv1.ClusterObjectDeployment{} + if err := reader.Get(ctx, req.NamespacedName, cod); err != nil { + return 0, false, client.IgnoreNotFound(err) + } + if !cod.DeletionTimestamp.IsZero() { + return 0, false, nil + } + var revisions ocv1.ClusterObjectSetList + if err := reader.List(ctx, &revisions, client.MatchingFields{clusterObjectSetGroupIndex: cod.Name}); err != nil { + return 0, false, err + } + latest := latestActiveRevision(revisions.Items) + if latest == nil || latest.Labels[labels.TemplateHashKey] != templateHash(cod.Spec.Template) { + // Creating the desired revision is outside the MVP deadline. + return 0, false, nil + } + remaining, hasDeadline := durationUntilDeploymentDeadline(clk, cod, latest) + return remaining, hasDeadline, nil + }, } - return nil } // setAvailable sets the deployment's Available condition. It is True when at least @@ -289,7 +345,16 @@ func setCondition(cod *ocv1.ClusterObjectDeployment, condType string, status met } func (c *ClusterObjectDeploymentReconciler) SetupWithManager(mgr ctrl.Manager) error { + if c.Clock == nil { + c.Clock = clock.RealClock{} + } return ctrl.NewControllerManagedBy(mgr). + WithOptions(controller.Options{ + RateLimiter: newCODDeadlineAwareRateLimiter( + workqueue.DefaultTypedControllerRateLimiter[ctrl.Request](), + mgr.GetClient(), c.Clock, + ), + }). For( &ocv1.ClusterObjectDeployment{}, builder.WithPredicates( @@ -446,13 +511,12 @@ func buildClusterObjectSet(cod *ocv1.ClusterObjectDeployment, revision int64, ha Annotations: maps.Clone(cod.Spec.Template.Metadata.Annotations), }, Spec: ocv1.ClusterObjectSetSpec{ - Group: cod.Name, - Revision: revision, - LifecycleState: ocv1.ClusterObjectSetLifecycleStateActive, - Phases: cod.Spec.Template.Spec.Phases, - ProgressDeadlineMinutes: cod.Spec.Template.Spec.ProgressDeadlineMinutes, - ProgressionProbes: cod.Spec.Template.Spec.ProgressionProbes, - CollisionProtection: cod.Spec.Template.Spec.CollisionProtection, + Group: cod.Name, + Revision: revision, + LifecycleState: ocv1.ClusterObjectSetLifecycleStateActive, + Phases: cod.Spec.Template.Spec.Phases, + ProgressionProbes: cod.Spec.Template.Spec.ProgressionProbes, + CollisionProtection: cod.Spec.Template.Spec.CollisionProtection, }, } } diff --git a/internal/object-controller/controllers/clusterobjectdeployment_controller_test.go b/internal/object-controller/controllers/clusterobjectdeployment_controller_test.go index a80fd2758f..f1c7284c8a 100644 --- a/internal/object-controller/controllers/clusterobjectdeployment_controller_test.go +++ b/internal/object-controller/controllers/clusterobjectdeployment_controller_test.go @@ -9,6 +9,7 @@ import ( "math" "strings" "testing" + "time" "github.com/stretchr/testify/require" apierrors "k8s.io/apimachinery/pkg/api/errors" @@ -111,20 +112,14 @@ func markCOSReady(t *testing.T, c client.Client, name string, ready bool) { require.NoError(t, c.Status().Update(t.Context(), cos)) } -// markCOSProgressDeadlineExceeded sets the named revision's Ready condition to -// False with reason ProgressDeadlineExceeded, mirroring what the ClusterObjectSet -// controller records when a revision does not roll out within its deadline. -func markCOSProgressDeadlineExceeded(t *testing.T, c client.Client, name string) { +// expireRevisionDeadline moves the revision's creation timestamp before the +// deployment's deadline without changing the revision's readiness condition. +func expireRevisionDeadline(t *testing.T, c client.Client, name string) { t.Helper() cos, ok := getCOS(t, c, name) require.True(t, ok) - apimeta.SetStatusCondition(&cos.Status.Conditions, metav1.Condition{ - Type: ocv1.ClusterObjectSetTypeReady, - Status: metav1.ConditionFalse, - Reason: ocv1.ReasonProgressDeadlineExceeded, - Message: "test", - }) - require.NoError(t, c.Status().Update(t.Context(), cos)) + cos.CreationTimestamp = metav1.NewTime(time.Now().Add(-time.Hour)) + require.NoError(t, c.Update(t.Context(), cos)) } func Test_ClusterObjectDeploymentReconciler_Available(t *testing.T) { @@ -208,7 +203,7 @@ func Test_ClusterObjectDeploymentReconciler_Available(t *testing.T) { require.NoError(t, err) // Revision 2 blows its deadline while revision 1 is still Active and Ready. - markCOSProgressDeadlineExceeded(t, c, clusterObjectDeploymentName+"-2") + expireRevisionDeadline(t, c, clusterObjectDeploymentName+"-2") _, err = reconcileCOD(t, c) require.NoError(t, err) @@ -235,7 +230,7 @@ func Test_ClusterObjectDeploymentReconciler_CreatesFirstRevision(t *testing.T) { require.Equal(t, int64(1), cos.Spec.Revision) require.Equal(t, ocv1.ClusterObjectSetLifecycleStateActive, cos.Spec.LifecycleState) require.Equal(t, ocv1.CollisionProtectionPrevent, cos.Spec.CollisionProtection) - require.Equal(t, int32(15), cos.Spec.ProgressDeadlineMinutes, "progressDeadlineMinutes should pass through to the revision") + require.Zero(t, cos.Spec.ProgressDeadlineMinutes, "the deployment enforces its own progress deadline") require.NotEmpty(t, cos.Labels[labels.TemplateHashKey]) ref := metav1.GetControllerOf(cos) @@ -410,7 +405,7 @@ func Test_ClusterObjectDeploymentReconciler_NoNewRevisionWhenTemplateUnchanged(t }, { name: "deadline-exceeded latest revision reports ProgressDeadlineExceeded", - prep: func(t *testing.T, c client.Client, cosName string) { markCOSProgressDeadlineExceeded(t, c, cosName) }, + prep: func(t *testing.T, c client.Client, cosName string) { expireRevisionDeadline(t, c, cosName) }, wantStatus: metav1.ConditionFalse, wantReason: ocv1.ClusterObjectDeploymentReasonProgressDeadlineExceeded, }, diff --git a/internal/object-controller/controllers/clusterobjectdeployment_deadline_test.go b/internal/object-controller/controllers/clusterobjectdeployment_deadline_test.go index cd18940bff..b95fdb90bf 100644 --- a/internal/object-controller/controllers/clusterobjectdeployment_deadline_test.go +++ b/internal/object-controller/controllers/clusterobjectdeployment_deadline_test.go @@ -4,6 +4,7 @@ package controllers_test import ( "context" + "errors" "fmt" "testing" "time" @@ -28,11 +29,32 @@ import ( "github.com/operator-framework/operator-controller/internal/testutil" ) -func Test_ClusterObjectDeploymentReconciler_DeadlineCoversEntireRevision(t *testing.T) { - creation := time.Date(2026, 10, 22, 12, 0, 0, 0, time.UTC) - clk := clocktesting.NewFakeClock(creation) +func newDeadlineDeploymentClient(t *testing.T, cod *ocv1.ClusterObjectDeployment, clk *clocktesting.FakeClock) client.WithWatch { + t.Helper() scheme := newScheme(t) require.NoError(t, corev1.AddToScheme(scheme)) + return testutil.WithIndexes(t, fake.NewClientBuilder().WithScheme(scheme), controllers.SetupIndexes). + WithStatusSubresource(&ocv1.ClusterObjectDeployment{}, &ocv1.ClusterObjectSet{}). + WithObjects(cod). + WithInterceptorFuncs(interceptor.Funcs{ + Create: func(ctx context.Context, cl client.WithWatch, obj client.Object, opts ...client.CreateOption) error { + obj.SetCreationTimestamp(metav1.NewTime(clk.Now())) + obj.SetUID(types.UID(obj.GetName())) + return cl.Create(ctx, obj, opts...) + }, + }).Build() +} + +func reconcileDeadlineDeployment(t *testing.T, c client.Client, clk *clocktesting.FakeClock) (ctrl.Result, error) { + t.Helper() + // Reconstruct the reconciler so timing cannot depend on process-local state. + r := &controllers.ClusterObjectDeploymentReconciler{Client: c, Scheme: newScheme(t), Clock: clk} + return r.Reconcile(t.Context(), ctrl.Request{NamespacedName: types.NamespacedName{Name: clusterObjectDeploymentName}}) +} + +func TestClusterObjectDeploymentReconcilerDeadlineCoversEntireRevision(t *testing.T) { + creation := time.Date(2026, 10, 22, 12, 0, 0, 0, time.UTC) + clk := clocktesting.NewFakeClock(creation) mockCtrl := gomock.NewController(t) cod := codFixture() @@ -50,24 +72,15 @@ func Test_ClusterObjectDeploymentReconciler_DeadlineCoversEntireRevision(t *test cod.Spec.Template.Spec.Phases = append(cod.Spec.Template.Spec.Phases, *phase) } - c := testutil.WithIndexes(t, fake.NewClientBuilder().WithScheme(scheme), controllers.SetupIndexes). - WithStatusSubresource(&ocv1.ClusterObjectDeployment{}, &ocv1.ClusterObjectSet{}). - WithObjects(cod). - WithInterceptorFuncs(interceptor.Funcs{ - Create: func(ctx context.Context, cl client.WithWatch, obj client.Object, opts ...client.CreateOption) error { - // Supply the creation metadata normally assigned by the API server. - obj.SetCreationTimestamp(metav1.NewTime(clk.Now())) - obj.SetUID(types.UID(obj.GetName())) - return cl.Create(ctx, obj, opts...) - }, - }).Build() - _, err := reconcileCOD(t, c) + c := newDeadlineDeploymentClient(t, cod, clk) + result, err := reconcileDeadlineDeployment(t, c, clk) require.NoError(t, err) + require.Equal(t, 10*time.Minute, result.RequeueAfter) cosName := cod.Name + "-1" cos, found := getCOS(t, c, cosName) require.True(t, found) require.True(t, cos.CreationTimestamp.Time.Equal(creation)) - require.Equal(t, int32(10), cos.Spec.ProgressDeadlineMinutes) + require.Zero(t, cos.Spec.ProgressDeadlineMinutes, "the COD owns the deadline budget") require.Len(t, cos.Spec.Phases, 5) require.Equal(t, metav1.ConditionTrue, progressing(t, c).Status) // Start after finalizer initialization; the fake client's metadata patch @@ -114,9 +127,10 @@ func Test_ClusterObjectDeploymentReconciler_DeadlineCoversEntireRevision(t *test } result, err := r.Reconcile(t.Context(), ctrl.Request{NamespacedName: types.NamespacedName{Name: cosName}}) require.NoError(t, err) - _, err = reconcileCOD(t, c) + deploymentResult, err := reconcileDeadlineDeployment(t, c, clk) require.NoError(t, err) - return result + require.Zero(t, result.RequeueAfter, "the COS does not schedule deadline checks") + return deploymentResult } // Four phase completions consume eight minutes of the same ten-minute budget. @@ -144,7 +158,7 @@ func Test_ClusterObjectDeploymentReconciler_DeadlineCoversEntireRevision(t *test ready := apimeta.FindStatusCondition(cos.Status.Conditions, ocv1.ClusterObjectSetTypeReady) require.NotNil(t, ready) require.Equal(t, metav1.ConditionFalse, ready.Status) - require.Equal(t, ocv1.ReasonProgressDeadlineExceeded, ready.Reason) + require.Equal(t, ocv1.ClusterObjectSetReasonObjectsNotReady, ready.Reason) condition := progressing(t, c) require.NotNil(t, condition) require.Equal(t, metav1.ConditionFalse, condition.Status) @@ -179,7 +193,7 @@ func Test_ClusterObjectDeploymentReconciler_DeadlineCoversEntireRevision(t *test require.NoError(t, c.Get(t.Context(), client.ObjectKeyFromObject(cod), cod)) cod.Spec.Template.Spec.Phases[0].Objects[0].Object.Object["data"] = map[string]interface{}{"version": "second"} require.NoError(t, c.Update(t.Context(), cod)) - _, err = reconcileCOD(t, c) + _, err = reconcileDeadlineDeployment(t, c, clk) require.NoError(t, err) cosName = cod.Name + "-2" cos, found = getCOS(t, c, cosName) @@ -191,3 +205,204 @@ func Test_ClusterObjectDeploymentReconciler_DeadlineCoversEntireRevision(t *test require.Equal(t, 10*time.Minute, reconcileRevision().RequeueAfter) require.Equal(t, metav1.ConditionTrue, progressing(t, c).Status) } + +func TestClusterObjectDeploymentReconcilerDeadlineWithoutCOSDeadlineCondition(t *testing.T) { + for _, tc := range []struct { + name string + status metav1.ConditionStatus + reason string + }{ + {name: "no Ready condition"}, + {name: "pending", status: metav1.ConditionUnknown, reason: ocv1.ClusterObjectSetReasonPending}, + {name: "preconditions not met", status: metav1.ConditionFalse, reason: ocv1.ClusterObjectSetReasonPreconditionsNotMet}, + {name: "invalid specification", status: metav1.ConditionFalse, reason: ocv1.ClusterObjectSetReasonInvalidSpecification}, + {name: "objects not ready", status: metav1.ConditionFalse, reason: ocv1.ClusterObjectSetReasonObjectsNotReady}, + } { + t.Run(tc.name, func(t *testing.T) { + cod := codFixture() + cod.Spec.Template.Spec.ProgressDeadlineMinutes = 10 + c := newCODClient(t, cod) + _, err := reconcileCOD(t, c) + require.NoError(t, err) + cos, found := getCOS(t, c, cod.Name+"-1") + require.True(t, found) + cos.CreationTimestamp = metav1.NewTime(time.Now().Add(-11 * time.Minute)) + require.NoError(t, c.Update(t.Context(), cos)) + if tc.reason != "" { + cos.Status.Conditions = []metav1.Condition{{ + Type: ocv1.ClusterObjectSetTypeReady, Status: tc.status, Reason: tc.reason, + }} + require.NoError(t, c.Status().Update(t.Context(), cos)) + } + + // Only reconcile the COD: it must enforce the revision deadline even + // when the COS has not reported its own ProgressDeadlineExceeded reason. + result, err := reconcileCOD(t, c) + require.NoError(t, err) + require.Zero(t, result.RequeueAfter) + condition := progressing(t, c) + require.NotNil(t, condition) + require.Equal(t, metav1.ConditionFalse, condition.Status) + require.Equal(t, ocv1.ClusterObjectDeploymentReasonProgressDeadlineExceeded, condition.Reason) + cos, found = getCOS(t, c, cos.Name) + require.True(t, found) + if tc.reason != "" { + require.Equal(t, tc.reason, apimeta.FindStatusCondition(cos.Status.Conditions, ocv1.ClusterObjectSetTypeReady).Reason) + } + }) + } +} + +func TestClusterObjectDeploymentReconcilerIgnoresCOSDeadlineCondition(t *testing.T) { + for _, deadline := range []int32{0, 10} { + t.Run(fmt.Sprintf("deadline %d", deadline), func(t *testing.T) { + clk := clocktesting.NewFakeClock(time.Date(2026, 10, 22, 12, 0, 0, 0, time.UTC)) + cod := codFixture() + cod.Spec.Template.Spec.ProgressDeadlineMinutes = deadline + c := newDeadlineDeploymentClient(t, cod, clk) + _, err := reconcileDeadlineDeployment(t, c, clk) + require.NoError(t, err) + cos, found := getCOS(t, c, cod.Name+"-1") + require.True(t, found) + cos.Status.Conditions = []metav1.Condition{{ + Type: ocv1.ClusterObjectSetTypeReady, Status: metav1.ConditionFalse, + Reason: ocv1.ReasonProgressDeadlineExceeded, + }} + require.NoError(t, c.Status().Update(t.Context(), cos)) + + result, err := reconcileDeadlineDeployment(t, c, clk) + require.NoError(t, err) + require.Equal(t, time.Duration(deadline)*time.Minute, result.RequeueAfter) + condition := progressing(t, c) + require.Equal(t, metav1.ConditionTrue, condition.Status) + require.Equal(t, ocv1.ClusterObjectDeploymentReasonNewClusterObjectSetProgressing, condition.Reason) + }) + } +} + +func TestClusterObjectDeploymentReconcilerDeadlineSchedulingAndRecovery(t *testing.T) { + start := time.Date(2026, 10, 22, 12, 0, 0, 0, time.UTC) + clk := clocktesting.NewFakeClock(start) + cod := codFixture() + cod.Spec.Template.Spec.ProgressDeadlineMinutes = 10 + c := newDeadlineDeploymentClient(t, cod, clk) + result, err := reconcileDeadlineDeployment(t, c, clk) + require.NoError(t, err) + require.Equal(t, 10*time.Minute, result.RequeueAfter) + cos, found := getCOS(t, c, cod.Name+"-1") + require.True(t, found) + cos.Status.Conditions = []metav1.Condition{{ + Type: ocv1.ClusterObjectSetTypeReady, Status: metav1.ConditionFalse, + Reason: ocv1.ClusterObjectSetReasonPreconditionsNotMet, Message: "referenced Secret is mutable", + }} + require.NoError(t, c.Status().Update(t.Context(), cos)) + + // A quiet COS need not report ProgressDeadlineExceeded for the COD to expire. + clk.Step(9 * time.Minute) + result, err = reconcileDeadlineDeployment(t, c, clk) + require.NoError(t, err) + require.Equal(t, time.Minute, result.RequeueAfter) + require.Equal(t, metav1.ConditionTrue, progressing(t, c).Status) + clk.Step(time.Minute) + result, err = reconcileDeadlineDeployment(t, c, clk) + require.NoError(t, err) + require.Zero(t, result.RequeueAfter) + condition := progressing(t, c) + require.Equal(t, metav1.ConditionFalse, condition.Status) + require.Equal(t, ocv1.ClusterObjectDeploymentReasonProgressDeadlineExceeded, condition.Reason) + transition := condition.LastTransitionTime + + // Partial phase progress neither extends the budget nor clears the timeout. + clk.Step(time.Minute) + cos.Status.ObservedPhases = []ocv1.ObservedPhase{{Name: "resources", ObjectCounts: ocv1.ObjectCounts{Total: 1, Ready: 1}}} + require.NoError(t, c.Status().Update(t.Context(), cos)) + result, err = reconcileDeadlineDeployment(t, c, clk) + require.NoError(t, err) + require.Zero(t, result.RequeueAfter) + require.Equal(t, transition, progressing(t, c).LastTransitionTime) + require.Equal(t, ocv1.ClusterObjectDeploymentReasonProgressDeadlineExceeded, progressing(t, c).Reason) + cos, found = getCOS(t, c, cos.Name) + require.True(t, found) + require.Equal(t, ocv1.ClusterObjectSetReasonPreconditionsNotMet, + apimeta.FindStatusCondition(cos.Status.Conditions, ocv1.ClusterObjectSetTypeReady).Reason) + + // The historical completion latch wins even over a stale COS deadline reason. + cos.Status.CompletedAt = metav1.NewTime(clk.Now()) + cos.Status.Conditions[0].Reason = ocv1.ReasonProgressDeadlineExceeded + require.NoError(t, c.Status().Update(t.Context(), cos)) + result, err = reconcileDeadlineDeployment(t, c, clk) + require.NoError(t, err) + require.Zero(t, result.RequeueAfter) + require.Equal(t, ocv1.ClusterObjectDeploymentReasonNewClusterObjectSetProgressed, progressing(t, c).Reason) + clk.Step(time.Hour) + markCOSReady(t, c, cos.Name, false) + result, err = reconcileDeadlineDeployment(t, c, clk) + require.NoError(t, err) + require.Zero(t, result.RequeueAfter) + require.Equal(t, ocv1.ClusterObjectDeploymentReasonNewClusterObjectSetProgressed, progressing(t, c).Reason) +} + +func TestClusterObjectDeploymentReconcilerDeadlineExcludesCreationRetries(t *testing.T) { + clk := clocktesting.NewFakeClock(time.Date(2026, 10, 22, 12, 0, 0, 0, time.UTC)) + cod := codFixture() + cod.Spec.Template.Spec.ProgressDeadlineMinutes = 10 + cod.CreationTimestamp = metav1.NewTime(clk.Now()) + creationErr := errors.New("revision creation denied") + denied := true + c := interceptor.NewClient(newDeadlineDeploymentClient(t, cod, clk), interceptor.Funcs{ + Create: func(ctx context.Context, cl client.WithWatch, obj client.Object, opts ...client.CreateOption) error { + if denied { + return creationErr + } + return cl.Create(ctx, obj, opts...) + }, + }) + for range 2 { + result, err := reconcileDeadlineDeployment(t, c, clk) + require.ErrorIs(t, err, creationErr) + require.Zero(t, result.RequeueAfter) + require.Equal(t, metav1.ConditionTrue, progressing(t, c).Status) + clk.Step(20 * time.Minute) + } + denied = false + result, err := reconcileDeadlineDeployment(t, c, clk) + require.NoError(t, err) + require.Equal(t, 10*time.Minute, result.RequeueAfter) + require.Equal(t, metav1.ConditionTrue, progressing(t, c).Status) +} + +func TestClusterObjectDeploymentReconcilerDeadlineRetainsAvailability(t *testing.T) { + clk := clocktesting.NewFakeClock(time.Date(2026, 10, 22, 12, 0, 0, 0, time.UTC)) + cod := codFixture() + cod.Spec.Template.Spec.ProgressDeadlineMinutes = 10 + c := newDeadlineDeploymentClient(t, cod, clk) + _, err := reconcileDeadlineDeployment(t, c, clk) + require.NoError(t, err) + markCOSReady(t, c, cod.Name+"-1", true) + cos, found := getCOS(t, c, cod.Name+"-1") + require.True(t, found) + cos.Status.CompletedAt = metav1.NewTime(clk.Now()) + require.NoError(t, c.Status().Update(t.Context(), cos)) + _, err = reconcileDeadlineDeployment(t, c, clk) + require.NoError(t, err) + + clk.Step(time.Hour) + require.NoError(t, c.Get(t.Context(), client.ObjectKeyFromObject(cod), cod)) + cod.Spec.Template.Spec.Phases[0].Objects[0].Object.Object["data"] = map[string]interface{}{"version": "second"} + require.NoError(t, c.Update(t.Context(), cod)) + result, err := reconcileDeadlineDeployment(t, c, clk) + require.NoError(t, err) + require.Equal(t, 10*time.Minute, result.RequeueAfter) + cos, found = getCOS(t, c, cod.Name+"-2") + require.True(t, found) + cos.Status.Conditions = []metav1.Condition{{ + Type: ocv1.ClusterObjectSetTypeReady, Status: metav1.ConditionFalse, Reason: ocv1.ClusterObjectSetReasonPreconditionsNotMet, + }} + require.NoError(t, c.Status().Update(t.Context(), cos)) + clk.Step(10 * time.Minute) + result, err = reconcileDeadlineDeployment(t, c, clk) + require.NoError(t, err) + require.Zero(t, result.RequeueAfter) + require.Equal(t, ocv1.ClusterObjectDeploymentReasonProgressDeadlineExceeded, progressing(t, c).Reason) + require.Equal(t, metav1.ConditionTrue, available(t, c).Status) +} diff --git a/internal/object-controller/controllers/progress_deadline.go b/internal/object-controller/controllers/progress_deadline.go index cc4addfecc..1684b030f3 100644 --- a/internal/object-controller/controllers/progress_deadline.go +++ b/internal/object-controller/controllers/progress_deadline.go @@ -16,17 +16,16 @@ import ( ) // deadlineAwareRateLimiter wraps a delegate rate limiter and caps the backoff -// duration to the time remaining until the COS progress deadline, ensuring +// duration to the time remaining until a revision's progress deadline, ensuring // that ProgressDeadlineExceeded is set promptly even during exponential backoff. // // After the deadline passes, it allows one immediate requeue (returning 0) so // the reconciler can set the ProgressDeadlineExceeded condition, then falls // back to the delegate's normal backoff. This avoids both tight-looping and -// coupling to the COS's status conditions. +// coupling to status conditions. type deadlineAwareRateLimiter struct { delegate workqueue.TypedRateLimiter[ctrl.Request] - client client.Reader - clock clock.Clock + remaining func(ctrl.Request) (time.Duration, bool, error) pastDeadline sync.Map } @@ -35,22 +34,33 @@ func newDeadlineAwareRateLimiter( c client.Reader, clk clock.Clock, ) *deadlineAwareRateLimiter { - return &deadlineAwareRateLimiter{delegate: delegate, client: c, clock: clk} + return &deadlineAwareRateLimiter{ + delegate: delegate, + remaining: func(req ctrl.Request) (time.Duration, bool, error) { + cos := &ocv1.ClusterObjectSet{} + if err := c.Get(context.Background(), req.NamespacedName, cos); err != nil { + return 0, false, client.IgnoreNotFound(err) + } + remaining, hasDeadline := durationUntilDeadline(clk, cos) + return remaining, hasDeadline, nil + }, + } } func (r *deadlineAwareRateLimiter) When(item ctrl.Request) time.Duration { backoff := r.delegate.When(item) - cos := &ocv1.ClusterObjectSet{} - if err := r.client.Get(context.Background(), item.NamespacedName, cos); err != nil { + remaining, hasDeadline, err := r.remaining(item) + if err != nil { + // An unknown deadline must not re-arm the immediate requeue after expiry. return backoff } - - remaining, hasDeadline := durationUntilDeadline(r.clock, cos) if !hasDeadline { + r.pastDeadline.Delete(item) return backoff } if remaining > 0 { + r.pastDeadline.Delete(item) if remaining < backoff { return remaining } diff --git a/internal/object-controller/controllers/progress_deadline_test.go b/internal/object-controller/controllers/progress_deadline_test.go index 98fc10d59a..7cb061af27 100644 --- a/internal/object-controller/controllers/progress_deadline_test.go +++ b/internal/object-controller/controllers/progress_deadline_test.go @@ -3,18 +3,24 @@ package controllers import ( + "context" + "errors" "testing" "time" "github.com/stretchr/testify/require" + apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" clocktesting "k8s.io/utils/clock/testing" ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/client/fake" + "sigs.k8s.io/controller-runtime/pkg/client/interceptor" ocv1 "github.com/operator-framework/operator-controller/api/v1" + "github.com/operator-framework/operator-controller/internal/shared/labels" ) func TestDurationUntilDeadline(t *testing.T) { @@ -261,3 +267,227 @@ func TestDeadlineAwareRateLimiter(t *testing.T) { require.Equal(t, time.Duration(0), limiter.When(req)) }) } + +func TestCODDeadlineAwareRateLimiter(t *testing.T) { + creation := time.Date(2026, 10, 22, 12, 0, 0, 0, time.UTC) + for _, tc := range []struct { + name string + elapsed time.Duration + backoff time.Duration + mutate func(*ocv1.ClusterObjectDeployment, *ocv1.ClusterObjectSet) + omitCOS bool + expectWait time.Duration + }{ + {name: "cap backoff at revision deadline", elapsed: 9 * time.Minute, backoff: 15 * time.Minute, expectWait: time.Minute}, + {name: "keep shorter backoff", elapsed: time.Minute, backoff: 30 * time.Second, expectWait: 30 * time.Second}, + {name: "expired revision wakes immediately", elapsed: 10 * time.Minute, backoff: 15 * time.Minute}, + {name: "creation pending has no deadline", elapsed: time.Hour, backoff: 15 * time.Minute, omitCOS: true, expectWait: 15 * time.Minute}, + { + name: "changed template waits for its own revision", elapsed: time.Hour, backoff: 15 * time.Minute, expectWait: 15 * time.Minute, + mutate: func(cod *ocv1.ClusterObjectDeployment, _ *ocv1.ClusterObjectSet) { + cod.Spec.Template.Spec.CollisionProtection = ocv1.CollisionProtectionIfNoController + }, + }, + { + name: "completed revision has no deadline", elapsed: time.Hour, backoff: 15 * time.Minute, expectWait: 15 * time.Minute, + mutate: func(_ *ocv1.ClusterObjectDeployment, cos *ocv1.ClusterObjectSet) { + cos.Status.CompletedAt = metav1.NewTime(creation.Add(time.Minute)) + }, + }, + { + name: "disabled deadline", elapsed: time.Hour, backoff: 15 * time.Minute, expectWait: 15 * time.Minute, + mutate: func(cod *ocv1.ClusterObjectDeployment, cos *ocv1.ClusterObjectSet) { + cod.Spec.Template.Spec.ProgressDeadlineMinutes = 0 + cos.Labels[labels.TemplateHashKey] = templateHash(cod.Spec.Template) + }, + }, + { + name: "creation metadata not yet available", elapsed: time.Hour, backoff: 15 * time.Minute, expectWait: 15 * time.Minute, + mutate: func(_ *ocv1.ClusterObjectDeployment, cos *ocv1.ClusterObjectSet) { + cos.CreationTimestamp = metav1.Time{} + }, + }, + { + name: "archived revision is ignored", elapsed: time.Hour, backoff: 15 * time.Minute, expectWait: 15 * time.Minute, + mutate: func(_ *ocv1.ClusterObjectDeployment, cos *ocv1.ClusterObjectSet) { + cos.Spec.LifecycleState = ocv1.ClusterObjectSetLifecycleStateArchived + }, + }, + { + name: "deleting deployment is ignored", elapsed: time.Hour, backoff: 15 * time.Minute, expectWait: 15 * time.Minute, + mutate: func(cod *ocv1.ClusterObjectDeployment, _ *ocv1.ClusterObjectSet) { + cod.Finalizers = []string{"test"} + cod.DeletionTimestamp = &metav1.Time{Time: creation} + }, + }, + } { + t.Run(tc.name, func(t *testing.T) { + cod, cos := deadlineRateLimiterObjects(creation) + if tc.mutate != nil { + tc.mutate(cod, cos) + } + objects := []client.Object{cod} + if !tc.omitCOS { + objects = append(objects, cos) + } + limiter := newCODDeadlineAwareRateLimiter( + &fixedRateLimiter{duration: tc.backoff}, + deadlineRateLimiterClient(t, objects...), + clocktesting.NewFakeClock(creation.Add(tc.elapsed)), + ) + require.Equal(t, tc.expectWait, limiter.When(ctrl.Request{NamespacedName: types.NamespacedName{Name: cod.Name}})) + }) + } + + t.Run("a new revision gets another deadline wakeup", func(t *testing.T) { + cod, cos := deadlineRateLimiterObjects(creation) + c := deadlineRateLimiterClient(t, cod, cos) + clk := clocktesting.NewFakeClock(creation.Add(10 * time.Minute)) + limiter := newCODDeadlineAwareRateLimiter(&fixedRateLimiter{duration: 15 * time.Minute}, c, clk) + req := ctrl.Request{NamespacedName: types.NamespacedName{Name: cod.Name}} + require.Zero(t, limiter.When(req)) + require.Equal(t, 15*time.Minute, limiter.When(req), "an expired rollout must not busy-loop") + + next := cos.DeepCopy() + next.Name = cod.Name + "-2" + next.ResourceVersion = "" + next.Spec.Revision = 2 + next.CreationTimestamp = metav1.NewTime(clk.Now()) + require.NoError(t, c.Create(t.Context(), next)) + require.Equal(t, 10*time.Minute, limiter.When(req)) + clk.Step(10 * time.Minute) + require.Zero(t, limiter.When(req)) + require.Equal(t, 15*time.Minute, limiter.When(req)) + }) +} + +func TestCODDeadlineAwareRateLimiter_ReadFailures(t *testing.T) { + for _, operation := range []string{"get", "list"} { + t.Run(operation, func(t *testing.T) { + creation := time.Date(2026, 10, 22, 12, 0, 0, 0, time.UTC) + cod, cos := deadlineRateLimiterObjects(creation) + readErr := errors.New("temporary API read failure") + failRead := false + reader := interceptor.NewClient(deadlineRateLimiterClient(t, cod, cos), interceptor.Funcs{ + Get: func(ctx context.Context, c client.WithWatch, key client.ObjectKey, obj client.Object, opts ...client.GetOption) error { + if failRead && operation == "get" { + return readErr + } + return c.Get(ctx, key, obj, opts...) + }, + List: func(ctx context.Context, c client.WithWatch, list client.ObjectList, opts ...client.ListOption) error { + if failRead && operation == "list" { + return readErr + } + return c.List(ctx, list, opts...) + }, + }) + limiter := newCODDeadlineAwareRateLimiter( + &fixedRateLimiter{duration: 15 * time.Minute}, reader, + clocktesting.NewFakeClock(creation.Add(10*time.Minute)), + ) + req := ctrl.Request{NamespacedName: types.NamespacedName{Name: cod.Name}} + require.Zero(t, limiter.When(req)) + for range 2 { + failRead = true + require.Equal(t, 15*time.Minute, limiter.When(req)) + failRead = false + require.Equal(t, 15*time.Minute, limiter.When(req), "read recovery must not re-arm the immediate deadline requeue") + } + }) + } +} + +func TestCODDeadlineAwareRateLimiter_DeadlineRemoved(t *testing.T) { + creation := time.Date(2026, 10, 22, 12, 0, 0, 0, time.UTC) + for _, tc := range []struct { + name string + getErr error + mutateCOD func(*ocv1.ClusterObjectDeployment) + mutateRevisions func(*ocv1.ClusterObjectSetList) + }{ + { + name: "deployment absent", + getErr: apierrors.NewNotFound(ocv1.GroupVersion.WithResource("clusterobjectdeployments").GroupResource(), "test-cod"), + }, + { + name: "deployment deleting", + mutateCOD: func(cod *ocv1.ClusterObjectDeployment) { + cod.DeletionTimestamp = &metav1.Time{Time: creation} + }, + }, + { + name: "revision absent", + mutateRevisions: func(revisions *ocv1.ClusterObjectSetList) { + revisions.Items = nil + }, + }, + { + name: "revision deleting", + mutateRevisions: func(revisions *ocv1.ClusterObjectSetList) { + revisions.Items[0].DeletionTimestamp = &metav1.Time{Time: creation} + }, + }, + } { + t.Run(tc.name, func(t *testing.T) { + cod, cos := deadlineRateLimiterObjects(creation) + removeDeadline := false + reader := interceptor.NewClient(deadlineRateLimiterClient(t, cod, cos), interceptor.Funcs{ + Get: func(ctx context.Context, c client.WithWatch, key client.ObjectKey, obj client.Object, opts ...client.GetOption) error { + if removeDeadline && tc.getErr != nil { + return tc.getErr + } + if err := c.Get(ctx, key, obj, opts...); err != nil { + return err + } + if removeDeadline && tc.mutateCOD != nil { + tc.mutateCOD(obj.(*ocv1.ClusterObjectDeployment)) + } + return nil + }, + List: func(ctx context.Context, c client.WithWatch, list client.ObjectList, opts ...client.ListOption) error { + if err := c.List(ctx, list, opts...); err != nil { + return err + } + if removeDeadline && tc.mutateRevisions != nil { + tc.mutateRevisions(list.(*ocv1.ClusterObjectSetList)) + } + return nil + }, + }) + limiter := newCODDeadlineAwareRateLimiter( + &fixedRateLimiter{duration: 15 * time.Minute}, reader, + clocktesting.NewFakeClock(creation.Add(10*time.Minute)), + ) + req := ctrl.Request{NamespacedName: types.NamespacedName{Name: cod.Name}} + require.Zero(t, limiter.When(req)) + require.Equal(t, 15*time.Minute, limiter.When(req)) + removeDeadline = true + require.Equal(t, 15*time.Minute, limiter.When(req)) + removeDeadline = false + require.Zero(t, limiter.When(req), "a definitively absent deadline must reset the immediate deadline requeue") + require.Equal(t, 15*time.Minute, limiter.When(req)) + }) + } +} + +func deadlineRateLimiterObjects(creation time.Time) (*ocv1.ClusterObjectDeployment, *ocv1.ClusterObjectSet) { + cod := &ocv1.ClusterObjectDeployment{ + ObjectMeta: metav1.ObjectMeta{Name: "test-cod"}, + Spec: ocv1.ClusterObjectDeploymentSpec{Template: testTemplate()}, + } + cod.Spec.Template.Spec.ProgressDeadlineMinutes = 10 + cos := buildClusterObjectSet(cod, 1, templateHash(cod.Spec.Template)) + cos.CreationTimestamp = metav1.NewTime(creation) + return cod, cos +} + +func deadlineRateLimiterClient(t *testing.T, objects ...client.Object) client.WithWatch { + t.Helper() + scheme := runtime.NewScheme() + require.NoError(t, ocv1.AddToScheme(scheme)) + return fake.NewClientBuilder().WithScheme(scheme).WithObjects(objects...). + WithIndex(&ocv1.ClusterObjectSet{}, clusterObjectSetGroupIndex, func(obj client.Object) []string { + return []string{obj.(*ocv1.ClusterObjectSet).Spec.Group} + }).Build() +} diff --git a/test/e2e/features/cod_progression_timeout.feature b/test/e2e/features/cod_progression_timeout.feature index cdb58906a9..61839e8911 100644 --- a/test/e2e/features/cod_progression_timeout.feature +++ b/test/e2e/features/cod_progression_timeout.feature @@ -3,16 +3,15 @@ Feature: ClusterObjectDeployment progression timeout As an OLM user applying a ClusterObjectDeployment, I want spec.template.spec.progressDeadlineMinutes - to be passed through to the ClusterObjectSet revisions it creates, so that a revision that does not - roll out within the deadline is reported as failed and the deployment is reported unavailable. + to bound the rollout of the desired ClusterObjectSet revision, so that a revision that does not + roll out within the deadline is reported as stalled while its readiness failures remain visible. Background: Given OLM is available - Scenario: ClusterObjectDeployment passes progressDeadlineMinutes through to its revision and times out + Scenario: ClusterObjectDeployment times out while its revision retains its readiness failure Given namespace "${TEST_NAMESPACE}" is available And min value for ClusterObjectDeployment .spec.template.spec.progressDeadlineMinutes is set to 1 - And min value for ClusterObjectSet .spec.progressDeadlineMinutes is set to 1 When resource is applied """ apiVersion: olm.operatorframework.io/v1 @@ -59,6 +58,6 @@ Feature: ClusterObjectDeployment progression timeout image: does-not-exist:latest """ Then ClusterObjectSet "${NAME}-1" has group "${NAME}" - And ClusterObjectSet "${NAME}-1" reports Ready as False with Reason ProgressDeadlineExceeded And ClusterObjectDeployment "${NAME}" reports Progressing as False with Reason ProgressDeadlineExceeded + And ClusterObjectSet "${NAME}-1" reports Ready as False with Reason ObjectsNotReady And ClusterObjectDeployment "${NAME}" reports Available as False with Reason Unavailable From 43b961f985be8aac0454dcf25ac693edf91c2a42 Mon Sep 17 00:00:00 2001 From: Fabricio Aguiar Date: Mon, 26 Oct 2026 14:54:11 +0000 Subject: [PATCH 3/3] chore(api): regenerate revision deadline documentation Regenerate apply configuration comments, API reference documentation, the COD CRD, and experimental manifests from the clarified deadline contract. Signed-off-by: Fabricio Aguiar rh-pre-commit.version: 2.3.2 rh-pre-commit.check-secrets: ENABLED --- .../api/v1/clusterobjectdeploymentstatus.go | 2 +- .../api/v1/clusterobjectdeploymenttemplatespec.go | 13 +++++++------ docs/api-reference/olmv1-api-reference.md | 4 ++-- ...atorframework.io_clusterobjectdeployments.yaml | 15 ++++++++------- manifests/experimental-e2e.yaml | 15 ++++++++------- manifests/experimental.yaml | 15 ++++++++------- 6 files changed, 34 insertions(+), 30 deletions(-) diff --git a/applyconfigurations/api/v1/clusterobjectdeploymentstatus.go b/applyconfigurations/api/v1/clusterobjectdeploymentstatus.go index effebcc2b5..2b93b2f5ab 100644 --- a/applyconfigurations/api/v1/clusterobjectdeploymentstatus.go +++ b/applyconfigurations/api/v1/clusterobjectdeploymentstatus.go @@ -33,7 +33,7 @@ type ClusterObjectDeploymentStatusApplyConfiguration struct { // rollout is incomplete, or False with reason NewClusterObjectSetProgressed after // completion. Completion remains recorded even if the completed revision later // becomes unhealthy. Progressing is False with reason ProgressDeadlineExceeded when - // the latest revision does not roll out within its spec.progressDeadlineMinutes. + // the latest revision does not roll out within spec.template.spec.progressDeadlineMinutes. // Progressing is False with reason InvalidSpecification when spec.template.metadata // contains labels or annotations that are not valid for the resulting // ClusterObjectSet; no new revision is created until the metadata is corrected. diff --git a/applyconfigurations/api/v1/clusterobjectdeploymenttemplatespec.go b/applyconfigurations/api/v1/clusterobjectdeploymenttemplatespec.go index 4b09cb2103..2c7bb4d347 100644 --- a/applyconfigurations/api/v1/clusterobjectdeploymenttemplatespec.go +++ b/applyconfigurations/api/v1/clusterobjectdeploymenttemplatespec.go @@ -31,12 +31,13 @@ type ClusterObjectDeploymentTemplateSpecApplyConfiguration struct { // at most 50 objects using the existing inline object or immutable Secret reference // representation. Phase, object, and probe order participate in template identity. Phases []ClusterObjectSetPhaseApplyConfiguration `json:"phases,omitempty"` - // progressDeadlineMinutes is an optional field that defines the maximum period - // of time in minutes after which an installation should be considered failed and - // require manual intervention. This functionality is disabled when no value - // is provided. The minimum period is 10 minutes, and the maximum is 720 minutes (12 hours). - // - // The value is passed through to each ClusterObjectSet revision created from this template. + // progressDeadlineMinutes is the maximum number of minutes a revision may take + // to roll out, measured from the ClusterObjectSet's creation timestamp. Time + // spent waiting for the ClusterObjectSet to be created is not counted. The + // deadline covers all phases together; completing a phase does not reset it. + // Once a revision has completed, the deadline no longer applies, even if it + // later becomes unhealthy. When omitted, no deadline is enforced. The minimum + // period is 10 minutes, and the maximum is 720 minutes (12 hours). ProgressDeadlineMinutes *int32 `json:"progressDeadlineMinutes,omitempty"` // progressionProbes supplies custom readiness probes for objects in every phase. // The existing ClusterObjectSet selector and assertion semantics apply. diff --git a/docs/api-reference/olmv1-api-reference.md b/docs/api-reference/olmv1-api-reference.md index 8fa0349426..25d2e89aca 100644 --- a/docs/api-reference/olmv1-api-reference.md +++ b/docs/api-reference/olmv1-api-reference.md @@ -458,7 +458,7 @@ _Appears in:_ | Field | Description | Default | Validation | | --- | --- | --- | --- | -| `conditions` _[Condition](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.31/#condition-v1-meta) array_ | conditions describes rollout progress and availability for the observed generation.
Progressing is True with reason NewClusterObjectSetProgressing while the desired
rollout is incomplete, or False with reason NewClusterObjectSetProgressed after
completion. Completion remains recorded even if the completed revision later
becomes unhealthy. Progressing is False with reason ProgressDeadlineExceeded when
the latest revision does not roll out within its spec.progressDeadlineMinutes.
Progressing is False with reason InvalidSpecification when spec.template.metadata
contains labels or annotations that are not valid for the resulting
ClusterObjectSet; no new revision is created until the metadata is corrected.
Available is True with reason Available when at least one active (non-archived)
ClusterObjectSet revision is Ready, so the deployment remains available across
rollouts while a previous revision is still Ready. It is False with reason
Unavailable when active revisions exist but none are Ready yet, or with reason
NoActiveRevision when no active revision exists. | | Optional: \{\}
| +| `conditions` _[Condition](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.31/#condition-v1-meta) array_ | conditions describes rollout progress and availability for the observed generation.
Progressing is True with reason NewClusterObjectSetProgressing while the desired
rollout is incomplete, or False with reason NewClusterObjectSetProgressed after
completion. Completion remains recorded even if the completed revision later
becomes unhealthy. Progressing is False with reason ProgressDeadlineExceeded when
the latest revision does not roll out within spec.template.spec.progressDeadlineMinutes.
Progressing is False with reason InvalidSpecification when spec.template.metadata
contains labels or annotations that are not valid for the resulting
ClusterObjectSet; no new revision is created until the metadata is corrected.
Available is True with reason Available when at least one active (non-archived)
ClusterObjectSet revision is Ready, so the deployment remains available across
rollouts while a previous revision is still Ready. It is False with reason
Unavailable when active revisions exist but none are Ready yet, or with reason
NoActiveRevision when no active revision exists. | | Optional: \{\}
| #### ClusterObjectDeploymentTemplate @@ -510,7 +510,7 @@ _Appears in:_ | Field | Description | Default | Validation | | --- | --- | --- | --- | | `phases` _[ClusterObjectSetPhase](#clusterobjectsetphase) array_ | phases is an ordered list of 1 to 20 uniquely named phases. Each phase contains
at most 50 objects using the existing inline object or immutable Secret reference
representation. Phase, object, and probe order participate in template identity. | | MaxItems: 20
MinItems: 1
Required: \{\}
| -| `progressDeadlineMinutes` _integer_ | progressDeadlineMinutes is an optional field that defines the maximum period
of time in minutes after which an installation should be considered failed and
require manual intervention. This functionality is disabled when no value
is provided. The minimum period is 10 minutes, and the maximum is 720 minutes (12 hours).
The value is passed through to each ClusterObjectSet revision created from this template. | | Maximum: 720
Minimum: 10
Optional: \{\}
| +| `progressDeadlineMinutes` _integer_ | progressDeadlineMinutes is the maximum number of minutes a revision may take
to roll out, measured from the ClusterObjectSet's creation timestamp. Time
spent waiting for the ClusterObjectSet to be created is not counted. The
deadline covers all phases together; completing a phase does not reset it.
Once a revision has completed, the deadline no longer applies, even if it
later becomes unhealthy. When omitted, no deadline is enforced. The minimum
period is 10 minutes, and the maximum is 720 minutes (12 hours). | | Maximum: 720
Minimum: 10
Optional: \{\}
| | `progressionProbes` _[ProgressionProbe](#progressionprobe) array_ | progressionProbes supplies custom readiness probes for objects in every phase.
The existing ClusterObjectSet selector and assertion semantics apply. | | MaxItems: 20
MinItems: 1
Optional: \{\}
| | `collisionProtection` _[CollisionProtection](#collisionprotection)_ | collisionProtection is the required default collision protection strategy for objects in the revision.
Objects and phases may override it. | | Enum: [Prevent IfNoController None]
Required: \{\}
| diff --git a/helm/olmv1/base/object-controller/crd/experimental/olm.operatorframework.io_clusterobjectdeployments.yaml b/helm/olmv1/base/object-controller/crd/experimental/olm.operatorframework.io_clusterobjectdeployments.yaml index 1012ca6122..1879b3eb4f 100644 --- a/helm/olmv1/base/object-controller/crd/experimental/olm.operatorframework.io_clusterobjectdeployments.yaml +++ b/helm/olmv1/base/object-controller/crd/experimental/olm.operatorframework.io_clusterobjectdeployments.yaml @@ -257,12 +257,13 @@ spec: x-kubernetes-list-type: map progressDeadlineMinutes: description: |- - progressDeadlineMinutes is an optional field that defines the maximum period - of time in minutes after which an installation should be considered failed and - require manual intervention. This functionality is disabled when no value - is provided. The minimum period is 10 minutes, and the maximum is 720 minutes (12 hours). - - The value is passed through to each ClusterObjectSet revision created from this template. + progressDeadlineMinutes is the maximum number of minutes a revision may take + to roll out, measured from the ClusterObjectSet's creation timestamp. Time + spent waiting for the ClusterObjectSet to be created is not counted. The + deadline covers all phases together; completing a phase does not reset it. + Once a revision has completed, the deadline no longer applies, even if it + later becomes unhealthy. When omitted, no deadline is enforced. The minimum + period is 10 minutes, and the maximum is 720 minutes (12 hours). format: int32 maximum: 720 minimum: 10 @@ -545,7 +546,7 @@ spec: rollout is incomplete, or False with reason NewClusterObjectSetProgressed after completion. Completion remains recorded even if the completed revision later becomes unhealthy. Progressing is False with reason ProgressDeadlineExceeded when - the latest revision does not roll out within its spec.progressDeadlineMinutes. + the latest revision does not roll out within spec.template.spec.progressDeadlineMinutes. Progressing is False with reason InvalidSpecification when spec.template.metadata contains labels or annotations that are not valid for the resulting ClusterObjectSet; no new revision is created until the metadata is corrected. diff --git a/manifests/experimental-e2e.yaml b/manifests/experimental-e2e.yaml index bd269df650..1a7f2ad2b7 100644 --- a/manifests/experimental-e2e.yaml +++ b/manifests/experimental-e2e.yaml @@ -1586,12 +1586,13 @@ spec: x-kubernetes-list-type: map progressDeadlineMinutes: description: |- - progressDeadlineMinutes is an optional field that defines the maximum period - of time in minutes after which an installation should be considered failed and - require manual intervention. This functionality is disabled when no value - is provided. The minimum period is 10 minutes, and the maximum is 720 minutes (12 hours). - - The value is passed through to each ClusterObjectSet revision created from this template. + progressDeadlineMinutes is the maximum number of minutes a revision may take + to roll out, measured from the ClusterObjectSet's creation timestamp. Time + spent waiting for the ClusterObjectSet to be created is not counted. The + deadline covers all phases together; completing a phase does not reset it. + Once a revision has completed, the deadline no longer applies, even if it + later becomes unhealthy. When omitted, no deadline is enforced. The minimum + period is 10 minutes, and the maximum is 720 minutes (12 hours). format: int32 maximum: 720 minimum: 10 @@ -1874,7 +1875,7 @@ spec: rollout is incomplete, or False with reason NewClusterObjectSetProgressed after completion. Completion remains recorded even if the completed revision later becomes unhealthy. Progressing is False with reason ProgressDeadlineExceeded when - the latest revision does not roll out within its spec.progressDeadlineMinutes. + the latest revision does not roll out within spec.template.spec.progressDeadlineMinutes. Progressing is False with reason InvalidSpecification when spec.template.metadata contains labels or annotations that are not valid for the resulting ClusterObjectSet; no new revision is created until the metadata is corrected. diff --git a/manifests/experimental.yaml b/manifests/experimental.yaml index 5272f9d9a8..9cd8e25eaf 100644 --- a/manifests/experimental.yaml +++ b/manifests/experimental.yaml @@ -1547,12 +1547,13 @@ spec: x-kubernetes-list-type: map progressDeadlineMinutes: description: |- - progressDeadlineMinutes is an optional field that defines the maximum period - of time in minutes after which an installation should be considered failed and - require manual intervention. This functionality is disabled when no value - is provided. The minimum period is 10 minutes, and the maximum is 720 minutes (12 hours). - - The value is passed through to each ClusterObjectSet revision created from this template. + progressDeadlineMinutes is the maximum number of minutes a revision may take + to roll out, measured from the ClusterObjectSet's creation timestamp. Time + spent waiting for the ClusterObjectSet to be created is not counted. The + deadline covers all phases together; completing a phase does not reset it. + Once a revision has completed, the deadline no longer applies, even if it + later becomes unhealthy. When omitted, no deadline is enforced. The minimum + period is 10 minutes, and the maximum is 720 minutes (12 hours). format: int32 maximum: 720 minimum: 10 @@ -1835,7 +1836,7 @@ spec: rollout is incomplete, or False with reason NewClusterObjectSetProgressed after completion. Completion remains recorded even if the completed revision later becomes unhealthy. Progressing is False with reason ProgressDeadlineExceeded when - the latest revision does not roll out within its spec.progressDeadlineMinutes. + the latest revision does not roll out within spec.template.spec.progressDeadlineMinutes. Progressing is False with reason InvalidSpecification when spec.template.metadata contains labels or annotations that are not valid for the resulting ClusterObjectSet; no new revision is created until the metadata is corrected.