feat: set availabilty to degraded when reconcile fails

Signed-off-by: Sunil Thaha <sthaha@redhat.com>
sustainable-computing-io · Mar 12, 2024 · 3c2c6ff · 3c2c6ff
1 parent 978ea73
commit 3c2c6ff
Show file tree

Hide file tree

Showing 3 changed files with 37 additions and 26 deletions.
diff --git a/pkg/api/v1alpha1/kepler_types.go b/pkg/api/v1alpha1/kepler_types.go
@@ -108,9 +108,10 @@ const (
 type ConditionStatus string
 
 const (
-	ConditionTrue    ConditionStatus = "True"
-	ConditionFalse   ConditionStatus = "False"
-	ConditionUnknown ConditionStatus = "Unknown"
+	ConditionTrue     ConditionStatus = "True"
+	ConditionFalse    ConditionStatus = "False"
+	ConditionUnknown  ConditionStatus = "Unknown"
+	ConditionDegraded ConditionStatus = "Degraded"
 )
 
 type Condition struct {

diff --git a/pkg/controllers/kepler.go b/pkg/controllers/kepler.go
@@ -220,18 +220,19 @@ func (r KeplerReconciler) setInvalidStatus(ctx context.Context, req ctrl.Request
 			return nil
 		}
 
+		now := metav1.Now()
 		invalidKepler.Status.Exporter.Conditions = []v1alpha1.Condition{{
 			Type:               v1alpha1.Reconciled,
 			Status:             v1alpha1.ConditionFalse,
 			ObservedGeneration: invalidKepler.Generation,
-			LastTransitionTime: metav1.Now(),
+			LastTransitionTime: now,
 			Reason:             v1alpha1.InvalidKeplerResource,
 			Message:            "Only a single instance of Kepler named kepler is reconciled",
 		}, {
 			Type:               v1alpha1.Available,
 			Status:             v1alpha1.ConditionUnknown,
 			ObservedGeneration: invalidKepler.Generation,
-			LastTransitionTime: metav1.Now(),
+			LastTransitionTime: now,
 			Reason:             v1alpha1.InvalidKeplerResource,
 			Message:            "This instance of Kepler is invalid",
 		}}

diff --git a/pkg/controllers/kepler_internal.go b/pkg/controllers/kepler_internal.go
@@ -189,33 +189,36 @@ func (r KeplerInternalReconciler) getInternal(ctx context.Context, req ctrl.Requ
 }
 
 func (r KeplerInternalReconciler) updateStatus(ctx context.Context, req ctrl.Request, recErr error) error {
+	logger := r.logger.WithValues("keplerinternal", req.Name, "action", "update-status")
+	logger.V(3).Info("Start of status update")
+	defer logger.V(3).Info("End of status update")
+
 	return retry.RetryOnConflict(retry.DefaultBackoff, func() error {
 
-		logger := r.logger.WithValues("keplerinternal", req.Name)
 		ki, _ := r.getInternal(ctx, req)
 		// may be deleted
 		if ki == nil || !ki.GetDeletionTimestamp().IsZero() {
 			// retry since some error has occurred
-			r.logger.V(6).Info("Reconcile has deleted kepler; skipping update")
+			r.logger.V(6).Info("Reconcile has deleted kepler-internal; skipping status update")
 			return nil
 		}
 
 		// sanitize the conditions so that all types are present and the order is predictable
 		ki.Status.Exporter.Conditions = sanitizeConditions(ki.Status.Exporter.Conditions)
 
-		updated := r.updateReconciledStatus(ctx, ki, recErr) ||
-			r.updateAvailableStatus(ctx, ki, recErr)
-
-		if !updated {
-			logger.V(6).Info("no changes to existing status; skipping update")
-			return nil
-		}
+		{
+			now := metav1.Now()
+			reconciledChanged := r.updateReconciledStatus(ctx, ki, recErr, now)
+			availableChanged := r.updateAvailableStatus(ctx, ki, recErr, now)
+			logger.V(6).Info("conditions updated", "reconciled", reconciledChanged, "available", availableChanged)
 
-		now := metav1.Now()
-		for i := range ki.Status.Exporter.Conditions {
-			ki.Status.Exporter.Conditions[i].LastTransitionTime = now
+			if !reconciledChanged && !availableChanged {
+				logger.V(6).Info("no changes to existing status; skipping update")
+				return nil
+			}
 		}
 
+		logger.V(3).Info("Updating status -> condition", "condition", ki.Status.Exporter.Conditions)
 		return r.Client.Status().Update(ctx, ki)
 
 	})
@@ -256,7 +259,7 @@ func sanitizeConditions(conditions []v1alpha1.Condition) []v1alpha1.Condition {
 	return conditions
 }
 
-func (r KeplerInternalReconciler) updateReconciledStatus(ctx context.Context, ki *v1alpha1.KeplerInternal, recErr error) bool {
+func (r KeplerInternalReconciler) updateReconciledStatus(ctx context.Context, ki *v1alpha1.KeplerInternal, recErr error, time metav1.Time) bool {
 
 	reconciled := v1alpha1.Condition{
 		Type:               v1alpha1.Reconciled,
@@ -272,7 +275,7 @@ func (r KeplerInternalReconciler) updateReconciledStatus(ctx context.Context, ki
 		reconciled.Message = recErr.Error()
 	}
 
-	return updateCondition(ki.Status.Exporter.Conditions, reconciled)
+	return updateCondition(ki.Status.Exporter.Conditions, reconciled, time)
 }
 
 func findCondition(conditions []v1alpha1.Condition, t v1alpha1.ConditionType) *v1alpha1.Condition {
@@ -285,7 +288,7 @@ func findCondition(conditions []v1alpha1.Condition, t v1alpha1.ConditionType) *v
 }
 
 // returns true if the condition has been updated
-func updateCondition(conditions []v1alpha1.Condition, latest v1alpha1.Condition) bool {
+func updateCondition(conditions []v1alpha1.Condition, latest v1alpha1.Condition, time metav1.Time) bool {
 
 	old := findCondition(conditions, latest.Type)
 	if old == nil {
@@ -303,15 +306,17 @@ func updateCondition(conditions []v1alpha1.Condition, latest v1alpha1.Condition)
 	old.Status = latest.Status
 	old.Reason = latest.Reason
 	old.Message = latest.Message
+	// NOTE: last transition time changes only if the status changes
+	old.LastTransitionTime = time
 	return true
 }
 
-func (r KeplerInternalReconciler) updateAvailableStatus(ctx context.Context, ki *v1alpha1.KeplerInternal, recErr error) bool {
+func (r KeplerInternalReconciler) updateAvailableStatus(ctx context.Context, ki *v1alpha1.KeplerInternal, recErr error, time metav1.Time) bool {
 	// get daemonset owned by kepler
 	dset := appsv1.DaemonSet{}
 	key := types.NamespacedName{Name: ki.DaemonsetName(), Namespace: ki.Namespace()}
 	if err := r.Client.Get(ctx, key, &dset); err != nil {
-		return updateCondition(ki.Status.Exporter.Conditions, availableConditionForGetError(err))
+		return updateCondition(ki.Status.Exporter.Conditions, availableConditionForGetError(err), time)
 	}
 
 	ds := dset.Status
@@ -323,14 +328,18 @@ func (r KeplerInternalReconciler) updateAvailableStatus(ctx context.Context, ki
 	ki.Status.Exporter.NumberAvailable = ds.NumberAvailable
 	ki.Status.Exporter.NumberUnavailable = ds.NumberUnavailable
 
-	c := availableCondition(&dset)
+	available := availableCondition(&dset)
+
 	if recErr == nil {
-		c.ObservedGeneration = ki.Generation
+		available.ObservedGeneration = ki.Generation
+	} else {
+		// failure to reconcile is a Degraded condition
+		available.Status = v1alpha1.ConditionDegraded
+		available.Reason = v1alpha1.ReconcileError
 	}
 
-	updated := updateCondition(ki.Status.Exporter.Conditions, c)
+	updated := updateCondition(ki.Status.Exporter.Conditions, available, time)
 
-	// update estimator status
 	estimatorStatus := v1alpha1.EstimatorStatus{
 		Status: v1alpha1.DeploymentNotInstalled,
 	}