Merge pull request #17 from gary-lgy/rebalance-unschedulable

cmd/controller-manager/app/core.go

@@ -24,6 +24,8 @@ import (
 	"k8s.io/klog/v2"
 
 	fedcorev1a1 "github.com/kubewharf/kubeadmiral/pkg/apis/core/v1alpha1"
+	"github.com/kubewharf/kubeadmiral/pkg/client/generic"
+	"github.com/kubewharf/kubeadmiral/pkg/controllers/automigration"
 	controllercontext "github.com/kubewharf/kubeadmiral/pkg/controllers/context"
 	"github.com/kubewharf/kubeadmiral/pkg/controllers/federate"
 	"github.com/kubewharf/kubeadmiral/pkg/controllers/federatedcluster"
@@ -190,3 +192,36 @@ func startFederateController(
 	go federateController.Run(ctx)
 	return nil
 }
+
+func startAutoMigrationController(
+	ctx context.Context,
+	controllerCtx *controllercontext.Context,
+	typeConfig *fedcorev1a1.FederatedTypeConfig,
+) error {
+	if typeConfig.Spec.AutoMigration == nil || !typeConfig.Spec.AutoMigration.Enabled {
+		klog.Infof("Auto migration controller disabled for FederatedTypeConfig %s", typeConfig.Name)
+		return nil
+	}
+
+	genericClient, err := generic.New(controllerCtx.RestConfig)
+	if err != nil {
+		return fmt.Errorf("error creating generic client: %w", err)
+	}
+
+	federatedAPIResource := typeConfig.GetFederatedType()
+	federatedGVR := schemautil.APIResourceToGVR(&federatedAPIResource)
+
+	federateController, err := automigration.NewAutoMigrationController(
+		controllerConfigFromControllerContext(controllerCtx),
+		typeConfig,
+		genericClient,
+		controllerCtx.KubeClientset,
+		controllerCtx.DynamicClientset.Resource(federatedGVR),
+		controllerCtx.DynamicInformerFactory.ForResource(federatedGVR),
+	)
+	if err != nil {
+		return fmt.Errorf("error creating auto-migration controller: %w", err)
+	}
+	go federateController.Run(ctx)
+	return nil
+}

cmd/controller-manager/app/ftcmanager.go

@@ -34,13 +34,15 @@ import (
 )
 
 const (
-	FederateControllerName = "federate-controller"
-	GlobalSchedulerName    = "global-scheduler"
+	FederateControllerName      = "federate-controller"
+	GlobalSchedulerName         = "global-scheduler"
+	AutoMigrationControllerName = "auto-migration-controller"
 )
 
 var knownFTCSubControllers = map[string]StartFTCSubControllerFunc{
-	GlobalSchedulerName:    startGlobalScheduler,
-	FederateControllerName: startFederateController,
+	GlobalSchedulerName:         startGlobalScheduler,
+	FederateControllerName:      startFederateController,
+	AutoMigrationControllerName: startAutoMigrationController,
 }
 
 // StartFTCSubControllerFunc is responsible for constructing and starting a FTC subcontroller. A FTC subcontroller is started/stopped
@@ -162,6 +164,7 @@ func (m *FederatedTypeConfigManager) processQueueItem(ctx context.Context) {
 			continue
 		}
 
+		// TODO[ftcmanager]: handle controllers that do not need to be started for certain FTCs
 		if err := startFunc(subControllerCtx, m.controllerCtx, typeConfig); err != nil {
 			keyedLogger.Error(err, "Failed to start subcontrolelr")
 		} else {

config/crds/core.kubeadmiral.io_federatedtypeconfigs.yaml

@@ -37,6 +37,15 @@ spec:
             type: object
           spec:
             properties:
+              autoMigration:
+                description: Configurations for auto migration.
+                properties:
+                  enabled:
+                    description: Whether or not to enable auto migration.
+                    type: boolean
+                required:
+                - enabled
+                type: object
               controllers:
                 description: The controllers that must run before the resource can
                   be propagated to member clusters. Each inner slice specifies a step.

config/crds/patches/core.kubeadmiral.io_clusterpropagationpolicies.yaml

@@ -0,0 +1 @@
+core.kubeadmiral.io_propagationpolicies.yaml

config/crds/patches/core.kubeadmiral.io_propagationpolicies.yaml

@@ -0,0 +1,3 @@
+# controller-gen does not respect {} as default value for a struct field
+# issue: https://github.com/kubernetes-sigs/controller-tools/issues/622
+- .spec.versions[0].schema.openAPIV3Schema.properties.spec.properties.replicaRescheduling.default = {}

config/sample/host/01-ftc.yaml

@@ -89,6 +89,8 @@ spec:
     version: v1alpha1
   statusAggregation: Enabled
   revisionHistory: Enabled
+  autoMigration:
+    enabled: true
   controllers:
     - - kubeadmiral.io/global-scheduler
     - - kubeadmiral.io/overridepolicy-controller

hack/generate-groups.sh

@@ -25,6 +25,8 @@ set -o pipefail
 
 CODEGEN_VERSION=${CODEGEN_VERSION:-"v0.19.0"}
 CONTROLLERGEN_VERSION=${CONTROLLERGEN_VERSION:-"v0.11.1"}
+YQ_VERSION=${YQ_VERSION:-"v4.33.1"}
+
 MODULE_NAME=${MODULE_NAME:-"github.com/kubewharf/kubeadmiral"}
 groups=(
   core/v1alpha1
@@ -34,6 +36,7 @@ groups=(
 # install code-generator binaries
 go install k8s.io/code-generator/cmd/{client-gen,lister-gen,informer-gen,deepcopy-gen}@${CODEGEN_VERSION}
 go install sigs.k8s.io/controller-tools/cmd/controller-gen@${CONTROLLERGEN_VERSION}
+go install github.com/mikefarah/yq/v4@${YQ_VERSION}
 
 # define variables
 GOBIN="$(go env GOBIN)"
@@ -55,6 +58,16 @@ function codegen::join() { local IFS="$1"; shift; echo "$*"; }
 # generate manifests
 echo "Generating manifests"
 ${GOBIN}/controller-gen crd paths=$(codegen::join ";" "${INPUT_DIRS[@]}") output:crd:artifacts:config=config/crds
+# apply CRD patches
+for patch_file in config/crds/patches/*.yaml; do
+    crd_file="config/crds/$(basename "${patch_file}")"
+    if [[ ! -f "$crd_file" ]]; then
+        echo "CRD patch file $patch_file does not have a corresponding CRD file" >&2
+        exit 1
+    fi
+    # the patch file should be an array of yq assignment commands
+    "${GOBIN}"/yq eval '.[]' "$patch_file" | xargs -I{} "${GOBIN}"/yq -i '{}' "$crd_file"
+done
 
 # generate deepcopy
 echo "Generating deepcopy funcs"

pkg/apis/core/v1alpha1/types_federatedtypeconfig.go

@@ -82,6 +82,7 @@ type FederatedTypeConfigSpec struct {
 	// resource.
 	// +optional
 	StatusType *APIResource `json:"statusType,omitempty"`
+
 	// Whether or not Status object should be populated.
 	// +optional
 	StatusCollection *StatusCollection `json:"statusCollection,omitempty"`
@@ -92,6 +93,10 @@ type FederatedTypeConfigSpec struct {
 	// Whether or not to plan the rollout process
 	// +optional
 	RolloutPlan *RolloutPlanMode `json:"rolloutPlan,omitempty"`
+	// Configurations for auto migration.
+	// +optional
+	AutoMigration *AutoMigrationConfig `json:"autoMigration,omitempty"`
+
 	// The controllers that must run before the resource can be propagated to member clusters.
 	// Each inner slice specifies a step. Step T must complete before step T+1 can commence.
 	// Controllers within each step can execute in parallel.
@@ -151,6 +156,11 @@ type RevisionHistoryMode string
 
 type RolloutPlanMode string
 
+type AutoMigrationConfig struct {
+	// Whether or not to enable auto migration.
+	Enabled bool `json:"enabled"`
+}
+
 // APIResource defines how to configure the dynamic client for an API resource.
 type APIResource struct {
 	// metav1.GroupVersion is not used since the json annotation of

pkg/apis/core/v1alpha1/types_propagationpolicy.go

@@ -97,6 +97,16 @@ type PropagationPolicySpec struct {
 	// to clusters where the leader is scheduled.
 	// +optional
 	DisableFollowerScheduling bool `json:"disableFollowerScheduling,omitempty"`
+
+	// Configures behaviors related to auto migration. If absent, auto migration will be disabled.
+	// +optional
+	AutoMigration *AutoMigration `json:"autoMigration,omitempty"`
+
+	// Configures behaviors related to replica rescheduling.
+	// +optional
+	// Default set via a post-generation patch.
+	// See patch file for details.
+	ReplicaRescheduling *ReplicaRescheduling `json:"replicaRescheduling,omitempty"`
 }
 
 type PropagationPolicyStatus struct {
@@ -142,3 +152,38 @@ type Preferences struct {
 	// +kubebuilder:validation:Minimum=0
 	Weight *int64 `json:"weight,omitempty"`
 }
+
+// Preferences regarding auto migration.
+type AutoMigration struct {
+	// When a replica should be subject to auto migration.
+	// +optional
+	// +kubebuilder:default:={podUnschedulableFor:"1m"}
+	Trigger AutoMigrationTrigger `json:"when"`
+
+	// Besides starting new replicas in other cluster(s), whether to keep the unschedulable replicas
+	// in the original cluster so we can go back to the desired state when the cluster recovers.
+	// +optional
+	// +kubebuilder:default:=false
+	KeepUnschedulableReplicas bool `json:"keepUnschedulableReplicas"`
+}
+
+// Criteria for determining when a replica is subject to auto migration.
+// +kubebuilder:validation:MinProperties:=1
+type AutoMigrationTrigger struct {
+	// A pod will be subject to auto migration if it remains unschedulable beyond this duration.
+	// Duration should be specified in a format that can be parsed by Go's time.ParseDuration.
+	// +optional
+	// +kubebuilder:validation:Format:=duration
+	PodUnschedulableDuration *metav1.Duration `json:"podUnschedulableFor,omitempty"`
+}
+
+// Preferences regarding replica rescheduling.
+type ReplicaRescheduling struct {
+	// If set to true, the scheduler will attempt to prevent migrating existing replicas during rescheduling.
+	// In order to do so, replica scheduling preferences might not be fully respected.
+	// If set to false, the scheduler will always rebalance the replicas based on the specified preferences, which might
+	// cause temporary service disruption.
+	// +optional
+	// +kubebuilder:default:=true
+	AvoidDisruption bool `json:"avoidDisruption"`
+}