From 5bf05f58695c2392c8487e56e3203498f5396b3a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gary=20Liu=20=28=E5=88=98=E5=B9=BF=E6=BA=90=29?= Date: Wed, 12 Apr 2023 09:06:16 +0000 Subject: [PATCH 01/10] test(scheduler): fix broken unit tests --- pkg/controllers/scheduler/core/generic_scheduler_test.go | 2 +- pkg/controllers/scheduler/framework/runtime/framework_test.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/controllers/scheduler/core/generic_scheduler_test.go b/pkg/controllers/scheduler/core/generic_scheduler_test.go index 78429b6f..4b72d6ab 100644 --- a/pkg/controllers/scheduler/core/generic_scheduler_test.go +++ b/pkg/controllers/scheduler/core/generic_scheduler_test.go @@ -88,7 +88,7 @@ func getFramework() framework.Framework { DefaultRegistry := runtime.Registry{ "NaiveReplicas": newNaiveReplicas, } - f, _ := runtime.NewFramework(DefaultRegistry) + f, _ := runtime.NewFramework(DefaultRegistry, nil) return f } diff --git a/pkg/controllers/scheduler/framework/runtime/framework_test.go b/pkg/controllers/scheduler/framework/runtime/framework_test.go index 282db057..8d746458 100644 --- a/pkg/controllers/scheduler/framework/runtime/framework_test.go +++ b/pkg/controllers/scheduler/framework/runtime/framework_test.go @@ -101,7 +101,7 @@ func TestRunFilterPlugins(t *testing.T) { for _, test := range tests { t.Run(test.name, func(t *testing.T) { - fwk, err := NewFramework(test.plugins) + fwk, err := NewFramework(test.plugins, nil) if err != nil { t.Errorf("unexpected error when creating framework: %v", err) } From e8e5518926c7ad10c9913fe8ad92715f21f5d150 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gary=20Liu=20=28=E5=88=98=E5=B9=BF=E6=BA=90=29?= Date: Wed, 12 Apr 2023 08:10:37 +0000 Subject: [PATCH 02/10] refactor(scheduler): scheduling unit Standardize GroupVersion{Kind, Resource}; add Labels and Annotations Co-authored-by: Lim Haw Jia --- .../plugins/apiresources/apiresources.go | 26 ++---- .../plugins/apiresources/apiresources_test.go | 13 ++- .../scheduler/framework/runtime/framework.go | 2 +- pkg/controllers/scheduler/framework/types.go | 12 ++- pkg/controllers/scheduler/framework/util.go | 4 - pkg/controllers/scheduler/scheduler_test.go | 91 +++++++++++++------ pkg/controllers/scheduler/schedulingunit.go | 48 +++++++--- 7 files changed, 126 insertions(+), 70 deletions(-) diff --git a/pkg/controllers/scheduler/framework/plugins/apiresources/apiresources.go b/pkg/controllers/scheduler/framework/plugins/apiresources/apiresources.go index 9362b3d8..055e432e 100644 --- a/pkg/controllers/scheduler/framework/plugins/apiresources/apiresources.go +++ b/pkg/controllers/scheduler/framework/plugins/apiresources/apiresources.go @@ -5,6 +5,8 @@ package apiresources import ( "context" + "k8s.io/apimachinery/pkg/runtime/schema" + fedcorev1a1 "github.com/kubewharf/kubeadmiral/pkg/apis/core/v1alpha1" "github.com/kubewharf/kubeadmiral/pkg/controllers/scheduler/framework" ) @@ -29,24 +31,16 @@ func (pl *APIResources) Filter(ctx context.Context, su *framework.SchedulingUnit return framework.NewResult(framework.Error, err.Error()) } - GVK := getGroupVersionKind(su) - enableGVKs := getEnabledGroupVersionKinds(cluster) - for _, enableGVK := range enableGVKs { - if enableGVK == GVK { + gvk := su.GroupVersion.WithKind(su.Kind) + for _, r := range cluster.Status.APIResourceTypes { + enabledGVK := schema.GroupVersionKind{ + Group: r.Group, + Version: r.Version, + Kind: r.Kind, + } + if enabledGVK == gvk { return framework.NewResult(framework.Success) } } return framework.NewResult(framework.Unschedulable, "No matched group version kind.") } - -func getEnabledGroupVersionKinds(cluster *fedcorev1a1.FederatedCluster) []string { - gvks := []string{} - for _, r := range cluster.Status.APIResourceTypes { - gvks = append(gvks, framework.GVKString(r)) - } - return gvks -} - -func getGroupVersionKind(su *framework.SchedulingUnit) string { - return su.GroupVersionKind -} diff --git a/pkg/controllers/scheduler/framework/plugins/apiresources/apiresources_test.go b/pkg/controllers/scheduler/framework/plugins/apiresources/apiresources_test.go index b0e2369d..e6d171e8 100644 --- a/pkg/controllers/scheduler/framework/plugins/apiresources/apiresources_test.go +++ b/pkg/controllers/scheduler/framework/plugins/apiresources/apiresources_test.go @@ -5,7 +5,9 @@ import ( "reflect" "testing" + appsv1 "k8s.io/api/apps/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime/schema" fedcorev1a1 "github.com/kubewharf/kubeadmiral/pkg/apis/core/v1alpha1" "github.com/kubewharf/kubeadmiral/pkg/controllers/scheduler/framework" @@ -22,10 +24,11 @@ func clusterWithAPIResource(clusterName string, resources []fedcorev1a1.APIResou } } -func suWithAPIResource(suName, gvk string) *framework.SchedulingUnit { +func suWithAPIResource(suName string, gvk schema.GroupVersionKind) *framework.SchedulingUnit { return &framework.SchedulingUnit{ - Name: suName, - GroupVersionKind: gvk, + Name: suName, + GroupVersion: gvk.GroupVersion(), + Kind: gvk.Kind, } } @@ -38,7 +41,7 @@ func TestAPIResourcesFilter(t *testing.T) { }{ { name: "the cluster has the resources for schedulingUnit", - su: suWithAPIResource("su1", "apps/v1/DaemonSet"), + su: suWithAPIResource("su1", appsv1.SchemeGroupVersion.WithKind("DaemonSet")), cluster: clusterWithAPIResource("clusterA", []fedcorev1a1.APIResource{ { Group: "apps", @@ -50,7 +53,7 @@ func TestAPIResourcesFilter(t *testing.T) { }, { name: "the cluster does not have the resources for schedulingUnit", - su: suWithAPIResource("su1", "apps/v1/DaemonSet"), + su: suWithAPIResource("su1", appsv1.SchemeGroupVersion.WithKind("DaemonSet")), cluster: clusterWithAPIResource("clusterA", []fedcorev1a1.APIResource{ { Group: "apps", diff --git a/pkg/controllers/scheduler/framework/runtime/framework.go b/pkg/controllers/scheduler/framework/runtime/framework.go index 68b4b43c..2ffe11e3 100644 --- a/pkg/controllers/scheduler/framework/runtime/framework.go +++ b/pkg/controllers/scheduler/framework/runtime/framework.go @@ -208,7 +208,7 @@ func (f *frameworkImpl) RunReplicasPlugin( if len(f.replicasPlugins) == 0 { return clusterReplicasList, framework.NewResult( framework.Success, - fmt.Sprintf("no replicas plugin register for this type %s", schedulingUnit.GroupVersionKind), + fmt.Sprintf("no replicas plugin registered in the framework"), ) } for _, plugin := range f.replicasPlugins { diff --git a/pkg/controllers/scheduler/framework/types.go b/pkg/controllers/scheduler/framework/types.go index e032655a..74a215f8 100644 --- a/pkg/controllers/scheduler/framework/types.go +++ b/pkg/controllers/scheduler/framework/types.go @@ -31,10 +31,14 @@ import ( ) type SchedulingUnit struct { - Name string - Namespace string - GroupVersionKind string - GroupVersionResource schema.GroupVersionResource + GroupVersion schema.GroupVersion + Kind string + Resource string + + Namespace string + Name string + Labels map[string]string + Annotations map[string]string // Only care about the requests resources // TODO(all), limit resources, Best Effort resources diff --git a/pkg/controllers/scheduler/framework/util.go b/pkg/controllers/scheduler/framework/util.go index 3a3922f4..a12dedd9 100644 --- a/pkg/controllers/scheduler/framework/util.go +++ b/pkg/controllers/scheduler/framework/util.go @@ -449,10 +449,6 @@ func getFilteredTaints(taints []corev1.Taint, inclusionFilter taintsFilterFunc) return filteredTaints } -func GVKString(r fedcorev1a1.APIResource) string { - return strings.Join([]string{r.Group, r.Version, r.Kind}, "/") -} - // DefaultNormalizeScore generates a Normalize Score function that can normalize the // scores to [0, maxPriority]. If reverse is set to true, it reverses the scores by // subtracting it from maxPriority. diff --git a/pkg/controllers/scheduler/scheduler_test.go b/pkg/controllers/scheduler/scheduler_test.go index ac971651..fed68d1c 100644 --- a/pkg/controllers/scheduler/scheduler_test.go +++ b/pkg/controllers/scheduler/scheduler_test.go @@ -17,26 +17,29 @@ limitations under the License. package scheduler import ( - "reflect" "testing" + "github.com/onsi/gomega" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/utils/pointer" fedcorev1a1 "github.com/kubewharf/kubeadmiral/pkg/apis/core/v1alpha1" + "github.com/kubewharf/kubeadmiral/pkg/controllers/common" "github.com/kubewharf/kubeadmiral/pkg/controllers/scheduler/framework" ) func TestGetSchedulingUnitWithAnnotationOverrides(t *testing.T) { tests := []struct { - name string - policy fedcorev1a1.GenericPropagationPolicy - annotations map[string]string - expectedResult *framework.SchedulingUnit + name string + policy fedcorev1a1.GenericPropagationPolicy + annotations map[string]string + templateObjectMeta *metav1.ObjectMeta + expectedResult *framework.SchedulingUnit }{ { name: "scheduling mode override", @@ -197,7 +200,7 @@ func TestGetSchedulingUnitWithAnnotationOverrides(t *testing.T) { ClusterSelector: map[string]string{ "label": "value1", }, - MaxClusters: pointer.Int64Ptr(5), + MaxClusters: pointer.Int64(5), }, }, annotations: map[string]string{ @@ -208,7 +211,7 @@ func TestGetSchedulingUnitWithAnnotationOverrides(t *testing.T) { ClusterSelector: map[string]string{ "label": "value1", }, - MaxClusters: pointer.Int64Ptr(10), + MaxClusters: pointer.Int64(10), }, }, { @@ -218,7 +221,7 @@ func TestGetSchedulingUnitWithAnnotationOverrides(t *testing.T) { ClusterSelector: map[string]string{ "label": "value1", }, - MaxClusters: pointer.Int64Ptr(5), + MaxClusters: pointer.Int64(5), Placements: []fedcorev1a1.Placement{ { ClusterName: "cluster1", @@ -250,7 +253,7 @@ func TestGetSchedulingUnitWithAnnotationOverrides(t *testing.T) { ClusterSelector: map[string]string{ "label": "value1", }, - MaxClusters: pointer.Int64Ptr(5), + MaxClusters: pointer.Int64(5), ClusterNames: map[string]struct{}{ "cluster1": {}, "cluster2": {}, @@ -260,7 +263,7 @@ func TestGetSchedulingUnitWithAnnotationOverrides(t *testing.T) { "cluster2": 2, }, MaxReplicas: map[string]int64{ - "cluster1": *pointer.Int64Ptr(10), + "cluster1": 10, }, Weights: map[string]int64{ "cluster1": 2, @@ -268,9 +271,32 @@ func TestGetSchedulingUnitWithAnnotationOverrides(t *testing.T) { }, }, }, + { + name: "template object meta", + policy: &fedcorev1a1.PropagationPolicy{ + Spec: fedcorev1a1.PropagationPolicySpec{ + SchedulingMode: fedcorev1a1.SchedulingModeDivide, + }, + }, + templateObjectMeta: &metav1.ObjectMeta{ + Namespace: metav1.NamespaceDefault, + Name: "test", + Labels: map[string]string{"label": "value1"}, + Annotations: map[string]string{"annotation": "value1"}, + }, + expectedResult: &framework.SchedulingUnit{ + SchedulingMode: fedcorev1a1.SchedulingModeDivide, + Namespace: metav1.NamespaceDefault, + Name: "test", + Labels: map[string]string{"label": "value1"}, + Annotations: map[string]string{"annotation": "value1"}, + }, + }, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { + g := gomega.NewWithT(t) + scheduler := Scheduler{ typeConfig: &fedcorev1a1.FederatedTypeConfig{ Spec: fedcorev1a1.FederatedTypeConfigSpec{ @@ -280,25 +306,35 @@ func TestGetSchedulingUnitWithAnnotationOverrides(t *testing.T) { }, }, } - obj := &unstructured.Unstructured{} + templateObjectMeta := test.templateObjectMeta + if templateObjectMeta == nil { + templateObjectMeta = &metav1.ObjectMeta{} + } + templateObjectMetaUns, err := runtime.DefaultUnstructuredConverter.ToUnstructured(templateObjectMeta) + g.Expect(err).NotTo(gomega.HaveOccurred()) + + obj := &unstructured.Unstructured{Object: make(map[string]interface{})} obj.SetAnnotations(test.annotations) + err = unstructured.SetNestedMap(obj.Object, templateObjectMetaUns, common.TemplatePath...) + g.Expect(err).NotTo(gomega.HaveOccurred()) + su, err := scheduler.schedulingUnitForFedObject(obj, test.policy) - if err != nil { - t.Errorf("unexpected error when getting scheduling unit: %v", err) - } + g.Expect(err).NotTo(gomega.HaveOccurred()) // override fields we don't want to test + su.GroupVersion = test.expectedResult.GroupVersion + su.Kind = test.expectedResult.Kind + su.Resource = test.expectedResult.Resource su.Name = test.expectedResult.Name su.Namespace = test.expectedResult.Namespace - su.GroupVersionKind = test.expectedResult.GroupVersionKind - su.GroupVersionResource = test.expectedResult.GroupVersionResource + su.Labels = test.expectedResult.Labels + su.Annotations = test.expectedResult.Annotations su.DesiredReplicas = test.expectedResult.DesiredReplicas su.CurrentClusters = test.expectedResult.CurrentClusters su.ResourceRequest = test.expectedResult.ResourceRequest + su.AvoidDisruption = test.expectedResult.AvoidDisruption - if !reflect.DeepEqual(su, test.expectedResult) { - t.Errorf("unexpected scheduling unit: %v want %v", su, test.expectedResult) - } + g.Expect(su).To(gomega.Equal(test.expectedResult)) }) } } @@ -309,7 +345,6 @@ func TestSchedulingMode(t *testing.T) { gvk schema.GroupVersionKind replicasSpecPath string expectedResult fedcorev1a1.SchedulingMode - expectedError string }{ "deployments should be able to use divide mode": { policy: &fedcorev1a1.PropagationPolicy{ @@ -345,6 +380,7 @@ func TestSchedulingMode(t *testing.T) { for name, test := range tests { t.Run(name, func(t *testing.T) { + g := gomega.NewWithT(t) scheduler := Scheduler{ typeConfig: &fedcorev1a1.FederatedTypeConfig{ ObjectMeta: metav1.ObjectMeta{ @@ -362,15 +398,14 @@ func TestSchedulingMode(t *testing.T) { }, }, } - obj := &unstructured.Unstructured{} + obj := &unstructured.Unstructured{Object: make(map[string]interface{})} + templateObjectMetaUns, err := runtime.DefaultUnstructuredConverter.ToUnstructured(&metav1.ObjectMeta{}) + g.Expect(err).NotTo(gomega.HaveOccurred()) + err = unstructured.SetNestedMap(obj.Object, templateObjectMetaUns, common.TemplatePath...) + g.Expect(err).NotTo(gomega.HaveOccurred()) su, err := scheduler.schedulingUnitForFedObject(obj, test.policy) - if err != nil && err.Error() != test.expectedError { - t.Errorf("unexpected error when getting scheduling unit: %v", err) - } else if err == nil { - if su.SchedulingMode != test.expectedResult { - t.Fatalf("expected schedulingMode to be %v, but got %v", test.expectedResult, su.SchedulingMode) - } - } + g.Expect(err).NotTo(gomega.HaveOccurred()) + g.Expect(su.SchedulingMode).To(gomega.Equal(test.expectedResult)) }) } } diff --git a/pkg/controllers/scheduler/schedulingunit.go b/pkg/controllers/scheduler/schedulingunit.go index 7e156187..8c2868cb 100644 --- a/pkg/controllers/scheduler/schedulingunit.go +++ b/pkg/controllers/scheduler/schedulingunit.go @@ -18,11 +18,13 @@ package scheduler import ( "encoding/json" + "fmt" "strconv" - "strings" corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/klog/v2" @@ -38,7 +40,11 @@ func (s *Scheduler) schedulingUnitForFedObject( policy fedcorev1a1.GenericPropagationPolicy, ) (*framework.SchedulingUnit, error) { targetType := s.typeConfig.GetTargetType() - targetGVK := strings.Join([]string{targetType.Group, targetType.Version, targetType.Kind}, "/") + + objectMeta, err := getTemplateObjectMeta(fedObject) + if err != nil { + return nil, fmt.Errorf("error retrieving object meta from template: %w", err) + } schedulingMode := getSchedulingModeFromPolicy(policy) schedulingModeOverride, exists := getSchedulingModeFromObject(fedObject) @@ -70,18 +76,16 @@ func (s *Scheduler) schedulingUnitForFedObject( } schedulingUnit := &framework.SchedulingUnit{ - Name: fedObject.GetName(), - Namespace: fedObject.GetNamespace(), - GroupVersionKind: targetGVK, - GroupVersionResource: schema.GroupVersionResource{ - Resource: targetType.Name, - Group: targetType.Group, - Version: targetType.Version, - }, - + GroupVersion: schema.GroupVersion{Group: targetType.Group, Version: targetType.Version}, + Kind: targetType.Kind, + Resource: targetType.Name, + Namespace: objectMeta.GetNamespace(), + Name: objectMeta.GetName(), + Labels: objectMeta.GetLabels(), + Annotations: objectMeta.GetAnnotations(), DesiredReplicas: desiredReplicasOption, CurrentClusters: currentReplicas, - AvoidDisruption: policy.GetSpec().ReplicaRescheduling.AvoidDisruption, + AvoidDisruption: true, } if autoMigration := policy.GetSpec().AutoMigration; autoMigration != nil { @@ -95,6 +99,10 @@ func (s *Scheduler) schedulingUnitForFedObject( } } + if replicaRescheduling := policy.GetSpec().ReplicaRescheduling; replicaRescheduling != nil { + schedulingUnit.AvoidDisruption = replicaRescheduling.AvoidDisruption + } + schedulingUnit.SchedulingMode = schedulingMode schedulingUnit.StickyCluster = getIsStickyClusterFromPolicy(policy) @@ -154,6 +162,22 @@ func (s *Scheduler) schedulingUnitForFedObject( return schedulingUnit, nil } +func getTemplateObjectMeta(fedObject *unstructured.Unstructured) (*metav1.ObjectMeta, error) { + templateContent, exists, err := unstructured.NestedMap(fedObject.Object, common.TemplatePath...) + if err != nil { + return nil, fmt.Errorf("error retrieving template: %w", err) + } + if !exists { + return nil, fmt.Errorf("template not found") + } + objectMeta := metav1.ObjectMeta{} + err = runtime.DefaultUnstructuredConverter.FromUnstructured(templateContent, &objectMeta) + if err != nil { + return nil, fmt.Errorf("template cannot be converted to unstructured: %w", err) + } + return &objectMeta, nil +} + func getCurrentReplicasFromObject( typeConfig *fedcorev1a1.FederatedTypeConfig, object *unstructured.Unstructured, From 6d5b8354eba1cad459b0869a99d0b90be5536ad7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gary=20Liu=20=28=E5=88=98=E5=B9=BF=E6=BA=90=29?= Date: Wed, 12 Apr 2023 03:44:58 +0000 Subject: [PATCH 03/10] feat(scheduler): add scheduler webhook plugin CRD Co-authored-by: Lim Haw Jia --- ..._schedulerpluginwebhookconfigurations.yaml | 110 +++++++++++++ pkg/apis/core/v1alpha1/register.go | 2 + ...pes_schedulerpluginwebhookconfiguration.go | 92 +++++++++++ .../core/v1alpha1/zz_generated.deepcopy.go | 118 ++++++++++++++ .../typed/core/v1alpha1/core_client.go | 5 + .../core/v1alpha1/fake/fake_core_client.go | 4 + ...ake_schedulerpluginwebhookconfiguration.go | 106 ++++++++++++ .../core/v1alpha1/generated_expansion.go | 2 + .../schedulerpluginwebhookconfiguration.go | 152 ++++++++++++++++++ .../core/v1alpha1/interface.go | 7 + .../schedulerpluginwebhookconfiguration.go | 73 +++++++++ .../informers/externalversions/generic.go | 2 + .../core/v1alpha1/expansion_generated.go | 4 + .../schedulerpluginwebhookconfiguration.go | 52 ++++++ 14 files changed, 729 insertions(+) create mode 100644 config/crds/core.kubeadmiral.io_schedulerpluginwebhookconfigurations.yaml create mode 100644 pkg/apis/core/v1alpha1/types_schedulerpluginwebhookconfiguration.go create mode 100644 pkg/client/clientset/versioned/typed/core/v1alpha1/fake/fake_schedulerpluginwebhookconfiguration.go create mode 100644 pkg/client/clientset/versioned/typed/core/v1alpha1/schedulerpluginwebhookconfiguration.go create mode 100644 pkg/client/informers/externalversions/core/v1alpha1/schedulerpluginwebhookconfiguration.go create mode 100644 pkg/client/listers/core/v1alpha1/schedulerpluginwebhookconfiguration.go diff --git a/config/crds/core.kubeadmiral.io_schedulerpluginwebhookconfigurations.yaml b/config/crds/core.kubeadmiral.io_schedulerpluginwebhookconfigurations.yaml new file mode 100644 index 00000000..2f77a84e --- /dev/null +++ b/config/crds/core.kubeadmiral.io_schedulerpluginwebhookconfigurations.yaml @@ -0,0 +1,110 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.11.1 + creationTimestamp: null + name: schedulerpluginwebhookconfigurations.core.kubeadmiral.io +spec: + group: core.kubeadmiral.io + names: + kind: SchedulerPluginWebhookConfiguration + listKind: SchedulerPluginWebhookConfigurationList + plural: schedulerpluginwebhookconfigurations + singular: schedulerpluginwebhookconfiguration + scope: Cluster + versions: + - name: v1alpha1 + schema: + openAPIV3Schema: + description: SchedulerPluginWebhookConfiguration is a webhook that can be + used as a scheduler plugin. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + properties: + filterPath: + description: Path for the filter call, empty if not supported. This + path is appended to the URLPrefix when issuing the filter call to + webhook. + type: string + httpTimeout: + default: 5s + description: HTTPTimeout specifies the timeout duration for a call + to the webhook. Timeout fails the scheduling of the workload. Defaults + to 5 seconds. + format: duration + type: string + payloadVersions: + description: PayloadVersions is an ordered list of preferred request + and response versions the webhook expects. The scheduler will try + to use the first version in the list which it supports. If none + of the versions specified in this list supported by the scheduler, + scheduling will fail for this object. + items: + type: string + minItems: 1 + type: array + scorePath: + description: Path for the score call, empty if not supported. This + verb is appended to the URLPrefix when issuing the score call to + webhook. + type: string + selectPath: + description: Path for the select call, empty if not supported. This + verb is appended to the URLPrefix when issuing the select call to + webhook. + type: string + tlsConfig: + description: TLSConfig specifies the transport layer security config. + properties: + caData: + description: CAData holds PEM-encoded bytes (typically read from + a root certificates bundle). + format: byte + type: string + certData: + description: CertData holds PEM-encoded bytes (typically read + from a client certificate file). + format: byte + type: string + insecure: + description: Server should be accessed without verifying the TLS + certificate. For testing only. + type: boolean + keyData: + description: KeyData holds PEM-encoded bytes (typically read from + a client certificate key file). + format: byte + type: string + serverName: + description: ServerName is passed to the server for SNI and is + used in the client to check server certificates against. If + ServerName is empty, the hostname used to contact the server + is used. + type: string + type: object + urlPrefix: + description: URLPrefix at which the webhook is available + type: string + required: + - payloadVersions + - urlPrefix + type: object + required: + - spec + type: object + served: true + storage: true diff --git a/pkg/apis/core/v1alpha1/register.go b/pkg/apis/core/v1alpha1/register.go index 05b52032..83d0cec8 100644 --- a/pkg/apis/core/v1alpha1/register.go +++ b/pkg/apis/core/v1alpha1/register.go @@ -69,6 +69,8 @@ func addKnownTypes(scheme *runtime.Scheme) error { &OverridePolicyList{}, &ClusterOverridePolicy{}, &ClusterOverridePolicyList{}, + &SchedulerPluginWebhookConfiguration{}, + &SchedulerPluginWebhookConfigurationList{}, ) metav1.AddToGroupVersion(scheme, SchemeGroupVersion) return nil diff --git a/pkg/apis/core/v1alpha1/types_schedulerpluginwebhookconfiguration.go b/pkg/apis/core/v1alpha1/types_schedulerpluginwebhookconfiguration.go new file mode 100644 index 00000000..be5dd4e1 --- /dev/null +++ b/pkg/apis/core/v1alpha1/types_schedulerpluginwebhookconfiguration.go @@ -0,0 +1,92 @@ +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +This file may have been modified by The KubeAdmiral Authors +("KubeAdmiral Modifications"). All KubeAdmiral Modifications +are Copyright 2023 The KubeAdmiral Authors. +*/ + +package v1alpha1 + +import ( + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +// +genclient +// +genclient:nonNamespaced +// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object +// +kubebuilder:resource:path=schedulerpluginwebhookconfigurations,singular=schedulerpluginwebhookconfiguration,scope=Cluster +// +kubebuilder:object:root=true + +// SchedulerPluginWebhookConfiguration is a webhook that can be used as a scheduler plugin. +type SchedulerPluginWebhookConfiguration struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata,omitempty"` + + Spec SchedulerPluginWebhookConfigurationSpec `json:"spec"` +} + +// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object + +// SchedulerPluginWebhookConfigurationList contains a list of SchedulerPluginWebhookConfiguration. +type SchedulerPluginWebhookConfigurationList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata,omitempty"` + Items []SchedulerPluginWebhookConfiguration `json:"items"` +} + +type SchedulerPluginWebhookConfigurationSpec struct { + // PayloadVersions is an ordered list of preferred request and response + // versions the webhook expects. + // The scheduler will try to use the first version in + // the list which it supports. If none of the versions specified in this list + // supported by the scheduler, scheduling will fail for this object. + // +kubebuilder:validation:Required + // +kubebuilder:validation:MinItems=1 + PayloadVersions []string `json:"payloadVersions"` + // URLPrefix at which the webhook is available + // +kubebuilder:validation:Required + URLPrefix string `json:"urlPrefix"` + // Path for the filter call, empty if not supported. This path is appended to the URLPrefix when issuing the filter call to webhook. + FilterPath string `json:"filterPath,omitempty"` + // Path for the score call, empty if not supported. This verb is appended to the URLPrefix when issuing the score call to webhook. + ScorePath string `json:"scorePath,omitempty"` + // Path for the select call, empty if not supported. This verb is appended to the URLPrefix when issuing the select call to webhook. + SelectPath string `json:"selectPath,omitempty"` + // TLSConfig specifies the transport layer security config. + TLSConfig *WebhookTLSConfig `json:"tlsConfig,omitempty"` + // HTTPTimeout specifies the timeout duration for a call to the webhook. Timeout fails the scheduling of the workload. + // Defaults to 5 seconds. + // +kubebuilder:default:="5s" + // +kubebuilder:validation:Format:=duration + HTTPTimeout metav1.Duration `json:"httpTimeout,omitempty"` +} + +// WebhookTLSConfig contains settings to enable TLS with the webhook server. +type WebhookTLSConfig struct { + // Server should be accessed without verifying the TLS certificate. For testing only. + Insecure bool `json:"insecure,omitempty"` + // ServerName is passed to the server for SNI and is used in the client to check server + // certificates against. If ServerName is empty, the hostname used to contact the + // server is used. + ServerName string `json:"serverName,omitempty"` + + // CertData holds PEM-encoded bytes (typically read from a client certificate file). + CertData []byte `json:"certData,omitempty"` + // KeyData holds PEM-encoded bytes (typically read from a client certificate key file). + KeyData []byte `json:"keyData,omitempty"` + // CAData holds PEM-encoded bytes (typically read from a root certificates bundle). + CAData []byte `json:"caData,omitempty"` +} diff --git a/pkg/apis/core/v1alpha1/zz_generated.deepcopy.go b/pkg/apis/core/v1alpha1/zz_generated.deepcopy.go index 37dbc191..1c1303a1 100644 --- a/pkg/apis/core/v1alpha1/zz_generated.deepcopy.go +++ b/pkg/apis/core/v1alpha1/zz_generated.deepcopy.go @@ -1150,6 +1150,93 @@ func (in *Resources) DeepCopy() *Resources { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *SchedulerPluginWebhookConfiguration) DeepCopyInto(out *SchedulerPluginWebhookConfiguration) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + in.Spec.DeepCopyInto(&out.Spec) + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new SchedulerPluginWebhookConfiguration. +func (in *SchedulerPluginWebhookConfiguration) DeepCopy() *SchedulerPluginWebhookConfiguration { + if in == nil { + return nil + } + out := new(SchedulerPluginWebhookConfiguration) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *SchedulerPluginWebhookConfiguration) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *SchedulerPluginWebhookConfigurationList) DeepCopyInto(out *SchedulerPluginWebhookConfigurationList) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ListMeta.DeepCopyInto(&out.ListMeta) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]SchedulerPluginWebhookConfiguration, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new SchedulerPluginWebhookConfigurationList. +func (in *SchedulerPluginWebhookConfigurationList) DeepCopy() *SchedulerPluginWebhookConfigurationList { + if in == nil { + return nil + } + out := new(SchedulerPluginWebhookConfigurationList) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *SchedulerPluginWebhookConfigurationList) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *SchedulerPluginWebhookConfigurationSpec) DeepCopyInto(out *SchedulerPluginWebhookConfigurationSpec) { + *out = *in + if in.PayloadVersions != nil { + in, out := &in.PayloadVersions, &out.PayloadVersions + *out = make([]string, len(*in)) + copy(*out, *in) + } + if in.TLSConfig != nil { + in, out := &in.TLSConfig, &out.TLSConfig + *out = new(WebhookTLSConfig) + (*in).DeepCopyInto(*out) + } + out.HTTPTimeout = in.HTTPTimeout + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new SchedulerPluginWebhookConfigurationSpec. +func (in *SchedulerPluginWebhookConfigurationSpec) DeepCopy() *SchedulerPluginWebhookConfigurationSpec { + if in == nil { + return nil + } + out := new(SchedulerPluginWebhookConfigurationSpec) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *SchedulingPlugin) DeepCopyInto(out *SchedulingPlugin) { *out = *in @@ -1327,3 +1414,34 @@ func (in *TypedRefCount) DeepCopy() *TypedRefCount { in.DeepCopyInto(out) return out } + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *WebhookTLSConfig) DeepCopyInto(out *WebhookTLSConfig) { + *out = *in + if in.CertData != nil { + in, out := &in.CertData, &out.CertData + *out = make([]byte, len(*in)) + copy(*out, *in) + } + if in.KeyData != nil { + in, out := &in.KeyData, &out.KeyData + *out = make([]byte, len(*in)) + copy(*out, *in) + } + if in.CAData != nil { + in, out := &in.CAData, &out.CAData + *out = make([]byte, len(*in)) + copy(*out, *in) + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new WebhookTLSConfig. +func (in *WebhookTLSConfig) DeepCopy() *WebhookTLSConfig { + if in == nil { + return nil + } + out := new(WebhookTLSConfig) + in.DeepCopyInto(out) + return out +} diff --git a/pkg/client/clientset/versioned/typed/core/v1alpha1/core_client.go b/pkg/client/clientset/versioned/typed/core/v1alpha1/core_client.go index 298628ce..d3554ac8 100644 --- a/pkg/client/clientset/versioned/typed/core/v1alpha1/core_client.go +++ b/pkg/client/clientset/versioned/typed/core/v1alpha1/core_client.go @@ -18,6 +18,7 @@ type CoreV1alpha1Interface interface { OverridePoliciesGetter PropagatedVersionsGetter PropagationPoliciesGetter + SchedulerPluginWebhookConfigurationsGetter SchedulingProfilesGetter } @@ -58,6 +59,10 @@ func (c *CoreV1alpha1Client) PropagationPolicies(namespace string) PropagationPo return newPropagationPolicies(c, namespace) } +func (c *CoreV1alpha1Client) SchedulerPluginWebhookConfigurations() SchedulerPluginWebhookConfigurationInterface { + return newSchedulerPluginWebhookConfigurations(c) +} + func (c *CoreV1alpha1Client) SchedulingProfiles() SchedulingProfileInterface { return newSchedulingProfiles(c) } diff --git a/pkg/client/clientset/versioned/typed/core/v1alpha1/fake/fake_core_client.go b/pkg/client/clientset/versioned/typed/core/v1alpha1/fake/fake_core_client.go index 09f02100..3bc3329e 100644 --- a/pkg/client/clientset/versioned/typed/core/v1alpha1/fake/fake_core_client.go +++ b/pkg/client/clientset/versioned/typed/core/v1alpha1/fake/fake_core_client.go @@ -44,6 +44,10 @@ func (c *FakeCoreV1alpha1) PropagationPolicies(namespace string) v1alpha1.Propag return &FakePropagationPolicies{c, namespace} } +func (c *FakeCoreV1alpha1) SchedulerPluginWebhookConfigurations() v1alpha1.SchedulerPluginWebhookConfigurationInterface { + return &FakeSchedulerPluginWebhookConfigurations{c} +} + func (c *FakeCoreV1alpha1) SchedulingProfiles() v1alpha1.SchedulingProfileInterface { return &FakeSchedulingProfiles{c} } diff --git a/pkg/client/clientset/versioned/typed/core/v1alpha1/fake/fake_schedulerpluginwebhookconfiguration.go b/pkg/client/clientset/versioned/typed/core/v1alpha1/fake/fake_schedulerpluginwebhookconfiguration.go new file mode 100644 index 00000000..673b47c2 --- /dev/null +++ b/pkg/client/clientset/versioned/typed/core/v1alpha1/fake/fake_schedulerpluginwebhookconfiguration.go @@ -0,0 +1,106 @@ +// Code generated by client-gen. DO NOT EDIT. + +package fake + +import ( + "context" + + v1alpha1 "github.com/kubewharf/kubeadmiral/pkg/apis/core/v1alpha1" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + labels "k8s.io/apimachinery/pkg/labels" + schema "k8s.io/apimachinery/pkg/runtime/schema" + types "k8s.io/apimachinery/pkg/types" + watch "k8s.io/apimachinery/pkg/watch" + testing "k8s.io/client-go/testing" +) + +// FakeSchedulerPluginWebhookConfigurations implements SchedulerPluginWebhookConfigurationInterface +type FakeSchedulerPluginWebhookConfigurations struct { + Fake *FakeCoreV1alpha1 +} + +var schedulerpluginwebhookconfigurationsResource = schema.GroupVersionResource{Group: "core.kubeadmiral.io", Version: "v1alpha1", Resource: "schedulerpluginwebhookconfigurations"} + +var schedulerpluginwebhookconfigurationsKind = schema.GroupVersionKind{Group: "core.kubeadmiral.io", Version: "v1alpha1", Kind: "SchedulerPluginWebhookConfiguration"} + +// Get takes name of the schedulerPluginWebhookConfiguration, and returns the corresponding schedulerPluginWebhookConfiguration object, and an error if there is any. +func (c *FakeSchedulerPluginWebhookConfigurations) Get(ctx context.Context, name string, options v1.GetOptions) (result *v1alpha1.SchedulerPluginWebhookConfiguration, err error) { + obj, err := c.Fake. + Invokes(testing.NewRootGetAction(schedulerpluginwebhookconfigurationsResource, name), &v1alpha1.SchedulerPluginWebhookConfiguration{}) + if obj == nil { + return nil, err + } + return obj.(*v1alpha1.SchedulerPluginWebhookConfiguration), err +} + +// List takes label and field selectors, and returns the list of SchedulerPluginWebhookConfigurations that match those selectors. +func (c *FakeSchedulerPluginWebhookConfigurations) List(ctx context.Context, opts v1.ListOptions) (result *v1alpha1.SchedulerPluginWebhookConfigurationList, err error) { + obj, err := c.Fake. + Invokes(testing.NewRootListAction(schedulerpluginwebhookconfigurationsResource, schedulerpluginwebhookconfigurationsKind, opts), &v1alpha1.SchedulerPluginWebhookConfigurationList{}) + if obj == nil { + return nil, err + } + + label, _, _ := testing.ExtractFromListOptions(opts) + if label == nil { + label = labels.Everything() + } + list := &v1alpha1.SchedulerPluginWebhookConfigurationList{ListMeta: obj.(*v1alpha1.SchedulerPluginWebhookConfigurationList).ListMeta} + for _, item := range obj.(*v1alpha1.SchedulerPluginWebhookConfigurationList).Items { + if label.Matches(labels.Set(item.Labels)) { + list.Items = append(list.Items, item) + } + } + return list, err +} + +// Watch returns a watch.Interface that watches the requested schedulerPluginWebhookConfigurations. +func (c *FakeSchedulerPluginWebhookConfigurations) Watch(ctx context.Context, opts v1.ListOptions) (watch.Interface, error) { + return c.Fake. + InvokesWatch(testing.NewRootWatchAction(schedulerpluginwebhookconfigurationsResource, opts)) +} + +// Create takes the representation of a schedulerPluginWebhookConfiguration and creates it. Returns the server's representation of the schedulerPluginWebhookConfiguration, and an error, if there is any. +func (c *FakeSchedulerPluginWebhookConfigurations) Create(ctx context.Context, schedulerPluginWebhookConfiguration *v1alpha1.SchedulerPluginWebhookConfiguration, opts v1.CreateOptions) (result *v1alpha1.SchedulerPluginWebhookConfiguration, err error) { + obj, err := c.Fake. + Invokes(testing.NewRootCreateAction(schedulerpluginwebhookconfigurationsResource, schedulerPluginWebhookConfiguration), &v1alpha1.SchedulerPluginWebhookConfiguration{}) + if obj == nil { + return nil, err + } + return obj.(*v1alpha1.SchedulerPluginWebhookConfiguration), err +} + +// Update takes the representation of a schedulerPluginWebhookConfiguration and updates it. Returns the server's representation of the schedulerPluginWebhookConfiguration, and an error, if there is any. +func (c *FakeSchedulerPluginWebhookConfigurations) Update(ctx context.Context, schedulerPluginWebhookConfiguration *v1alpha1.SchedulerPluginWebhookConfiguration, opts v1.UpdateOptions) (result *v1alpha1.SchedulerPluginWebhookConfiguration, err error) { + obj, err := c.Fake. + Invokes(testing.NewRootUpdateAction(schedulerpluginwebhookconfigurationsResource, schedulerPluginWebhookConfiguration), &v1alpha1.SchedulerPluginWebhookConfiguration{}) + if obj == nil { + return nil, err + } + return obj.(*v1alpha1.SchedulerPluginWebhookConfiguration), err +} + +// Delete takes name of the schedulerPluginWebhookConfiguration and deletes it. Returns an error if one occurs. +func (c *FakeSchedulerPluginWebhookConfigurations) Delete(ctx context.Context, name string, opts v1.DeleteOptions) error { + _, err := c.Fake. + Invokes(testing.NewRootDeleteAction(schedulerpluginwebhookconfigurationsResource, name), &v1alpha1.SchedulerPluginWebhookConfiguration{}) + return err +} + +// DeleteCollection deletes a collection of objects. +func (c *FakeSchedulerPluginWebhookConfigurations) DeleteCollection(ctx context.Context, opts v1.DeleteOptions, listOpts v1.ListOptions) error { + action := testing.NewRootDeleteCollectionAction(schedulerpluginwebhookconfigurationsResource, listOpts) + + _, err := c.Fake.Invokes(action, &v1alpha1.SchedulerPluginWebhookConfigurationList{}) + return err +} + +// Patch applies the patch and returns the patched schedulerPluginWebhookConfiguration. +func (c *FakeSchedulerPluginWebhookConfigurations) Patch(ctx context.Context, name string, pt types.PatchType, data []byte, opts v1.PatchOptions, subresources ...string) (result *v1alpha1.SchedulerPluginWebhookConfiguration, err error) { + obj, err := c.Fake. + Invokes(testing.NewRootPatchSubresourceAction(schedulerpluginwebhookconfigurationsResource, name, pt, data, subresources...), &v1alpha1.SchedulerPluginWebhookConfiguration{}) + if obj == nil { + return nil, err + } + return obj.(*v1alpha1.SchedulerPluginWebhookConfiguration), err +} diff --git a/pkg/client/clientset/versioned/typed/core/v1alpha1/generated_expansion.go b/pkg/client/clientset/versioned/typed/core/v1alpha1/generated_expansion.go index 91cf89bd..ad30121e 100644 --- a/pkg/client/clientset/versioned/typed/core/v1alpha1/generated_expansion.go +++ b/pkg/client/clientset/versioned/typed/core/v1alpha1/generated_expansion.go @@ -18,4 +18,6 @@ type PropagatedVersionExpansion interface{} type PropagationPolicyExpansion interface{} +type SchedulerPluginWebhookConfigurationExpansion interface{} + type SchedulingProfileExpansion interface{} diff --git a/pkg/client/clientset/versioned/typed/core/v1alpha1/schedulerpluginwebhookconfiguration.go b/pkg/client/clientset/versioned/typed/core/v1alpha1/schedulerpluginwebhookconfiguration.go new file mode 100644 index 00000000..389821a7 --- /dev/null +++ b/pkg/client/clientset/versioned/typed/core/v1alpha1/schedulerpluginwebhookconfiguration.go @@ -0,0 +1,152 @@ +// Code generated by client-gen. DO NOT EDIT. + +package v1alpha1 + +import ( + "context" + "time" + + v1alpha1 "github.com/kubewharf/kubeadmiral/pkg/apis/core/v1alpha1" + scheme "github.com/kubewharf/kubeadmiral/pkg/client/clientset/versioned/scheme" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + types "k8s.io/apimachinery/pkg/types" + watch "k8s.io/apimachinery/pkg/watch" + rest "k8s.io/client-go/rest" +) + +// SchedulerPluginWebhookConfigurationsGetter has a method to return a SchedulerPluginWebhookConfigurationInterface. +// A group's client should implement this interface. +type SchedulerPluginWebhookConfigurationsGetter interface { + SchedulerPluginWebhookConfigurations() SchedulerPluginWebhookConfigurationInterface +} + +// SchedulerPluginWebhookConfigurationInterface has methods to work with SchedulerPluginWebhookConfiguration resources. +type SchedulerPluginWebhookConfigurationInterface interface { + Create(ctx context.Context, schedulerPluginWebhookConfiguration *v1alpha1.SchedulerPluginWebhookConfiguration, opts v1.CreateOptions) (*v1alpha1.SchedulerPluginWebhookConfiguration, error) + Update(ctx context.Context, schedulerPluginWebhookConfiguration *v1alpha1.SchedulerPluginWebhookConfiguration, opts v1.UpdateOptions) (*v1alpha1.SchedulerPluginWebhookConfiguration, error) + Delete(ctx context.Context, name string, opts v1.DeleteOptions) error + DeleteCollection(ctx context.Context, opts v1.DeleteOptions, listOpts v1.ListOptions) error + Get(ctx context.Context, name string, opts v1.GetOptions) (*v1alpha1.SchedulerPluginWebhookConfiguration, error) + List(ctx context.Context, opts v1.ListOptions) (*v1alpha1.SchedulerPluginWebhookConfigurationList, error) + Watch(ctx context.Context, opts v1.ListOptions) (watch.Interface, error) + Patch(ctx context.Context, name string, pt types.PatchType, data []byte, opts v1.PatchOptions, subresources ...string) (result *v1alpha1.SchedulerPluginWebhookConfiguration, err error) + SchedulerPluginWebhookConfigurationExpansion +} + +// schedulerPluginWebhookConfigurations implements SchedulerPluginWebhookConfigurationInterface +type schedulerPluginWebhookConfigurations struct { + client rest.Interface +} + +// newSchedulerPluginWebhookConfigurations returns a SchedulerPluginWebhookConfigurations +func newSchedulerPluginWebhookConfigurations(c *CoreV1alpha1Client) *schedulerPluginWebhookConfigurations { + return &schedulerPluginWebhookConfigurations{ + client: c.RESTClient(), + } +} + +// Get takes name of the schedulerPluginWebhookConfiguration, and returns the corresponding schedulerPluginWebhookConfiguration object, and an error if there is any. +func (c *schedulerPluginWebhookConfigurations) Get(ctx context.Context, name string, options v1.GetOptions) (result *v1alpha1.SchedulerPluginWebhookConfiguration, err error) { + result = &v1alpha1.SchedulerPluginWebhookConfiguration{} + err = c.client.Get(). + Resource("schedulerpluginwebhookconfigurations"). + Name(name). + VersionedParams(&options, scheme.ParameterCodec). + Do(ctx). + Into(result) + return +} + +// List takes label and field selectors, and returns the list of SchedulerPluginWebhookConfigurations that match those selectors. +func (c *schedulerPluginWebhookConfigurations) List(ctx context.Context, opts v1.ListOptions) (result *v1alpha1.SchedulerPluginWebhookConfigurationList, err error) { + var timeout time.Duration + if opts.TimeoutSeconds != nil { + timeout = time.Duration(*opts.TimeoutSeconds) * time.Second + } + result = &v1alpha1.SchedulerPluginWebhookConfigurationList{} + err = c.client.Get(). + Resource("schedulerpluginwebhookconfigurations"). + VersionedParams(&opts, scheme.ParameterCodec). + Timeout(timeout). + Do(ctx). + Into(result) + return +} + +// Watch returns a watch.Interface that watches the requested schedulerPluginWebhookConfigurations. +func (c *schedulerPluginWebhookConfigurations) Watch(ctx context.Context, opts v1.ListOptions) (watch.Interface, error) { + var timeout time.Duration + if opts.TimeoutSeconds != nil { + timeout = time.Duration(*opts.TimeoutSeconds) * time.Second + } + opts.Watch = true + return c.client.Get(). + Resource("schedulerpluginwebhookconfigurations"). + VersionedParams(&opts, scheme.ParameterCodec). + Timeout(timeout). + Watch(ctx) +} + +// Create takes the representation of a schedulerPluginWebhookConfiguration and creates it. Returns the server's representation of the schedulerPluginWebhookConfiguration, and an error, if there is any. +func (c *schedulerPluginWebhookConfigurations) Create(ctx context.Context, schedulerPluginWebhookConfiguration *v1alpha1.SchedulerPluginWebhookConfiguration, opts v1.CreateOptions) (result *v1alpha1.SchedulerPluginWebhookConfiguration, err error) { + result = &v1alpha1.SchedulerPluginWebhookConfiguration{} + err = c.client.Post(). + Resource("schedulerpluginwebhookconfigurations"). + VersionedParams(&opts, scheme.ParameterCodec). + Body(schedulerPluginWebhookConfiguration). + Do(ctx). + Into(result) + return +} + +// Update takes the representation of a schedulerPluginWebhookConfiguration and updates it. Returns the server's representation of the schedulerPluginWebhookConfiguration, and an error, if there is any. +func (c *schedulerPluginWebhookConfigurations) Update(ctx context.Context, schedulerPluginWebhookConfiguration *v1alpha1.SchedulerPluginWebhookConfiguration, opts v1.UpdateOptions) (result *v1alpha1.SchedulerPluginWebhookConfiguration, err error) { + result = &v1alpha1.SchedulerPluginWebhookConfiguration{} + err = c.client.Put(). + Resource("schedulerpluginwebhookconfigurations"). + Name(schedulerPluginWebhookConfiguration.Name). + VersionedParams(&opts, scheme.ParameterCodec). + Body(schedulerPluginWebhookConfiguration). + Do(ctx). + Into(result) + return +} + +// Delete takes name of the schedulerPluginWebhookConfiguration and deletes it. Returns an error if one occurs. +func (c *schedulerPluginWebhookConfigurations) Delete(ctx context.Context, name string, opts v1.DeleteOptions) error { + return c.client.Delete(). + Resource("schedulerpluginwebhookconfigurations"). + Name(name). + Body(&opts). + Do(ctx). + Error() +} + +// DeleteCollection deletes a collection of objects. +func (c *schedulerPluginWebhookConfigurations) DeleteCollection(ctx context.Context, opts v1.DeleteOptions, listOpts v1.ListOptions) error { + var timeout time.Duration + if listOpts.TimeoutSeconds != nil { + timeout = time.Duration(*listOpts.TimeoutSeconds) * time.Second + } + return c.client.Delete(). + Resource("schedulerpluginwebhookconfigurations"). + VersionedParams(&listOpts, scheme.ParameterCodec). + Timeout(timeout). + Body(&opts). + Do(ctx). + Error() +} + +// Patch applies the patch and returns the patched schedulerPluginWebhookConfiguration. +func (c *schedulerPluginWebhookConfigurations) Patch(ctx context.Context, name string, pt types.PatchType, data []byte, opts v1.PatchOptions, subresources ...string) (result *v1alpha1.SchedulerPluginWebhookConfiguration, err error) { + result = &v1alpha1.SchedulerPluginWebhookConfiguration{} + err = c.client.Patch(pt). + Resource("schedulerpluginwebhookconfigurations"). + Name(name). + SubResource(subresources...). + VersionedParams(&opts, scheme.ParameterCodec). + Body(data). + Do(ctx). + Into(result) + return +} diff --git a/pkg/client/informers/externalversions/core/v1alpha1/interface.go b/pkg/client/informers/externalversions/core/v1alpha1/interface.go index 89caf2cf..29d7e46c 100644 --- a/pkg/client/informers/externalversions/core/v1alpha1/interface.go +++ b/pkg/client/informers/externalversions/core/v1alpha1/interface.go @@ -24,6 +24,8 @@ type Interface interface { PropagatedVersions() PropagatedVersionInformer // PropagationPolicies returns a PropagationPolicyInformer. PropagationPolicies() PropagationPolicyInformer + // SchedulerPluginWebhookConfigurations returns a SchedulerPluginWebhookConfigurationInformer. + SchedulerPluginWebhookConfigurations() SchedulerPluginWebhookConfigurationInformer // SchedulingProfiles returns a SchedulingProfileInformer. SchedulingProfiles() SchedulingProfileInformer } @@ -79,6 +81,11 @@ func (v *version) PropagationPolicies() PropagationPolicyInformer { return &propagationPolicyInformer{factory: v.factory, namespace: v.namespace, tweakListOptions: v.tweakListOptions} } +// SchedulerPluginWebhookConfigurations returns a SchedulerPluginWebhookConfigurationInformer. +func (v *version) SchedulerPluginWebhookConfigurations() SchedulerPluginWebhookConfigurationInformer { + return &schedulerPluginWebhookConfigurationInformer{factory: v.factory, tweakListOptions: v.tweakListOptions} +} + // SchedulingProfiles returns a SchedulingProfileInformer. func (v *version) SchedulingProfiles() SchedulingProfileInformer { return &schedulingProfileInformer{factory: v.factory, tweakListOptions: v.tweakListOptions} diff --git a/pkg/client/informers/externalversions/core/v1alpha1/schedulerpluginwebhookconfiguration.go b/pkg/client/informers/externalversions/core/v1alpha1/schedulerpluginwebhookconfiguration.go new file mode 100644 index 00000000..9d4e0f7f --- /dev/null +++ b/pkg/client/informers/externalversions/core/v1alpha1/schedulerpluginwebhookconfiguration.go @@ -0,0 +1,73 @@ +// Code generated by informer-gen. DO NOT EDIT. + +package v1alpha1 + +import ( + "context" + time "time" + + corev1alpha1 "github.com/kubewharf/kubeadmiral/pkg/apis/core/v1alpha1" + versioned "github.com/kubewharf/kubeadmiral/pkg/client/clientset/versioned" + internalinterfaces "github.com/kubewharf/kubeadmiral/pkg/client/informers/externalversions/internalinterfaces" + v1alpha1 "github.com/kubewharf/kubeadmiral/pkg/client/listers/core/v1alpha1" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + runtime "k8s.io/apimachinery/pkg/runtime" + watch "k8s.io/apimachinery/pkg/watch" + cache "k8s.io/client-go/tools/cache" +) + +// SchedulerPluginWebhookConfigurationInformer provides access to a shared informer and lister for +// SchedulerPluginWebhookConfigurations. +type SchedulerPluginWebhookConfigurationInformer interface { + Informer() cache.SharedIndexInformer + Lister() v1alpha1.SchedulerPluginWebhookConfigurationLister +} + +type schedulerPluginWebhookConfigurationInformer struct { + factory internalinterfaces.SharedInformerFactory + tweakListOptions internalinterfaces.TweakListOptionsFunc +} + +// NewSchedulerPluginWebhookConfigurationInformer constructs a new informer for SchedulerPluginWebhookConfiguration type. +// Always prefer using an informer factory to get a shared informer instead of getting an independent +// one. This reduces memory footprint and number of connections to the server. +func NewSchedulerPluginWebhookConfigurationInformer(client versioned.Interface, resyncPeriod time.Duration, indexers cache.Indexers) cache.SharedIndexInformer { + return NewFilteredSchedulerPluginWebhookConfigurationInformer(client, resyncPeriod, indexers, nil) +} + +// NewFilteredSchedulerPluginWebhookConfigurationInformer constructs a new informer for SchedulerPluginWebhookConfiguration type. +// Always prefer using an informer factory to get a shared informer instead of getting an independent +// one. This reduces memory footprint and number of connections to the server. +func NewFilteredSchedulerPluginWebhookConfigurationInformer(client versioned.Interface, resyncPeriod time.Duration, indexers cache.Indexers, tweakListOptions internalinterfaces.TweakListOptionsFunc) cache.SharedIndexInformer { + return cache.NewSharedIndexInformer( + &cache.ListWatch{ + ListFunc: func(options v1.ListOptions) (runtime.Object, error) { + if tweakListOptions != nil { + tweakListOptions(&options) + } + return client.CoreV1alpha1().SchedulerPluginWebhookConfigurations().List(context.TODO(), options) + }, + WatchFunc: func(options v1.ListOptions) (watch.Interface, error) { + if tweakListOptions != nil { + tweakListOptions(&options) + } + return client.CoreV1alpha1().SchedulerPluginWebhookConfigurations().Watch(context.TODO(), options) + }, + }, + &corev1alpha1.SchedulerPluginWebhookConfiguration{}, + resyncPeriod, + indexers, + ) +} + +func (f *schedulerPluginWebhookConfigurationInformer) defaultInformer(client versioned.Interface, resyncPeriod time.Duration) cache.SharedIndexInformer { + return NewFilteredSchedulerPluginWebhookConfigurationInformer(client, resyncPeriod, cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc}, f.tweakListOptions) +} + +func (f *schedulerPluginWebhookConfigurationInformer) Informer() cache.SharedIndexInformer { + return f.factory.InformerFor(&corev1alpha1.SchedulerPluginWebhookConfiguration{}, f.defaultInformer) +} + +func (f *schedulerPluginWebhookConfigurationInformer) Lister() v1alpha1.SchedulerPluginWebhookConfigurationLister { + return v1alpha1.NewSchedulerPluginWebhookConfigurationLister(f.Informer().GetIndexer()) +} diff --git a/pkg/client/informers/externalversions/generic.go b/pkg/client/informers/externalversions/generic.go index b7f82547..adde4a0e 100644 --- a/pkg/client/informers/externalversions/generic.go +++ b/pkg/client/informers/externalversions/generic.go @@ -53,6 +53,8 @@ func (f *sharedInformerFactory) ForResource(resource schema.GroupVersionResource return &genericInformer{resource: resource.GroupResource(), informer: f.Core().V1alpha1().PropagatedVersions().Informer()}, nil case v1alpha1.SchemeGroupVersion.WithResource("propagationpolicies"): return &genericInformer{resource: resource.GroupResource(), informer: f.Core().V1alpha1().PropagationPolicies().Informer()}, nil + case v1alpha1.SchemeGroupVersion.WithResource("schedulerpluginwebhookconfigurations"): + return &genericInformer{resource: resource.GroupResource(), informer: f.Core().V1alpha1().SchedulerPluginWebhookConfigurations().Informer()}, nil case v1alpha1.SchemeGroupVersion.WithResource("schedulingprofiles"): return &genericInformer{resource: resource.GroupResource(), informer: f.Core().V1alpha1().SchedulingProfiles().Informer()}, nil diff --git a/pkg/client/listers/core/v1alpha1/expansion_generated.go b/pkg/client/listers/core/v1alpha1/expansion_generated.go index 9578d669..3227125f 100644 --- a/pkg/client/listers/core/v1alpha1/expansion_generated.go +++ b/pkg/client/listers/core/v1alpha1/expansion_generated.go @@ -46,6 +46,10 @@ type PropagationPolicyListerExpansion interface{} // PropagationPolicyNamespaceLister. type PropagationPolicyNamespaceListerExpansion interface{} +// SchedulerPluginWebhookConfigurationListerExpansion allows custom methods to be added to +// SchedulerPluginWebhookConfigurationLister. +type SchedulerPluginWebhookConfigurationListerExpansion interface{} + // SchedulingProfileListerExpansion allows custom methods to be added to // SchedulingProfileLister. type SchedulingProfileListerExpansion interface{} diff --git a/pkg/client/listers/core/v1alpha1/schedulerpluginwebhookconfiguration.go b/pkg/client/listers/core/v1alpha1/schedulerpluginwebhookconfiguration.go new file mode 100644 index 00000000..e0eddd7e --- /dev/null +++ b/pkg/client/listers/core/v1alpha1/schedulerpluginwebhookconfiguration.go @@ -0,0 +1,52 @@ +// Code generated by lister-gen. DO NOT EDIT. + +package v1alpha1 + +import ( + v1alpha1 "github.com/kubewharf/kubeadmiral/pkg/apis/core/v1alpha1" + "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/labels" + "k8s.io/client-go/tools/cache" +) + +// SchedulerPluginWebhookConfigurationLister helps list SchedulerPluginWebhookConfigurations. +// All objects returned here must be treated as read-only. +type SchedulerPluginWebhookConfigurationLister interface { + // List lists all SchedulerPluginWebhookConfigurations in the indexer. + // Objects returned here must be treated as read-only. + List(selector labels.Selector) (ret []*v1alpha1.SchedulerPluginWebhookConfiguration, err error) + // Get retrieves the SchedulerPluginWebhookConfiguration from the index for a given name. + // Objects returned here must be treated as read-only. + Get(name string) (*v1alpha1.SchedulerPluginWebhookConfiguration, error) + SchedulerPluginWebhookConfigurationListerExpansion +} + +// schedulerPluginWebhookConfigurationLister implements the SchedulerPluginWebhookConfigurationLister interface. +type schedulerPluginWebhookConfigurationLister struct { + indexer cache.Indexer +} + +// NewSchedulerPluginWebhookConfigurationLister returns a new SchedulerPluginWebhookConfigurationLister. +func NewSchedulerPluginWebhookConfigurationLister(indexer cache.Indexer) SchedulerPluginWebhookConfigurationLister { + return &schedulerPluginWebhookConfigurationLister{indexer: indexer} +} + +// List lists all SchedulerPluginWebhookConfigurations in the indexer. +func (s *schedulerPluginWebhookConfigurationLister) List(selector labels.Selector) (ret []*v1alpha1.SchedulerPluginWebhookConfiguration, err error) { + err = cache.ListAll(s.indexer, selector, func(m interface{}) { + ret = append(ret, m.(*v1alpha1.SchedulerPluginWebhookConfiguration)) + }) + return ret, err +} + +// Get retrieves the SchedulerPluginWebhookConfiguration from the index for a given name. +func (s *schedulerPluginWebhookConfigurationLister) Get(name string) (*v1alpha1.SchedulerPluginWebhookConfiguration, error) { + obj, exists, err := s.indexer.GetByKey(name) + if err != nil { + return nil, err + } + if !exists { + return nil, errors.NewNotFound(v1alpha1.Resource("schedulerpluginwebhookconfiguration"), name) + } + return obj.(*v1alpha1.SchedulerPluginWebhookConfiguration), nil +} From d37a54f67443d87f9f606f904cca09a2316fe419 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gary=20Liu=20=28=E5=88=98=E5=B9=BF=E6=BA=90=29?= Date: Wed, 12 Apr 2023 06:17:43 +0000 Subject: [PATCH 04/10] feat(scheduler): add webhook plugin payload types Co-authored-by: Lim Haw Jia --- pkg/apis/schedulerwebhook/v1alpha1/types.go | 102 ++++++++++++++++++++ 1 file changed, 102 insertions(+) create mode 100644 pkg/apis/schedulerwebhook/v1alpha1/types.go diff --git a/pkg/apis/schedulerwebhook/v1alpha1/types.go b/pkg/apis/schedulerwebhook/v1alpha1/types.go new file mode 100644 index 00000000..9986df3c --- /dev/null +++ b/pkg/apis/schedulerwebhook/v1alpha1/types.go @@ -0,0 +1,102 @@ +/* +Copyright 2023 The KubeAdmiral Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1alpha1 + +import ( + corev1 "k8s.io/api/core/v1" + + fedcorev1a1 "github.com/kubewharf/kubeadmiral/pkg/apis/core/v1alpha1" +) + +// PayloadVersion is the version of the payload that is used to communicate with the scheduler webhook. +const PayloadVersion = "v1alpha1" + +// SchedulingUnit represents an object that is being scheduled. +type SchedulingUnit struct { + APIVersion string `json:"apiVersion"` + Kind string `json:"kind"` + Resource string `json:"resource"` + + Namespace string `json:"namespace,omitempty"` + Name string `json:"name"` + Labels map[string]string `json:"labels,omitempty"` + Annotations map[string]string `json:"annotations,omitempty"` + + // SchedulingMode is the scheduling mode that will be used for scheduling the Kubernetes resource. It can be + // Duplicate or Divide. + SchedulingMode fedcorev1a1.SchedulingMode `json:"schedulingMode"` + // DesiredReplicas is the number of replicas requested in the template of the Kubernetes resource. If SchedulingMode + // is Duplicated, this is the number of replicas that will be propgated to each selected cluster. If SchedulingMode + // is Divide, this is the total number of replicas to distribute to all selected member clusters. + // If the object is not replica-based, this field will be nil. + DesiredReplicas *int64 `json:"desiredReplicas,omitempty"` + // ResourceRequest is the list of resources that will be requested by each replica of the Kubernetes resource. + ResourceRequest corev1.ResourceList `json:"resourceRequest,omitempty"` + + // CurrentClusters is the list of clusters that the object is currently scheduled to + CurrentClusters []string `json:"currentClusters"` + // CurrentReplicaDistribution is the number of replicas scheduled to each cluster in Divide mode. Note that this map + // will be empty if SchedulingMode is set to Duplicate. + CurrentReplicaDistribution map[string]int64 `json:"currentReplicaDistribution,omitempty"` + + // ClusterSelector is the ClusterSelector set in the PropagationPolicy. + ClusterSelector map[string]string `json:"clusterSelector,omitempty"` + // ClusterAffinity is the ClusterAffinity set in the PropagationPolicy. + ClusterAffinity []fedcorev1a1.ClusterSelectorTerm `json:"clusterAffinity,omitempty"` + // Tolerations is the Tolerations set in the PropagationPolicy. + Tolerations []corev1.Toleration `json:"tolerations,omitempty"` + // MaxClusters is the max clusters set in the PropgationPolicy. + MaxClusters *int64 `json:"maxClusters,omitempty"` + // Placements is the placements set in the PropgationPolicy. + Placements []fedcorev1a1.Placement `json:"placements,omitempty"` +} + +type FilterRequest struct { + SchedulingUnit SchedulingUnit `json:"schedulingUnit"` + Cluster fedcorev1a1.FederatedCluster `json:"cluster"` +} + +// TODO: allow webhook to return reason when we use reasons in the framework +type FilterResponse struct { + Selected bool `json:"selected"` + Error string `json:"error"` +} + +type ScoreRequest struct { + SchedulingUnit SchedulingUnit `json:"schedulingUnit"` + Cluster fedcorev1a1.FederatedCluster `json:"cluster"` +} + +type ScoreResponse struct { + Score int64 `json:"score"` + Error string `json:"error"` +} + +type ClusterScore struct { + Cluster fedcorev1a1.FederatedCluster `json:"cluster"` + Score int64 `json:"score"` +} + +type SelectRequest struct { + SchedulingUnit SchedulingUnit `json:"schedulingUnit"` + ClusterScores []ClusterScore `json:"clusterScores"` +} + +type SelectResponse struct { + SelectedClusterNames []string `json:"selectedClusterNames"` + Error string `json:"error"` +} From f90e69384ec4790f5d15f77b080f769f84f1512d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gary=20Liu=20=28=E5=88=98=E5=B9=BF=E6=BA=90=29?= Date: Fri, 14 Apr 2023 03:02:22 +0000 Subject: [PATCH 05/10] feat(scheduler): webhook plugin Co-authored-by: Lim Haw Jia --- .../extensions/webhook/v1alpha1/adapter.go | 92 +++++++ .../extensions/webhook/v1alpha1/plugin.go | 233 ++++++++++++++++++ 2 files changed, 325 insertions(+) create mode 100644 pkg/controllers/scheduler/extensions/webhook/v1alpha1/adapter.go create mode 100644 pkg/controllers/scheduler/extensions/webhook/v1alpha1/plugin.go diff --git a/pkg/controllers/scheduler/extensions/webhook/v1alpha1/adapter.go b/pkg/controllers/scheduler/extensions/webhook/v1alpha1/adapter.go new file mode 100644 index 00000000..b009cf48 --- /dev/null +++ b/pkg/controllers/scheduler/extensions/webhook/v1alpha1/adapter.go @@ -0,0 +1,92 @@ +/* +Copyright 2023 The KubeAdmiral Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1alpha1 + +import ( + fedcorev1a1 "github.com/kubewharf/kubeadmiral/pkg/apis/core/v1alpha1" + schedwebhookv1a1 "github.com/kubewharf/kubeadmiral/pkg/apis/schedulerwebhook/v1alpha1" + "github.com/kubewharf/kubeadmiral/pkg/controllers/scheduler/framework" +) + +func ConvertSchedulingUnit(su *framework.SchedulingUnit) *schedwebhookv1a1.SchedulingUnit { + currentClusters := []string{} + currentReplicaDistribution := map[string]int64{} + for cluster, replicas := range su.CurrentClusters { + currentClusters = append(currentClusters, cluster) + if su.SchedulingMode == fedcorev1a1.SchedulingModeDivide { + if replicas == nil { + // NOTE(hawjia): this should never happen, we should probably redesign the internal scheduling unit to + // make this case impossible. + currentReplicaDistribution[cluster] = 0 + } else { + currentReplicaDistribution[cluster] = *replicas + } + } + } + + placements := []fedcorev1a1.Placement{} + for cluster := range su.ClusterNames { + var weight *int64 + if w, ok := su.Weights[cluster]; ok { + weight = &w + } + + var maxReplicas *int64 + if max, ok := su.MaxReplicas[cluster]; ok { + maxReplicas = &max + } + + placement := fedcorev1a1.Placement{ + ClusterName: cluster, + Preferences: fedcorev1a1.Preferences{ + MinReplicas: su.MinReplicas[cluster], + MaxReplicas: maxReplicas, + Weight: weight, + }, + } + + placements = append(placements, placement) + } + + var affinity []fedcorev1a1.ClusterSelectorTerm + if su.Affinity != nil && su.Affinity.ClusterAffinity != nil && + su.Affinity.ClusterAffinity.RequiredDuringSchedulingIgnoredDuringExecution != nil { + affinity = su.Affinity.ClusterAffinity.RequiredDuringSchedulingIgnoredDuringExecution.ClusterSelectorTerms + } + + return &schedwebhookv1a1.SchedulingUnit{ + APIVersion: su.GroupVersion.String(), + Kind: su.Kind, + Resource: su.Resource, + + Namespace: su.Namespace, + Name: su.Name, + Labels: su.Labels, + Annotations: su.Annotations, + + SchedulingMode: su.SchedulingMode, + DesiredReplicas: su.DesiredReplicas, + ResourceRequest: su.ResourceRequest.ResourceList(), + CurrentClusters: currentClusters, + CurrentReplicaDistribution: currentReplicaDistribution, + ClusterSelector: su.ClusterSelector, + ClusterAffinity: affinity, + Tolerations: su.Tolerations, + MaxClusters: su.MaxClusters, + Placements: placements, + } +} diff --git a/pkg/controllers/scheduler/extensions/webhook/v1alpha1/plugin.go b/pkg/controllers/scheduler/extensions/webhook/v1alpha1/plugin.go new file mode 100644 index 00000000..4d96efd7 --- /dev/null +++ b/pkg/controllers/scheduler/extensions/webhook/v1alpha1/plugin.go @@ -0,0 +1,233 @@ +/* +Copyright 2023 The KubeAdmiral Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1alpha1 + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "net/http" + "net/url" + "time" + + "k8s.io/klog/v2" + + fedcorev1a1 "github.com/kubewharf/kubeadmiral/pkg/apis/core/v1alpha1" + schedwebhookv1a1 "github.com/kubewharf/kubeadmiral/pkg/apis/schedulerwebhook/v1alpha1" + "github.com/kubewharf/kubeadmiral/pkg/controllers/scheduler/framework" +) + +var _ framework.FilterPlugin = &WebhookPlugin{} +var _ framework.ScorePlugin = &WebhookPlugin{} +var _ framework.SelectPlugin = &WebhookPlugin{} + +type HTTPClient interface { + Do(req *http.Request) (*http.Response, error) +} + +type WebhookPlugin struct { + name string + urlPrefix string + filterPath string + scorePath string + selectPath string + client HTTPClient +} + +func NewWebhookPlugin( + name string, + urlPrefix string, + filterPath string, + scorePath string, + selectPath string, + client HTTPClient, +) *WebhookPlugin { + return &WebhookPlugin{ + name: name, + urlPrefix: urlPrefix, + filterPath: filterPath, + scorePath: scorePath, + selectPath: selectPath, + client: client, + } +} + +func (p *WebhookPlugin) Name() string { + return p.name +} + +func (p *WebhookPlugin) doRequest( + ctx context.Context, + path string, + body any, + response any, +) error { + url, err := url.JoinPath(p.urlPrefix, path) + if err != nil { + return fmt.Errorf("failed to join URL path: %w", err) + } + bodyBytes, err := json.Marshal(body) + if err != nil { + return fmt.Errorf("failed to marshal request: %w", err) + } + req, err := http.NewRequestWithContext(ctx, http.MethodPost, url, bytes.NewReader(bodyBytes)) + if err != nil { + return fmt.Errorf("failed to create request: %w", err) + } + req.Header.Add("Content-Type", "application/json") + req.Header.Add("Accept", "application/json") + req.Header.Add("User-Agent", "kubeadmiral-scheduler") + + logger := klog.FromContext(ctx) + logger.V(4).Info("Sending request to webhook") + start := time.Now() + + httpResp, err := p.client.Do(req) + if err != nil { + logger.Error(err, "Webhook request failed") + return fmt.Errorf("request failed: %w", err) + } + defer httpResp.Body.Close() + + err = json.NewDecoder(httpResp.Body).Decode(response) + if err != nil { + return fmt.Errorf("failed to decode response: %w", err) + } + logger.V(6).WithValues("response", response, "duration", time.Since(start)).Info("Webhook response received") + return nil +} + +func (p *WebhookPlugin) Filter( + ctx context.Context, + su *framework.SchedulingUnit, + cluster *fedcorev1a1.FederatedCluster, +) *framework.Result { + if p.filterPath == "" { + return framework.NewResult(framework.Error, "filter is not supported by the webhook") + } + + logger := klog.FromContext(ctx).WithValues("plugin", p.name, "pluginType", "Webhook", "stage", "Filter") + ctx = klog.NewContext(ctx, logger) + + req := schedwebhookv1a1.FilterRequest{ + SchedulingUnit: *ConvertSchedulingUnit(su), + Cluster: *cluster, + } + resp := schedwebhookv1a1.FilterResponse{} + if err := p.doRequest(ctx, p.filterPath, &req, &resp); err != nil { + return framework.NewResult(framework.Error, err.Error()) + } + + if len(resp.Error) > 0 { + return framework.NewResult(framework.Error, resp.Error) + } + + if resp.Selected { + return framework.NewResult(framework.Success) + } else { + return framework.NewResult(framework.Unschedulable) + } +} + +func (p *WebhookPlugin) Score( + ctx context.Context, + su *framework.SchedulingUnit, + cluster *fedcorev1a1.FederatedCluster, +) (int64, *framework.Result) { + if p.scorePath == "" { + return 0, framework.NewResult(framework.Error, "score is not supported by the webhook") + } + + logger := klog.FromContext(ctx).WithValues("plugin", p.name, "pluginType", "Webhook", "stage", "Score") + ctx = klog.NewContext(ctx, logger) + + req := schedwebhookv1a1.ScoreRequest{ + SchedulingUnit: *ConvertSchedulingUnit(su), + Cluster: *cluster, + } + resp := schedwebhookv1a1.ScoreResponse{} + if err := p.doRequest(ctx, p.scorePath, &req, &resp); err != nil { + return 0, framework.NewResult(framework.Error, err.Error()) + } + + if len(resp.Error) > 0 { + return 0, framework.NewResult(framework.Error, resp.Error) + } + + return resp.Score, framework.NewResult(framework.Success) +} + +func (p *WebhookPlugin) NormalizeScore(ctx context.Context, scores framework.ClusterScoreList) *framework.Result { + // TODO: should we enforce normalization for all plugins by default? + return framework.DefaultNormalizeScore(framework.MaxClusterScore, false, scores) +} + +func (p *WebhookPlugin) ScoreExtensions() framework.ScoreExtensions { + return p +} + +func (p *WebhookPlugin) SelectClusters( + ctx context.Context, + su *framework.SchedulingUnit, + clusterScores framework.ClusterScoreList, +) ([]*fedcorev1a1.FederatedCluster, *framework.Result) { + if p.selectPath == "" { + return nil, framework.NewResult(framework.Error, "select is not supported by the webhook") + } + + logger := klog.FromContext(ctx).WithValues("plugin", p.name, "pluginType", "Webhook", "stage", "Select") + ctx = klog.NewContext(ctx, logger) + + req := schedwebhookv1a1.SelectRequest{ + SchedulingUnit: *ConvertSchedulingUnit(su), + ClusterScores: []schedwebhookv1a1.ClusterScore{}, + } + + clusterMap := map[string]*fedcorev1a1.FederatedCluster{} + for _, score := range clusterScores { + clusterMap[score.Cluster.Name] = score.Cluster + + req.ClusterScores = append(req.ClusterScores, schedwebhookv1a1.ClusterScore{ + Cluster: *score.Cluster, + Score: score.Score, + }) + } + + resp := schedwebhookv1a1.SelectResponse{} + if err := p.doRequest(ctx, p.selectPath, &req, &resp); err != nil { + return nil, framework.NewResult(framework.Error, err.Error()) + } + + if len(resp.Error) > 0 { + return nil, framework.NewResult(framework.Error, resp.Error) + } + + selectedClusters := []*fedcorev1a1.FederatedCluster{} + for _, clusterName := range resp.SelectedClusterNames { + if cluster, ok := clusterMap[clusterName]; ok { + selectedClusters = append(selectedClusters, cluster) + } else { + return nil, framework.NewResult( + framework.Error, + fmt.Sprintf("cluster %q was not in the request sent to the webhook", clusterName), + ) + } + } + + return selectedClusters, framework.NewResult(framework.Success) +} From 373d9210f8a372e245c789ab7f94cb1416aef016 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gary=20Liu=20=28=E5=88=98=E5=B9=BF=E6=BA=90=29?= Date: Thu, 13 Apr 2023 10:40:25 +0000 Subject: [PATCH 06/10] feat(testing): semantic equality gomega matcher --- .../custommatchers/semantic_equality.go | 52 +++++++++++++++++++ 1 file changed, 52 insertions(+) create mode 100644 test/gomega/custommatchers/semantic_equality.go diff --git a/test/gomega/custommatchers/semantic_equality.go b/test/gomega/custommatchers/semantic_equality.go new file mode 100644 index 00000000..78a33630 --- /dev/null +++ b/test/gomega/custommatchers/semantic_equality.go @@ -0,0 +1,52 @@ +/* +Copyright 2023 The KubeAdmiral Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package custommatchers + +import ( + "fmt" + + "github.com/google/go-cmp/cmp" + "github.com/onsi/gomega/format" + "k8s.io/apimachinery/pkg/api/equality" +) + +type SemanticEqualityMatcher struct { + Expected interface{} +} + +func SemanticallyEqual(expected interface{}) *SemanticEqualityMatcher { + return &SemanticEqualityMatcher{ + Expected: expected, + } +} + +func (matcher *SemanticEqualityMatcher) Match(actual interface{}) (success bool, err error) { + return equality.Semantic.DeepEqual(actual, matcher.Expected), nil +} + +func (matcher *SemanticEqualityMatcher) FailureMessage(actual interface{}) (message string) { + return fmt.Sprintf( + "Expected\n%s\nto be semantically equal to\n%s\ndiff:\n%s", + format.Object(actual, 1), + format.Object(matcher.Expected, 1), + cmp.Diff(actual, matcher.Expected), + ) +} + +func (matcher *SemanticEqualityMatcher) NegatedFailureMessage(actual interface{}) (message string) { + return format.Message(actual, "not to be semantically equal to", matcher.Expected) +} From 7a31c4f26bea1fc806008723f06557f4eef72ff6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gary=20Liu=20=28=E5=88=98=E5=B9=BF=E6=BA=90=29?= Date: Fri, 14 Apr 2023 03:08:02 +0000 Subject: [PATCH 07/10] test(scheduler): webhook plugin Co-authored-by: Lim Haw Jia --- .../webhook/v1alpha1/adapter_test.go | 469 ++++++++++++++++++ .../webhook/v1alpha1/plugin_test.go | 376 ++++++++++++++ 2 files changed, 845 insertions(+) create mode 100644 pkg/controllers/scheduler/extensions/webhook/v1alpha1/adapter_test.go create mode 100644 pkg/controllers/scheduler/extensions/webhook/v1alpha1/plugin_test.go diff --git a/pkg/controllers/scheduler/extensions/webhook/v1alpha1/adapter_test.go b/pkg/controllers/scheduler/extensions/webhook/v1alpha1/adapter_test.go new file mode 100644 index 00000000..46e77a82 --- /dev/null +++ b/pkg/controllers/scheduler/extensions/webhook/v1alpha1/adapter_test.go @@ -0,0 +1,469 @@ +/* +Copyright 2023 The KubeAdmiral Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1alpha1 + +import ( + "sort" + "testing" + + "github.com/onsi/gomega" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/utils/pointer" + + fedcorev1a1 "github.com/kubewharf/kubeadmiral/pkg/apis/core/v1alpha1" + schedwebhookv1a1 "github.com/kubewharf/kubeadmiral/pkg/apis/schedulerwebhook/v1alpha1" + "github.com/kubewharf/kubeadmiral/pkg/controllers/scheduler/framework" + "github.com/kubewharf/kubeadmiral/test/gomega/custommatchers" +) + +func TestConvertSchedulingUnit(t *testing.T) { + tests := []struct { + name string + su *framework.SchedulingUnit + expectedResult *schedwebhookv1a1.SchedulingUnit + }{ + { + name: "Deployment duplicate mode", + su: &framework.SchedulingUnit{ + Name: "test", + Namespace: "test", + GroupVersion: schema.GroupVersion{ + Group: "apps", + Version: "v1", + }, + Kind: "Deployment", + Resource: "deployments", + Labels: map[string]string{ + "test-label-1-name": "test-label-1-value", + "test-label-2-name": "test-label-2-value", + }, + Annotations: map[string]string{ + "test-annotation-1-name": "test-annotation-1-value", + "test-annotation-2-name": "test-annotation-2-value", + }, + DesiredReplicas: pointer.Int64(10), + ResourceRequest: framework.Resource{ + MilliCPU: 2000, + Memory: 1000, + ScalarResources: map[corev1.ResourceName]int64{ + "customresource": 2, + }, + }, + CurrentClusters: map[string]*int64{ + "cluster1": nil, + "cluster2": nil, + }, + SchedulingMode: fedcorev1a1.SchedulingModeDuplicate, + StickyCluster: false, + ClusterSelector: map[string]string{ + "test-label-1-name": "test-label-1-value", + "test-label-2-name": "test-label-2-value", + }, + ClusterNames: map[string]struct{}{ + "cluster1": {}, + "cluster2": {}, + "cluster3": {}, + }, + Affinity: &framework.Affinity{ + ClusterAffinity: &framework.ClusterAffinity{ + RequiredDuringSchedulingIgnoredDuringExecution: &framework.ClusterSelector{ + ClusterSelectorTerms: []fedcorev1a1.ClusterSelectorTerm{ + { + MatchExpressions: []fedcorev1a1.ClusterSelectorRequirement{ + { + Key: "test", + Operator: fedcorev1a1.ClusterSelectorOpExists, + Values: []string{"a", "b", "c"}, + }, + }, + }, + }, + }, + }, + }, + Tolerations: []corev1.Toleration{ + { + Key: "test", + Operator: corev1.TolerationOpEqual, + Value: "test", + Effect: corev1.TaintEffectNoSchedule, + }, + }, + MaxClusters: pointer.Int64(5), + }, + expectedResult: &schedwebhookv1a1.SchedulingUnit{ + APIVersion: "apps/v1", + Kind: "Deployment", + Resource: "deployments", + Name: "test", + Namespace: "test", + Labels: map[string]string{ + "test-label-1-name": "test-label-1-value", + "test-label-2-name": "test-label-2-value", + }, + Annotations: map[string]string{ + "test-annotation-1-name": "test-annotation-1-value", + "test-annotation-2-name": "test-annotation-2-value", + }, + SchedulingMode: fedcorev1a1.SchedulingModeDuplicate, + DesiredReplicas: pointer.Int64(10), + ResourceRequest: map[corev1.ResourceName]resource.Quantity{ + corev1.ResourceCPU: *resource.NewMilliQuantity(2000, resource.DecimalSI), + corev1.ResourceMemory: *resource.NewQuantity(1000, resource.BinarySI), + corev1.ResourceEphemeralStorage: *resource.NewQuantity(0, resource.BinarySI), + "customresource": *resource.NewQuantity(2, resource.DecimalSI), + }, + CurrentClusters: []string{"cluster1", "cluster2"}, + CurrentReplicaDistribution: nil, + ClusterSelector: map[string]string{ + "test-label-1-name": "test-label-1-value", + "test-label-2-name": "test-label-2-value", + }, + ClusterAffinity: []fedcorev1a1.ClusterSelectorTerm{ + { + MatchExpressions: []fedcorev1a1.ClusterSelectorRequirement{ + { + Key: "test", + Operator: fedcorev1a1.ClusterSelectorOpExists, + Values: []string{"a", "b", "c"}, + }, + }, + }, + }, + Tolerations: []corev1.Toleration{ + { + Key: "test", + Operator: corev1.TolerationOpEqual, + Value: "test", + Effect: corev1.TaintEffectNoSchedule, + }, + }, + MaxClusters: pointer.Int64(5), + Placements: []fedcorev1a1.Placement{ + { + ClusterName: "cluster1", + }, + { + ClusterName: "cluster2", + }, + { + ClusterName: "cluster3", + }, + }, + }, + }, + { + name: "Deployment duplicate mode with nil values", + su: &framework.SchedulingUnit{ + Name: "test", + Namespace: "test", + GroupVersion: schema.GroupVersion{ + Group: "apps", + Version: "v1", + }, + Kind: "Deployment", + Resource: "deployments", + Labels: map[string]string{ + "test-label-1-name": "test-label-1-value", + "test-label-2-name": "test-label-2-value", + }, + Annotations: map[string]string{ + "test-annotation-1-name": "test-annotation-1-value", + "test-annotation-2-name": "test-annotation-2-value", + }, + DesiredReplicas: nil, + ResourceRequest: framework.Resource{}, + CurrentClusters: nil, + SchedulingMode: fedcorev1a1.SchedulingModeDuplicate, + StickyCluster: false, + ClusterSelector: nil, + ClusterNames: nil, + Affinity: nil, + Tolerations: nil, + MaxClusters: nil, + }, + expectedResult: &schedwebhookv1a1.SchedulingUnit{ + APIVersion: "apps/v1", + Kind: "Deployment", + Resource: "deployments", + Name: "test", + Namespace: "test", + Labels: map[string]string{ + "test-label-1-name": "test-label-1-value", + "test-label-2-name": "test-label-2-value", + }, + Annotations: map[string]string{ + "test-annotation-1-name": "test-annotation-1-value", + "test-annotation-2-name": "test-annotation-2-value", + }, + SchedulingMode: fedcorev1a1.SchedulingModeDuplicate, + DesiredReplicas: nil, + ResourceRequest: map[corev1.ResourceName]resource.Quantity{ + corev1.ResourceCPU: *resource.NewMilliQuantity(0, resource.DecimalSI), + corev1.ResourceMemory: *resource.NewQuantity(0, resource.BinarySI), + corev1.ResourceEphemeralStorage: *resource.NewQuantity(0, resource.BinarySI), + }, + CurrentClusters: nil, + CurrentReplicaDistribution: nil, + ClusterSelector: nil, + ClusterAffinity: nil, + Tolerations: nil, + MaxClusters: nil, + Placements: nil, + }, + }, + { + name: "Deployment divide mode", + su: &framework.SchedulingUnit{ + Name: "test", + Namespace: "test", + GroupVersion: schema.GroupVersion{ + Group: "apps", + Version: "v1", + }, + Kind: "Deployment", + Resource: "deployments", + Labels: map[string]string{ + "test-label-1-name": "test-label-1-value", + "test-label-2-name": "test-label-2-value", + }, + Annotations: map[string]string{ + "test-annotation-1-name": "test-annotation-1-value", + "test-annotation-2-name": "test-annotation-2-value", + }, + DesiredReplicas: pointer.Int64(10), + ResourceRequest: framework.Resource{ + MilliCPU: 2000, + Memory: 1000, + ScalarResources: map[corev1.ResourceName]int64{ + "customresource": 2, + }, + }, + CurrentClusters: map[string]*int64{ + "cluster1": pointer.Int64(5), + "cluster2": pointer.Int64(7), + }, + SchedulingMode: fedcorev1a1.SchedulingModeDivide, + StickyCluster: false, + ClusterSelector: map[string]string{ + "test-label-1-name": "test-label-1-value", + "test-label-2-name": "test-label-2-value", + }, + ClusterNames: map[string]struct{}{ + "cluster1": {}, + "cluster2": {}, + "cluster3": {}, + }, + Affinity: &framework.Affinity{ + ClusterAffinity: &framework.ClusterAffinity{ + RequiredDuringSchedulingIgnoredDuringExecution: &framework.ClusterSelector{ + ClusterSelectorTerms: []fedcorev1a1.ClusterSelectorTerm{ + { + MatchExpressions: []fedcorev1a1.ClusterSelectorRequirement{ + { + Key: "test", + Operator: fedcorev1a1.ClusterSelectorOpExists, + Values: []string{"a", "b", "c"}, + }, + }, + }, + }, + }, + }, + }, + Tolerations: []corev1.Toleration{ + { + Key: "test", + Operator: corev1.TolerationOpEqual, + Value: "test", + Effect: corev1.TaintEffectNoSchedule, + }, + }, + MaxClusters: pointer.Int64(5), + }, + expectedResult: &schedwebhookv1a1.SchedulingUnit{ + APIVersion: "apps/v1", + Kind: "Deployment", + Resource: "deployments", + Name: "test", + Namespace: "test", + Labels: map[string]string{ + "test-label-1-name": "test-label-1-value", + "test-label-2-name": "test-label-2-value", + }, + Annotations: map[string]string{ + "test-annotation-1-name": "test-annotation-1-value", + "test-annotation-2-name": "test-annotation-2-value", + }, + SchedulingMode: fedcorev1a1.SchedulingModeDivide, + DesiredReplicas: pointer.Int64(10), + ResourceRequest: map[corev1.ResourceName]resource.Quantity{ + corev1.ResourceCPU: *resource.NewMilliQuantity(2000, resource.DecimalSI), + corev1.ResourceMemory: *resource.NewQuantity(1000, resource.BinarySI), + corev1.ResourceEphemeralStorage: *resource.NewQuantity(0, resource.BinarySI), + "customresource": *resource.NewQuantity(2, resource.DecimalSI), + }, + CurrentClusters: []string{"cluster1", "cluster2"}, + CurrentReplicaDistribution: map[string]int64{ + "cluster1": 5, + "cluster2": 7, + }, + ClusterSelector: map[string]string{ + "test-label-1-name": "test-label-1-value", + "test-label-2-name": "test-label-2-value", + }, + ClusterAffinity: []fedcorev1a1.ClusterSelectorTerm{ + { + MatchExpressions: []fedcorev1a1.ClusterSelectorRequirement{ + { + Key: "test", + Operator: fedcorev1a1.ClusterSelectorOpExists, + Values: []string{"a", "b", "c"}, + }, + }, + }, + }, + Tolerations: []corev1.Toleration{ + { + Key: "test", + Operator: corev1.TolerationOpEqual, + Value: "test", + Effect: corev1.TaintEffectNoSchedule, + }, + }, + MaxClusters: pointer.Int64(5), + Placements: []fedcorev1a1.Placement{ + { + ClusterName: "cluster1", + }, + { + ClusterName: "cluster2", + }, + { + ClusterName: "cluster3", + }, + }, + }, + }, + { + name: "Secret duplicate mode", + su: &framework.SchedulingUnit{ + Name: "test", + Namespace: "test", + GroupVersion: schema.GroupVersion{ + Group: "core", + Version: "v1", + }, + Kind: "Secret", + Resource: "secrets", + Labels: map[string]string{ + "test-label-1-name": "test-label-1-value", + "test-label-2-name": "test-label-2-value", + }, + Annotations: map[string]string{ + "test-annotation-1-name": "test-annotation-1-value", + "test-annotation-2-name": "test-annotation-2-value", + }, + DesiredReplicas: nil, + ResourceRequest: framework.Resource{}, + CurrentClusters: map[string]*int64{ + "cluster1": nil, + "cluster2": nil, + "cluster3": nil, + "cluster4": nil, + "cluster5": nil, + }, + SchedulingMode: fedcorev1a1.SchedulingModeDuplicate, + StickyCluster: false, + ClusterSelector: map[string]string{}, + ClusterNames: nil, + Affinity: &framework.Affinity{}, + Tolerations: []corev1.Toleration{ + { + Key: "test", + Operator: corev1.TolerationOpEqual, + Value: "test", + Effect: corev1.TaintEffectNoExecute, + }, + }, + MaxClusters: nil, + }, + expectedResult: &schedwebhookv1a1.SchedulingUnit{ + APIVersion: "core/v1", + Kind: "Secret", + Resource: "secrets", + Name: "test", + Namespace: "test", + Labels: map[string]string{ + "test-label-1-name": "test-label-1-value", + "test-label-2-name": "test-label-2-value", + }, + Annotations: map[string]string{ + "test-annotation-1-name": "test-annotation-1-value", + "test-annotation-2-name": "test-annotation-2-value", + }, + SchedulingMode: fedcorev1a1.SchedulingModeDuplicate, + DesiredReplicas: nil, + ResourceRequest: map[corev1.ResourceName]resource.Quantity{ + corev1.ResourceCPU: *resource.NewMilliQuantity(0, resource.DecimalSI), + corev1.ResourceMemory: *resource.NewQuantity(0, resource.BinarySI), + corev1.ResourceEphemeralStorage: *resource.NewQuantity(0, resource.BinarySI), + }, + CurrentClusters: []string{"cluster1", "cluster2", "cluster3", "cluster4", "cluster5"}, + CurrentReplicaDistribution: nil, + ClusterSelector: map[string]string{}, + ClusterAffinity: []fedcorev1a1.ClusterSelectorTerm{}, + Tolerations: []corev1.Toleration{ + { + Key: "test", + Operator: corev1.TolerationOpEqual, + Value: "test", + Effect: corev1.TaintEffectNoExecute, + }, + }, + MaxClusters: nil, + Placements: nil, + }, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + converted := ConvertSchedulingUnit(test.su) + + // we need to sort currentClusters due to go's maps being unordered + sort.Slice(converted.CurrentClusters, func(i, j int) bool { + return converted.CurrentClusters[i] < converted.CurrentClusters[j] + }) + sort.Slice(test.expectedResult.CurrentClusters, func(i, j int) bool { + return test.expectedResult.CurrentClusters[i] < test.expectedResult.CurrentClusters[j] + }) + // we need to sort placements due to go's maps being unordered + sort.Slice(converted.Placements, func(i, j int) bool { + return converted.Placements[i].ClusterName < converted.Placements[j].ClusterName + }) + sort.Slice(test.expectedResult.Placements, func(i, j int) bool { + return test.expectedResult.Placements[i].ClusterName < test.expectedResult.Placements[j].ClusterName + }) + + g := gomega.NewWithT(t) + g.Expect(converted).To(custommatchers.SemanticallyEqual(test.expectedResult)) + }) + } +} diff --git a/pkg/controllers/scheduler/extensions/webhook/v1alpha1/plugin_test.go b/pkg/controllers/scheduler/extensions/webhook/v1alpha1/plugin_test.go new file mode 100644 index 00000000..701b2541 --- /dev/null +++ b/pkg/controllers/scheduler/extensions/webhook/v1alpha1/plugin_test.go @@ -0,0 +1,376 @@ +/* +Copyright 2023 The KubeAdmiral Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1alpha1 + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "testing" + + "github.com/go-logr/logr" + "github.com/onsi/gomega" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/klog/v2" + + fedcorev1a1 "github.com/kubewharf/kubeadmiral/pkg/apis/core/v1alpha1" + schedwebhookv1a1 "github.com/kubewharf/kubeadmiral/pkg/apis/schedulerwebhook/v1alpha1" + "github.com/kubewharf/kubeadmiral/pkg/controllers/scheduler/framework" + "github.com/kubewharf/kubeadmiral/test/gomega/custommatchers" +) + +const ( + filterPath = "filter" + scorePath = "score" + selectPath = "select" +) + +var ( + sampleHttpError = fmt.Errorf("I'm a teapot") + sampleWebhookError = "rejected: kubeadmiral is too weak" +) + +type fakeHTTPClient struct { + roundTrip func(req *http.Request) *http.Response + err error +} + +func (f *fakeHTTPClient) Do(req *http.Request) (*http.Response, error) { + if f.err != nil { + return nil, f.err + } + return f.roundTrip(req), nil +} + +var _ HTTPClient = &fakeHTTPClient{} + +func doTest[T any]( + t *testing.T, + path string, + checkReq func(g gomega.Gomega, req *T), + httpErr error, + resp any, + checkPluginResult func(g gomega.Gomega, plugin *WebhookPlugin), +) { + t.Helper() + g := gomega.NewWithT(t) + + client := &fakeHTTPClient{ + err: httpErr, + roundTrip: func(httpReq *http.Request) *http.Response { + g.Expect(httpReq.Method).To(gomega.Equal(http.MethodPost)) + g.Expect(httpReq.Header.Get("Content-Type")).To(gomega.Equal("application/json")) + g.Expect(httpReq.Header.Get("Accept")).To(gomega.Equal("application/json")) + g.Expect(httpReq.Header.Get("User-Agent")).To(gomega.Equal("kubeadmiral-scheduler")) + g.Expect(httpReq.URL.Path).To(gomega.Equal(path)) + + req := new(T) + err := json.NewDecoder(httpReq.Body).Decode(req) + g.Expect(err).NotTo(gomega.HaveOccurred()) + + // Verify the plugin sends the request correctly + checkReq(g, req) + + respBytes, err := json.Marshal(&resp) + g.Expect(err).NotTo(gomega.HaveOccurred()) + return &http.Response{ + StatusCode: http.StatusOK, + Body: io.NopCloser(bytes.NewReader(respBytes)), + } + }, + } + + plugin := NewWebhookPlugin( + "test", + "", + filterPath, + scorePath, + selectPath, + client, + ) + + // Verify the plugin processes webhook responses correctly + checkPluginResult(g, plugin) +} + +func TestFilter(t *testing.T) { + testCases := map[string]struct { + su *framework.SchedulingUnit + cluster *fedcorev1a1.FederatedCluster + httpErr error + selected bool + err string + }{ + "webhook selects cluster": { + su: getSampleSchedulingUnit(), + cluster: getSampleCluster("test"), + selected: true, + err: "", + }, + "webhook does not select clcuster": { + su: getSampleSchedulingUnit(), + cluster: getSampleCluster("test"), + selected: false, + err: "", + }, + "webhook returns error": { + su: getSampleSchedulingUnit(), + cluster: getSampleCluster("test"), + selected: false, + err: sampleWebhookError, + }, + "http error": { + su: getSampleSchedulingUnit(), + cluster: getSampleCluster("test"), + httpErr: sampleHttpError, + }, + } + + for name, tc := range testCases { + t.Run(name, func(t *testing.T) { + doTest( + t, + filterPath, + func(g gomega.Gomega, req *schedwebhookv1a1.FilterRequest) { + g.Expect(req.SchedulingUnit).To(custommatchers.SemanticallyEqual(*ConvertSchedulingUnit(tc.su))) + g.Expect(req.Cluster).To(custommatchers.SemanticallyEqual(*tc.cluster)) + }, + tc.httpErr, + schedwebhookv1a1.FilterResponse{ + Selected: tc.selected, + Error: tc.err, + }, + func(g gomega.Gomega, plugin *WebhookPlugin) { + result := plugin.Filter(getPluginContext(), tc.su, tc.cluster) + if tc.httpErr != nil { + g.Expect(result.Message()).To(gomega.Equal(fmt.Errorf("request failed: %w", tc.httpErr).Error())) + } else { + g.Expect(result.Message()).To(gomega.Equal(tc.err)) + } + g.Expect(result.IsSuccess()).To(gomega.Equal(tc.selected)) + }, + ) + }) + } +} + +func TestScore(t *testing.T) { + testCases := map[string]struct { + su *framework.SchedulingUnit + cluster *fedcorev1a1.FederatedCluster + httpErr error + score int64 + err string + }{ + "webhook returns score": { + su: getSampleSchedulingUnit(), + cluster: getSampleCluster("test"), + score: 5, + err: "", + }, + "webhook returns error": { + su: getSampleSchedulingUnit(), + cluster: getSampleCluster("test"), + err: sampleWebhookError, + }, + "http error": { + su: getSampleSchedulingUnit(), + cluster: getSampleCluster("test"), + httpErr: sampleHttpError, + }, + } + + for name, tc := range testCases { + t.Run(name, func(t *testing.T) { + doTest( + t, + scorePath, + func(g gomega.Gomega, req *schedwebhookv1a1.FilterRequest) { + g.Expect(req.SchedulingUnit).To(custommatchers.SemanticallyEqual(*ConvertSchedulingUnit(tc.su))) + g.Expect(req.Cluster).To(custommatchers.SemanticallyEqual(*tc.cluster)) + }, + tc.httpErr, + schedwebhookv1a1.ScoreResponse{ + Score: tc.score, + Error: tc.err, + }, + func(g gomega.Gomega, plugin *WebhookPlugin) { + score, result := plugin.Score(getPluginContext(), tc.su, tc.cluster) + if tc.httpErr != nil { + g.Expect(result.Message()).To(gomega.Equal(fmt.Errorf("request failed: %w", tc.httpErr).Error())) + } else { + g.Expect(result.Message()).To(gomega.Equal(tc.err)) + } + g.Expect(result.IsSuccess()).To(gomega.Equal(tc.httpErr == nil && tc.err == "")) + g.Expect(score).To(gomega.Equal(tc.score)) + }, + ) + }) + } +} + +func TestSelect(t *testing.T) { + clusters := []*fedcorev1a1.FederatedCluster{ + getSampleCluster("cluster1"), + getSampleCluster("cluster2"), + getSampleCluster("cluster3"), + } + clusterScores := make(framework.ClusterScoreList, 0, 3) + for i, cluster := range clusters { + clusterScores = append(clusterScores, framework.ClusterScore{ + Cluster: cluster, + Score: int64(i + 1), + }) + } + + testCases := map[string]struct { + su *framework.SchedulingUnit + httpErr error + expectedClusters []*fedcorev1a1.FederatedCluster + err string + }{ + "webhook selects clusters": { + su: getSampleSchedulingUnit(), + expectedClusters: clusters[:2], + err: "", + }, + "webhook returns error": { + su: getSampleSchedulingUnit(), + expectedClusters: nil, + err: sampleWebhookError, + }, + "http error": { + su: getSampleSchedulingUnit(), + expectedClusters: nil, + httpErr: sampleHttpError, + }, + } + + for name, tc := range testCases { + t.Run(name, func(t *testing.T) { + selectedClusterNames := make([]string, 0, len(tc.expectedClusters)) + for _, cluster := range tc.expectedClusters { + selectedClusterNames = append(selectedClusterNames, cluster.Name) + } + + doTest( + t, + selectPath, + func(g gomega.Gomega, req *schedwebhookv1a1.SelectRequest) { + g.Expect(req.SchedulingUnit).To(custommatchers.SemanticallyEqual(*ConvertSchedulingUnit(tc.su))) + expectedClusterScores := make([]schedwebhookv1a1.ClusterScore, 0, len(clusterScores)) + for _, cluster := range clusterScores { + expectedClusterScores = append(expectedClusterScores, schedwebhookv1a1.ClusterScore{ + Cluster: *cluster.Cluster, + Score: cluster.Score, + }) + } + g.Expect(req.ClusterScores).To(custommatchers.SemanticallyEqual(expectedClusterScores)) + }, + tc.httpErr, + schedwebhookv1a1.SelectResponse{ + SelectedClusterNames: selectedClusterNames, + Error: tc.err, + }, + func(g gomega.Gomega, plugin *WebhookPlugin) { + selectedClusters, result := plugin.SelectClusters(getPluginContext(), tc.su, clusterScores) + if tc.httpErr != nil { + g.Expect(result.Message()).To(gomega.Equal(fmt.Errorf("request failed: %w", tc.httpErr).Error())) + } else { + g.Expect(result.Message()).To(gomega.Equal(tc.err)) + } + g.Expect(result.IsSuccess()).To(gomega.Equal(tc.httpErr == nil && tc.err == "")) + g.Expect(selectedClusters).To(custommatchers.SemanticallyEqual(tc.expectedClusters)) + }, + ) + }) + } +} + +func getSampleSchedulingUnit() *framework.SchedulingUnit { + return &framework.SchedulingUnit{ + Name: "test", + Namespace: "test", + GroupVersion: schema.GroupVersion{ + Group: "apps", + Version: "v1", + }, + Kind: "Deployment", + Resource: "deployments", + Labels: map[string]string{ + "test-label-1-name": "test-label-1-value", + "test-label-2-name": "test-label-2-value", + }, + Annotations: map[string]string{ + "test-annotation-1-name": "test-annotation-1-value", + "test-annotation-2-name": "test-annotation-2-value", + }, + DesiredReplicas: nil, + ResourceRequest: framework.Resource{}, + CurrentClusters: nil, + SchedulingMode: fedcorev1a1.SchedulingModeDuplicate, + StickyCluster: false, + ClusterSelector: nil, + ClusterNames: nil, + Affinity: nil, + Tolerations: nil, + MaxClusters: nil, + } +} + +func getSampleCluster(name string) *fedcorev1a1.FederatedCluster { + return &fedcorev1a1.FederatedCluster{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + }, + Spec: fedcorev1a1.FederatedClusterSpec{ + APIEndpoint: "https://test-cluster", + SecretRef: fedcorev1a1.LocalSecretReference{ + Name: "test-cluster-secret", + }, + Taints: []corev1.Taint{ + { + Key: "test-taint-1-name", + Value: "test-taint-1-value", + Effect: corev1.TaintEffectNoSchedule, + }, + }, + }, + Status: fedcorev1a1.FederatedClusterStatus{ + Resources: fedcorev1a1.Resources{ + Allocatable: corev1.ResourceList{ + corev1.ResourceCPU: *resource.NewMilliQuantity(10000, resource.DecimalSI), + corev1.ResourceMemory: *resource.NewQuantity(10000, resource.BinarySI), + }, + Available: corev1.ResourceList{ + corev1.ResourceCPU: *resource.NewMilliQuantity(5000, resource.DecimalSI), + corev1.ResourceMemory: *resource.NewQuantity(5000, resource.BinarySI), + }, + }, + }, + } +} + +func getPluginContext() context.Context { + return klog.NewContext(context.Background(), logr.Discard()) +} From 080097b5cb30a197c78d6c9747dd56a6e1e6bba7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gary=20Liu=20=28=E5=88=98=E5=B9=BF=E6=BA=90=29?= Date: Thu, 13 Apr 2023 07:11:13 +0000 Subject: [PATCH 08/10] feat(scheduler-plugin): listen to webhook config and cache plugins --- cmd/controller-manager/app/core.go | 1 + pkg/controllers/scheduler/constants.go | 5 +- pkg/controllers/scheduler/scheduler.go | 28 +++++- pkg/controllers/scheduler/webhook.go | 128 +++++++++++++++++++++++++ 4 files changed, 159 insertions(+), 3 deletions(-) create mode 100644 pkg/controllers/scheduler/webhook.go diff --git a/cmd/controller-manager/app/core.go b/cmd/controller-manager/app/core.go index ffa8d891..a98e2508 100644 --- a/cmd/controller-manager/app/core.go +++ b/cmd/controller-manager/app/core.go @@ -150,6 +150,7 @@ func startGlobalScheduler( controllerCtx.FedInformerFactory.Core().V1alpha1().ClusterPropagationPolicies(), controllerCtx.FedInformerFactory.Core().V1alpha1().FederatedClusters(), controllerCtx.FedInformerFactory.Core().V1alpha1().SchedulingProfiles(), + controllerCtx.FedInformerFactory.Core().V1alpha1().SchedulerPluginWebhookConfigurations(), controllerCtx.Metrics, controllerCtx.WorkerCount, ) diff --git a/pkg/controllers/scheduler/constants.go b/pkg/controllers/scheduler/constants.go index 3a577456..54776924 100644 --- a/pkg/controllers/scheduler/constants.go +++ b/pkg/controllers/scheduler/constants.go @@ -44,8 +44,9 @@ const ( DefaultSchedulingMode = fedcorev1a1.SchedulingModeDuplicate - EventReasonScheduleFederatedObject = "ScheduleFederatedObject" - EventReasonInvalidFollowsObject = "InvalidFollowsObject" + EventReasonScheduleFederatedObject = "ScheduleFederatedObject" + EventReasonInvalidFollowsObject = "InvalidFollowsObject" + EventReasonWebhookConfigurationError = "WebhookConfigurationError" SchedulingTriggerHashAnnotation = common.DefaultPrefix + "scheduling-trigger-hash" ) diff --git a/pkg/controllers/scheduler/scheduler.go b/pkg/controllers/scheduler/scheduler.go index 3d565311..0747f8b9 100644 --- a/pkg/controllers/scheduler/scheduler.go +++ b/pkg/controllers/scheduler/scheduler.go @@ -21,6 +21,8 @@ package scheduler import ( "context" "fmt" + "reflect" + "sync" "time" corev1 "k8s.io/api/core/v1" @@ -81,6 +83,9 @@ type Scheduler struct { schedulingProfileLister fedcorev1a1listers.SchedulingProfileLister schedulingProfileSynced cache.InformerSynced + webhookConfigurationSynced cache.InformerSynced + webhookPlugins sync.Map + worker worker.ReconcileWorker eventRecorder record.EventRecorder @@ -100,6 +105,7 @@ func NewScheduler( clusterPropagationPolicyInformer fedcorev1a1informers.ClusterPropagationPolicyInformer, clusterInformer fedcorev1a1informers.FederatedClusterInformer, schedulingProfileInformer fedcorev1a1informers.SchedulingProfileInformer, + webhookConfigurationInformer fedcorev1a1informers.SchedulerPluginWebhookConfigurationInformer, metrics stats.Metrics, workerCount int, ) (*Scheduler, error) { @@ -158,6 +164,25 @@ func NewScheduler( s.schedulingProfileLister = schedulingProfileInformer.Lister() s.schedulingProfileSynced = schedulingProfileInformer.Informer().HasSynced + s.webhookConfigurationSynced = webhookConfigurationInformer.Informer().HasSynced + webhookConfigurationInformer.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{ + AddFunc: func(obj interface{}) { + s.cacheWebhookPlugin(obj.(*fedcorev1a1.SchedulerPluginWebhookConfiguration)) + }, + UpdateFunc: func(oldUntyped, newUntyped interface{}) { + oldConfig := oldUntyped.(*fedcorev1a1.SchedulerPluginWebhookConfiguration) + newConfig := newUntyped.(*fedcorev1a1.SchedulerPluginWebhookConfiguration) + if oldConfig.Spec.URLPrefix != newConfig.Spec.URLPrefix || + oldConfig.Spec.HTTPTimeout != newConfig.Spec.HTTPTimeout || + !reflect.DeepEqual(oldConfig.Spec.TLSConfig, newConfig.Spec.TLSConfig) { + s.cacheWebhookPlugin(newConfig) + } + }, + DeleteFunc: func(obj interface{}) { + s.webhookPlugins.Delete(obj.(*fedcorev1a1.SchedulerPluginWebhookConfiguration).Name) + }, + }) + s.algorithm = core.NewSchedulerAlgorithm(clusterInformer.Informer().GetStore()) return s, nil @@ -172,6 +197,7 @@ func (s *Scheduler) Run(ctx context.Context) { s.clusterPropagationPolicySynced, s.clusterSynced, s.schedulingProfileSynced, + s.webhookConfigurationSynced, } if s.typeConfig.GetNamespaced() { cachesSynced = append(cachesSynced, s.propagationPolicySynced) @@ -188,7 +214,7 @@ func (s *Scheduler) Run(ctx context.Context) { func (s *Scheduler) reconcile(qualifiedName common.QualifiedName) (status worker.Result) { _ = s.metrics.Rate("scheduler.throughput", 1) key := qualifiedName.String() - keyedLogger := s.logger.WithValues("control-loop", "reconcile", "key", key) + keyedLogger := s.logger.WithValues("origin", "reconcile", "key", key) startTime := time.Now() keyedLogger.V(3).Info("Start reconcile") diff --git a/pkg/controllers/scheduler/webhook.go b/pkg/controllers/scheduler/webhook.go new file mode 100644 index 00000000..27b7f321 --- /dev/null +++ b/pkg/controllers/scheduler/webhook.go @@ -0,0 +1,128 @@ +/* +Copyright 2023 The KubeAdmiral Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package scheduler + +import ( + "fmt" + "net/http" + + corev1 "k8s.io/api/core/v1" + utilnet "k8s.io/apimachinery/pkg/util/net" + utilruntime "k8s.io/apimachinery/pkg/util/runtime" + "k8s.io/apimachinery/pkg/util/sets" + "k8s.io/client-go/rest" + + fedcorev1a1 "github.com/kubewharf/kubeadmiral/pkg/apis/core/v1alpha1" + schedwebhookv1a1 "github.com/kubewharf/kubeadmiral/pkg/apis/schedulerwebhook/v1alpha1" + pluginv1a1 "github.com/kubewharf/kubeadmiral/pkg/controllers/scheduler/extensions/webhook/v1alpha1" + "github.com/kubewharf/kubeadmiral/pkg/controllers/scheduler/framework" + "github.com/kubewharf/kubeadmiral/pkg/controllers/scheduler/framework/runtime" +) + +// SchedulerSupportedPayloadVersions is the list of payload versions supported by the scheduler. +var SchedulerSupportedPayloadVersions = sets.New( + schedwebhookv1a1.PayloadVersion, +) + +func (s *Scheduler) cacheWebhookPlugin(config *fedcorev1a1.SchedulerPluginWebhookConfiguration) { + logger := s.logger.WithValues("origin", "webhookEventHandler", "name", config.Name) + + // Find the most preferred payload version that is supported by the scheduler. + var payloadVersion string + for _, version := range config.Spec.PayloadVersions { + if SchedulerSupportedPayloadVersions.Has(version) { + payloadVersion = version + break + } + } + if len(payloadVersion) == 0 { + msg := fmt.Sprintf( + "Failed to resolve payload version: no supported payload version found, webhook supports %v, scheduler supports %v", + config.Spec.PayloadVersions, SchedulerSupportedPayloadVersions.UnsortedList(), + ) + logger.Error(nil, msg) + s.eventRecorder.Event( + config, + corev1.EventTypeWarning, + EventReasonWebhookConfigurationError, + msg, + ) + return + } + + transport, err := makeTransport(&config.Spec) + if err != nil { + logger.Error(err, "Failed to create webhook transport") + s.eventRecorder.Eventf( + config, + corev1.EventTypeWarning, + EventReasonWebhookConfigurationError, + "Failed to create webhook transport: %v", + err, + ) + return + } + client := &http.Client{ + Transport: transport, + Timeout: config.Spec.HTTPTimeout.Duration, + } + + var plugin framework.Plugin + switch payloadVersion { + case schedwebhookv1a1.PayloadVersion: + plugin = pluginv1a1.NewWebhookPlugin( + config.Name, config.Spec.URLPrefix, + config.Spec.FilterPath, config.Spec.ScorePath, config.Spec.SelectPath, + client, + ) + default: + // this should not happen + utilruntime.HandleError(fmt.Errorf("unknown payload version %q", payloadVersion)) + return + } + s.webhookPlugins.Store(config.Name, plugin) +} + +func makeTransport(config *fedcorev1a1.SchedulerPluginWebhookConfigurationSpec) (http.RoundTripper, error) { + var restConfig rest.Config + if config.TLSConfig != nil { + restConfig.TLSClientConfig.Insecure = config.TLSConfig.Insecure + restConfig.TLSClientConfig.ServerName = config.TLSConfig.ServerName + restConfig.TLSClientConfig.CertData = config.TLSConfig.CertData + restConfig.TLSClientConfig.KeyData = config.TLSConfig.KeyData + restConfig.TLSClientConfig.CAData = config.TLSConfig.CAData + } + tlsConfig, err := rest.TLSConfigFor(&restConfig) + if err != nil { + return nil, fmt.Errorf("error creating TLS config: %w", err) + } + return utilnet.SetTransportDefaults(&http.Transport{TLSClientConfig: tlsConfig}), nil +} + +func (s *Scheduler) webhookPluginRegistry() runtime.Registry { + registry := runtime.Registry{} + + s.webhookPlugins.Range(func(name, plugin any) bool { + _ = registry.Register(name.(string), func(_ framework.Handle) (framework.Plugin, error) { + return plugin.(framework.Plugin), nil + }) + + return true + }) + + return registry +} From 42818fd5d2f7c6700bd5080808db2ea4a7909801 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gary=20Liu=20=28=E5=88=98=E5=B9=BF=E6=BA=90=29?= Date: Fri, 14 Apr 2023 03:53:35 +0000 Subject: [PATCH 09/10] chore(lint): clarify importas regex --- .golangci.yml | 38 ++++++++++++++++++++++---------------- 1 file changed, 22 insertions(+), 16 deletions(-) diff --git a/.golangci.yml b/.golangci.yml index b85bc4d3..563e8c37 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -146,38 +146,44 @@ linters-settings: - pkg: "k8s.io/apimachinery/pkg/apis/meta/v1" alias: metav1 - - pkg: k8s.io/api/(?P[\w\d]+)/(?Pv\d+)((?P\w)\w+(?P\d+))? + # regex for api version: + # (?Pv\d+)((?P\w)\w+(?P\d+))? + # e.g. v1alpha2 + # s1: v1 + # s2: a + # s3: 2 + - pkg: k8s.io/api/(?P[\w\d]+)/(?Pv\d+)((?P\w)\w+(?P\d+))? # corev1, appsv1... - alias: ${group}${v1}${v2}${v3} + alias: ${group}${s1}${s2}${s3} - pkg: "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" alias: apiextensionsv1 - - pkg: github.com/kubewharf/kubeadmiral/pkg/apis/(?P[\w\d]+)/(?Pv\d+)((?P\w)\w+(?P\d+))? + - pkg: github.com/kubewharf/kubeadmiral/pkg/apis/(?P[\w\d]+)/(?Pv\d+)((?P\w)\w+(?P\d+))? # fedcorev1a1 - alias: fed${group}${v1}${v2}${v3} + alias: fed${group}${s1}${s2}${s3} - - pkg: k8s.io/client-go/kubernetes/typed/(?P[\w\d]+)/(?Pv\d+)((?P\w)\w+(?P\d+))? - alias: ${group}${v1}${v2}${v3}client + - pkg: k8s.io/client-go/kubernetes/typed/(?P[\w\d]+)/(?Pv\d+)((?P\w)\w+(?P\d+))? + alias: ${group}${s1}${s2}${s3}client - pkg: github.com/kubewharf/kubeadmiral/pkg/client/clientset/versioned alias: fedclient - pkg: github.com/kubewharf/kubeadmiral/pkg/client/clientset/versioned/scheme alias: fedscheme - - pkg: github.com/kubewharf/kubeadmiral/pkg/client/clientset/versioned/typed/(?P[\w\d]+)/(?Pv\d+)((?P\w)\w+(?P\d+))? + - pkg: github.com/kubewharf/kubeadmiral/pkg/client/clientset/versioned/typed/(?P[\w\d]+)/(?Pv\d+)((?P\w)\w+(?P\d+))? # fedcorev1a1client - alias: fed${group}${v1}${v2}${v3}client + alias: fed${group}${s1}${s2}${s3}client - - pkg: k8s.io/client-go/informers/(?P[\w\d]+)/(?Pv\d+)((?P\w)\w+(?P\d+))? - alias: ${group}${v1}${v2}${v3}informers + - pkg: k8s.io/client-go/informers/(?P[\w\d]+)/(?Pv\d+)((?P\w)\w+(?P\d+))? + alias: ${group}${s1}${s2}${s3}informers - pkg: github.com/kubewharf/kubeadmiral/pkg/client/informers/externalversions alias: fedinformers - - pkg: github.com/kubewharf/kubeadmiral/pkg/client/informers/externalversions/(?P[\w\d]+)/(?Pv\d+)((?P\w)\w+(?P\d+))? + - pkg: github.com/kubewharf/kubeadmiral/pkg/client/informers/externalversions/(?P[\w\d]+)/(?Pv\d+)((?P\w)\w+(?P\d+))? # fedcorev1a1informers - alias: fed${group}${v1}${v2}${v3}informers + alias: fed${group}${s1}${s2}${s3}informers - - pkg: k8s.io/client-go/listers/(?P[\w\d]+)/(?Pv\d+)((?P\w)\w+(?P\d+))? - alias: ${group}${v1}${v2}${v3}listers - - pkg: github.com/kubewharf/kubeadmiral/pkg/client/listers/(?P[\w\d]+)/(?Pv\d+)((?P\w)\w+(?P\d+))? + - pkg: k8s.io/client-go/listers/(?P[\w\d]+)/(?Pv\d+)((?P\w)\w+(?P\d+))? + alias: ${group}${s1}${s2}${s3}listers + - pkg: github.com/kubewharf/kubeadmiral/pkg/client/listers/(?P[\w\d]+)/(?Pv\d+)((?P\w)\w+(?P\d+))? # fedcorev1a1listers - alias: fed${group}${v1}${v2}${v3}listers + alias: fed${group}${s1}${s2}${s3}listers govet: # Settings per analyzer. settings: From b39463c7c505467a0af87508bc81cd89819e3948 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gary=20Liu=20=28=E5=88=98=E5=B9=BF=E6=BA=90=29?= Date: Fri, 14 Apr 2023 03:54:28 +0000 Subject: [PATCH 10/10] chore(lint): allow pkg/apis/schedulerwebhook/v1alpha1 to be imported as schedwebhookv1a1 --- .golangci.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.golangci.yml b/.golangci.yml index 563e8c37..7d8d8c19 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -157,6 +157,9 @@ linters-settings: alias: ${group}${s1}${s2}${s3} - pkg: "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" alias: apiextensionsv1 + - pkg: github.com/kubewharf/kubeadmiral/pkg/apis/schedulerwebhook/(?Pv\d+)((?P\w)\w+(?P\d+))? + # schedwebhookv1a1 + alias: schedwebhook${s1}${s2}${s3} - pkg: github.com/kubewharf/kubeadmiral/pkg/apis/(?P[\w\d]+)/(?Pv\d+)((?P\w)\w+(?P\d+))? # fedcorev1a1 alias: fed${group}${s1}${s2}${s3}