From bb8b4813df3d6be504bf7838fdc9ddd8ff5065c7 Mon Sep 17 00:00:00 2001 From: Sebastian Widmer Date: Mon, 22 Jul 2024 10:36:46 +0200 Subject: [PATCH] Adds `forbidParallelRuns` configuration option --- api/v1beta1/schedulercanary_types.go | 4 ++ ...onitoring.appuio.io_schedulercanaries.yaml | 5 ++ controllers/schedulercanary_controller.go | 19 ++++++ .../schedulercanary_controller_test.go | 65 +++++++++++++++++++ 4 files changed, 93 insertions(+) diff --git a/api/v1beta1/schedulercanary_types.go b/api/v1beta1/schedulercanary_types.go index 442da2e..917e46d 100644 --- a/api/v1beta1/schedulercanary_types.go +++ b/api/v1beta1/schedulercanary_types.go @@ -42,6 +42,10 @@ type SchedulerCanarySpec struct { // The default is 1 minute. Interval metav1.Duration `json:"interval,omitempty"` + // ForbidParallelRuns will prevent the creation of a new canary pod if there is already a canary pod running. + // The default is false. + ForbidParallelRuns bool `json:"forbidParallelRuns,omitempty"` + // PodTemplate is the pod template to use for the canary pods. PodTemplate corev1.PodTemplateSpec `json:"podTemplate,omitempty"` } diff --git a/config/crd/bases/monitoring.appuio.io_schedulercanaries.yaml b/config/crd/bases/monitoring.appuio.io_schedulercanaries.yaml index 584c50d..6ca23b8 100644 --- a/config/crd/bases/monitoring.appuio.io_schedulercanaries.yaml +++ b/config/crd/bases/monitoring.appuio.io_schedulercanaries.yaml @@ -43,6 +43,11 @@ spec: spec: description: SchedulerCanarySpec defines the desired state of SchedulerCanary properties: + forbidParallelRuns: + description: |- + ForbidParallelRuns will prevent the creation of a new canary pod if there is already a canary pod running. + The default is false. + type: boolean interval: description: |- Interval is the interval at which a canary pod will be created. diff --git a/controllers/schedulercanary_controller.go b/controllers/schedulercanary_controller.go index 149c559..7a11d0a 100644 --- a/controllers/schedulercanary_controller.go +++ b/controllers/schedulercanary_controller.go @@ -18,8 +18,10 @@ package controllers import ( "context" + "fmt" "time" + corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" ctrl "sigs.k8s.io/controller-runtime" @@ -68,6 +70,22 @@ func (r *SchedulerCanaryReconciler) Reconcile(ctx context.Context, req ctrl.Requ return ctrl.Result{Requeue: true, RequeueAfter: rqi}, nil } + if instance.Spec.ForbidParallelRuns { + // Check if there is already a canary pod running. + pods := &corev1.PodList{} + if err := r.Client.List(ctx, pods, client.InNamespace(instance.Namespace), client.MatchingLabels{instanceLabel: instance.Name}); err != nil { + return ctrl.Result{}, fmt.Errorf("forbidParallelRuns: error checking for already running pods, failed to list pods: %w", err) + } + if len(pods.Items) > 0 { + podNames := make([]string, len(pods.Items)) + for i, pod := range pods.Items { + podNames[i] = pod.Name + } + l.Info("ForbidParallelRuns: already running pods found, skipping pod creation", "pods", podNames) + return ctrl.Result{}, nil + } + } + if err := r.createCanaryPod(ctx, instance); err != nil { return ctrl.Result{}, err } @@ -85,6 +103,7 @@ func (r *SchedulerCanaryReconciler) Reconcile(ctx context.Context, req ctrl.Requ func (r *SchedulerCanaryReconciler) SetupWithManager(mgr ctrl.Manager) error { return ctrl.NewControllerManagedBy(mgr). For(&monitoringv1beta1.SchedulerCanary{}). + Owns(&corev1.Pod{}). Complete(r) } diff --git a/controllers/schedulercanary_controller_test.go b/controllers/schedulercanary_controller_test.go index 2c958fe..35b1502 100644 --- a/controllers/schedulercanary_controller_test.go +++ b/controllers/schedulercanary_controller_test.go @@ -70,6 +70,71 @@ var _ = Describe("SchedulerCanary controller", func() { }) + Context("ForbidParallelRuns", func() { + BeforeEach(func() { + ctx := context.Background() + + schedulerCanary := &monitoringv1beta1.SchedulerCanary{ + ObjectMeta: metav1.ObjectMeta{ + Name: "my-canary", + Namespace: "default", + }, + Spec: monitoringv1beta1.SchedulerCanarySpec{ + Interval: metav1.Duration{Duration: time.Millisecond}, + MaxPodCompletionTimeout: metav1.Duration{Duration: time.Minute}, + ForbidParallelRuns: true, + PodTemplate: corev1.PodTemplateSpec{ + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "scheduler-canary", + Image: "busybox", + }, + }, + }, + }, + }, + } + Expect(k8sClient.Create(ctx, schedulerCanary)).Should(Succeed()) + }) + + It("ForbidParallelRuns: should not create more than one pod", func() { + ctx := context.Background() + + Eventually(func() (int, error) { + pods := &corev1.PodList{} + + err := k8sClient.List(ctx, pods, client.InNamespace("default"), client.MatchingLabels(map[string]string{ + instanceLabel: "my-canary", + })) + + return len(pods.Items), err + }, "10s", "250ms").Should(BeNumerically(">=", 1)) + + Consistently(func() (int, error) { + pods := &corev1.PodList{} + + err := k8sClient.List(ctx, pods, client.InNamespace("default"), client.MatchingLabels(map[string]string{ + instanceLabel: "my-canary", + })) + + return len(pods.Items), err + }, "5s", "250ms").Should(Equal(1)) + + Expect(k8sClient.DeleteAllOf(ctx, &corev1.Pod{}, client.InNamespace("default"))).Should(Succeed()) + + Eventually(func() (int, error) { + pods := &corev1.PodList{} + + err := k8sClient.List(ctx, pods, client.InNamespace("default"), client.MatchingLabels(map[string]string{ + instanceLabel: "my-canary", + })) + + return len(pods.Items), err + }, "10s", "250ms").Should(BeNumerically(">=", 1)) + }) + }) + AfterEach(func() { ctx := context.Background()