add some tests

Signed-off-by: Xuhui zhang <xuhui@juicedata.io>
juicedata · Oct 16, 2024 · 9b8b464 · 9b8b464
1 parent d763ac1
commit 9b8b464
Show file tree

Hide file tree

Showing 12 changed files with 349 additions and 7 deletions.
diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml
@@ -43,8 +43,11 @@ jobs:
 
     - name: Set up kind
       uses: helm/kind-action@v1
+      with:
+        config: test/e2e/config/kind-config.yaml
+        cluster_name: e2e
 
     - name: Run e2e tests
       env:
-        KIND_CLUSTER: chart-testing
+        KIND_CLUSTER: e2e
       run: make test-e2e
diff --git a/api/v1/cachegroup_types.go b/api/v1/cachegroup_types.go
@@ -89,6 +89,15 @@ type CacheGroupWorkerTemplate struct {
 	// +optional
 	Volumes []corev1.Volume `json:"volumes,omitempty"`
 
+	// Set DNS policy for the pod.
+	// Defaults to "ClusterFirst".
+	// Valid values are 'ClusterFirstWithHostNet', 'ClusterFirst', 'Default' or 'None'.
+	// DNS parameters given in DNSConfig will be merged with the policy selected with DNSPolicy.
+	// To have DNS options set along with hostNetwork, you have to specify DNS policy
+	// explicitly to 'ClusterFirstWithHostNet'.
+	// +optional
+	DNSPolicy *corev1.DNSPolicy `json:"dnsPolicy,omitempty"`
+
 	Opts []string `json:"opts,omitempty"`
 }
 

diff --git a/api/v1/zz_generated.deepcopy.go b/api/v1/zz_generated.deepcopy.go
diff --git a/config/crd/bases/juicefs.io_cachegroups.yaml b/config/crd/bases/juicefs.io_cachegroups.yaml
@@ -77,6 +77,8 @@ spec:
                   overwrite:
                     items:
                       properties:
+                        dnsPolicy:
+                          type: string
                         env:
                           items:
                             properties:
@@ -1469,6 +1471,8 @@ spec:
                     type: array
                   template:
                     properties:
+                      dnsPolicy:
+                        type: string
                       env:
                         items:
                           properties:

diff --git a/config/manager/manager.yaml b/config/manager/manager.yaml
@@ -60,6 +60,7 @@ spec:
       containers:
       - command:
         - /manager
+        - controller
         args:
           - --leader-elect
           - --health-probe-bind-address=:8081

diff --git a/pkg/builder/pod.go b/pkg/builder/pod.go
@@ -166,6 +166,9 @@ func NewCacheGroupWorker(ctx context.Context, cg *juicefsiov1.CacheGroup, secret
 	} else {
 		worker.Spec.HostNetwork = true
 	}
+	if spec.DNSPolicy != nil {
+		worker.Spec.DNSPolicy = *spec.DNSPolicy
+	}
 	worker.Spec.Tolerations = spec.Tolerations
 	worker.Spec.SchedulerName = spec.SchedulerName
 	worker.Spec.ServiceAccountName = spec.ServiceAccountName
@@ -258,6 +261,9 @@ func MergeCacheGrouopWorkerTemplate(template *juicefsiov1.CacheGroupWorkerTempla
 	if overwrite.Opts != nil {
 		template.Opts = overwrite.Opts
 	}
+	if overwrite.DNSPolicy != nil {
+		template.DNSPolicy = overwrite.DNSPolicy
+	}
 }
 
 func genAuthCmdWithSecret(ctx context.Context, secrets map[string]string) []string {

diff --git a/pkg/utils/secret.go b/pkg/utils/secret.go
@@ -54,6 +54,6 @@ func ValidateSecret(secrets *corev1.Secret) error {
 		return fmt.Errorf("token is missing")
 	}
 
-	// TOOD: validate other fields, like ak/sk, bucket, etc.
+	// TODO: validate other fields, like ak/sk, bucket, etc.
 	return nil
 }
diff --git a/test/e2e/config/e2e-test-cachegroup.yaml b/test/e2e/config/e2e-test-cachegroup.yaml
@@ -0,0 +1,26 @@
+
+apiVersion: juicefs.io/v1
+kind: CacheGroup
+metadata:
+  name: e2e-test-cachegroup
+spec:
+  secretRef:
+    name: juicefs-secret
+  worker:
+    template:
+      nodeSelector:
+        juicefs.io/cg-worker: "true"
+      image: juicedata/mount:ee-5.1.1-1faf43b
+      dnsPolicy: ClusterFirstWithHostNet
+      hostNetwork: true
+      tolerations:
+      - key: node-role.kubernetes.io/control-plane
+        operator: Exists
+        effect: NoSchedule
+      resources:
+        requests:
+          cpu: 100m
+          memory: 128Mi
+        limits:
+          cpu: 200m
+          memory: 1Gi
diff --git a/test/e2e/config/kind-config.yaml b/test/e2e/config/kind-config.yaml
@@ -0,0 +1,6 @@
+kind: Cluster
+apiVersion: kind.x-k8s.io/v1alpha4
+nodes:
+- role: control-plane
+- role: worker
+- role: worker
diff --git a/test/e2e/config/minio.yaml b/test/e2e/config/minio.yaml
@@ -0,0 +1,32 @@
+# Deploys a new Namespace for the MinIO Pod
+apiVersion: v1
+kind: Pod
+metadata:
+  labels:
+    app: minio
+  name: minio
+  namespace: default 
+spec:
+  containers:
+  - name: minio
+    image: quay.io/minio/minio:latest
+    command:
+    - /bin/bash
+    - -c
+    args: 
+    - minio server /data --console-address :9090
+---
+apiVersion: v1
+kind: Service
+metadata:
+  labels:
+    app: minio
+  name: minio
+  namespace: default
+spec:
+  ports:
+  - port: 9000
+    protocol: TCP
+    targetPort: 9000
+  selector:
+    app: minio
diff --git a/test/e2e/e2e_test.go b/test/e2e/e2e_test.go
@@ -24,6 +24,7 @@ import (
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
 
+	"github.com/juicedata/juicefs-cache-group-operator/pkg/common"
 	"github.com/juicedata/juicefs-cache-group-operator/test/utils"
 )
 
@@ -37,9 +38,15 @@ var _ = Describe("controller", Ordered, func() {
 		// By("installing the cert-manager")
 		// Expect(utils.InstallCertManager()).To(Succeed())
 
+		By("installing the minio")
+		Expect(utils.InstallMinio()).To(Succeed())
+
 		By("creating manager namespace")
 		cmd := exec.Command("kubectl", "create", "ns", namespace)
 		_, _ = utils.Run(cmd)
+
+		By("creating the secret")
+		Expect(utils.CreateSecret(namespace)).To(Succeed())
 	})
 
 	AfterAll(func() {
@@ -49,6 +56,12 @@ var _ = Describe("controller", Ordered, func() {
 		// By("uninstalling the cert-manager bundle")
 		// utils.UninstallCertManager()
 
+		By("uninstalling the minio")
+		utils.UninstallMinio()
+
+		By("deleting the secret")
+		utils.DeleteSecret(namespace)
+
 		By("removing manager namespace")
 		cmd := exec.Command("kubectl", "delete", "ns", namespace)
 		_, _ = utils.Run(cmd)
@@ -116,6 +129,177 @@ var _ = Describe("controller", Ordered, func() {
 				return nil
 			}
 			EventuallyWithOffset(1, verifyControllerUp, time.Minute, time.Second).Should(Succeed())
+		})
+	})
+
+	Context("CacheGroup Controller", func() {
+		cgName := "e2e-test-cachegroup"
+
+		BeforeEach(func() {
+			cmd := exec.Command("kubectl", "label", "nodes", "--all", "juicefs.io/cg-worker-", "--overwrite")
+			_, err := utils.Run(cmd)
+			ExpectWithOffset(2, err).NotTo(HaveOccurred())
+
+			cmd = exec.Command("kubectl", "apply", "-f", "test/e2e/config/e2e-test-cachegroup.yaml", "-n", namespace)
+			_, err = utils.Run(cmd)
+			Expect(err).NotTo(HaveOccurred())
+		})
+
+		AfterEach(func() {
+			cmd := exec.Command("kubectl", "delete", "cachegroups.juicefs.io", cgName, "-n", namespace)
+			_, err := utils.Run(cmd)
+			ExpectWithOffset(2, err).NotTo(HaveOccurred())
+
+			// vetify workers is deleted
+			verifyWorkerDeleted := func() error {
+				cmd := exec.Command("kubectl", "get", "pods", "-l", "juicefs.io/cache-group="+cgName, "-n", namespace, "--no-headers", "--ignore-not-found=true")
+				result, err := utils.Run(cmd)
+				if err != nil {
+					return fmt.Errorf("check worker pods failed, %+v", err)
+				}
+				if len(utils.GetNonEmptyLines(string(result))) != 0 {
+					return fmt.Errorf("worker pods still exists")
+				}
+				return nil
+			}
+			Eventually(verifyWorkerDeleted, time.Minute, time.Second).Should(Succeed())
+		})
+
+		RegisterFailHandler(func(message string, callerSkip ...int) {
+			// print logs
+			cmd := exec.Command("kubectl", "logs", "-l", "control-plane=controller-manager", "-n", namespace)
+			log, _ := utils.Run(cmd)
+			fmt.Println(string(log))
+			Fail(message, callerSkip...)
+		})
+
+		It("should reconcile the CacheGroup", func() {
+			By("validating that the CacheGroup resource is reconciled")
+			verifyCacheGroupReconciled := func() error {
+				cmd := exec.Command("kubectl", "get", "cachegroups.juicefs.io", cgName, "-o", "jsonpath={.status.phase}", "-n", namespace)
+				status, err := utils.Run(cmd)
+				ExpectWithOffset(2, err).NotTo(HaveOccurred())
+
+				if string(status) != "Waiting" {
+					return fmt.Errorf("CacheGroup resource in %s status", status)
+				}
+				return nil
+			}
+			Eventually(verifyCacheGroupReconciled, time.Minute, time.Second).Should(Succeed())
+
+			cmd := exec.Command("kubectl", "label", "nodes", "--all", "juicefs.io/cg-worker=true", "--overwrite")
+			_, err := utils.Run(cmd)
+			ExpectWithOffset(1, err).NotTo(HaveOccurred())
+
+			// validatting workers is created
+			verifyWorkerCreated := func() error {
+				cmd = exec.Command("kubectl", "get", "pods", "-l", "juicefs.io/cache-group="+cgName, "-n", namespace, "--no-headers")
+				result, err := utils.Run(cmd)
+				if err != nil {
+					return fmt.Errorf("worker pods not created")
+				}
+				if len(utils.GetNonEmptyLines(string(result))) != 3 {
+					return fmt.Errorf("expect 3 worker pods created, but got %d", len(utils.GetNonEmptyLines(string(result))))
+				}
+				return nil
+			}
+			Eventually(verifyWorkerCreated, 5*time.Minute, 3*time.Second).Should(Succeed())
+
+			By("validating cg status is up to date")
+			verifyCgStatusUpToDate := func() error {
+				// Validate pod status
+				cmd := exec.Command("kubectl", "get",
+					"cachegroups.juicefs.io", cgName, "-o", "jsonpath={.status.phase}",
+					"-n", namespace,
+				)
+				status, err := utils.Run(cmd)
+				ExpectWithOffset(2, err).NotTo(HaveOccurred())
+				if string(status) != "Ready" {
+					return fmt.Errorf("cg expect Ready status, but got %s", status)
+				}
+
+				cmd = exec.Command("kubectl", "get",
+					"cachegroups.juicefs.io", cgName, "-o", "jsonpath={.status.readyWorker}",
+					"-n", namespace,
+				)
+				readyWorker, err := utils.Run(cmd)
+				ExpectWithOffset(2, err).NotTo(HaveOccurred())
+				if string(readyWorker) != "3" {
+					return fmt.Errorf("cg expect has 3 readyWorker status, but got %s", readyWorker)
+				}
+				cmd = exec.Command("kubectl", "get",
+					"cachegroups.juicefs.io", cgName, "-o", "jsonpath={.status.expectWorker}",
+					"-n", namespace,
+				)
+				expectWorker, err := utils.Run(cmd)
+				ExpectWithOffset(2, err).NotTo(HaveOccurred())
+				if string(expectWorker) != "3" {
+					return fmt.Errorf("cg expect has 3 expectWorker status, but got %s", expectWorker)
+				}
+				return nil
+			}
+			Eventually(verifyCgStatusUpToDate, time.Minute, time.Second).Should(Succeed())
+		})
+
+		It("should reconcile the worker with node labels update", func() {
+			nodeName := utils.GetKindNodeName("worker")
+			cmd := exec.Command("kubectl", "label", "nodes", nodeName, "juicefs.io/cg-worker=true", "--overwrite")
+			_, err := utils.Run(cmd)
+			ExpectWithOffset(1, err).NotTo(HaveOccurred())
+
+			By("validating worker created")
+			verifyWorkerCreated := func() error {
+				expectWorkerName := common.GenWorkerName(cgName, nodeName)
+				cmd = exec.Command("kubectl", "wait", "pod/"+expectWorkerName,
+					"--for", "condition=Ready",
+					"--namespace", namespace,
+					"--timeout", "2m",
+				)
+				_, err = utils.Run(cmd)
+				ExpectWithOffset(1, err).NotTo(HaveOccurred())
+
+				return nil
+			}
+			Eventually(verifyWorkerCreated, time.Minute, time.Second).Should(Succeed())
+
+			By("validating worker nodename")
+			verifyWorkerNodeName := func() error {
+				expectWorkerName := common.GenWorkerName(cgName, nodeName)
+
+				// Validate pod nodeName
+				cmd = exec.Command("kubectl", "get",
+					"pods", expectWorkerName, "-o", "jsonpath={.spec.nodeName}",
+					"-n", namespace,
+				)
+				expectNodeName, err := utils.Run(cmd)
+				ExpectWithOffset(1, err).NotTo(HaveOccurred())
+				ExpectWithOffset(1, string(expectNodeName)).Should(Equal(nodeName))
+				return nil
+			}
+			Eventually(verifyWorkerNodeName, time.Minute, 3*time.Second).Should(Succeed())
+
+			By("validating that the clean redundant worker")
+			verifyCleanRedundantWorker := func() error {
+				cmd := exec.Command("kubectl", "label", "nodes", nodeName, "juicefs.io/cg-worker-", "--overwrite")
+				_, err := utils.Run(cmd)
+				ExpectWithOffset(1, err).NotTo(HaveOccurred())
+				expectWorkerName := common.GenWorkerName(cgName, nodeName)
+				// Validate pod deleted
+				cmd = exec.Command("kubectl", "get",
+					"pods", expectWorkerName,
+					"-n", namespace,
+					"--ignore-not-found=true",
+				)
+				output, err := utils.Run(cmd)
+				if err != nil {
+					return fmt.Errorf("check worker pods failed, %+v", err)
+				}
+				if len(utils.GetNonEmptyLines(string(output))) != 0 {
+					return fmt.Errorf("worker pod still exists, worker: %s, output: %s", expectWorkerName, string(output))
+				}
+				return nil
+			}
+			Eventually(verifyCleanRedundantWorker, 2*time.Minute, 3*time.Second).Should(Succeed())
 
 		})
 	})