add some tests

Signed-off-by: Xuhui zhang <xuhui@juicedata.io>

.github/workflows/ci.yaml

@@ -43,6 +43,8 @@ jobs:
 
     - name: Set up kind
       uses: helm/kind-action@v1
+      with:
+        config: test/e2e/config/kind-config.yaml
 
     - name: Run e2e tests
       env:

api/v1/cachegroup_types.go

@@ -89,6 +89,15 @@ type CacheGroupWorkerTemplate struct {
 	// +optional
 	Volumes []corev1.Volume `json:"volumes,omitempty"`
 
+	// Set DNS policy for the pod.
+	// Defaults to "ClusterFirst".
+	// Valid values are 'ClusterFirstWithHostNet', 'ClusterFirst', 'Default' or 'None'.
+	// DNS parameters given in DNSConfig will be merged with the policy selected with DNSPolicy.
+	// To have DNS options set along with hostNetwork, you have to specify DNS policy
+	// explicitly to 'ClusterFirstWithHostNet'.
+	// +optional
+	DNSPolicy *corev1.DNSPolicy `json:"dnsPolicy,omitempty"`
+
 	Opts []string `json:"opts,omitempty"`
 }
 

config/crd/bases/juicefs.io_cachegroups.yaml

@@ -77,6 +77,8 @@ spec:
                   overwrite:
                     items:
                       properties:
+                        dnsPolicy:
+                          type: string
                         env:
                           items:
                             properties:
@@ -1469,6 +1471,8 @@ spec:
                     type: array
                   template:
                     properties:
+                      dnsPolicy:
+                        type: string
                       env:
                         items:
                           properties:

config/manager/manager.yaml

@@ -60,6 +60,7 @@ spec:
       containers:
       - command:
         - /manager
+        - controller
         args:
           - --leader-elect
           - --health-probe-bind-address=:8081

config/rbac/role.yaml

@@ -17,6 +17,7 @@ rules:
   resources:
   - pods
   verbs:
+  - create
   - delete
   - get
   - list

internal/controller/cachegroup_controller.go

@@ -47,7 +47,7 @@ type CacheGroupReconciler struct {
 // +kubebuilder:rbac:groups=juicefs.io,resources=cachegroups/status,verbs=get;update;patch
 // +kubebuilder:rbac:groups=juicefs.io,resources=cachegroups/finalizers,verbs=update
 // +kubebuilder:rbac:groups="",resources=nodes,verbs=get;list;watch
-// +kubebuilder:rbac:groups="",resources=pods,verbs=get;list;delete
+// +kubebuilder:rbac:groups="",resources=pods,verbs=get;list;delete;create
 // +kubebuilder:rbac:groups="",resources=pods/exec,verbs=create
 
 // Reconcile is part of the main kubernetes reconciliation loop which aims to

pkg/builder/pod.go

@@ -177,6 +177,9 @@ func NewCacheGroupWorker(ctx context.Context, cg *juicefsiov1.CacheGroup, nodeNa
 	} else {
 		worker.Spec.HostNetwork = true
 	}
+	if spec.DNSPolicy != nil {
+		worker.Spec.DNSPolicy = *spec.DNSPolicy
+	}
 	worker.Spec.Tolerations = spec.Tolerations
 	worker.Spec.SchedulerName = spec.SchedulerName
 	worker.Spec.ServiceAccountName = spec.ServiceAccountName
@@ -269,4 +272,7 @@ func MergeCacheGrouopWorkerTemplate(template *juicefsiov1.CacheGroupWorkerTempla
 	if overwrite.Opts != nil {
 		template.Opts = overwrite.Opts
 	}
+	if overwrite.DNSPolicy != nil {
+		template.DNSPolicy = overwrite.DNSPolicy
+	}
 }

test/e2e/config/e2e-test-cachegroup.yaml

@@ -0,0 +1,22 @@
+
+apiVersion: juicefs.io/v1
+kind: CacheGroup
+metadata:
+  name: e2e-test-cachegroup
+spec:
+  secretRef:
+    name: juicefs-secret
+  worker:
+    template:
+      nodeSelector:
+        juicefs.io/cg-worker: "true"
+      image: juicedata/mount:ee-5.1.1-1faf43b
+      dnsPolicy: ClusterFirstWithHostNet
+      hostNetwork: true
+      resources:
+        requests:
+          cpu: 100m
+          memory: 128Mi
+        limits:
+          cpu: 200m
+          memory: 1Gi

test/e2e/config/kind-config.yaml

@@ -0,0 +1,6 @@
+kind: Cluster
+apiVersion: kind.x-k8s.io/v1alpha4
+nodes:
+- role: control-plane
+- role: worker
+- role: worker

test/e2e/config/minio.yaml

@@ -0,0 +1,32 @@
+# Deploys a new Namespace for the MinIO Pod
+apiVersion: v1
+kind: Pod
+metadata:
+  labels:
+    app: minio
+  name: minio
+  namespace: default 
+spec:
+  containers:
+  - name: minio
+    image: quay.io/minio/minio:latest
+    command:
+    - /bin/bash
+    - -c
+    args: 
+    - minio server /data --console-address :9090
+---
+apiVersion: v1
+kind: Service
+metadata:
+  labels:
+    app: minio
+  name: minio
+  namespace: default
+spec:
+  ports:
+  - port: 9090
+    protocol: TCP
+    targetPort: 9090
+  selector:
+    app: minio

test/e2e/e2e_test.go

@@ -24,6 +24,7 @@ import (
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
 
+	"github.com/juicedata/juicefs-cache-group-operator/pkg/common"
 	"github.com/juicedata/juicefs-cache-group-operator/test/utils"
 )
 
@@ -37,9 +38,15 @@ var _ = Describe("controller", Ordered, func() {
 		// By("installing the cert-manager")
 		// Expect(utils.InstallCertManager()).To(Succeed())
 
+		By("installing the minio")
+		Expect(utils.InstallMinio()).To(Succeed())
+
 		By("creating manager namespace")
 		cmd := exec.Command("kubectl", "create", "ns", namespace)
 		_, _ = utils.Run(cmd)
+
+		By("creating the secret")
+		Expect(utils.CreateSecret(namespace)).To(Succeed())
 	})
 
 	AfterAll(func() {
@@ -49,6 +56,12 @@ var _ = Describe("controller", Ordered, func() {
 		// By("uninstalling the cert-manager bundle")
 		// utils.UninstallCertManager()
 
+		By("uninstalling the minio")
+		utils.UninstallMinio()
+
+		By("deleting the secret")
+		utils.DeleteSecret(namespace)
+
 		By("removing manager namespace")
 		cmd := exec.Command("kubectl", "delete", "ns", namespace)
 		_, _ = utils.Run(cmd)
@@ -116,6 +129,135 @@ var _ = Describe("controller", Ordered, func() {
 				return nil
 			}
 			EventuallyWithOffset(1, verifyControllerUp, time.Minute, time.Second).Should(Succeed())
+		})
+	})
+
+	Context("CacheGroup Controller", func() {
+		cgName := "e2e-test-cachegroup"
+
+		BeforeEach(func() {
+			cmd := exec.Command("kubectl", "label", "nodes", "--all", "juicefs.io/cg-worker-", "--overwrite")
+			_, err := utils.Run(cmd)
+			ExpectWithOffset(2, err).NotTo(HaveOccurred())
+
+			cmd = exec.Command("kubectl", "apply", "-f", "test/e2e/config/e2e-test-cachegroup.yaml", "-n", namespace)
+			_, err = utils.Run(cmd)
+			Expect(err).NotTo(HaveOccurred())
+		})
+
+		AfterEach(func() {
+			cmd := exec.Command("kubectl", "delete", "cachegroups.juicefs.io", cgName, "-n", namespace)
+			_, err := utils.Run(cmd)
+			ExpectWithOffset(2, err).NotTo(HaveOccurred())
+		})
+
+		It("should reconcile the CacheGroup", func() {
+			By("validating that the CacheGroup resource is reconciled")
+			verifyCacheGroupReconciled := func() error {
+				cmd := exec.Command("kubectl", "get", "cachegroups.juicefs.io", cgName, "-o", "jsonpath={.status.phase}", "-n", namespace)
+				status, err := utils.Run(cmd)
+				ExpectWithOffset(2, err).NotTo(HaveOccurred())
+
+				if string(status) != "Waiting" {
+					return fmt.Errorf("CacheGroup resource in %s status", status)
+				}
+				return nil
+			}
+			Eventually(verifyCacheGroupReconciled, time.Minute, time.Second).Should(Succeed())
+
+			// By("validating cg status is up to date")
+			// verifyCgStatusUpToDate := func() error {
+			// 	// Validate pod status
+			// 	cmd := exec.Command("kubectl", "get",
+			// 		"cachegroups.juicefs.io", cgName, "-o", "jsonpath={.status.phase}",
+			// 		"-n", namespace,
+			// 	)
+			// 	status, err := utils.Run(cmd)
+			// 	ExpectWithOffset(2, err).NotTo(HaveOccurred())
+			// 	if string(status) != "Ready" {
+			// 		return fmt.Errorf("cg expect Ready status, but got %s", status)
+			// 	}
+
+			// 	cmd = exec.Command("kubectl", "get",
+			// 		"cachegroups.juicefs.io", cgName, "-o", "jsonpath={.status.readyWorker}",
+			// 		"-n", namespace,
+			// 	)
+			// 	readyWorker, err := utils.Run(cmd)
+			// 	ExpectWithOffset(2, err).NotTo(HaveOccurred())
+			// 	if string(readyWorker) != "1" {
+			// 		return fmt.Errorf("cg expect has 1 readyWorker status, but got %s", readyWorker)
+			// 	}
+			// 	cmd = exec.Command("kubectl", "get",
+			// 		"cachegroups.juicefs.io", cgName, "-o", "jsonpath={.status.expectWorker}",
+			// 		"-n", namespace,
+			// 	)
+			// 	expectWorker, err := utils.Run(cmd)
+			// 	ExpectWithOffset(2, err).NotTo(HaveOccurred())
+			// 	if string(expectWorker) != "1" {
+			// 		return fmt.Errorf("cg expect has 1 expectWorker status, but got %s", expectWorker)
+			// 	}
+			// 	return nil
+			// }
+			// Eventually(verifyCgStatusUpToDate, time.Minute, time.Second).Should(Succeed())
+
+		})
+
+		It("should reconcile the worker with node labels update", func() {
+			cmd := exec.Command("kubectl", "label", "nodes", "kind-worker", "juicefs.io/cg-worker=true", "--overwrite")
+			_, err := utils.Run(cmd)
+			ExpectWithOffset(1, err).NotTo(HaveOccurred())
+
+			By("validating worker created")
+			verifyWorkerCreated := func() error {
+				expectWorkerName := common.GenWorkerName(cgName, "kind-worker")
+				cmd = exec.Command("kubectl", "wait", "pod/"+expectWorkerName,
+					"--for", "condition=Ready",
+					"--namespace", namespace,
+					"--timeout", "2m",
+				)
+				_, err = utils.Run(cmd)
+				ExpectWithOffset(1, err).NotTo(HaveOccurred())
+
+				return nil
+			}
+			Eventually(verifyWorkerCreated, time.Minute, time.Second).Should(Succeed())
+
+			By("validating worker nodename")
+			verifyWorkerNodeName := func() error {
+				expectWorkerName := common.GenWorkerName(cgName, "kind-worker")
+
+				// Validate pod nodeName
+				cmd = exec.Command("kubectl", "get",
+					"pods", expectWorkerName, "-o", "jsonpath={.spec.nodeName}",
+					"-n", namespace,
+				)
+				nodeName, err := utils.Run(cmd)
+				ExpectWithOffset(1, err).NotTo(HaveOccurred())
+				ExpectWithOffset(1, string(nodeName)).Should(Equal("kind-worker"))
+				return nil
+			}
+			Eventually(verifyWorkerNodeName, time.Minute, time.Second).Should(Succeed())
+
+			By("validating that the clean redundant worker")
+			verifyCleanRedundantWorker := func() error {
+				cmd := exec.Command("kubectl", "label", "nodes", "kind-worker", "juicefs.io/cg-worker-", "--overwrite")
+				_, err := utils.Run(cmd)
+				ExpectWithOffset(1, err).NotTo(HaveOccurred())
+
+				expectWorkerName := common.GenWorkerName(cgName, "kind-worker")
+
+				// Validate pod deleted
+				cmd = exec.Command("kubectl", "get",
+					"pods", expectWorkerName,
+					"-n", namespace,
+				)
+				_, err = utils.Run(cmd)
+				if err == nil {
+					return fmt.Errorf("worker pod %s still exists", expectWorkerName)
+				}
+				return nil
+			}
+			Eventually(verifyCleanRedundantWorker, time.Minute, time.Second).Should(Succeed())
 
 		})
 	})