[test] Add Cilium BGP LB e2e test

.github/workflows/ci.yml

@@ -156,6 +156,9 @@ jobs:
       - name: Run E2E Tests
         run: devbox run e2e-test
 
+      - name: Run Cilium BGP e2e test
+        run: devbox run e2e-test-bgp
+
       - name: Cleanup Resources
         if: always()
         run: devbox run cleanup-cluster

Makefile

@@ -195,6 +195,43 @@ e2e-test:
 	LINODE_TOKEN=$(LINODE_TOKEN) \
 	chainsaw test e2e/test --parallel 2
 
+.PHONY: e2e-test-bgp
+e2e-test-bgp:
+	KUBECONFIG=$(KUBECONFIG_PATH) kubectl -n kube-system patch daemonset ccm-linode --patch '\
+	spec:\
+	  template:\
+	    spec:\
+	      containers:\
+	      - name: ccm-linode\
+	        args:\
+	        - --leader-elect-resource-lock=leases\
+	        - --v=3\
+	        - --secure-port=10253\
+	        - --webhook-secure-port=0\
+	        - --enable-route-controller=true\
+	        - --vpc-name=capl-cluster\
+	        - --configure-cloud-routes=true\
+	        - --cluster-cidr=10.0.0.0/8\
+			- --load-balancer-type=cilium-bgp\
+        	- --bgp-node-selector=cilium-bgp-peering=true\
+        	- --ip-holder-suffix=e2e-test
+	KUBECONFIG=$(KUBECONFIG_PATH) kubectl patch clusterrole ccm-linode-clusterrole --type='json' \
+	-p='[{\
+		"op": "add",\
+		"path": "/rules/-",\
+		"value": {\
+			"apiGroups": ["cilium.io"],\
+			"resources": ["ciliumloadbalancerippools", "ciliumbgppeeringpolicies"],\
+			"verbs": ["get", "list", "watch", "create", "update", "patch", "delete"]\
+		}\
+	}]'
+	CLUSTER_NAME=$(CLUSTER_NAME) \
+	MGMT_KUBECONFIG=$(MGMT_KUBECONFIG_PATH) \
+	KUBECONFIG=$(KUBECONFIG_PATH) \
+	REGION=$(LINODE_REGION) \
+	LINODE_TOKEN=$(LINODE_TOKEN) \
+	chainsaw test e2e/test/lb-cilium-bgp
+
 #####################################################################
 # OS / ARCH
 #####################################################################

devbox.json

@@ -21,6 +21,7 @@
     "scripts": {
       "mgmt-and-capl-cluster": "make mgmt-and-capl-cluster",
       "e2e-test": "make e2e-test",
+      "e2e-test-bgp": "make e2e-test-bgp",
       "cleanup-cluster": "make cleanup-cluster"
     }
   },

e2e/lb-cilium-bgp/chainsaw-test.yaml

@@ -0,0 +1,113 @@
+# yaml-language-server: $schema=https://raw.githubusercontent.com/kyverno/chainsaw/main/.schemas/json/test-chainsaw-v1alpha1.json
+apiVersion: chainsaw.kyverno.io/v1alpha1
+kind: Test
+metadata:
+  name: cilium-bgp-test
+spec:
+  namespace: "cilium-bgp-test"
+  steps:
+    - name: Check if CCM is deployed
+      try:
+        - assert:
+            file: ../test/assert-ccm-resources.yaml
+    - name: Create a pod and service with load balancer type cilium-bgp
+      try:
+        - apply:
+            file: create-pod-service.yaml
+      catch:
+        - describe:
+            apiVersion: v1
+            kind: Pod
+        - describe:
+            apiVersion: v1
+            kind: Service
+    - name: Verify CiliumLoadBalancerIPPool creation
+      try:
+        - assert:
+            resource:
+              apiVersion: cilium.io/v2alpha1
+              kind: CiliumLoadBalancerIPPool
+              metadata:
+                name: cilium-bgp-test-test-bgp-svc-pool
+              spec:
+                disabled: false
+    - name: Verify CiliumBGPPeeringPolicy
+      try:
+        - assert:
+            resource:
+              apiVersion: cilium.io/v2alpha1
+              kind: CiliumBGPPeeringPolicy
+              metadata:
+                name: linode-ccm-bgp-peering
+              spec:
+                nodeSelector:
+                  matchLabels:
+                    cilium-bgp-peering: "true"
+    - name: Check LoadBalancer IP assignment
+      try:
+        - assert:
+            resource:
+              apiVersion: v1
+              kind: Service
+              metadata:
+                name: test-bgp-svc
+              status:
+                conditions:
+                - status: "True"
+                  type: cilium.io/IPAMRequestSatisfied
+    - name: Verify IP sharing on labeled nodes
+      try:
+        - script:
+            content: |
+              set -e
+
+              # Get the LoadBalancer IP
+              LB_IP=$(kubectl get svc test-bgp-svc -n cilium-bgp-test -o jsonpath='{.status.loadBalancer.ingress[0].ip}')
+
+              # Get nodes with BGP label
+              BGP_NODES=$(kubectl get nodes -l cilium-bgp-peering=true -o name)
+
+              if [ -z "$BGP_NODES" ]; then
+                echo "No nodes found with label cilium-bgp-peering=true"
+                exit 1
+              fi
+
+              # Check if IP is shared on each BGP node
+              for node in $BGP_NODES; do
+                NODE_ID=$(kubectl get $node -o jsonpath='{.spec.providerID}' | sed 's|linode://||')
+                echo "Node ID: $NODE_ID"
+
+                SHARED_IPS=$(curl -H "Authorization: Bearer $LINODE_TOKEN" \
+                            "https://api.linode.com/v4/linode/instances/$NODE_ID/ips" | \
+                            jq -r '.ipv4.shared[].address')
+                echo "shared IPs: $SHARED_IPS"
+
+                if ! echo "$SHARED_IPS" | grep -q "$LB_IP"; then
+                  echo "LoadBalancer IP $LB_IP not found in shared IPs of node $node"
+                  exit 1
+                fi
+
+              # Check if the nanode has the shared IP
+              NANODE_ID=$(curl -H "Authorization: Bearer $LINODE_TOKEN" \
+                "https://api.linode.com/v4/linode/instances" | \
+                jq -r '.data[] | select(.label | endswith("e2e-test")) | .id')
+
+              NANODE_IPS=$(curl -H "Authorization: Bearer $LINODE_TOKEN" \
+                "https://api.linode.com/v4/linode/instances/$NANODE_ID/ips" | \
+                jq -r '.ipv4.public[].address')
+
+              if ! echo "$NANODE_IPS" | grep -q "$LB_IP"; then
+                echo "LoadBalancer IP not found in nanode (IP placeholder) IPs"
+                exit 1
+              fi
+
+              # Delete the nanode
+              curl -X DELETE -H "Authorization: Bearer $LINODE_TOKEN" \
+                "https://api.linode.com/v4/linode/instances/$NANODE_ID"
+
+              done
+            check:
+              ($error == null): true
+              (contains($stdout, 'LoadBalancer IP not found in shared IPs of node')): false
+              (contains($stdout, 'LoadBalancer IP not found in nanode IPs')): false
+

e2e/lb-cilium-bgp/create-pod-service.yaml

@@ -0,0 +1,24 @@
+apiVersion: v1
+kind: Pod
+metadata:
+  name: test-pod-1
+  labels:
+    app: test-bgp
+spec:
+  containers:
+    - name: nginx
+      image: nginx:latest
+      ports:
+        - containerPort: 80
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: test-bgp-svc
+spec:
+  type: LoadBalancer
+  ports:
+    - port: 80
+      targetPort: 80
+  selector:
+    app: test-bgp