Add minikube CI testing

.github/workflows/test.yaml

@@ -18,10 +18,12 @@ jobs:
       - uses: actions/setup-go@v4
         with:
           go-version: '1.21'
-      - name: Create k8s Kind Cluster
-        uses: helm/kind-action@v1.7.0
-        with:
-          install_only: true
+      - name: Install Minikube
+        run: |
+          wget https://storage.googleapis.com/minikube/releases/latest/minikube-linux-amd64
+          sudo cp minikube-linux-amd64 /usr/local/bin/minikube
+          sudo chmod +x /usr/local/bin/minikube      
+
       - uses: azure/setup-helm@v3
       - name: Run e2e
         run: |

Dockerfile

@@ -1,5 +1,5 @@
 # Build the manager binary
-FROM --platform=${BUILDPLATFORM:-linux/amd64} golang:1.21 as builder
+FROM --platform=${BUILDPLATFORM:-linux/amd64} docker.io/golang:1.21 as builder
 
 ARG TARGETPLATFORM
 ARG BUILDPLATFORM

DockerfileDebug

@@ -1,4 +1,4 @@
-FROM golang:1.21 as builder
+FROM docker.io/golang:1.21 as builder
 
 WORKDIR /code
 
@@ -8,7 +8,7 @@ RUN go mod download
 RUN go install github.com/go-delve/delve/cmd/dlv@latest
 RUN go build -gcflags="all=-N -l"  -o /app main.go
 
-FROM nicolaka/netshoot:latest
+FROM docker.io/ubuntu:22.04
 ENV GOTRACEBACK=crash
 WORKDIR /
 COPY --from=builder /app .

DockerfileTestGrow

@@ -1,4 +1,4 @@
-FROM ubuntu:22.04
+FROM docker.io/ubuntu:22.04
 RUN apt-get update -y && apt-get install watch
 LABEL org.opencontainers.image.source="https://github.com/jmcgrath207/k8s-ephemeral-storage-metrics"
 

DockerfileTestShrink

@@ -1,4 +1,4 @@
-FROM ubuntu:22.04
+FROM docker.io/ubuntu:22.04
 RUN apt-get update -y && apt-get install watch
 LABEL org.opencontainers.image.source="https://github.com/jmcgrath207/k8s-ephemeral-storage-metrics"
 

Makefile

@@ -12,7 +12,10 @@ $(LOCALBIN):
 
 
 ginkgo:
-	test -s $(LOCALBIN)/ginkgo || GOBIN=$(LOCALBIN) go install github.com/onsi/ginkgo/v2/ginkgo@latest
+	test -s $(LOCALBIN)/ginkgo || GOBIN=$(LOCALBIN) go install github.com/onsi/ginkgo/v2/ginkgo@v2.13.1
+
+crane:
+	test -s $(LOCALBIN)/crane || GOBIN=$(LOCALBIN) go install github.com/google/go-containerregistry/cmd/crane@latest
 
 .PHONY: fmt
 fmt: ## Run go fmt against code.
@@ -27,8 +30,14 @@ helm-docs:
 	$(LOCALBIN)/helm-docs  --template-files "${GITROOT}/chart/README.md.gotmpl"
 	cat "${GITROOT}/Header.md" "${GITROOT}/chart/README.md" > "${GITROOT}/README.md"
 
-new_kind:
-	./scripts/create_kind.sh
+minikube_new:
+	./scripts/create-minikube.sh
+
+minikube_scale_up:
+	minikube node add
+
+minikube_scale_down:
+	minikube node delete m02
 
 init: fmt vet
 
@@ -41,7 +50,10 @@ deploy_e2e_debug: init
 deploy_local: init
 	./scripts/deploy.sh
 
-deploy_e2e: init ginkgo new_kind
+deploy_e2e: init ginkgo crane minikube_new
+	ENV='e2e' ./scripts/deploy.sh
+
+deploy_e2e_dirty: init
 	ENV='e2e' ./scripts/deploy.sh
 
 release-docker:

chart/templates/test_deployments.yaml

@@ -15,8 +15,8 @@ spec:
         name: shrink-test
     spec:
       containers:
-        - image: ghcr.io/jmcgrath207/k8s-ephemeral-storage-shrink-test:latest
-          imagePullPolicy: "{{ .Values.dev.image.imagePullPolicy }}"
+        - image: "{{ .Values.dev.shrink.image }}"
+          imagePullPolicy: "{{ .Values.dev.shrink.imagePullPolicy }}"
           name: shrink-test
           resources:
             requests:
@@ -54,8 +54,8 @@ spec:
         name: grow-test
     spec:
       containers:
-        - image: ghcr.io/jmcgrath207/k8s-ephemeral-storage-grow-test:latest
-          imagePullPolicy: "{{ .Values.dev.image.imagePullPolicy }}"
+        - image: "{{ .Values.dev.grow.image }}"
+          imagePullPolicy: "{{ .Values.dev.grow.imagePullPolicy }}"
           name: grow-test
           resources:
             requests:

chart/values.yaml

@@ -36,10 +36,14 @@ prometheus:
 pprof: false
 
 
-# -- For local development of kind and/or deploy grow and shrink test pods
+# -- For local development or testing that will deploy grow and shrink pods and debug service
 dev:
   enabled: false
-  image:
+  shrink:
+    image: ghcr.io/jmcgrath207/k8s-ephemeral-storage-shrink-test:latest
+    imagePullPolicy: IfNotPresent
+  grow:
+    image: ghcr.io/jmcgrath207/k8s-ephemeral-storage-grow-test:latest
     imagePullPolicy: IfNotPresent
 
 podAnnotations: {}

scripts/create-minikube.sh

@@ -0,0 +1,12 @@
+#!/bin/bash
+
+minikube delete || true
+c=$(docker ps -q) && [[ $c ]] && docker kill $c
+docker network prune -f
+minikube start \
+  --kubernetes-version="${K8S_VERSION}" \
+  --insecure-registry "10.0.0.0/24"
+minikube addons enable registry
+
+# Add Service Monitor CRD
+kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.65.1/example/prometheus-operator-crd/monitoring.coreos.com_servicemonitors.yaml

scripts/deploy.sh

@@ -5,15 +5,52 @@
 
 set -e
 
-source scripts/helpers.sh
+CURRENT_DIR="$(realpath $(dirname ${BASH_SOURCE[0]}))"
+cd "${CURRENT_DIR}" || true
 
+source helpers.sh
 
 function main() {
   local image_tag
   local dockerfile
+  local registry
+  local image
+  local common_set_values
+  local common_set_values_arr
+  local grow_repo_image
+  local shrink_repo_image
+  local e2e_values_arr
 
   trap 'trap_func' EXIT ERR
 
+  while [ "$(kubectl get pods -n kube-system -l kubernetes.io/minikube-addons=registry -o=jsonpath='{.items[*].status.phase}')" != "Running Running" ]; do
+    echo "waiting for minikube registry and proxy pod to start. Sleep 10" && sleep 10
+  done
+  # Need both. External to push and internal for pods to pull from registry in cluster
+  external_registry="$(minikube ip):5000"
+  internal_registry="$(kubectl get service -n kube-system registry --template='{{.spec.clusterIP}}')"
+
+  ### Build and Load Images ###
+
+  grow_repo_image="k8s-ephemeral-storage-grow-test:latest"
+
+  docker build  --build-arg TARGETOS=linux --build-arg TARGETARCH=amd64 -f ../DockerfileTestGrow \
+  -t "${external_registry}/${grow_repo_image}" -t "${internal_registry}/${grow_repo_image}" ../.
+
+  docker save "${external_registry}/${grow_repo_image}" > /tmp/image.tar
+  ${LOCALBIN}/crane push --insecure /tmp/image.tar "${external_registry}/${grow_repo_image}"
+  rm /tmp/image.tar
+
+
+  shrink_repo_image="k8s-ephemeral-storage-shrink-test:latest"
+
+  docker build --build-arg TARGETOS=linux --build-arg TARGETARCH=amd64 -f ../DockerfileTestShrink \
+  -t "${external_registry}/${shrink_repo_image}" -t "${internal_registry}/${shrink_repo_image}" ../.
+
+  docker save "${external_registry}/${shrink_repo_image}" > /tmp/image.tar
+  ${LOCALBIN}/crane push --insecure /tmp/image.tar "${external_registry}/${shrink_repo_image}"
+  rm /tmp/image.tar
+
   if [[ $ENV == "debug" ]]; then
     image_tag="debug-latest"
     dockerfile="DockerfileDebug"
@@ -22,53 +59,53 @@ function main() {
     dockerfile="Dockerfile"
   fi
 
-  common_set_values_arr=(
-  "image.repository=local.io/local/$DEPLOYMENT_NAME",
-  "image.tag=${image_tag}",
-  "image.imagePullPolicy=Never",
-  "deploy_type=Deployment",
-  "log_level=debug"
-  "dev.enabled=true",
-  "dev.image.imagePullPolicy=Never"
-  "metrics.adjusted_polling_rate=true"
-  "pprof=true"
-  )
+  # Main image
+  main_repo_image="${DEPLOYMENT_NAME}:${image_tag}"
+  docker build --build-arg TARGETOS=linux --build-arg TARGETARCH=amd64 -f ../${dockerfile} \
+  -t "${external_registry}/${main_repo_image}" -t "${internal_registry}/${main_repo_image}" ../.
 
-  docker build --build-arg TARGETOS=linux --build-arg TARGETARCH=amd64 -f DockerfileTestGrow -t ghcr.io/jmcgrath207/k8s-ephemeral-storage-grow-test:latest .
-  kind load docker-image -v 9 --name "${DEPLOYMENT_NAME}-cluster" --nodes "${DEPLOYMENT_NAME}-cluster-worker" "ghcr.io/jmcgrath207/k8s-ephemeral-storage-grow-test:latest"
+  docker save "${external_registry}/${main_repo_image}" > /tmp/image.tar
+  ${LOCALBIN}/crane push --insecure /tmp/image.tar "${external_registry}/${main_repo_image}"
+  rm /tmp/image.tar
 
-  docker build --build-arg TARGETOS=linux --build-arg TARGETARCH=amd64 -f DockerfileTestShrink -t ghcr.io/jmcgrath207/k8s-ephemeral-storage-shrink-test:latest .
-  kind load docker-image -v 9 --name "${DEPLOYMENT_NAME}-cluster" --nodes "${DEPLOYMENT_NAME}-cluster-worker" "ghcr.io/jmcgrath207/k8s-ephemeral-storage-shrink-test:latest"
 
-  if [[ $ENV =~ "e2e"   ]]; then
+  ### Install Chart ###
+
+  common_set_values_arr=(
+    "image.repository=${internal_registry}/$DEPLOYMENT_NAME"
+    "image.tag=${image_tag}"
+    "deploy_type=Deployment"
+    "log_level=debug"
+    "dev.enabled=true"
+    "dev.shrink.image=${internal_registry}/${shrink_repo_image}"
+    "dev.grow.image=${internal_registry}/${grow_repo_image}"
+    "metrics.adjusted_polling_rate=true"
+    "pprof=true"
+  )
+
+  if [[ $ENV =~ "e2e" ]]; then
 
     e2e_values_arr=("interval=5")
     common_set_values_arr+=("${e2e_values_arr[@]}")
   fi
 
   common_set_values=$(echo ${common_set_values_arr[*]} | sed 's/, /,/g' | sed 's/ /,/g')
 
-  docker build --build-arg TARGETOS=linux --build-arg TARGETARCH=amd64 -f ${dockerfile} -t local.io/local/$DEPLOYMENT_NAME:$image_tag .
-  kind load docker-image -v 9 --name "${DEPLOYMENT_NAME}-cluster" --nodes "${DEPLOYMENT_NAME}-cluster-worker" "local.io/local/${DEPLOYMENT_NAME}:${image_tag}"
-
-
-
-  # Install Par Chart
-  helm upgrade --install $DEPLOYMENT_NAME ./chart \
+  helm upgrade --install "${DEPLOYMENT_NAME}" ../chart \
     --set "${common_set_values}" \
     --create-namespace \
-    --namespace $DEPLOYMENT_NAME
+    --namespace "${DEPLOYMENT_NAME}"
 
-  # Patch deploy so Kind image upload to work.
+
+  # Patch deploy so minikube image upload works.
   if [[ $ENV == "debug" ]]; then
     # Disable for Debugging of Delve.
-    kubectl patch deployments.apps -n $DEPLOYMENT_NAME k8s-ephemeral-storage-metrics -p \
+    kubectl patch deployments.apps -n "${DEPLOYMENT_NAME}" k8s-ephemeral-storage-metrics -p \
       '{ "spec": {"template": { "spec":{"securityContext": null, "containers":[{"name":"metrics", "livenessProbe": null, "readinessProbe": null, "securityContext": null, "command": null, "args": null  }]}}}}'
   fi
 
-
-  # kill dangling port forwards if found.
-  # Exporter Porter
+  # Kill dangling port forwards if found.
+  # Main Exporter Port
   sudo ss -aK '( dport = :9100 or sport = :9100 )' | true
   # Prometheus Port
   sudo ss -aK '( dport = :9090 or sport = :9090 )' | true
@@ -81,31 +118,22 @@ function main() {
     printf "\n\n" && while :; do kubectl port-forward -n $DEPLOYMENT_NAME service/k8s-ephemeral-storage-metrics 9100:9100 || sleep 5; done
   ) &
 
-  # Start Prometheus Port Forward
-  (
-    sleep 10
-    printf "\n\n" && while :; do kubectl port-forward -n $DEPLOYMENT_NAME service/prometheus-operated  9090:9090 || sleep 5; done
-  ) &
-  # Start Pprof Forward
-  (
-    sleep 10
-    printf "\n\n" && while :; do kubectl port-forward -n $DEPLOYMENT_NAME service/pprof 6060:6060 || sleep 5; done
-  ) &
+  # Wait until main pod comes up
+  while [ "$(kubectl get pods -n $DEPLOYMENT_NAME -l app.kubernetes.io/name=k8s-ephemeral-storage-metrics -o=jsonpath='{.items[*].status.phase}')" != "Running" ]; do
+    echo "waiting for k8s-ephemeral-storage-metrics  pod to start. Sleep 10" && sleep 10
+  done
 
   if [[ $ENV == "debug" ]]; then
     # Background log following for manager
     (
       sleep 10
-      printf "\n\n" && while :; do kubectl logs -n $DEPLOYMENT_NAME -l app.kubernetes.io/name=k8s-ephemeral-storage-metrics  -f || sleep 5; done
+      printf "\n\n" && while :; do kubectl logs -n $DEPLOYMENT_NAME -l app.kubernetes.io/name=k8s-ephemeral-storage-metrics -f || sleep 5; done
     ) &
 
-    while [ "$(kubectl get pods -n $DEPLOYMENT_NAME -l app.kubernetes.io/name=k8s-ephemeral-storage-metrics -o=jsonpath='{.items[*].status.phase}')" != "Running" ]; do
-        echo  "waiting for k8s-ephemeral-storage-metrics  pod to start. Sleep 10" && sleep 10
-    done
     kubectl port-forward -n $DEPLOYMENT_NAME services/debug 30002:30002
 
   elif [[ $ENV == "e2e" ]]; then
-    ${LOCALBIN}/ginkgo -v -r ./tests/e2e/...
+    ${LOCALBIN}/ginkgo -v -r ../tests/e2e/...
   elif [[ $ENV == "e2e-debug" ]]; then
     sleep infinity
   else

tests/e2e/deployment_test.go

@@ -109,7 +109,7 @@ func checkPrometheus(checkSlice []string) {
 
 func WatchContainerPercentage() {
 	status := 0
-	re := regexp.MustCompile(`ephemeral_storage_container_limit_percentage{container="grow-test",node_name="ephemeral-metrics-cluster-worker".+,pod_namespace="ephemeral-metrics"}\s+(.+)`)
+	re := regexp.MustCompile(`ephemeral_storage_container_limit_percentage{container="grow-test",node_name="minikube".+,pod_namespace="ephemeral-metrics"}\s+(.+)`)
 	output := requestPrometheusString()
 	match := re.FindAllStringSubmatch(output, -1)
 	floatValue, _ := strconv.ParseFloat(match[0][1], 64)
@@ -132,7 +132,7 @@ func WatchDeadPod() {
 
 func WatchNodePercentage() {
 	status := 0
-	re := regexp.MustCompile(`ephemeral_storage_node_percentage\{node_name="ephemeral-metrics-cluster-control-plane"}\s+(.+)`)
+	re := regexp.MustCompile(`ephemeral_storage_node_percentage\{node_name="minikube"}\s+(.+)`)
 	output := requestPrometheusString()
 	match := re.FindAllStringSubmatch(output, -1)
 	floatValue, _ := strconv.ParseFloat(match[0][1], 64)
@@ -145,7 +145,7 @@ func WatchNodePercentage() {
 func WatchPollingRate(pollRateUpper float64, pollingRateLower float64, timeout time.Duration) {
 	status := 0
 	startTime := time.Now()
-	re := regexp.MustCompile(`ephemeral_storage_adjusted_polling_rate\{node_name="ephemeral-metrics-cluster-control-plane"}\s+(.+)`)
+	re := regexp.MustCompile(`ephemeral_storage_adjusted_polling_rate\{node_name="minikube"}\s+(.+)`)
 	for {
 		elapsed := time.Since(startTime)
 		if elapsed >= timeout {
@@ -211,14 +211,14 @@ func WatchEphemeralPodSize(podname string, sizeChange float64, timeout time.Dura
 var _ = ginkgo.Describe("Test Metrics\n", func() {
 
 	ginkgo.Context("Observe labels\n", func() {
-		ginkgo.Specify("\nReturn A Record IP addresses and Proxy IP address", func() {
+		ginkgo.Specify("\nMake sure all metrics are in the exporter", func() {
 			var checkSlice []string
 			checkSlice = append(checkSlice, "ephemeral_storage_pod_usage",
 				"ephemeral_storage_node_available",
 				"ephemeral_storage_node_capacity",
 				"ephemeral_storage_node_percentage",
 				"pod_name=\"k8s-ephemeral-storage", "ephemeral_storage_adjusted_polling_rate",
-				"node_name=\"ephemeral-metrics-cluster-worker", "node_name=\"ephemeral-metrics-cluster-control-plane",
+				"node_name=\"minikube",
 				"ephemeral_storage_container_limit_percentage")
 			checkPrometheus(checkSlice)
 		})