add observability and many-pods test chart

Makefile

@@ -51,19 +51,19 @@ deploy_local: init
 	./scripts/deploy.sh
 
 deploy_observability:
-	./scripts/deploy_observability.sh
+	ENV='observability' ./scripts/deploy.sh
 
 deploy_e2e: init ginkgo crane minikube_new
 	ENV='e2e' ./scripts/deploy.sh
 
 deploy_e2e_dirty: init
 	ENV='e2e' ./scripts/deploy.sh
 
-deploy_test_chart:
-	helm install test ./tests/chart/test -n test --create-namespace
+deploy_many_pods:
+	helm install many-pods ./tests/charts/many-pods -n many-pods --create-namespace
 
-destroy_test_chart:
-	helm delete -n test test
+destroy_many_pods:
+	helm delete -n many-pods many-pods
 
 release-docker:
 	GITHUB_TOKEN="${GITHUB_TOKEN}" VERSION="${VERSION}" ./scripts/release-docker.sh

scripts/deploy.sh

@@ -130,41 +130,23 @@ function main() {
     printf "\n\n" && while :; do kubectl port-forward -n $DEPLOYMENT_NAME service/k8s-ephemeral-storage-metrics 9100:9100 || kill_main_exporter_port && sleep 5; done
   ) &
 
-  # Wait until main pod comes up
-  while [ "$(kubectl get pods -n $DEPLOYMENT_NAME -l app.kubernetes.io/name=k8s-ephemeral-storage-metrics -o=jsonpath='{.items[*].status.phase}')" != "Running" ]; do
-    echo "Waiting for k8s-ephemeral-storage-metrics pod to start. Sleep 10" && sleep 10
-  done
-
-  # Wait until grow-test comes up
-  while [ "$(kubectl get pods -n $DEPLOYMENT_NAME -l name=grow-test -o=jsonpath='{.items[*].status.phase}')" != "Running" ]; do
-    echo "Waiting for grow-test pod to start. Sleep 10" && sleep 10
-  done
-
-  # Wait until shrink-test comes up
-  while [ "$(kubectl get pods -n $DEPLOYMENT_NAME -l name=shrink-test -o=jsonpath='{.items[*].status.phase}')" != "Running" ]; do
-    echo "Waiting for shrink-test pod to start. Sleep 10" && sleep 10
-  done
+  wait_pods
 
   if [[ $ENV == "debug" ]]; then
-    # Background log following for manager
-    (
-      sleep 10
-      printf "\n\n" && while :; do kubectl logs -n $DEPLOYMENT_NAME -l app.kubernetes.io/name=k8s-ephemeral-storage-metrics -f || sleep 5; done
-    ) &
-
+    follow_main_logs
     kubectl port-forward -n $DEPLOYMENT_NAME services/debug 30002:30002
 
   elif [[ $ENV == "e2e" ]]; then
     ${LOCALBIN}/ginkgo -v -r ../tests/e2e/...
   elif [[ $ENV == "e2e-debug" ]]; then
     sleep infinity
+  elif [[ $ENV == "observability" ]]; then
+    deploy_observability
+    follow_main_logs
+    sleep infinity
   else
-    # Assume make local deploy
-    # Background log following for manager
-    (
-      sleep 10
-      printf "\n\n" && while :; do kubectl logs -n $DEPLOYMENT_NAME -l app.kubernetes.io/name=k8s-ephemeral-storage-metrics -f || sleep 5; done
-    ) &
+    # Assume ENV=local deploy
+    follow_main_logs
     sleep infinity
   fi
 }

scripts/helpers.sh

@@ -10,27 +10,100 @@ function kill_main_exporter_port {
 function trap_func() {
   set +e
   {
-	helm delete $DEPLOYMENT_NAME -n $DEPLOYMENT_NAME
-	jobs -p | xargs kill -SIGSTOP
-	jobs -p | xargs kill -9
-  # Kill dangling port forwards if found.
-  kill_main_exporter_port
-	# Debug Port
-	sudo ss -aK '( dport = :30002 or sport = :30002 )'
-  # Prometheus Port
-  sudo ss -aK '( dport = :9090 or sport = :9090 )' || true
-  # Pprof Port
-  sudo ss -aK '( dport = :6060 or sport = :6060 )' || true
-	sudo ss -aK '( dport = :9000 or sport = :9000 )' || true
-	sudo ss -aK '( dport = :3000 or sport = :3000 )' || true
-	sudo ss -aK '( dport = :4040 or sport = :4040 )' || true
-	} &> /dev/null
+    helm delete $DEPLOYMENT_NAME -n $DEPLOYMENT_NAME
+    helm delete observability -n observability || true
+    jobs -p | xargs kill -SIGSTOP
+    jobs -p | xargs kill -9
+    # Kill dangling port forwards if found.
+    kill_main_exporter_port
+    # Debug Port
+    sudo ss -aK '( dport = :30002 or sport = :30002 )' || true
+    # Prometheus
+    sudo ss -aK '( dport = :9090 or sport = :9090 )' || true
+    # Pprof
+    sudo ss -aK '( dport = :6060 or sport = :6060 )' || true
+    # Prometheus
+    sudo ss -aK '( dport = :9000 or sport = :9000 )' || true
+    # Grafana
+    sudo ss -aK '( dport = :3000 or sport = :3000 )' || true
+    # Pryoscope
+    sudo ss -aK '( dport = :4040 or sport = :4040 )' || true
+  } &>/dev/null
 }
 
-
 function add_test_clients() {
   while [ "$(kubectl get pods -n $DEPLOYMENT_NAME -l k8s-app=$DEPLOYMENT_NAME -o=jsonpath='{.items[*].status.phase}')" != "Running" ]; do
-    echo  "waiting for metrics pod to start. Sleep 10" && sleep 10
-done
-	kubectl apply -f tests/resources/debug_service.yaml
+    echo "waiting for metrics pod to start. Sleep 10" && sleep 10
+  done
+  kubectl apply -f tests/resources/debug_service.yaml
+}
+
+function deploy_observability() {
+  local chart_path
+  chart_path="../tests/charts/observability"
+
+  cd "${chart_path}" || exit
+  helm dep up
+  helm upgrade --install observability . \
+    --create-namespace \
+    --namespace observability
+
+  helm repo add metrics-server https://kubernetes-sigs.github.io/metrics-server/
+  helm repo update
+  helm upgrade --install --set args={--kubelet-insecure-tls} metrics-server metrics-server/metrics-server --namespace kube-system
+
+  # Start Pprof Forward
+  (
+    sleep 10
+    printf "\n\n" && while :; do kubectl port-forward -n $DEPLOYMENT_NAME service/pprof 6060:6060 || sleep 5; done
+  ) &
+
+  # Start Prometheus Port Forward
+  (
+    sleep 10
+    printf "\n\n" && while :; do kubectl port-forward -n observability service/prometheus-operated 9090:9090 || sleep 5; done
+  ) &
+
+  # Start Grafana Port Forward
+  (
+    sleep 10
+    printf "\n\n" && while :; do kubectl port-forward -n observability service/grafana-service 3000:3000 || sleep 5; done
+  ) &
+
+  # Start Pyroscope Port Forward
+  (
+    sleep 10
+    printf "\n\n" && while :; do kubectl port-forward -n observability service/observability-pyroscope 4040:4040 || sleep 5; done
+  ) &
+}
+
+function follow_main_logs() {
+
+  # Background log following for manager
+  (
+    sleep 10
+    printf "\n\n" && while :; do kubectl logs -n $DEPLOYMENT_NAME -l app.kubernetes.io/name=k8s-ephemeral-storage-metrics -f || sleep 5; done
+  ) &
+}
+
+function wait_pods() {
+
+  # Wait until main pod comes up
+  while [ "$(kubectl get pods -n $DEPLOYMENT_NAME -l app.kubernetes.io/name=k8s-ephemeral-storage-metrics -o=jsonpath='{.items[*].status.phase}')" != "Running" ]; do
+    echo "Waiting for k8s-ephemeral-storage-metrics pod to start. Sleep 10" && sleep 10
+  done
+
+  # Wait until grow-test comes up
+  while [ "$(kubectl get pods -n $DEPLOYMENT_NAME -l name=grow-test -o=jsonpath='{.items[*].status.phase}')" != "Running" ]; do
+    echo "Waiting for grow-test pod to start. Sleep 10" && sleep 10
+  done
+
+  # Wait until shrink-test comes up
+  while [ "$(kubectl get pods -n $DEPLOYMENT_NAME -l name=shrink-test -o=jsonpath='{.items[*].status.phase}')" != "Running" ]; do
+    echo "Waiting for shrink-test pod to start. Sleep 10" && sleep 10
+  done
+
+  if [[ $ENV == "observability" ]]; then
+    echo ""
+  fi
 }

tests/chart/test/Chart.yaml → tests/charts/many-pods/Chart.yaml

@@ -1,5 +1,5 @@
 apiVersion: v2
-name: test
+name: many-pods
 description: A Helm chart for Kubernetes
 
 # A chart can be either an 'application' or a 'library' chart.

tests/charts/many-pods/README.md

@@ -0,0 +1,83 @@
+## Helm Install
+
+```bash
+helm repo add k8s-ephemeral-storage-metrics https://jmcgrath207.github.io/k8s-ephemeral-storage-metrics/chart
+helm repo update
+helm upgrade --install my-deployment k8s-ephemeral-storage-metrics/k8s-ephemeral-storage-metrics
+```
+
+## Values
+
+| Key | Type | Default | Description |
+|-----|------|---------|-------------|
+| affinity | object | `{}` |  |
+| autoscaling.enabled | bool | `false` |  |
+| autoscaling.maxReplicas | int | `100` |  |
+| autoscaling.minReplicas | int | `1` |  |
+| autoscaling.targetCPUUtilizationPercentage | int | `80` |  |
+| fullnameOverride | string | `""` |  |
+| image.pullPolicy | string | `"IfNotPresent"` |  |
+| image.repository | string | `"nginx"` |  |
+| image.tag | string | `""` |  |
+| imagePullSecrets | list | `[]` |  |
+| ingress.annotations | object | `{}` |  |
+| ingress.className | string | `""` |  |
+| ingress.enabled | bool | `false` |  |
+| ingress.hosts[0].host | string | `"chart-example.local"` |  |
+| ingress.hosts[0].paths[0].path | string | `"/"` |  |
+| ingress.hosts[0].paths[0].pathType | string | `"ImplementationSpecific"` |  |
+| ingress.tls | list | `[]` |  |
+| livenessProbe.httpGet.path | string | `"/"` |  |
+| livenessProbe.httpGet.port | string | `"http"` |  |
+| nameOverride | string | `""` |  |
+| nodeSelector | object | `{}` |  |
+| podAnnotations | object | `{}` |  |
+| podLabels | object | `{}` |  |
+| podSecurityContext | object | `{}` |  |
+| readinessProbe.httpGet.path | string | `"/"` |  |
+| readinessProbe.httpGet.port | string | `"http"` |  |
+| replicaCount | int | `5` |  |
+| resources | object | `{}` |  |
+| securityContext | object | `{}` |  |
+| service.port | int | `80` |  |
+| service.type | string | `"ClusterIP"` |  |
+| serviceAccount.annotations | object | `{}` |  |
+| serviceAccount.automount | bool | `true` |  |
+| serviceAccount.create | bool | `true` |  |
+| serviceAccount.name | string | `""` |  |
+| tolerations | list | `[]` |  |
+| volumeMounts | list | `[]` |  |
+| volumes | list | `[]` |  |
+
+## Contribute
+
+### Start Kind
+```bash
+make new_kind
+```
+
+### Run locally
+```bash
+make deploy_local
+```
+
+### Run locally with Delve Debug
+```bash
+make deploy_debug
+```
+Then connect to `localhost:30002` with [delve](https://github.com/go-delve/delve) or your IDE.
+
+### Run e2e Test
+```bash
+make deploy_e2e
+```
+
+### Debug e2e
+```bash
+make deploy_e2e_debug
+```
+Then run a debug against [deployment_test.go](tests/e2e/deployment_test.go)
+
+## License
+
+This project is licensed under the [MIT License](https://opensource.org/licenses/MIT). See the `LICENSE` file for more details.

observability/.helmignore → tests/charts/observability/.helmignore

@@ -20,4 +20,5 @@
 .idea/
 *.tmproj
 .vscode/
-README.md.gotmpl
+README.md.gotmpl
+Chart.lock