From ab9f046f8bcf8dfc9cbf6b8699bc2a3cc823e392 Mon Sep 17 00:00:00 2001
From: John McGrath <8764013+jmcgrath207@users.noreply.github.com>
Date: Sun, 31 Mar 2024 17:27:25 -0500
Subject: [PATCH] add observability and many-pods test charts (#75)

---
 Makefile                                      |  11 +-
 chart/templates/DeployType.yaml               |   2 +
 scripts/create_kind.sh                        |  18 ---
 scripts/deploy.sh                             |  37 ++----
 scripts/helpers.sh                            | 106 +++++++++++++++---
 .../test => charts/many-pods}/.helmignore     |   0
 .../test => charts/many-pods}/Chart.yaml      |   2 +-
 tests/charts/many-pods/README.md              |  83 ++++++++++++++
 .../many-pods}/templates/_helpers.tpl         |   0
 .../many-pods}/templates/deployment.yaml      |   0
 .../many-pods}/templates/serviceaccount.yaml  |   0
 .../templates/tests/test-connection.yaml      |   0
 .../test => charts/many-pods}/values.yaml     |   0
 tests/charts/observability/.helmignore        |  24 ++++
 tests/charts/observability/Chart.yaml         |  20 ++++
 .../observability/templates/grafana-cr.yaml   |  15 +++
 .../templates/grafana-dashboard-cr.yaml       |  39 +++++++
 .../templates/grafana-datasource-cr.yaml      |  19 ++++
 tests/charts/observability/values.yaml        |  33 ++++++
 19 files changed, 343 insertions(+), 66 deletions(-)
 rename tests/{chart/test => charts/many-pods}/.helmignore (100%)
 rename tests/{chart/test => charts/many-pods}/Chart.yaml (98%)
 create mode 100644 tests/charts/many-pods/README.md
 rename tests/{chart/test => charts/many-pods}/templates/_helpers.tpl (100%)
 rename tests/{chart/test => charts/many-pods}/templates/deployment.yaml (100%)
 rename tests/{chart/test => charts/many-pods}/templates/serviceaccount.yaml (100%)
 rename tests/{chart/test => charts/many-pods}/templates/tests/test-connection.yaml (100%)
 rename tests/{chart/test => charts/many-pods}/values.yaml (100%)
 create mode 100644 tests/charts/observability/.helmignore
 create mode 100644 tests/charts/observability/Chart.yaml
 create mode 100644 tests/charts/observability/templates/grafana-cr.yaml
 create mode 100644 tests/charts/observability/templates/grafana-dashboard-cr.yaml
 create mode 100644 tests/charts/observability/templates/grafana-datasource-cr.yaml
 create mode 100644 tests/charts/observability/values.yaml

diff --git a/Makefile b/Makefile
index 894f884..3a211f2 100644
--- a/Makefile
+++ b/Makefile
@@ -50,17 +50,20 @@ deploy_e2e_debug: init
 deploy_local: init
 	./scripts/deploy.sh
 
+deploy_observability:
+	ENV='observability' ./scripts/deploy.sh
+
 deploy_e2e: init ginkgo crane minikube_new
 	ENV='e2e' ./scripts/deploy.sh
 
 deploy_e2e_dirty: init
 	ENV='e2e' ./scripts/deploy.sh
 
-deploy_test_chart:
-	helm install test ./tests/chart/test -n test --create-namespace
+deploy_many_pods:
+	helm install many-pods ./tests/charts/many-pods -n many-pods --create-namespace
 
-destroy_test_chart:
-	helm delete -n test test
+destroy_many_pods:
+	helm delete -n many-pods many-pods
 
 release-docker:
 	GITHUB_TOKEN="${GITHUB_TOKEN}" VERSION="${VERSION}" ./scripts/release-docker.sh
diff --git a/chart/templates/DeployType.yaml b/chart/templates/DeployType.yaml
index de40bc7..06443d7 100644
--- a/chart/templates/DeployType.yaml
+++ b/chart/templates/DeployType.yaml
@@ -44,6 +44,8 @@ spec:
             - name: debug
               containerPort: 30002
               protocol: TCP
+              {{ end }}
+            {{ if .Values.pprof }}
             - name: pprof
               containerPort: 6060
               protocol: TCP
diff --git a/scripts/create_kind.sh b/scripts/create_kind.sh
index d294e25..7b54a03 100755
--- a/scripts/create_kind.sh
+++ b/scripts/create_kind.sh
@@ -23,21 +23,3 @@ kubectl get nodes -o wide
 
 # Deploy Service Monitor CRD
 kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.65.1/example/prometheus-operator-crd/monitoring.coreos.com_servicemonitors.yaml
-
-if ! [[  $ENV =~ "e2e" ]]; then
-  # Deploy Prometheus
-  helm repo add prometheus-community https://prometheus-community.github.io/helm-charts
-  helm repo update
-  helm upgrade --install kube-prometheus-stack prometheus-community/kube-prometheus-stack --version 46.8.0 -n "${DEPLOYMENT_NAME}" --create-namespace \
-  --set grafana.enabled=false \
-  --set kubeApiServer.enabled=false \
-  --set kubernetesServiceMonitors.enabled=false \
-  --set kubelet.enabled=false \
-  --set kubeControllerManager.enabled=false \
-  --set kube-state-metrics.enabled=false \
-  --set prometheus-node-exporter.enabled=false \
-  --set alertmanager.enabled=false\
-  --set prometheus.prometheusSpec.serviceMonitorSelectorNilUsesHelmValues=false \
-  --set prometheus.prometheusSpec.podMonitorSelectorNilUsesHelmValues=false \
-  --set prometheus.prometheusSpec.probeSelectorNilUsesHelmValues=false
-fi
diff --git a/scripts/deploy.sh b/scripts/deploy.sh
index 44f18d3..5c38871 100755
--- a/scripts/deploy.sh
+++ b/scripts/deploy.sh
@@ -1,7 +1,4 @@
 #!/bin/bash
-#
-# Brief description of your script
-# Copyright 2023 john
 
 set -e
 
@@ -133,41 +130,23 @@ function main() {
     printf "\n\n" && while :; do kubectl port-forward -n $DEPLOYMENT_NAME service/k8s-ephemeral-storage-metrics 9100:9100 || kill_main_exporter_port && sleep 5; done
   ) &
 
-  # Wait until main pod comes up
-  while [ "$(kubectl get pods -n $DEPLOYMENT_NAME -l app.kubernetes.io/name=k8s-ephemeral-storage-metrics -o=jsonpath='{.items[*].status.phase}')" != "Running" ]; do
-    echo "Waiting for k8s-ephemeral-storage-metrics pod to start. Sleep 10" && sleep 10
-  done
-
-  # Wait until grow-test comes up
-  while [ "$(kubectl get pods -n $DEPLOYMENT_NAME -l name=grow-test -o=jsonpath='{.items[*].status.phase}')" != "Running" ]; do
-    echo "Waiting for grow-test pod to start. Sleep 10" && sleep 10
-  done
-
-  # Wait until shrink-test comes up
-  while [ "$(kubectl get pods -n $DEPLOYMENT_NAME -l name=shrink-test -o=jsonpath='{.items[*].status.phase}')" != "Running" ]; do
-    echo "Waiting for shrink-test pod to start. Sleep 10" && sleep 10
-  done
+  wait_pods
 
   if [[ $ENV == "debug" ]]; then
-    # Background log following for manager
-    (
-      sleep 10
-      printf "\n\n" && while :; do kubectl logs -n $DEPLOYMENT_NAME -l app.kubernetes.io/name=k8s-ephemeral-storage-metrics -f || sleep 5; done
-    ) &
-
+    follow_main_logs
     kubectl port-forward -n $DEPLOYMENT_NAME services/debug 30002:30002
 
   elif [[ $ENV == "e2e" ]]; then
     ${LOCALBIN}/ginkgo -v -r ../tests/e2e/...
   elif [[ $ENV == "e2e-debug" ]]; then
     sleep infinity
+  elif [[ $ENV == "observability" ]]; then
+    deploy_observability
+    follow_main_logs
+    sleep infinity
   else
-    # Assume make local deploy
-    # Background log following for manager
-    (
-      sleep 10
-      printf "\n\n" && while :; do kubectl logs -n $DEPLOYMENT_NAME -l app.kubernetes.io/name=k8s-ephemeral-storage-metrics -f || sleep 5; done
-    ) &
+    # Assume ENV=local deploy
+    follow_main_logs
     sleep infinity
   fi
 }
diff --git a/scripts/helpers.sh b/scripts/helpers.sh
index b5a4883..324cbe2 100755
--- a/scripts/helpers.sh
+++ b/scripts/helpers.sh
@@ -10,22 +10,100 @@ function kill_main_exporter_port {
 function trap_func() {
   set +e
   {
-	helm delete $DEPLOYMENT_NAME -n $DEPLOYMENT_NAME
-	jobs -p | xargs kill -SIGSTOP
-	jobs -p | xargs kill -9
-  # Kill dangling port forwards if found.
-  kill_main_exporter_port
-  # Prometheus Port
-  sudo ss -aK '( dport = :9090 or sport = :9090 )' || true
-  # Pprof Port
-  sudo ss -aK '( dport = :6060 or sport = :6060 )' || true
-	} &> /dev/null
+    helm delete $DEPLOYMENT_NAME -n $DEPLOYMENT_NAME
+    helm delete observability -n observability || true
+    jobs -p | xargs kill -SIGSTOP
+    jobs -p | xargs kill -9
+    # Kill dangling port forwards if found.
+    kill_main_exporter_port
+    # Debug Port
+    sudo ss -aK '( dport = :30002 or sport = :30002 )' || true
+    # Prometheus
+    sudo ss -aK '( dport = :9090 or sport = :9090 )' || true
+    # Pprof
+    sudo ss -aK '( dport = :6060 or sport = :6060 )' || true
+    # Prometheus
+    sudo ss -aK '( dport = :9000 or sport = :9000 )' || true
+    # Grafana
+    sudo ss -aK '( dport = :3000 or sport = :3000 )' || true
+    # Pryoscope
+    sudo ss -aK '( dport = :4040 or sport = :4040 )' || true
+  } &>/dev/null
 }
 
-
 function add_test_clients() {
   while [ "$(kubectl get pods -n $DEPLOYMENT_NAME -l k8s-app=$DEPLOYMENT_NAME -o=jsonpath='{.items[*].status.phase}')" != "Running" ]; do
-    echo  "waiting for metrics pod to start. Sleep 10" && sleep 10
-done
-	kubectl apply -f tests/resources/debug_service.yaml
+    echo "waiting for metrics pod to start. Sleep 10" && sleep 10
+  done
+  kubectl apply -f tests/resources/debug_service.yaml
+}
+
+function deploy_observability() {
+  local chart_path
+  chart_path="../tests/charts/observability"
+
+  cd "${chart_path}" || exit
+  helm dep up
+  helm upgrade --install observability . \
+    --create-namespace \
+    --namespace observability
+
+  helm repo add metrics-server https://kubernetes-sigs.github.io/metrics-server/
+  helm repo update
+  helm upgrade --install --set args={--kubelet-insecure-tls} metrics-server metrics-server/metrics-server --namespace kube-system
+
+  # Start Pprof Forward
+  (
+    sleep 10
+    printf "\n\n" && while :; do kubectl port-forward -n $DEPLOYMENT_NAME service/pprof 6060:6060 || sleep 5; done
+  ) &
+
+  # Start Prometheus Port Forward
+  (
+    sleep 10
+    printf "\n\n" && while :; do kubectl port-forward -n observability service/prometheus-operated 9090:9090 || sleep 5; done
+  ) &
+
+  # Start Grafana Port Forward
+  (
+    sleep 10
+    printf "\n\n" && while :; do kubectl port-forward -n observability service/grafana-service 3000:3000 || sleep 5; done
+  ) &
+
+  # Start Pyroscope Port Forward
+  (
+    sleep 10
+    printf "\n\n" && while :; do kubectl port-forward -n observability service/observability-pyroscope 4040:4040 || sleep 5; done
+  ) &
+}
+
+function follow_main_logs() {
+
+  # Background log following for manager
+  (
+    sleep 10
+    printf "\n\n" && while :; do kubectl logs -n $DEPLOYMENT_NAME -l app.kubernetes.io/name=k8s-ephemeral-storage-metrics -f || sleep 5; done
+  ) &
+}
+
+function wait_pods() {
+
+  # Wait until main pod comes up
+  while [ "$(kubectl get pods -n $DEPLOYMENT_NAME -l app.kubernetes.io/name=k8s-ephemeral-storage-metrics -o=jsonpath='{.items[*].status.phase}')" != "Running" ]; do
+    echo "Waiting for k8s-ephemeral-storage-metrics pod to start. Sleep 10" && sleep 10
+  done
+
+  # Wait until grow-test comes up
+  while [ "$(kubectl get pods -n $DEPLOYMENT_NAME -l name=grow-test -o=jsonpath='{.items[*].status.phase}')" != "Running" ]; do
+    echo "Waiting for grow-test pod to start. Sleep 10" && sleep 10
+  done
+
+  # Wait until shrink-test comes up
+  while [ "$(kubectl get pods -n $DEPLOYMENT_NAME -l name=shrink-test -o=jsonpath='{.items[*].status.phase}')" != "Running" ]; do
+    echo "Waiting for shrink-test pod to start. Sleep 10" && sleep 10
+  done
+
+  if [[ $ENV == "observability" ]]; then
+    echo ""
+  fi
 }
diff --git a/tests/chart/test/.helmignore b/tests/charts/many-pods/.helmignore
similarity index 100%
rename from tests/chart/test/.helmignore
rename to tests/charts/many-pods/.helmignore
diff --git a/tests/chart/test/Chart.yaml b/tests/charts/many-pods/Chart.yaml
similarity index 98%
rename from tests/chart/test/Chart.yaml
rename to tests/charts/many-pods/Chart.yaml
index 3ebad47..92971fd 100644
--- a/tests/chart/test/Chart.yaml
+++ b/tests/charts/many-pods/Chart.yaml
@@ -1,5 +1,5 @@
 apiVersion: v2
-name: test
+name: many-pods
 description: A Helm chart for Kubernetes
 
 # A chart can be either an 'application' or a 'library' chart.
diff --git a/tests/charts/many-pods/README.md b/tests/charts/many-pods/README.md
new file mode 100644
index 0000000..ad50d35
--- /dev/null
+++ b/tests/charts/many-pods/README.md
@@ -0,0 +1,83 @@
+## Helm Install
+
+```bash
+helm repo add k8s-ephemeral-storage-metrics https://jmcgrath207.github.io/k8s-ephemeral-storage-metrics/chart
+helm repo update
+helm upgrade --install my-deployment k8s-ephemeral-storage-metrics/k8s-ephemeral-storage-metrics
+```
+
+## Values
+
+| Key | Type | Default | Description |
+|-----|------|---------|-------------|
+| affinity | object | `{}` |  |
+| autoscaling.enabled | bool | `false` |  |
+| autoscaling.maxReplicas | int | `100` |  |
+| autoscaling.minReplicas | int | `1` |  |
+| autoscaling.targetCPUUtilizationPercentage | int | `80` |  |
+| fullnameOverride | string | `""` |  |
+| image.pullPolicy | string | `"IfNotPresent"` |  |
+| image.repository | string | `"nginx"` |  |
+| image.tag | string | `""` |  |
+| imagePullSecrets | list | `[]` |  |
+| ingress.annotations | object | `{}` |  |
+| ingress.className | string | `""` |  |
+| ingress.enabled | bool | `false` |  |
+| ingress.hosts[0].host | string | `"chart-example.local"` |  |
+| ingress.hosts[0].paths[0].path | string | `"/"` |  |
+| ingress.hosts[0].paths[0].pathType | string | `"ImplementationSpecific"` |  |
+| ingress.tls | list | `[]` |  |
+| livenessProbe.httpGet.path | string | `"/"` |  |
+| livenessProbe.httpGet.port | string | `"http"` |  |
+| nameOverride | string | `""` |  |
+| nodeSelector | object | `{}` |  |
+| podAnnotations | object | `{}` |  |
+| podLabels | object | `{}` |  |
+| podSecurityContext | object | `{}` |  |
+| readinessProbe.httpGet.path | string | `"/"` |  |
+| readinessProbe.httpGet.port | string | `"http"` |  |
+| replicaCount | int | `5` |  |
+| resources | object | `{}` |  |
+| securityContext | object | `{}` |  |
+| service.port | int | `80` |  |
+| service.type | string | `"ClusterIP"` |  |
+| serviceAccount.annotations | object | `{}` |  |
+| serviceAccount.automount | bool | `true` |  |
+| serviceAccount.create | bool | `true` |  |
+| serviceAccount.name | string | `""` |  |
+| tolerations | list | `[]` |  |
+| volumeMounts | list | `[]` |  |
+| volumes | list | `[]` |  |
+
+## Contribute
+
+### Start Kind
+```bash
+make new_kind
+```
+
+### Run locally
+```bash
+make deploy_local
+```
+
+### Run locally with Delve Debug
+```bash
+make deploy_debug
+```
+Then connect to `localhost:30002` with [delve](https://github.com/go-delve/delve) or your IDE.
+
+### Run e2e Test
+```bash
+make deploy_e2e
+```
+
+### Debug e2e
+```bash
+make deploy_e2e_debug
+```
+Then run a debug against [deployment_test.go](tests/e2e/deployment_test.go)
+
+## License
+
+This project is licensed under the [MIT License](https://opensource.org/licenses/MIT). See the `LICENSE` file for more details.
\ No newline at end of file
diff --git a/tests/chart/test/templates/_helpers.tpl b/tests/charts/many-pods/templates/_helpers.tpl
similarity index 100%
rename from tests/chart/test/templates/_helpers.tpl
rename to tests/charts/many-pods/templates/_helpers.tpl
diff --git a/tests/chart/test/templates/deployment.yaml b/tests/charts/many-pods/templates/deployment.yaml
similarity index 100%
rename from tests/chart/test/templates/deployment.yaml
rename to tests/charts/many-pods/templates/deployment.yaml
diff --git a/tests/chart/test/templates/serviceaccount.yaml b/tests/charts/many-pods/templates/serviceaccount.yaml
similarity index 100%
rename from tests/chart/test/templates/serviceaccount.yaml
rename to tests/charts/many-pods/templates/serviceaccount.yaml
diff --git a/tests/chart/test/templates/tests/test-connection.yaml b/tests/charts/many-pods/templates/tests/test-connection.yaml
similarity index 100%
rename from tests/chart/test/templates/tests/test-connection.yaml
rename to tests/charts/many-pods/templates/tests/test-connection.yaml
diff --git a/tests/chart/test/values.yaml b/tests/charts/many-pods/values.yaml
similarity index 100%
rename from tests/chart/test/values.yaml
rename to tests/charts/many-pods/values.yaml
diff --git a/tests/charts/observability/.helmignore b/tests/charts/observability/.helmignore
new file mode 100644
index 0000000..eb928d6
--- /dev/null
+++ b/tests/charts/observability/.helmignore
@@ -0,0 +1,24 @@
+# Patterns to ignore when building packages.
+# This supports shell glob matching, relative path matching, and
+# negation (prefixed with !). Only one pattern per line.
+.DS_Store
+# Common VCS dirs
+.git/
+.gitignore
+.bzr/
+.bzrignore
+.hg/
+.hgignore
+.svn/
+# Common backup files
+*.swp
+*.bak
+*.tmp
+*~
+# Various IDEs
+.project
+.idea/
+*.tmproj
+.vscode/
+README.md.gotmpl
+Chart.lock
\ No newline at end of file
diff --git a/tests/charts/observability/Chart.yaml b/tests/charts/observability/Chart.yaml
new file mode 100644
index 0000000..513a11a
--- /dev/null
+++ b/tests/charts/observability/Chart.yaml
@@ -0,0 +1,20 @@
+apiVersion: v2
+name: observability
+version: 1.0.0
+appVersion: 1.0.0
+kubeVersion: ">=1.21.0-0"
+description: chart for observability testing
+
+dependencies:
+  - name: grafana-operator
+    version: v5.6.2
+    repository: oci://ghcr.io/grafana/helm-charts
+    condition: grafana-operator.enabled
+  - name: pyroscope
+    version: 1.4.0
+    repository: https://grafana.github.io/helm-charts
+    condition: pyroscope.enabled
+  - name: kube-prometheus-stack
+    version: 46.8.0
+    repository: https://prometheus-community.github.io/helm-charts
+    condition: kube-prometheus-stack.enabled
diff --git a/tests/charts/observability/templates/grafana-cr.yaml b/tests/charts/observability/templates/grafana-cr.yaml
new file mode 100644
index 0000000..c41519a
--- /dev/null
+++ b/tests/charts/observability/templates/grafana-cr.yaml
@@ -0,0 +1,15 @@
+apiVersion: grafana.integreatly.org/v1beta1
+kind: Grafana
+metadata:
+  name: grafana
+  labels:
+    dashboards: "grafana"
+spec:
+  config:
+    log:
+      mode: "console"
+    auth:
+      disable_login_form: "false"
+    security:
+      admin_user: admin
+      admin_password: admin
\ No newline at end of file
diff --git a/tests/charts/observability/templates/grafana-dashboard-cr.yaml b/tests/charts/observability/templates/grafana-dashboard-cr.yaml
new file mode 100644
index 0000000..691133e
--- /dev/null
+++ b/tests/charts/observability/templates/grafana-dashboard-cr.yaml
@@ -0,0 +1,39 @@
+apiVersion: grafana.integreatly.org/v1beta1
+kind: GrafanaDashboard
+metadata:
+  name: grafanadashboard-sample
+spec:
+  resyncPeriod: 30s
+  instanceSelector:
+    matchLabels:
+      dashboards: "grafana"
+  json: >
+    {
+      "id": null,
+      "title": "Simple Dashboard",
+      "tags": [],
+      "style": "dark",
+      "timezone": "browser",
+      "editable": true,
+      "hideControls": false,
+      "graphTooltip": 1,
+      "panels": [],
+      "time": {
+        "from": "now-6h",
+        "to": "now"
+      },
+      "timepicker": {
+        "time_options": [],
+        "refresh_intervals": []
+      },
+      "templating": {
+        "list": []
+      },
+      "annotations": {
+        "list": []
+      },
+      "refresh": "5s",
+      "schemaVersion": 17,
+      "version": 0,
+      "links": []
+    }
\ No newline at end of file
diff --git a/tests/charts/observability/templates/grafana-datasource-cr.yaml b/tests/charts/observability/templates/grafana-datasource-cr.yaml
new file mode 100644
index 0000000..e2aa2ef
--- /dev/null
+++ b/tests/charts/observability/templates/grafana-datasource-cr.yaml
@@ -0,0 +1,19 @@
+
+apiVersion: grafana.integreatly.org/v1beta1
+kind: GrafanaDatasource
+metadata:
+  name: grafanadatasource-sample1
+spec:
+  instanceSelector:
+    matchLabels:
+      dashboards: "grafana"
+  datasource:
+    name: prom
+    type: prometheus
+    access: proxy
+    url: http://prometheus-operated:9090
+    isDefault: true
+    jsonData:
+      'tlsSkipVerify': true
+      'timeInterval': "5s"
+    editable: true
\ No newline at end of file
diff --git a/tests/charts/observability/values.yaml b/tests/charts/observability/values.yaml
new file mode 100644
index 0000000..52f2655
--- /dev/null
+++ b/tests/charts/observability/values.yaml
@@ -0,0 +1,33 @@
+# Chart used for observability testing.
+
+
+# https://grafana.github.io/grafana-operator/docs/examples/basic/readme/
+grafana-operator:
+  enabled: true
+
+pyroscope:
+  enabled: true
+
+kube-prometheus-stack:
+  enabled: true
+  grafana:
+    enabled: false
+  kubeApiServer:
+    enabled: false
+  kubernetesServiceMonitors:
+    enabled: false
+  kubelet:
+    enabled: false
+  kubeControllerManager:
+    enabled: false
+  kube-state-metrics:
+    enabled: false
+  prometheus-node-exporter:
+    enabled: false
+  alertmanager:
+    enabled: false
+  prometheus:
+    prometheusSpec:
+      serviceMonitorSelectorNilUsesHelmValues: false
+      podMonitorSelectorNilUsesHelmValues: false
+      probeSelectorNilUsesHelmValues: false