Skip to content

Commit

Permalink
add observability and many-pods test charts (#75)
Browse files Browse the repository at this point in the history
  • Loading branch information
jmcgrath207 authored Mar 31, 2024
1 parent 9787c14 commit ab9f046
Show file tree
Hide file tree
Showing 19 changed files with 343 additions and 66 deletions.
11 changes: 7 additions & 4 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -50,17 +50,20 @@ deploy_e2e_debug: init
deploy_local: init
./scripts/deploy.sh

deploy_observability:
ENV='observability' ./scripts/deploy.sh

deploy_e2e: init ginkgo crane minikube_new
ENV='e2e' ./scripts/deploy.sh

deploy_e2e_dirty: init
ENV='e2e' ./scripts/deploy.sh

deploy_test_chart:
helm install test ./tests/chart/test -n test --create-namespace
deploy_many_pods:
helm install many-pods ./tests/charts/many-pods -n many-pods --create-namespace

destroy_test_chart:
helm delete -n test test
destroy_many_pods:
helm delete -n many-pods many-pods

release-docker:
GITHUB_TOKEN="${GITHUB_TOKEN}" VERSION="${VERSION}" ./scripts/release-docker.sh
Expand Down
2 changes: 2 additions & 0 deletions chart/templates/DeployType.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,8 @@ spec:
- name: debug
containerPort: 30002
protocol: TCP
{{ end }}
{{ if .Values.pprof }}
- name: pprof
containerPort: 6060
protocol: TCP
Expand Down
18 changes: 0 additions & 18 deletions scripts/create_kind.sh
Original file line number Diff line number Diff line change
Expand Up @@ -23,21 +23,3 @@ kubectl get nodes -o wide

# Deploy Service Monitor CRD
kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.65.1/example/prometheus-operator-crd/monitoring.coreos.com_servicemonitors.yaml

if ! [[ $ENV =~ "e2e" ]]; then
# Deploy Prometheus
helm repo add prometheus-community https://prometheus-community.github.io/helm-charts
helm repo update
helm upgrade --install kube-prometheus-stack prometheus-community/kube-prometheus-stack --version 46.8.0 -n "${DEPLOYMENT_NAME}" --create-namespace \
--set grafana.enabled=false \
--set kubeApiServer.enabled=false \
--set kubernetesServiceMonitors.enabled=false \
--set kubelet.enabled=false \
--set kubeControllerManager.enabled=false \
--set kube-state-metrics.enabled=false \
--set prometheus-node-exporter.enabled=false \
--set alertmanager.enabled=false\
--set prometheus.prometheusSpec.serviceMonitorSelectorNilUsesHelmValues=false \
--set prometheus.prometheusSpec.podMonitorSelectorNilUsesHelmValues=false \
--set prometheus.prometheusSpec.probeSelectorNilUsesHelmValues=false
fi
37 changes: 8 additions & 29 deletions scripts/deploy.sh
Original file line number Diff line number Diff line change
@@ -1,7 +1,4 @@
#!/bin/bash
#
# Brief description of your script
# Copyright 2023 john

set -e

Expand Down Expand Up @@ -133,41 +130,23 @@ function main() {
printf "\n\n" && while :; do kubectl port-forward -n $DEPLOYMENT_NAME service/k8s-ephemeral-storage-metrics 9100:9100 || kill_main_exporter_port && sleep 5; done
) &

# Wait until main pod comes up
while [ "$(kubectl get pods -n $DEPLOYMENT_NAME -l app.kubernetes.io/name=k8s-ephemeral-storage-metrics -o=jsonpath='{.items[*].status.phase}')" != "Running" ]; do
echo "Waiting for k8s-ephemeral-storage-metrics pod to start. Sleep 10" && sleep 10
done

# Wait until grow-test comes up
while [ "$(kubectl get pods -n $DEPLOYMENT_NAME -l name=grow-test -o=jsonpath='{.items[*].status.phase}')" != "Running" ]; do
echo "Waiting for grow-test pod to start. Sleep 10" && sleep 10
done

# Wait until shrink-test comes up
while [ "$(kubectl get pods -n $DEPLOYMENT_NAME -l name=shrink-test -o=jsonpath='{.items[*].status.phase}')" != "Running" ]; do
echo "Waiting for shrink-test pod to start. Sleep 10" && sleep 10
done
wait_pods

if [[ $ENV == "debug" ]]; then
# Background log following for manager
(
sleep 10
printf "\n\n" && while :; do kubectl logs -n $DEPLOYMENT_NAME -l app.kubernetes.io/name=k8s-ephemeral-storage-metrics -f || sleep 5; done
) &

follow_main_logs
kubectl port-forward -n $DEPLOYMENT_NAME services/debug 30002:30002

elif [[ $ENV == "e2e" ]]; then
${LOCALBIN}/ginkgo -v -r ../tests/e2e/...
elif [[ $ENV == "e2e-debug" ]]; then
sleep infinity
elif [[ $ENV == "observability" ]]; then
deploy_observability
follow_main_logs
sleep infinity
else
# Assume make local deploy
# Background log following for manager
(
sleep 10
printf "\n\n" && while :; do kubectl logs -n $DEPLOYMENT_NAME -l app.kubernetes.io/name=k8s-ephemeral-storage-metrics -f || sleep 5; done
) &
# Assume ENV=local deploy
follow_main_logs
sleep infinity
fi
}
Expand Down
106 changes: 92 additions & 14 deletions scripts/helpers.sh
Original file line number Diff line number Diff line change
Expand Up @@ -10,22 +10,100 @@ function kill_main_exporter_port {
function trap_func() {
set +e
{
helm delete $DEPLOYMENT_NAME -n $DEPLOYMENT_NAME
jobs -p | xargs kill -SIGSTOP
jobs -p | xargs kill -9
# Kill dangling port forwards if found.
kill_main_exporter_port
# Prometheus Port
sudo ss -aK '( dport = :9090 or sport = :9090 )' || true
# Pprof Port
sudo ss -aK '( dport = :6060 or sport = :6060 )' || true
} &> /dev/null
helm delete $DEPLOYMENT_NAME -n $DEPLOYMENT_NAME
helm delete observability -n observability || true
jobs -p | xargs kill -SIGSTOP
jobs -p | xargs kill -9
# Kill dangling port forwards if found.
kill_main_exporter_port
# Debug Port
sudo ss -aK '( dport = :30002 or sport = :30002 )' || true
# Prometheus
sudo ss -aK '( dport = :9090 or sport = :9090 )' || true
# Pprof
sudo ss -aK '( dport = :6060 or sport = :6060 )' || true
# Prometheus
sudo ss -aK '( dport = :9000 or sport = :9000 )' || true
# Grafana
sudo ss -aK '( dport = :3000 or sport = :3000 )' || true
# Pryoscope
sudo ss -aK '( dport = :4040 or sport = :4040 )' || true
} &>/dev/null
}


function add_test_clients() {
while [ "$(kubectl get pods -n $DEPLOYMENT_NAME -l k8s-app=$DEPLOYMENT_NAME -o=jsonpath='{.items[*].status.phase}')" != "Running" ]; do
echo "waiting for metrics pod to start. Sleep 10" && sleep 10
done
kubectl apply -f tests/resources/debug_service.yaml
echo "waiting for metrics pod to start. Sleep 10" && sleep 10
done
kubectl apply -f tests/resources/debug_service.yaml
}

function deploy_observability() {
local chart_path
chart_path="../tests/charts/observability"

cd "${chart_path}" || exit
helm dep up
helm upgrade --install observability . \
--create-namespace \
--namespace observability

helm repo add metrics-server https://kubernetes-sigs.github.io/metrics-server/
helm repo update
helm upgrade --install --set args={--kubelet-insecure-tls} metrics-server metrics-server/metrics-server --namespace kube-system

# Start Pprof Forward
(
sleep 10
printf "\n\n" && while :; do kubectl port-forward -n $DEPLOYMENT_NAME service/pprof 6060:6060 || sleep 5; done
) &

# Start Prometheus Port Forward
(
sleep 10
printf "\n\n" && while :; do kubectl port-forward -n observability service/prometheus-operated 9090:9090 || sleep 5; done
) &

# Start Grafana Port Forward
(
sleep 10
printf "\n\n" && while :; do kubectl port-forward -n observability service/grafana-service 3000:3000 || sleep 5; done
) &

# Start Pyroscope Port Forward
(
sleep 10
printf "\n\n" && while :; do kubectl port-forward -n observability service/observability-pyroscope 4040:4040 || sleep 5; done
) &
}

function follow_main_logs() {

# Background log following for manager
(
sleep 10
printf "\n\n" && while :; do kubectl logs -n $DEPLOYMENT_NAME -l app.kubernetes.io/name=k8s-ephemeral-storage-metrics -f || sleep 5; done
) &
}

function wait_pods() {

# Wait until main pod comes up
while [ "$(kubectl get pods -n $DEPLOYMENT_NAME -l app.kubernetes.io/name=k8s-ephemeral-storage-metrics -o=jsonpath='{.items[*].status.phase}')" != "Running" ]; do
echo "Waiting for k8s-ephemeral-storage-metrics pod to start. Sleep 10" && sleep 10
done

# Wait until grow-test comes up
while [ "$(kubectl get pods -n $DEPLOYMENT_NAME -l name=grow-test -o=jsonpath='{.items[*].status.phase}')" != "Running" ]; do
echo "Waiting for grow-test pod to start. Sleep 10" && sleep 10
done

# Wait until shrink-test comes up
while [ "$(kubectl get pods -n $DEPLOYMENT_NAME -l name=shrink-test -o=jsonpath='{.items[*].status.phase}')" != "Running" ]; do
echo "Waiting for shrink-test pod to start. Sleep 10" && sleep 10
done

if [[ $ENV == "observability" ]]; then
echo ""
fi
}
File renamed without changes.
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
apiVersion: v2
name: test
name: many-pods
description: A Helm chart for Kubernetes

# A chart can be either an 'application' or a 'library' chart.
Expand Down
83 changes: 83 additions & 0 deletions tests/charts/many-pods/README.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,83 @@
## Helm Install

```bash
helm repo add k8s-ephemeral-storage-metrics https://jmcgrath207.github.io/k8s-ephemeral-storage-metrics/chart
helm repo update
helm upgrade --install my-deployment k8s-ephemeral-storage-metrics/k8s-ephemeral-storage-metrics
```

## Values

| Key | Type | Default | Description |
|-----|------|---------|-------------|
| affinity | object | `{}` | |
| autoscaling.enabled | bool | `false` | |
| autoscaling.maxReplicas | int | `100` | |
| autoscaling.minReplicas | int | `1` | |
| autoscaling.targetCPUUtilizationPercentage | int | `80` | |
| fullnameOverride | string | `""` | |
| image.pullPolicy | string | `"IfNotPresent"` | |
| image.repository | string | `"nginx"` | |
| image.tag | string | `""` | |
| imagePullSecrets | list | `[]` | |
| ingress.annotations | object | `{}` | |
| ingress.className | string | `""` | |
| ingress.enabled | bool | `false` | |
| ingress.hosts[0].host | string | `"chart-example.local"` | |
| ingress.hosts[0].paths[0].path | string | `"/"` | |
| ingress.hosts[0].paths[0].pathType | string | `"ImplementationSpecific"` | |
| ingress.tls | list | `[]` | |
| livenessProbe.httpGet.path | string | `"/"` | |
| livenessProbe.httpGet.port | string | `"http"` | |
| nameOverride | string | `""` | |
| nodeSelector | object | `{}` | |
| podAnnotations | object | `{}` | |
| podLabels | object | `{}` | |
| podSecurityContext | object | `{}` | |
| readinessProbe.httpGet.path | string | `"/"` | |
| readinessProbe.httpGet.port | string | `"http"` | |
| replicaCount | int | `5` | |
| resources | object | `{}` | |
| securityContext | object | `{}` | |
| service.port | int | `80` | |
| service.type | string | `"ClusterIP"` | |
| serviceAccount.annotations | object | `{}` | |
| serviceAccount.automount | bool | `true` | |
| serviceAccount.create | bool | `true` | |
| serviceAccount.name | string | `""` | |
| tolerations | list | `[]` | |
| volumeMounts | list | `[]` | |
| volumes | list | `[]` | |

## Contribute

### Start Kind
```bash
make new_kind
```

### Run locally
```bash
make deploy_local
```

### Run locally with Delve Debug
```bash
make deploy_debug
```
Then connect to `localhost:30002` with [delve](https://github.com/go-delve/delve) or your IDE.

### Run e2e Test
```bash
make deploy_e2e
```

### Debug e2e
```bash
make deploy_e2e_debug
```
Then run a debug against [deployment_test.go](tests/e2e/deployment_test.go)

## License

This project is licensed under the [MIT License](https://opensource.org/licenses/MIT). See the `LICENSE` file for more details.
File renamed without changes.
File renamed without changes.
File renamed without changes.
24 changes: 24 additions & 0 deletions tests/charts/observability/.helmignore
Original file line number Diff line number Diff line change
@@ -0,0 +1,24 @@
# Patterns to ignore when building packages.
# This supports shell glob matching, relative path matching, and
# negation (prefixed with !). Only one pattern per line.
.DS_Store
# Common VCS dirs
.git/
.gitignore
.bzr/
.bzrignore
.hg/
.hgignore
.svn/
# Common backup files
*.swp
*.bak
*.tmp
*~
# Various IDEs
.project
.idea/
*.tmproj
.vscode/
README.md.gotmpl
Chart.lock
20 changes: 20 additions & 0 deletions tests/charts/observability/Chart.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,20 @@
apiVersion: v2
name: observability
version: 1.0.0
appVersion: 1.0.0
kubeVersion: ">=1.21.0-0"
description: chart for observability testing

dependencies:
- name: grafana-operator
version: v5.6.2
repository: oci://ghcr.io/grafana/helm-charts
condition: grafana-operator.enabled
- name: pyroscope
version: 1.4.0
repository: https://grafana.github.io/helm-charts
condition: pyroscope.enabled
- name: kube-prometheus-stack
version: 46.8.0
repository: https://prometheus-community.github.io/helm-charts
condition: kube-prometheus-stack.enabled
15 changes: 15 additions & 0 deletions tests/charts/observability/templates/grafana-cr.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,15 @@
apiVersion: grafana.integreatly.org/v1beta1
kind: Grafana
metadata:
name: grafana
labels:
dashboards: "grafana"
spec:
config:
log:
mode: "console"
auth:
disable_login_form: "false"
security:
admin_user: admin
admin_password: admin
Loading

0 comments on commit ab9f046

Please sign in to comment.