From 70be14698443783f054f3ccccdaa717ffd83948f Mon Sep 17 00:00:00 2001 From: John McGrath <8764013+jmcgrath207@users.noreply.github.com> Date: Fri, 29 Mar 2024 03:02:30 -0500 Subject: [PATCH] expanded e2e test and fixed aggressive pod polling (#70) --- Makefile | 6 + main.go | 14 +-- scripts/deploy.sh | 11 +- tests/chart/test/.helmignore | 23 ++++ tests/chart/test/Chart.yaml | 24 ++++ tests/chart/test/templates/_helpers.tpl | 62 ++++++++++ tests/chart/test/templates/deployment.yaml | 68 +++++++++++ .../chart/test/templates/serviceaccount.yaml | 13 +++ .../test/templates/tests/test-connection.yaml | 15 +++ tests/chart/test/values.yaml | 107 ++++++++++++++++++ tests/e2e/deployment_test.go | 54 ++++++--- 11 files changed, 368 insertions(+), 29 deletions(-) create mode 100644 tests/chart/test/.helmignore create mode 100644 tests/chart/test/Chart.yaml create mode 100644 tests/chart/test/templates/_helpers.tpl create mode 100644 tests/chart/test/templates/deployment.yaml create mode 100644 tests/chart/test/templates/serviceaccount.yaml create mode 100644 tests/chart/test/templates/tests/test-connection.yaml create mode 100644 tests/chart/test/values.yaml diff --git a/Makefile b/Makefile index 659978f..55b3cb9 100644 --- a/Makefile +++ b/Makefile @@ -56,6 +56,12 @@ deploy_e2e: init ginkgo crane minikube_new deploy_e2e_dirty: init ENV='e2e' ./scripts/deploy.sh +deploy_test_chart: + helm install test ./tests/chart/test -n test --create-namespace + +destroy_test_chart: + helm delete -n test test + release-docker: GITHUB_TOKEN="${GITHUB_TOKEN}" VERSION="${VERSION}" ./scripts/release-docker.sh diff --git a/main.go b/main.go index 6eda5f9..917a540 100644 --- a/main.go +++ b/main.go @@ -231,15 +231,11 @@ func podWatch() { podDataWaitGroup.Wait() stopCh := make(chan struct{}) defer close(stopCh) - sharedInformerFactory := informers.NewSharedInformerFactory(clientset, 2*time.Second) + sharedInformerFactory := informers.NewSharedInformerFactory(clientset, time.Duration(sampleInterval)*time.Second) podInformer := sharedInformerFactory.Core().V1().Pods().Informer() // Define event handlers for Pod events eventHandler := cache.ResourceEventHandlerFuncs{ - AddFunc: func(obj interface{}) { - p := obj.(*v1.Pod) - getPodData(*p) - }, UpdateFunc: func(oldObj, newObj interface{}) { p := newObj.(*v1.Pod) getPodData(*p) @@ -263,15 +259,9 @@ func podWatch() { // Start the informer to begin watching for Pod events go sharedInformerFactory.Start(stopCh) - // Use a ticker to trigger the watcher every 15 seconds - ticker := time.NewTicker(time.Duration(sampleInterval) * time.Second) - defer ticker.Stop() - - // TODO: make this more event driven instead of polling for { + time.Sleep(time.Duration(sampleInterval) * time.Second) select { - case <-ticker.C: - log.Debug().Msg("Watching podWatch for Pod events...") case <-stopCh: log.Error().Msg("Watcher podWatch stopped.") os.Exit(1) diff --git a/scripts/deploy.sh b/scripts/deploy.sh index 617d631..44f18d3 100755 --- a/scripts/deploy.sh +++ b/scripts/deploy.sh @@ -21,6 +21,7 @@ function main() { local external_registry local internal_registry local status_code + local time_tag trap 'trap_func' EXIT ERR @@ -55,7 +56,9 @@ function main() { ### Build and Load Images ### - grow_repo_image="k8s-ephemeral-storage-grow-test:latest" + time_tag=$(date +%Y%m%d%H%M%S) + + grow_repo_image="k8s-ephemeral-storage-grow-test:${time_tag}" docker build --build-arg TARGETOS=linux --build-arg TARGETARCH=amd64 -f ../DockerfileTestGrow \ -t "${external_registry}/${grow_repo_image}" -t "${internal_registry}/${grow_repo_image}" ../. @@ -64,7 +67,7 @@ function main() { "${LOCALBIN}/crane" push --insecure /tmp/image.tar "${external_registry}/${grow_repo_image}" rm /tmp/image.tar - shrink_repo_image="k8s-ephemeral-storage-shrink-test:latest" + shrink_repo_image="k8s-ephemeral-storage-shrink-test:${time_tag}" docker build --build-arg TARGETOS=linux --build-arg TARGETARCH=amd64 -f ../DockerfileTestShrink \ -t "${external_registry}/${shrink_repo_image}" -t "${internal_registry}/${shrink_repo_image}" ../. @@ -74,10 +77,10 @@ function main() { rm /tmp/image.tar if [[ $ENV == "debug" ]]; then - image_tag="debug-latest" + image_tag="debug-${time_tag}" dockerfile="DockerfileDebug" else - image_tag="latest" + image_tag="${time_tag}" dockerfile="Dockerfile" fi diff --git a/tests/chart/test/.helmignore b/tests/chart/test/.helmignore new file mode 100644 index 0000000..0e8a0eb --- /dev/null +++ b/tests/chart/test/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/tests/chart/test/Chart.yaml b/tests/chart/test/Chart.yaml new file mode 100644 index 0000000..3ebad47 --- /dev/null +++ b/tests/chart/test/Chart.yaml @@ -0,0 +1,24 @@ +apiVersion: v2 +name: test +description: A Helm chart for Kubernetes + +# A chart can be either an 'application' or a 'library' chart. +# +# Application charts are a collection of templates that can be packaged into versioned archives +# to be deployed. +# +# Library charts provide useful utilities or functions for the chart developer. They're included as +# a dependency of application charts to inject those utilities and functions into the rendering +# pipeline. Library charts do not define any templates and therefore cannot be deployed. +type: application + +# This is the chart version. This version number should be incremented each time you make changes +# to the chart and its templates, including the app version. +# Versions are expected to follow Semantic Versioning (https://semver.org/) +version: 0.1.0 + +# This is the version number of the application being deployed. This version number should be +# incremented each time you make changes to the application. Versions are not expected to +# follow Semantic Versioning. They should reflect the version the application is using. +# It is recommended to use it with quotes. +appVersion: "1.16.0" diff --git a/tests/chart/test/templates/_helpers.tpl b/tests/chart/test/templates/_helpers.tpl new file mode 100644 index 0000000..7286a2d --- /dev/null +++ b/tests/chart/test/templates/_helpers.tpl @@ -0,0 +1,62 @@ +{{/* +Expand the name of the chart. +*/}} +{{- define "test.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "test.fullname" -}} +{{- if .Values.fullnameOverride }} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- $name := default .Chart.Name .Values.nameOverride }} +{{- if contains $name .Release.Name }} +{{- .Release.Name | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} +{{- end }} +{{- end }} +{{- end }} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "test.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Common labels +*/}} +{{- define "test.labels" -}} +helm.sh/chart: {{ include "test.chart" . }} +{{ include "test.selectorLabels" . }} +{{- if .Chart.AppVersion }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end }} + +{{/* +Selector labels +*/}} +{{- define "test.selectorLabels" -}} +app.kubernetes.io/name: {{ include "test.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end }} + +{{/* +Create the name of the service account to use +*/}} +{{- define "test.serviceAccountName" -}} +{{- if .Values.serviceAccount.create }} +{{- default (include "test.fullname" .) .Values.serviceAccount.name }} +{{- else }} +{{- default "default" .Values.serviceAccount.name }} +{{- end }} +{{- end }} diff --git a/tests/chart/test/templates/deployment.yaml b/tests/chart/test/templates/deployment.yaml new file mode 100644 index 0000000..739327e --- /dev/null +++ b/tests/chart/test/templates/deployment.yaml @@ -0,0 +1,68 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "test.fullname" . }} + labels: + {{- include "test.labels" . | nindent 4 }} +spec: + {{- if not .Values.autoscaling.enabled }} + replicas: {{ .Values.replicaCount }} + {{- end }} + selector: + matchLabels: + {{- include "test.selectorLabels" . | nindent 6 }} + template: + metadata: + {{- with .Values.podAnnotations }} + annotations: + {{- toYaml . | nindent 8 }} + {{- end }} + labels: + {{- include "test.labels" . | nindent 8 }} + {{- with .Values.podLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + spec: + {{- with .Values.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} + serviceAccountName: {{ include "test.serviceAccountName" . }} + securityContext: + {{- toYaml .Values.podSecurityContext | nindent 8 }} + containers: + - name: {{ .Chart.Name }} + securityContext: + {{- toYaml .Values.securityContext | nindent 12 }} + image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}" + imagePullPolicy: {{ .Values.image.pullPolicy }} + ports: + - name: http + containerPort: {{ .Values.service.port }} + protocol: TCP + livenessProbe: + {{- toYaml .Values.livenessProbe | nindent 12 }} + readinessProbe: + {{- toYaml .Values.readinessProbe | nindent 12 }} + resources: + {{- toYaml .Values.resources | nindent 12 }} + {{- with .Values.volumeMounts }} + volumeMounts: + {{- toYaml . | nindent 12 }} + {{- end }} + {{- with .Values.volumes }} + volumes: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} diff --git a/tests/chart/test/templates/serviceaccount.yaml b/tests/chart/test/templates/serviceaccount.yaml new file mode 100644 index 0000000..0fc7571 --- /dev/null +++ b/tests/chart/test/templates/serviceaccount.yaml @@ -0,0 +1,13 @@ +{{- if .Values.serviceAccount.create -}} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "test.serviceAccountName" . }} + labels: + {{- include "test.labels" . | nindent 4 }} + {{- with .Values.serviceAccount.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +automountServiceAccountToken: {{ .Values.serviceAccount.automount }} +{{- end }} diff --git a/tests/chart/test/templates/tests/test-connection.yaml b/tests/chart/test/templates/tests/test-connection.yaml new file mode 100644 index 0000000..f78ec6d --- /dev/null +++ b/tests/chart/test/templates/tests/test-connection.yaml @@ -0,0 +1,15 @@ +apiVersion: v1 +kind: Pod +metadata: + name: "{{ include "test.fullname" . }}-test-connection" + labels: + {{- include "test.labels" . | nindent 4 }} + annotations: + "helm.sh/hook": test +spec: + containers: + - name: wget + image: busybox + command: ['wget'] + args: ['{{ include "test.fullname" . }}:{{ .Values.service.port }}'] + restartPolicy: Never diff --git a/tests/chart/test/values.yaml b/tests/chart/test/values.yaml new file mode 100644 index 0000000..5c1331b --- /dev/null +++ b/tests/chart/test/values.yaml @@ -0,0 +1,107 @@ +# Default values for test. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +replicaCount: 5 + +image: + repository: nginx + pullPolicy: IfNotPresent + # Overrides the image tag whose default is the chart appVersion. + tag: "" + +imagePullSecrets: [] +nameOverride: "" +fullnameOverride: "" + +serviceAccount: + # Specifies whether a service account should be created + create: true + # Automatically mount a ServiceAccount's API credentials? + automount: true + # Annotations to add to the service account + annotations: {} + # The name of the service account to use. + # If not set and create is true, a name is generated using the fullname template + name: "" + +podAnnotations: {} +podLabels: {} + +podSecurityContext: {} + # fsGroup: 2000 + +securityContext: {} + # capabilities: + # drop: + # - ALL + # readOnlyRootFilesystem: true + # runAsNonRoot: true + # runAsUser: 1000 + +service: + type: ClusterIP + port: 80 + +ingress: + enabled: false + className: "" + annotations: {} + # kubernetes.io/ingress.class: nginx + # kubernetes.io/tls-acme: "true" + hosts: + - host: chart-example.local + paths: + - path: / + pathType: ImplementationSpecific + tls: [] + # - secretName: chart-example-tls + # hosts: + # - chart-example.local + +resources: {} + # We usually recommend not to specify default resources and to leave this as a conscious + # choice for the user. This also increases chances charts run on environments with little + # resources, such as Minikube. If you do want to specify resources, uncomment the following + # lines, adjust them as necessary, and remove the curly braces after 'resources:'. + # limits: + # cpu: 100m + # memory: 128Mi + # requests: + # cpu: 100m + # memory: 128Mi + +livenessProbe: + httpGet: + path: / + port: http +readinessProbe: + httpGet: + path: / + port: http + +autoscaling: + enabled: false + minReplicas: 1 + maxReplicas: 100 + targetCPUUtilizationPercentage: 80 + # targetMemoryUtilizationPercentage: 80 + +# Additional volumes on the output Deployment definition. +volumes: [] +# - name: foo +# secret: +# secretName: mysecret +# optional: false + +# Additional volumeMounts on the output Deployment definition. +volumeMounts: [] +# - name: foo +# mountPath: "/etc/foo" +# readOnly: true + +nodeSelector: {} + +tolerations: [] + +affinity: {} diff --git a/tests/e2e/deployment_test.go b/tests/e2e/deployment_test.go index 58267a4..3e31801 100644 --- a/tests/e2e/deployment_test.go +++ b/tests/e2e/deployment_test.go @@ -16,6 +16,8 @@ import ( var httpClient *http.Client +type getPodSize func(podName string) float64 + func requestPrometheusString() string { var resp *http.Response @@ -190,7 +192,7 @@ func WatchPollingRate(pollRateUpper float64, pollingRateLower float64, timeout t } -func getPodSize(podName string) float64 { +func getPodUsageSize(podName string) float64 { output := requestPrometheusString() re := regexp.MustCompile(fmt.Sprintf(`ephemeral_storage_pod_usage.+pod_name="%s.+\}\s(.+)`, podName)) match := re.FindAllStringSubmatch(output, 2) @@ -198,7 +200,25 @@ func getPodSize(podName string) float64 { return currentPodSize } -func WatchEphemeralPodSize(podName string, desiredSizeChange float64, timeout time.Duration) { +func getContainerLimitPercentage(podName string) float64 { + output := requestPrometheusString() + re := regexp.MustCompile(fmt.Sprintf(`ephemeral_storage_container_limit_percentage.+pod_name="%s.+\}\s(.+)`, podName)) + match := re.FindAllStringSubmatch(output, 2) + currentPodSize, _ := strconv.ParseFloat(match[0][1], 64) + return currentPodSize +} + +func getContainerVolumeLimitPercentage(podName string) float64 { + output := requestPrometheusString() + re := regexp.MustCompile( + fmt.Sprintf(`ephemeral_storage_container_volume_limit_percentage.+container="%s",mount_path="\/cache".+\}\s(.+)`, + podName)) + match := re.FindAllStringSubmatch(output, 2) + currentPodSize, _ := strconv.ParseFloat(match[0][1], 64) + return currentPodSize +} + +func WatchEphemeralPodSize(podName string, desiredSizeChange float64, timeout time.Duration, getPodSize getPodSize) { // Watch Prometheus Metrics until the ephemeral storage shrinks or grows to a certain desiredSizeChange. var currentPodSize float64 var targetSizeChange float64 @@ -280,31 +300,39 @@ var _ = ginkgo.Describe("Test Metrics\n", func() { }) ginkgo.Context("Observe change in ephemeral_storage_pod_usage metric\n", func() { ginkgo.Specify("\nWatch Pod grow to 100000 Bytes", func() { - WatchEphemeralPodSize("grow-test", 100000, time.Second*90) + WatchEphemeralPodSize("grow-test", 100000, time.Second*90, getPodUsageSize) }) ginkgo.Specify("\nWatch Pod shrink to 100000 Bytes", func() { // Shrinking of ephemeral_storage reflects slower from Node API up to 5 minutes. - // Wait until it's reporting correctly, and start testing with the minimum of 10mb of data + // Wait until it's reporting correctly, and start testing with the minimum of 11mb of data // since the shrink container adds 12mb then decrements 12k a second. // Ex. /api/v1/nodes/minikube/proxy/stats/summary for { - currentPodSize := getPodSize("shrink-test") - if currentPodSize >= 10000000.0 { + currentPodSize := getPodUsageSize("shrink-test") + if currentPodSize >= 11000000.0 { break } time.Sleep(time.Second * 5) } - WatchEphemeralPodSize("shrink-test", 100000, time.Second*90) + WatchEphemeralPodSize("shrink-test", 100000, time.Second*180, getPodUsageSize) }) }) - // TODO: needs grow and shrink test - ginkgo.Context("Observe change in ephemeral_storage_container_volume_limit_percentage metric\n", func() { - ginkgo.GinkgoWriter.Printf("Noop Pass") + ginkgo.Context("Observe change in ephemeral_storage_container_limit_percentage metric\n", func() { + ginkgo.Specify("\nWatch Pod grow to 0.2 percent", func() { + WatchEphemeralPodSize("grow-test", 0.2, time.Second*90, getContainerLimitPercentage) + }) + ginkgo.Specify("\nWatch Pod shrink to 0.2 percent", func() { + WatchEphemeralPodSize("shrink-test", 0.2, time.Second*180, getContainerLimitPercentage) + }) }) - // TODO: needs grow and shrink test - ginkgo.Context("Observe change in ephemeral_storage_container_limit_percentage metric\n", func() { - ginkgo.GinkgoWriter.Printf("Noop Pass") + ginkgo.Context("Observe change in ephemeral_storage_container_volume_limit_percentage metric\n", func() { + ginkgo.Specify("\nWatch Pod grow to 0.2 percent", func() { + WatchEphemeralPodSize("grow-test", 0.2, time.Second*90, getContainerVolumeLimitPercentage) + }) + ginkgo.Specify("\nWatch Pod shrink to 0.2 percent", func() { + WatchEphemeralPodSize("shrink-test", 0.2, time.Second*180, getContainerVolumeLimitPercentage) + }) }) ginkgo.Context("\nMake sure percentage is not over 100", func() { ginkgo.Specify("\nTest ephemeral_storage_node_percentage", func() {