From 15eb95b38909c2e2dde77939756c79d829e89f35 Mon Sep 17 00:00:00 2001
From: Tom Martensen <tmartens@redhat.com>
Date: Fri, 29 Sep 2023 08:21:54 +0200
Subject: [PATCH 01/19] wip: secrets + Helm charts

---
 Makefile                                      | 177 +++---------------
 chart/infra-server/static/flavors.yaml        |   2 +-
 chart/infra-server/static/oidc.yaml           |   2 +-
 chart/infra-server/templates/certificate.yaml |   2 +-
 .../templates/demo-certifier.yaml             |   2 +-
 chart/infra-server/templates/deployment.yaml  |   4 +-
 chart/infra-server/templates/ingress.yaml     |   2 +-
 chart/infra-server/templates/namespace.yaml   |   5 -
 chart/infra-server/templates/secrets.yaml     |   4 +-
 scripts/deploy/helm.sh                        |  58 ++++++
 10 files changed, 89 insertions(+), 169 deletions(-)
 delete mode 100644 chart/infra-server/templates/namespace.yaml
 create mode 100755 scripts/deploy/helm.sh

diff --git a/Makefile b/Makefile
index 3010b627a..a2c03f9dc 100644
--- a/Makefile
+++ b/Makefile
@@ -1,6 +1,8 @@
 SHELL := /usr/bin/env bash
 export GO111MODULE=on
 
+# Add check that cluster name is not infra-prod or deployment
+
 .PHONY: all
 all: image
 
@@ -14,6 +16,13 @@ else
 	VERSION := $(TAG)
 endif
 
+LOCAL_VALUES_FILE=chart/infra-server/configuration/infra-values-${ENVIRONMENT}.yaml
+LOCAL_COMBINED_VALUES_FILE=chart/infra-server/configuration/infra-values-from-files-${ENVIRONMENT}.yaml
+
+ifeq '$(SECRET_VERSION)' ''
+SECRET_VERSION := latest
+endif
+
 .PHONY: tag
 tag:
 	@echo $(VERSION)
@@ -194,46 +203,19 @@ proto-generated-srcs: protoc-tools
 		--swagger_out=logtostderr=true:$(PROTO_OUTPUT_DIR) \
 		$(PROTO_FILES)
 
+
 ##########
 ## Kube ##
 ##########
-dev_context = gke_stackrox-infra_us-west2_infra-development
-prod_context = gke_stackrox-infra_us-west2_infra-production
-this_context = $(shell kubectl config current-context)
 
 ## Meta
 .PHONY: pre-check
 pre-check:
-ifndef DEPLOYMENT
-	$(error DEPLOYMENT is undefined)
-endif
 ifndef ENVIRONMENT
 	$(error ENVIRONMENT is undefined)
-endif
-	@if [[ "${DEPLOYMENT}" == "local" && ("${this_context}" == "${dev_context}" || "${this_context}" == "${prod_context}") ]]; then \
-		echo "Your kube context is not set to a local infra!"; \
-		exit 1; \
-	fi
-	@if [[ "${DEPLOYMENT}" == "development" && "${this_context}" != "${dev_context}" ]]; then \
-		echo -e "Your kube context is not set to development infra:\n\tkubectl config use-context ${dev_context}"; \
-		exit 1; \
-	fi
-	@if [[ "${DEPLOYMENT}" == "production" && "${this_context}" != "${prod_context}" ]]; then \
-		echo -e "Your kube context is not set to production infra:\n\tkubectl config use-context ${prod_context}"; \
-		exit 1; \
-	fi
-
-.PHONY: setup-kc
-setup-kc: pre-check
-	$(info DEPLOYMENT: ${DEPLOYMENT}, ENVIRONMENT: ${ENVIRONMENT})
-ifeq ($(DEPLOYMENT), local)
-kc=kubectl
-else ifeq ($(DEPLOYMENT), development)
-kc=kubectl --context gke_stackrox-infra_us-west2_infra-development
-else ifeq ($(DEPLOYMENT), production)
-kc=kubectl --context gke_stackrox-infra_us-west2_infra-production
 endif
 
+# TODO: this needs to be re-done for GCP secrets manger
 ## Configuration
 .PHONY: configuration-download
 configuration-download:
@@ -255,95 +237,11 @@ configuration-upload:
 create-consolidated-values:
 	@./scripts/create-consolidated-values.sh
 
-## Render
-.PHONY: clean-render
-clean-render:
-	@rm -rf chart-rendered
-
-.PHONY: render
-render: pre-check clean-render create-consolidated-values
-	@if [[ ! -e chart/infra-server/configuration ]]; then \
-		echo chart/infra-server/configuration is absent. Try:; \
-		echo make configuration-download; \
-		exit 1; \
-	fi
-	@mkdir -p chart-rendered
-	helm template chart/infra-server \
-	    --output-dir chart-rendered \
-		--set deployment="${DEPLOYMENT}" \
-		--set tag="$(VERSION)" \
-		--values chart/infra-server/configuration/${ENVIRONMENT}-values.yaml \
-		--values chart/infra-server/configuration/${ENVIRONMENT}-values-from-files.yaml
-
-.PHONY: render-local
-render-local:
-	DEPLOYMENT=local ENVIRONMENT=development make render
-
-.PHONY: render-development
-render-development:
-	DEPLOYMENT=development ENVIRONMENT=development make render
-
-.PHONY: render-production
-render-production:
-	DEPLOYMENT=production ENVIRONMENT=production make render
-
 ## Common install targets
 bounce-infra-pods: setup-kc
 	$(kc) -n infra rollout restart deploy/infra-server-deployment
 	$(kc) -n infra rollout status deploy/infra-server-deployment --watch --timeout=3m
 
-install-common: setup-kc
-	@if ! $(kc) get ns argo 2> /dev/null; then \
-		$(kc) create namespace argo; \
-	fi
-	$(kc) apply -n argo -f https://github.com/argoproj/argo-workflows/releases/download/v3.3.9/install.yaml;
-	@if ! $(kc) get ns infra 2> /dev/null; then \
-		$(kc) apply -f chart/infra-server/templates/namespace.yaml; \
-	fi
-
-## Install (without write)
-install: setup-kc install-common
-	gsutil cat gs://infra-configuration/latest/configuration/$(ENVIRONMENT)-values.yaml \
-               gs://infra-configuration/latest/configuration/$(ENVIRONMENT)-values-from-files.yaml | \
-	helm template chart/infra-server \
-		--set deployment="$(DEPLOYMENT)" \
-		--set tag="$(VERSION)" \
-		--values - | \
-	$(kc) apply -R \
-	    -f -
-	@sleep 10
-	make bounce-infra-pods
-
-.PHONY: install-local
-install-local:
-	DEPLOYMENT=local ENVIRONMENT=development make install-common install
-
-.PHONY: install-development
-install-development:
-	DEPLOYMENT=development ENVIRONMENT=development make install-common install
-
-.PHONY: install-production
-install-production:
-	DEPLOYMENT=production ENVIRONMENT=production make install-common install
-
-## Install (with rendered)
-.PHONY: install-with-rendered
-install-with-rendered: setup-kc install-common
-	$(kc) apply -R \
-	    -f chart-rendered/infra-server
-
-.PHONY: install-local
-install-local-with-rendered:
-	DEPLOYMENT=local ENVIRONMENT=development make render install-with-rendered
-
-.PHONY: install-development
-install-development-with-rendered:
-	DEPLOYMENT=development ENVIRONMENT=development make render install-with-rendered
-
-.PHONY: install-production
-install-production-with-rendered:
-	DEPLOYMENT=production ENVIRONMENT=production make render install-with-rendered
-
 ## Diff
 .PHONY: diff
 diff: setup-kc
@@ -355,48 +253,6 @@ diff: setup-kc
 		--values - | \
 	$(kc) diff -R -f -
 
-.PHONY: diff-local
-diff-local:
-	DEPLOYMENT=local ENVIRONMENT=development make diff
-
-.PHONY: diff-development
-diff-development:
-	DEPLOYMENT=development ENVIRONMENT=development make diff
-
-.PHONY: diff-production
-diff-production:
-	DEPLOYMENT=production ENVIRONMENT=production make diff
-
-## Clean
-.PHONY: clean-infra
-clean-infra:
-	$(kc) delete namespace infra || true
-
-.PHONY: clean-argo
-clean-argo:
-	$(kc) delete namespace argo || true
-
-.PHONY: clean-local
-clean-local:
-	DEPLOYMENT=local ENVIRONMENT=development make setup-kc clean-infra clean-argo
-
-.PHONY: clean-development
-clean-development:
-	DEPLOYMENT=development ENVIRONMENT=development make setup-kc clean-infra
-
-## Deploy
-.PHONY: deploy-local
-deploy-local: push install-local
-	@echo "All done!"
-
-.PHONY: deploy-development
-deploy-development: push install-development
-	@echo "All done!"
-
-.PHONY: deploy-production
-deploy-production: push install-production
-	@echo "All done!"
-
 ##########
 ## Misc ##
 ##########
@@ -413,3 +269,14 @@ update-version:
 	@perl -p -i -e 's#image: (${image_regex}):(.*)#image: \1:${image_version}#g' \
 		./chart/infra-server/static/*.yaml
 	@git diff --name-status ./chart/infra-server/static/*.yaml
+
+.PHONY: install-argo
+install-argo:
+	helm repo add argo https://argoproj.github.io/argo-helm
+	helm upgrade \
+		argo-workflows \
+		argo/argo-workflows \
+		--version 0.16.9 \
+		--install \
+		--create-namespace \
+		--namespace argo
diff --git a/chart/infra-server/static/flavors.yaml b/chart/infra-server/static/flavors.yaml
index d38fe4a55..ac4624af9 100644
--- a/chart/infra-server/static/flavors.yaml
+++ b/chart/infra-server/static/flavors.yaml
@@ -735,7 +735,7 @@
     - name: data
       description: An archive that includes ssh keys to connect to cluster nodes
 
-{{ if ne .Values.deployment "production" -}}
+{{ if ne .Values.environment "production" -}}
 #################
 #  URL artifact #
 #################
diff --git a/chart/infra-server/static/oidc.yaml b/chart/infra-server/static/oidc.yaml
index 701ad62ba..004dee7a9 100644
--- a/chart/infra-server/static/oidc.yaml
+++ b/chart/infra-server/static/oidc.yaml
@@ -2,7 +2,7 @@ issuer: "https://auth.stage.redhat.com/auth/realms/EmployeeIDP"
 clientID: "rhacs-dev-infra"
 clientSecret: "clientSecret-example"
 sessionSecret: "sessionSecret-example"
-endpoint: {{ if eq .Values.deployment "local" }}localhost:8443{{ else }}dev.infra.rox.systems{{ end }}
+endpoint: {{ if eq .Values.environment "local" }}localhost:8443{{ else }}dev.infra.rox.systems{{ end }}
 accessTokenClaims:
   - value: 'https://auth.stage.redhat.com/auth/realms/EmployeeIDP'
     op: 'eq'
diff --git a/chart/infra-server/templates/certificate.yaml b/chart/infra-server/templates/certificate.yaml
index 587b51cd7..6ffbefe27 100644
--- a/chart/infra-server/templates/certificate.yaml
+++ b/chart/infra-server/templates/certificate.yaml
@@ -1,4 +1,4 @@
-{{ if ne .Values.deployment "local" -}}
+{{ if ne .Values.environment "local" -}}
 ---
 
 apiVersion: networking.gke.io/v1
diff --git a/chart/infra-server/templates/demo-certifier.yaml b/chart/infra-server/templates/demo-certifier.yaml
index 03aec7574..df9d66995 100644
--- a/chart/infra-server/templates/demo-certifier.yaml
+++ b/chart/infra-server/templates/demo-certifier.yaml
@@ -1,4 +1,4 @@
-{{ if eq .Values.deployment "production" -}}
+{{ if eq .Values.environment "production" -}}
 
 apiVersion: v1
 kind: Secret
diff --git a/chart/infra-server/templates/deployment.yaml b/chart/infra-server/templates/deployment.yaml
index 70da8484b..0f1f840b3 100644
--- a/chart/infra-server/templates/deployment.yaml
+++ b/chart/infra-server/templates/deployment.yaml
@@ -30,7 +30,7 @@ spec:
             - name: GOOGLE_APPLICATION_CREDENTIALS
               value: /configuration/google-credentials.json
             - name: TEST_MODE
-              value: "{{ eq .Values.deployment "local" }}"
+              value: "{{ eq .Values.environment "local" }}"
           readinessProbe:
             httpGet:
               path: /
@@ -56,7 +56,7 @@ spec:
           secret:
             secretName: infra-server-secrets
 
-{{ if eq .Values.deployment "local" }}
+{{ if eq .Values.environment "development" }}
 
       imagePullSecrets:
         - name: infra-image-registry-pull-secret
diff --git a/chart/infra-server/templates/ingress.yaml b/chart/infra-server/templates/ingress.yaml
index 856091131..c6af9442a 100644
--- a/chart/infra-server/templates/ingress.yaml
+++ b/chart/infra-server/templates/ingress.yaml
@@ -1,4 +1,4 @@
-{{ if ne .Values.deployment "local" -}}
+{{ if ne .Values.environment "local" -}}
 apiVersion: networking.k8s.io/v1
 kind: Ingress
 
diff --git a/chart/infra-server/templates/namespace.yaml b/chart/infra-server/templates/namespace.yaml
deleted file mode 100644
index 70432fc7f..000000000
--- a/chart/infra-server/templates/namespace.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-apiVersion: v1
-kind: Namespace
-
-metadata:
-  name: infra
diff --git a/chart/infra-server/templates/secrets.yaml b/chart/infra-server/templates/secrets.yaml
index 1252d070b..6f3f74ad8 100644
--- a/chart/infra-server/templates/secrets.yaml
+++ b/chart/infra-server/templates/secrets.yaml
@@ -70,7 +70,7 @@ data:
   workflow-osd-gcp.yaml: |-
     {{- .Files.Get "static/workflow-osd-gcp.yaml" | b64enc | nindent 4 }}
 
-{{ if ne .Values.deployment "production" }}
+{{ if ne .Values.environment "production" }}
 
   test-url-artifact.yaml: |-
     {{- .Files.Get "static/test-url-artifact.yaml" | b64enc | nindent 4 }}
@@ -80,7 +80,7 @@ data:
 
 {{ end }}
 
-{{ if eq .Values.deployment "local" }}
+{{ if eq .Values.environment "development" }}
 ---
 
 apiVersion: v1
diff --git a/scripts/deploy/helm.sh b/scripts/deploy/helm.sh
new file mode 100755
index 000000000..409837c3d
--- /dev/null
+++ b/scripts/deploy/helm.sh
@@ -0,0 +1,58 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+TASK="$1"
+TAG="$2"
+ENVIRONMENT="$3"
+SECRET_VERSION="$4"
+
+check_not_empty() {
+    for V in "$@"; do
+        typeset -n VAR="$V"
+        if [ -z "${VAR:-}" ]; then
+            echo "ERROR: Variable $V is not set or empty"
+            exit 1
+        fi
+    done
+}
+
+template() {
+    helm template \
+		infra-server \
+		chart/infra-server \
+		--debug \
+		--namespace infra \
+		--set tag="${TAG}" \
+        --set environment="${ENVIRONMENT}" \
+		--values - \
+    < <(gcloud secrets versions access "${SECRET_VERSION}" \
+        --secret "infra-values-${ENVIRONMENT}" \
+        --project stackrox-infra \
+    && gcloud secrets versions access "${SECRET_VERSION}" \
+        --secret "infra-values-from-files-${ENVIRONMENT}" \
+        --project stackrox-infra \
+    )
+}
+
+deploy() {
+    helm upgrade \
+        infra-server \
+        chart/infra-server \
+        --install \
+        --create-namespace \
+        --namespace infra \
+        --set tag="${TAG}" \
+        --set environment="${ENVIRONMENT}" \
+        --values - \
+    < <(gcloud secrets versions access "${SECRET_VERSION}" \
+        --secret "infra-values-${ENVIRONMENT}" \
+        --project stackrox-infra \
+    && gcloud secrets versions access "${SECRET_VERSION}" \
+        --secret "infra-values-from-files-${ENVIRONMENT}" \
+        --project stackrox-infra \
+    )
+}
+
+check_not_empty TAG ENVIRONMENT SECRET_VERSION
+eval "$TASK"

From 08468479f54731338c7146f640e540ab9cd2e64d Mon Sep 17 00:00:00 2001
From: Tom Martensen <tmartens@redhat.com>
Date: Fri, 29 Sep 2023 15:21:08 +0200
Subject: [PATCH 02/19] fix and stuff

---
 Makefile               | 224 ++++++++++++++++++++---------------------
 scripts/deploy/helm.sh |  19 ++++
 2 files changed, 128 insertions(+), 115 deletions(-)

diff --git a/Makefile b/Makefile
index a2c03f9dc..1b2456c83 100644
--- a/Makefile
+++ b/Makefile
@@ -1,7 +1,7 @@
 SHELL := /usr/bin/env bash
 export GO111MODULE=on
 
-# Add check that cluster name is not infra-prod or deployment
+# TODO: Add check that cluster name is not infra-prod or deployment
 
 .PHONY: all
 all: image
@@ -32,92 +32,6 @@ IMAGE=us.gcr.io/stackrox-infra/infra-server:$(VERSION)
 image-name:
 	@echo $(IMAGE)
 
-#############
-## Linting ##
-#############
-
-.PHONY: argo-workflow-lint
-argo-workflow-lint:
-	@argo lint ./chart/infra-server/static/workflow*.yaml
-
-.PHONY: shellcheck
-shellcheck:
-	@shellcheck -x -- **/*.{bats,sh}
-
-###########
-## Build ##
-###########
-
-# server - Builds the infra-server binary
-# When run locally, a Darwin binary is built and installed into the user's GOPATH bin.
-# When run in CI, a Darwin and Linux binary is built.
-.PHONY: server
-server:
-	@echo "+ $@"
-	GOARCH=amd64 GOOS=linux ./scripts/go-build -o bin/infra-server-linux-amd64 ./cmd/infra-server
-
-# cli - Builds the infractl client binary
-# When run in CI or when preparing an image, a Darwin and Linux binary is built.
-.PHONY: cli
-cli:
-	@echo "+ $@"
-	GOARCH=amd64 GOOS=darwin ./scripts/go-build -o bin/infractl-darwin-amd64 ./cmd/infractl
-	GOARCH=arm64 GOOS=darwin ./scripts/go-build -o bin/infractl-darwin-arm64 ./cmd/infractl
-	GOARCH=amd64 GOOS=linux  ./scripts/go-build -o bin/infractl-linux-amd64  ./cmd/infractl
-
-# cli-local - Builds the infractl client binary
-# When run locally, a Darwin binary is built and installed into the user's GOPATH bin.
-.PHONY: cli-local
-cli-local:
-	@echo "+ $@"
-	./scripts/go-build -o $(GOPATH)/bin/infractl  ./cmd/infractl
-
-.PHONY: ui
-ui:
-	@echo "+ $@"
-	@make -C ui all
-
-.PHONY: image
-image:
-	docker build . -t $(IMAGE) -f image/Dockerfile --secret id=npmrc,src=${HOME}/.npmrc
-
-.PHONY: push
-push:
-	docker push $(IMAGE) | cat
-
-#############
-## Testing ##
-#############
-
-.PHONY: unit-test
-unit-test: proto-generated-srcs
-	@echo "+ $@"
-	@go test -v ./...
-
-.PHONY: go-e2e-tests
-go-e2e-tests: proto-generated-srcs
-	@kubectl apply -f workflows/
-	@go test ./test/e2e/... -tags=e2e -v -parallel 5 -count 1 -cover -timeout 1h
-
-# Assuming a local dev infra server is running and accessible via a port-forward
-# i.e. nohup kubectl -n infra port-forward svc/infra-server-service 8443:8443 &
-.PHONY: pull-infractl-from-dev-server
-pull-infractl-from-dev-server:
-	@mkdir -p bin
-	@rm -f bin/infractl
-	set -o pipefail; \
-	curl --retry 3 --insecure --silent --show-error --fail --location https://localhost:8443/v1/cli/linux/amd64/upgrade \
-          | jq -r ".result.fileChunk" \
-          | base64 -d \
-          > bin/infractl
-	chmod +x bin/infractl
-	bin/infractl -k -e localhost:8443 version
-
-.PHONY: e2e-tests
-e2e-tests:
-	@kubectl apply -f "workflows/*.yaml"
-	@bats --jobs 5 --no-parallelize-within-files --recursive .
-
 ##############
 ## Protobuf ##
 ##############
@@ -203,6 +117,91 @@ proto-generated-srcs: protoc-tools
 		--swagger_out=logtostderr=true:$(PROTO_OUTPUT_DIR) \
 		$(PROTO_FILES)
 
+###########
+## Build ##
+###########
+
+# server - Builds the infra-server binary
+# When run locally, a Darwin binary is built and installed into the user's GOPATH bin.
+# When run in CI, a Darwin and Linux binary is built.
+.PHONY: server
+server:
+	@echo "+ $@"
+	GOARCH=amd64 GOOS=linux ./scripts/go-build -o bin/infra-server-linux-amd64 ./cmd/infra-server
+
+# cli - Builds the infractl client binary
+# When run in CI or when preparing an image, a Darwin and Linux binary is built.
+.PHONY: cli
+cli:
+	@echo "+ $@"
+	GOARCH=amd64 GOOS=darwin ./scripts/go-build -o bin/infractl-darwin-amd64 ./cmd/infractl
+	GOARCH=arm64 GOOS=darwin ./scripts/go-build -o bin/infractl-darwin-arm64 ./cmd/infractl
+	GOARCH=amd64 GOOS=linux  ./scripts/go-build -o bin/infractl-linux-amd64  ./cmd/infractl
+
+# cli-local - Builds the infractl client binary
+# When run locally, a Darwin binary is built and installed into the user's GOPATH bin.
+.PHONY: cli-local
+cli-local:
+	@echo "+ $@"
+	./scripts/go-build -o $(GOPATH)/bin/infractl  ./cmd/infractl
+
+.PHONY: ui
+ui:
+	@echo "+ $@"
+	@make -C ui all
+
+.PHONY: image
+image:
+	docker build . -t $(IMAGE) -f image/Dockerfile --secret id=npmrc,src=${HOME}/.npmrc
+
+.PHONY: push
+push:
+	docker push $(IMAGE) | cat
+
+#############
+## Linting ##
+#############
+
+.PHONY: argo-workflow-lint
+argo-workflow-lint:
+	@argo lint ./chart/infra-server/static/workflow*.yaml
+
+.PHONY: shellcheck
+shellcheck:
+	@shellcheck -x -- **/*.{bats,sh}
+
+#############
+## Testing ##
+#############
+
+.PHONY: unit-test
+unit-test: proto-generated-srcs
+	@echo "+ $@"
+	@go test -v ./...
+
+.PHONY: bats-e2e-tests
+bats-e2e-tests:
+	@kubectl apply -f "workflows/*.yaml"
+	@bats --jobs 5 --no-parallelize-within-files --recursive .
+
+.PHONY: go-e2e-tests
+go-e2e-tests: proto-generated-srcs
+	@kubectl apply -f workflows/
+	@go test ./test/e2e/... -tags=e2e -v -parallel 5 -count 1 -cover -timeout 1h
+
+# Assuming a local dev infra server is running and accessible via a port-forward
+# i.e. nohup kubectl -n infra port-forward svc/infra-server-service 8443:8443 &
+.PHONY: pull-infractl-from-dev-server
+pull-infractl-from-dev-server:
+	@mkdir -p bin
+	@rm -f bin/infractl
+	set -o pipefail; \
+	curl --retry 3 --insecure --silent --show-error --fail --location https://localhost:8443/v1/cli/linux/amd64/upgrade \
+          | jq -r ".result.fileChunk" \
+          | base64 -d \
+          > bin/infractl
+	chmod +x bin/infractl
+	bin/infractl -k -e localhost:8443 version
 
 ##########
 ## Kube ##
@@ -215,13 +214,14 @@ ifndef ENVIRONMENT
 	$(error ENVIRONMENT is undefined)
 endif
 
-# TODO: this needs to be re-done for GCP secrets manger
+# TODO: this needs to be re-done for GCP secrets manager
 ## Configuration
 .PHONY: configuration-download
 configuration-download:
 	@echo "Downloading configuration from gs://infra-configuration"
 	gsutil -m cp -R "gs://infra-configuration/latest/configuration" "chart/infra-server/"
 
+# TODO: this needs to be re-done for GCP secrets manager
 .PHONY: configuration-upload
 configuration-upload: CONST_DATESTAMP := $(shell date '+%Y-%m-%d-%H-%M-%S')
 configuration-upload:
@@ -230,6 +230,7 @@ configuration-upload:
 	@echo "Uploading configuration to gs://infra-configuration/latest/"
 	gsutil -m cp -R chart/infra-server/configuration "gs://infra-configuration/latest/"
 
+# TODO: this needs to be re-done for GCP secrets manager
 # Combines configuration/{development,production} files into single helm value.yaml files
 # (configuration/{development,production}-values-from-files.yaml) that can be used in template
 # rendering.
@@ -238,38 +239,31 @@ create-consolidated-values:
 	@./scripts/create-consolidated-values.sh
 
 ## Common install targets
-bounce-infra-pods: setup-kc
+.PHONY: bounce-infra-pods
+bounce-infra-pods:
 	$(kc) -n infra rollout restart deploy/infra-server-deployment
 	$(kc) -n infra rollout status deploy/infra-server-deployment --watch --timeout=3m
 
+# TODO: this needs to check the context
 ## Diff
 .PHONY: diff
-diff: setup-kc
-	gsutil cat gs://infra-configuration/latest/configuration/$(ENVIRONMENT)-values.yaml \
-               gs://infra-configuration/latest/configuration/$(ENVIRONMENT)-values-from-files.yaml | \
-	helm template chart/infra-server \
-		--set deployment="$(DEPLOYMENT)" \
-		--set tag="$(VERSION)" \
-		--values - | \
-	$(kc) diff -R -f -
-
-##########
-## Misc ##
-##########
-.PHONY: gotags
-gotags:
-	@gotags -R . > tags
-	@echo "GoTags written to $(PWD)/tags"
-
-.PHONY: update-version
-update-version: image_regex   := gcr.io/stackrox-infra/automation-flavors/.*
-update-version: image_version := 0.2.16
-update-version:
-	@echo 'Updating automation-flavor image versions to "${image_version}"'
-	@perl -p -i -e 's#image: (${image_regex}):(.*)#image: \1:${image_version}#g' \
-		./chart/infra-server/static/*.yaml
-	@git diff --name-status ./chart/infra-server/static/*.yaml
-
+diff:
+	@./scripts/deploy/helm.sh diff $(VERSION) $(SECRET_VERSION)
+
+# TODO: this needs to check the context
+.PHONY: deploy
+deploy:
+	@./scripts/deploy/helm deploy $(VERSION) $(SECRET_VERSION)
+
+# TODO: this needs to check the context
+.PHONY: template
+template:
+	@./scripts/deploy/helm template $(VERSION) $(SECRET_VERSION)
+
+##################
+## Dependencies ##
+##################
+# TODO: this needs to check the context
 .PHONY: install-argo
 install-argo:
 	helm repo add argo https://argoproj.github.io/argo-helm
diff --git a/scripts/deploy/helm.sh b/scripts/deploy/helm.sh
index 409837c3d..fd1a4c4f9 100755
--- a/scripts/deploy/helm.sh
+++ b/scripts/deploy/helm.sh
@@ -54,5 +54,24 @@ deploy() {
     )
 }
 
+diff() {
+    helm template \
+		infra-server \
+		chart/infra-server \
+		--debug \
+		--namespace infra \
+		--set tag="${TAG}" \
+        --set environment="${ENVIRONMENT}" \
+		--values - \
+    < <(gcloud secrets versions access "${SECRET_VERSION}" \
+        --secret "infra-values-${ENVIRONMENT}" \
+        --project stackrox-infra \
+    && gcloud secrets versions access "${SECRET_VERSION}" \
+        --secret "infra-values-from-files-${ENVIRONMENT}" \
+        --project stackrox-infra \
+    ) | \
+	kubectl diff -R -f -
+}
+
 check_not_empty TAG ENVIRONMENT SECRET_VERSION
 eval "$TASK"

From a1a3b2afb0a8042f577a0f2c4a6fdf92acbfd3f7 Mon Sep 17 00:00:00 2001
From: Tom Martensen <tmartens@redhat.com>
Date: Wed, 4 Oct 2023 13:38:51 +0200
Subject: [PATCH 03/19] progress

---
 Makefile               | 47 ++++++++++++++++++++++++------------------
 scripts/deploy/helm.sh | 31 +++++++++++++++-------------
 2 files changed, 44 insertions(+), 34 deletions(-)

diff --git a/Makefile b/Makefile
index 1b2456c83..71ffd3ad2 100644
--- a/Makefile
+++ b/Makefile
@@ -1,8 +1,6 @@
 SHELL := /usr/bin/env bash
 export GO111MODULE=on
 
-# TODO: Add check that cluster name is not infra-prod or deployment
-
 .PHONY: all
 all: image
 
@@ -206,6 +204,9 @@ pull-infractl-from-dev-server:
 ##########
 ## Kube ##
 ##########
+dev_context = gke_stackrox-infra_us-west2_infra-development
+prod_context = gke_stackrox-infra_us-west2_infra-production
+this_context = $(shell kubectl config current-context)
 
 ## Meta
 .PHONY: pre-check
@@ -213,6 +214,14 @@ pre-check:
 ifndef ENVIRONMENT
 	$(error ENVIRONMENT is undefined)
 endif
+	@if [[ "${ENVIRONMENT}" == "development" && "${this_context}" == "${prod_context}" ]]; then \
+		echo -e "Your kube context is not set to a development infra. Use the following for dev cluster or set it to your PR cluster\n\tkubectl config use-context ${dev_context}\n"; \
+		exit 1; \
+	fi
+	@if [[ "${ENVIRONMENT}" == "production" && "${this_context}" != "${prod_context}" ]]; then \
+		echo -e "Your kube context is not set to production infra:\n\tkubectl config use-context ${prod_context}"; \
+		exit 1; \
+	fi
 
 # TODO: this needs to be re-done for GCP secrets manager
 ## Configuration
@@ -238,34 +247,32 @@ configuration-upload:
 create-consolidated-values:
 	@./scripts/create-consolidated-values.sh
 
-## Common install targets
-.PHONY: bounce-infra-pods
-bounce-infra-pods:
-	$(kc) -n infra rollout restart deploy/infra-server-deployment
-	$(kc) -n infra rollout status deploy/infra-server-deployment --watch --timeout=3m
+## Render template
+.PHONY: template
+template: pre-check
+	@./scripts/deploy/helm template $(VERSION) $(SECRET_VERSION)
+
+## Deploy
+.PHONY: deploy
+deploy: pre-check
+	@./scripts/deploy/helm deploy $(VERSION) $(SECRET_VERSION)
 
-# TODO: this needs to check the context
 ## Diff
 .PHONY: diff
-diff:
+diff: pre-check
 	@./scripts/deploy/helm.sh diff $(VERSION) $(SECRET_VERSION)
 
-# TODO: this needs to check the context
-.PHONY: deploy
-deploy:
-	@./scripts/deploy/helm deploy $(VERSION) $(SECRET_VERSION)
-
-# TODO: this needs to check the context
-.PHONY: template
-template:
-	@./scripts/deploy/helm template $(VERSION) $(SECRET_VERSION)
+## Bounce pods
+.PHONY: bounce-infra-pods
+bounce-infra-pods:
+	$(kc) -n infra rollout restart deploy/infra-server-deployment
+	$(kc) -n infra rollout status deploy/infra-server-deployment --watch --timeout=3m
 
 ##################
 ## Dependencies ##
 ##################
-# TODO: this needs to check the context
 .PHONY: install-argo
-install-argo:
+install-argo: pre-check
 	helm repo add argo https://argoproj.github.io/argo-helm
 	helm upgrade \
 		argo-workflows \
diff --git a/scripts/deploy/helm.sh b/scripts/deploy/helm.sh
index fd1a4c4f9..b4715cbf1 100755
--- a/scripts/deploy/helm.sh
+++ b/scripts/deploy/helm.sh
@@ -4,8 +4,11 @@ set -euo pipefail
 
 TASK="$1"
 TAG="$2"
-ENVIRONMENT="$3"
-SECRET_VERSION="$4"
+SECRET_VERSION="$3"
+
+PROJECT="stackrox-infra"
+RELEASE_NAMESPACE="infra"
+RELEASE_NAME="infra-server"
 
 check_not_empty() {
     for V in "$@"; do
@@ -19,56 +22,56 @@ check_not_empty() {
 
 template() {
     helm template \
-		infra-server \
+		"${RELEASE_NAME}" \
 		chart/infra-server \
 		--debug \
-		--namespace infra \
+		--namespace "${RELEASE_NAMESPACE}" \
 		--set tag="${TAG}" \
         --set environment="${ENVIRONMENT}" \
 		--values - \
     < <(gcloud secrets versions access "${SECRET_VERSION}" \
         --secret "infra-values-${ENVIRONMENT}" \
-        --project stackrox-infra \
+        --project "${PROJECT}" \
     && gcloud secrets versions access "${SECRET_VERSION}" \
         --secret "infra-values-from-files-${ENVIRONMENT}" \
-        --project stackrox-infra \
+        --project "${PROJECT}" \
     )
 }
 
 deploy() {
     helm upgrade \
-        infra-server \
+        "${RELEASE_NAME}" \
         chart/infra-server \
         --install \
         --create-namespace \
-        --namespace infra \
+        --namespace "${RELEASE_NAMESPACE}" \
         --set tag="${TAG}" \
         --set environment="${ENVIRONMENT}" \
         --values - \
     < <(gcloud secrets versions access "${SECRET_VERSION}" \
         --secret "infra-values-${ENVIRONMENT}" \
-        --project stackrox-infra \
+        --project "${PROJECT}" \
     && gcloud secrets versions access "${SECRET_VERSION}" \
         --secret "infra-values-from-files-${ENVIRONMENT}" \
-        --project stackrox-infra \
+        --project "${PROJECT}" \
     )
 }
 
 diff() {
     helm template \
-		infra-server \
+		"${RELEASE_NAME}" \
 		chart/infra-server \
 		--debug \
-		--namespace infra \
+		--namespace "${RELEASE_NAMESPACE}" \
 		--set tag="${TAG}" \
         --set environment="${ENVIRONMENT}" \
 		--values - \
     < <(gcloud secrets versions access "${SECRET_VERSION}" \
         --secret "infra-values-${ENVIRONMENT}" \
-        --project stackrox-infra \
+        --project "${PROJECT}" \
     && gcloud secrets versions access "${SECRET_VERSION}" \
         --secret "infra-values-from-files-${ENVIRONMENT}" \
-        --project stackrox-infra \
+        --project "${PROJECT}" \
     ) | \
 	kubectl diff -R -f -
 }

From 70e5c127c8e14edf68af8d6d2a20cc0fc93d762f Mon Sep 17 00:00:00 2001
From: Tom Martensen <tmartens@redhat.com>
Date: Tue, 10 Oct 2023 15:14:40 +0200
Subject: [PATCH 04/19] add secret management script

---
 scripts/deploy/secrets.sh | 100 ++++++++++++++++++++++++++++++++++++++
 1 file changed, 100 insertions(+)
 create mode 100755 scripts/deploy/secrets.sh

diff --git a/scripts/deploy/secrets.sh b/scripts/deploy/secrets.sh
new file mode 100755
index 000000000..e377a6d15
--- /dev/null
+++ b/scripts/deploy/secrets.sh
@@ -0,0 +1,100 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+TASK="$1"
+ENVIRONMENT="$2"
+SECRET_VERSION="$3"
+
+PROJECT="stackrox-infra"
+
+check_not_empty() {
+    for V in "$@"; do
+        typeset -n VAR="$V"
+        if [ -z "${VAR:-}" ]; then
+            echo "ERROR: Variable $V is not set or empty"
+            exit 1
+        fi
+    done
+}
+
+# Downloads secrets files for an ENVIRONMENT.
+download_secrets() {
+    mkdir -p chart/infra-server/configuration
+    gcloud secrets versions access "${SECRET_VERSION}" \
+        --secret "infra-values-${ENVIRONMENT}" \
+        --project "${PROJECT}" \
+    > "chart/infra-server/configuration/${ENVIRONMENT}-values.yaml"
+
+    gcloud secrets versions access "${SECRET_VERSION}" \
+        --secret "infra-values-from-files-${ENVIRONMENT}" \
+        --project "${PROJECT}" \
+    > "chart/infra-server/configuration/${ENVIRONMENT}-values-from-files.yaml"
+}
+
+# Uploads secrets files for an ENVIRONMENT.
+upload_secrets() {
+    gcloud secrets versions add \
+    "infra-values-${ENVIRONMENT}" \
+    --project "${PROJECT}" \
+    --data-file "chart/infra-server/configuration/${ENVIRONMENT}-values.yaml"
+
+   gcloud secrets versions add \
+   "infra-values-from-files-${ENVIRONMENT}" \
+    --project "${PROJECT}" \
+    --data-file "chart/infra-server/configuration/${ENVIRONMENT}-values-from-files.yaml"
+}
+
+# Shows all available keys in a secrets file.
+show_available_secret_files() {
+    yq 'keys' "chart/infra-server/configuration/${ENVIRONMENT}-values-from-files.yaml"
+}
+
+# Downloads secrets, asks for which secret file to show, and displayed decoded value.
+show() {
+    download_secrets
+    show_available_secret_files
+
+    echo "> Secret file to show:"
+    read -r secret_name
+
+    echo "> Contents:"
+    yq \
+        ".${secret_name}" \
+        "chart/infra-server/configuration/${ENVIRONMENT}-values-from-files.yaml" \
+    | base64 --decode
+
+}
+
+# Downloads secrets, asks for which secret file to change and what to, and uploads new values.
+edit() {
+    download_secrets
+    show_available_secret_files
+
+    echo "> Secret file to change:"
+    read -r secret_name
+
+    echo "> Enter new value. Type 'EOF' on a line by itself to finish:"
+    new_value=""
+
+    while IFS= read -r line; do
+        if [ "$line" = "EOF" ]; then
+            break
+        fi
+        new_value+="$line\n"
+    done
+
+    yq eval \
+        --inplace ".${secret_name} = \"$(echo -e -n "${new_value}" | base64)\"" \
+        "chart/infra-server/configuration/${ENVIRONMENT}-values-from-files.yaml"
+    upload_secrets
+}
+
+# Revert downloads a specific secrets version, and uploads it as the latest
+revert() {
+    download_secrets
+    upload_secrets
+}
+
+check_not_empty ENVIRONMENT SECRET_VERSION
+eval "$TASK"

From b33dcf8bb72631d360f43f0fdbedaa6114d4e82c Mon Sep 17 00:00:00 2001
From: Tom Martensen <tmartens@redhat.com>
Date: Wed, 11 Oct 2023 11:10:55 +0200
Subject: [PATCH 05/19] finish stuff for development

---
 .github/workflows/PR.yaml                     |  2 +-
 Makefile                                      | 65 +++++++++----------
 chart/infra-server/static/oidc.yaml           | 12 ----
 chart/infra-server/templates/certificate.yaml |  2 +-
 chart/infra-server/templates/deployment.yaml  |  2 +-
 chart/infra-server/templates/ingress.yaml     |  2 +-
 scripts/create-consolidated-values.sh         | 41 ------------
 scripts/deploy/helm.sh                        | 55 +++++++++++-----
 scripts/deploy/secrets.sh                     |  4 +-
 9 files changed, 73 insertions(+), 112 deletions(-)
 delete mode 100644 chart/infra-server/static/oidc.yaml
 delete mode 100755 scripts/create-consolidated-values.sh

diff --git a/.github/workflows/PR.yaml b/.github/workflows/PR.yaml
index c509991b5..4c48278c0 100644
--- a/.github/workflows/PR.yaml
+++ b/.github/workflows/PR.yaml
@@ -97,7 +97,7 @@ jobs:
 
       - name: Deploy infra to dev cluster
         run: |
-          make install-local
+          ENVIRONMENT=development TEST_MODE=true make helm-deploy
           sleep 10 # wait for old pods to disappear so the svc port-forward doesn't connect to them
           kubectl -n infra port-forward svc/infra-server-service 8443:8443 &
           sleep 10
diff --git a/Makefile b/Makefile
index 71ffd3ad2..8bdadfa4a 100644
--- a/Makefile
+++ b/Makefile
@@ -223,50 +223,45 @@ endif
 		exit 1; \
 	fi
 
-# TODO: this needs to be re-done for GCP secrets manager
-## Configuration
-.PHONY: configuration-download
-configuration-download:
-	@echo "Downloading configuration from gs://infra-configuration"
-	gsutil -m cp -R "gs://infra-configuration/latest/configuration" "chart/infra-server/"
-
-# TODO: this needs to be re-done for GCP secrets manager
-.PHONY: configuration-upload
-configuration-upload: CONST_DATESTAMP := $(shell date '+%Y-%m-%d-%H-%M-%S')
-configuration-upload:
-	@echo "Uploading configuration to gs://infra-configuration/${CONST_DATESTAMP}"
-	gsutil -m cp -R chart/infra-server/configuration "gs://infra-configuration/${CONST_DATESTAMP}/"
-	@echo "Uploading configuration to gs://infra-configuration/latest/"
-	gsutil -m cp -R chart/infra-server/configuration "gs://infra-configuration/latest/"
-
-# TODO: this needs to be re-done for GCP secrets manager
-# Combines configuration/{development,production} files into single helm value.yaml files
-# (configuration/{development,production}-values-from-files.yaml) that can be used in template
-# rendering.
-.PHONY: create-consolidated-values
-create-consolidated-values:
-	@./scripts/create-consolidated-values.sh
-
 ## Render template
-.PHONY: template
-template: pre-check
-	@./scripts/deploy/helm template $(VERSION) $(SECRET_VERSION)
+.PHONY: helm-template
+helm-template: pre-check
+	@./scripts/deploy/helm template $(VERSION) $(ENVIRONMENT) $(SECRET_VERSION)
 
 ## Deploy
-.PHONY: deploy
-deploy: pre-check
-	@./scripts/deploy/helm deploy $(VERSION) $(SECRET_VERSION)
+.PHONY: helm-deploy
+helm-deploy: pre-check
+	@./scripts/deploy/helm deploy $(VERSION) $(ENVIRONMENT) $(SECRET_VERSION)
 
 ## Diff
-.PHONY: diff
-diff: pre-check
-	@./scripts/deploy/helm.sh diff $(VERSION) $(SECRET_VERSION)
+.PHONY: helm-diff
+helm-diff: pre-check
+	@./scripts/deploy/helm.sh diff $(VERSION) $(ENVIRONMENT) $(SECRET_VERSION)
 
 ## Bounce pods
 .PHONY: bounce-infra-pods
 bounce-infra-pods:
-	$(kc) -n infra rollout restart deploy/infra-server-deployment
-	$(kc) -n infra rollout status deploy/infra-server-deployment --watch --timeout=3m
+	kubectl -n infra rollout restart deploy/infra-server-deployment
+	kubectl -n infra rollout status deploy/infra-server-deployment --watch --timeout=3m
+
+#############
+## Secrets ##
+#############
+.PHONY: secrets-download
+secrets-download: pre-check
+	@./scripts/deploy/secrets.sh download_secrets $(ENVIRONMENT)
+
+.PHONY: secrets-upload
+secrets-upload: pre-check
+	@./scripts/deploy/secrets.sh upload_secrets $(ENVIRONMENT) $(SECRET_VERSION)
+
+.PHONY: secrets-show
+secrets-show: pre-check
+	@./scripts/deploy/secrets.sh show $(ENVIRONMENT) $(SECRET_VERSION)
+
+.PHONY: secrets-edit
+secrets-edit: pre-check
+	./scripts/deploy/secrets.sh edit $(ENVIRONMENT) $(SECRET_VERSION)
 
 ##################
 ## Dependencies ##
diff --git a/chart/infra-server/static/oidc.yaml b/chart/infra-server/static/oidc.yaml
deleted file mode 100644
index 004dee7a9..000000000
--- a/chart/infra-server/static/oidc.yaml
+++ /dev/null
@@ -1,12 +0,0 @@
-issuer: "https://auth.stage.redhat.com/auth/realms/EmployeeIDP"
-clientID: "rhacs-dev-infra"
-clientSecret: "clientSecret-example"
-sessionSecret: "sessionSecret-example"
-endpoint: {{ if eq .Values.environment "local" }}localhost:8443{{ else }}dev.infra.rox.systems{{ end }}
-accessTokenClaims:
-  - value: 'https://auth.stage.redhat.com/auth/realms/EmployeeIDP'
-    op: 'eq'
-    path: 'iss'
-  - value: 'rover-group-example'
-    op: 'in'
-    path: 'realm_access.roles'
diff --git a/chart/infra-server/templates/certificate.yaml b/chart/infra-server/templates/certificate.yaml
index 6ffbefe27..ae08599eb 100644
--- a/chart/infra-server/templates/certificate.yaml
+++ b/chart/infra-server/templates/certificate.yaml
@@ -1,4 +1,4 @@
-{{ if ne .Values.environment "local" -}}
+{{- if .Capabilities.APIVersions.Has "networking.gke.io/v1/ManagedCertificate" -}}
 ---
 
 apiVersion: networking.gke.io/v1
diff --git a/chart/infra-server/templates/deployment.yaml b/chart/infra-server/templates/deployment.yaml
index 0f1f840b3..574f493c1 100644
--- a/chart/infra-server/templates/deployment.yaml
+++ b/chart/infra-server/templates/deployment.yaml
@@ -30,7 +30,7 @@ spec:
             - name: GOOGLE_APPLICATION_CREDENTIALS
               value: /configuration/google-credentials.json
             - name: TEST_MODE
-              value: "{{ eq .Values.environment "local" }}"
+              value: "{{ eq .Values.testMode true }}"
           readinessProbe:
             httpGet:
               path: /
diff --git a/chart/infra-server/templates/ingress.yaml b/chart/infra-server/templates/ingress.yaml
index c6af9442a..9e16b3e4f 100644
--- a/chart/infra-server/templates/ingress.yaml
+++ b/chart/infra-server/templates/ingress.yaml
@@ -1,4 +1,4 @@
-{{ if ne .Values.environment "local" -}}
+{{- if .Capabilities.APIVersions.Has "networking.gke.io/v1/ManagedCertificate" -}}
 apiVersion: networking.k8s.io/v1
 kind: Ingress
 
diff --git a/scripts/create-consolidated-values.sh b/scripts/create-consolidated-values.sh
deleted file mode 100755
index 17eb2e76c..000000000
--- a/scripts/create-consolidated-values.sh
+++ /dev/null
@@ -1,41 +0,0 @@
-#!/usr/bin/env bash
-
-ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")"/.. && pwd)"
-source "$ROOT/scripts/lib.sh"
-
-set -euo pipefail
-
-create_consolidated_values() {
-    if [[ "$#" -ne 1 ]]; then
-        die "missing args. usage: create_consolidated_values <environment>"
-    fi
-    local environment="$1"
-
-    info "Creating a combined values file for chart/infra-server/configuration/$environment files"
-
-    if [[ ! -e "$ROOT/chart/infra-server/configuration" ]]; then
-        die "chart/infra-server/configuration is missing. Download the configuration with 'make configuration-download'"
-    fi
-
-    local values_file="$ROOT/chart/infra-server/configuration/$environment-values-from-files.yaml"
-    rm -f "$values_file"
-
-    {
-        echo "# This is a helm values file that combines the contents of the $environment configuration files."
-        echo "# It is updated by each render-* make target. Changes made here will be lost."
-        echo
-    } >> "$values_file"
-
-    pushd "$ROOT/chart/infra-server/configuration/$environment" > /dev/null
-    while IFS='' read -r cfg_file; do
-      local helm_safe_key="${cfg_file//[.-]/_}"
-      helm_safe_key="${helm_safe_key////__}"
-
-      echo "$helm_safe_key: $(base64 < "$cfg_file" | tr -d '\n')" >> "$values_file"
-      echo >> "$values_file"
-    done < <(find . -type f -not -name '*.md' -not -name '*.DS_Store' | cut -c3-)
-    popd > /dev/null
-}
-
-create_consolidated_values "development"
-create_consolidated_values "production"
diff --git a/scripts/deploy/helm.sh b/scripts/deploy/helm.sh
index b4715cbf1..f87394d3c 100755
--- a/scripts/deploy/helm.sh
+++ b/scripts/deploy/helm.sh
@@ -4,7 +4,12 @@ set -euo pipefail
 
 TASK="$1"
 TAG="$2"
-SECRET_VERSION="$3"
+ENVIRONMENT="$3"
+SECRET_VERSION="${4:-latest}"
+
+# Enables TEST_MODE in chart.
+# Cannot use CI, because then CD with GHA would not be possible.
+TEST_MODE="${TEST_MODE:-false}"
 
 PROJECT="stackrox-infra"
 RELEASE_NAMESPACE="infra"
@@ -21,14 +26,18 @@ check_not_empty() {
 }
 
 template() {
-    helm template \
-		"${RELEASE_NAME}" \
-		chart/infra-server \
-		--debug \
-		--namespace "${RELEASE_NAMESPACE}" \
-		--set tag="${TAG}" \
+    # Need to use helm upgrade --dry-run to have .Capabilities context available
+    helm upgrade \
+        "${RELEASE_NAME}" \
+        chart/infra-server \
+        --install \
+        --create-namespace \
+        --dry-run \
+        --namespace "${RELEASE_NAMESPACE}" \
+        --set tag="${TAG}" \
         --set environment="${ENVIRONMENT}" \
-		--values - \
+        --set testMode="${TEST_MODE}" \
+        --values - \
     < <(gcloud secrets versions access "${SECRET_VERSION}" \
         --secret "infra-values-${ENVIRONMENT}" \
         --project "${PROJECT}" \
@@ -38,15 +47,19 @@ template() {
     )
 }
 
+# deploy upgrades the Helm release with
 deploy() {
     helm upgrade \
         "${RELEASE_NAME}" \
         chart/infra-server \
         --install \
         --create-namespace \
+        --timeout 5m \
+        --wait \
         --namespace "${RELEASE_NAMESPACE}" \
         --set tag="${TAG}" \
         --set environment="${ENVIRONMENT}" \
+        --set testMode="${TEST_MODE}" \
         --values - \
     < <(gcloud secrets versions access "${SECRET_VERSION}" \
         --secret "infra-values-${ENVIRONMENT}" \
@@ -57,24 +70,30 @@ deploy() {
     )
 }
 
+# diff renders the Helm chart and compares the deployed resources to show what would change on next deployment.
 diff() {
-    helm template \
-		"${RELEASE_NAME}" \
-		chart/infra-server \
-		--debug \
-		--namespace "${RELEASE_NAMESPACE}" \
-		--set tag="${TAG}" \
+    # Need to use helm upgrade --dry-run to have .Capabilities context available
+    helm upgrade \
+        "${RELEASE_NAME}" \
+        chart/infra-server \
+        --install \
+        --create-namespace \
+        --dry-run \
+        --namespace "${RELEASE_NAMESPACE}" \
+        --set tag="${TAG}" \
         --set environment="${ENVIRONMENT}" \
-		--values - \
+        --set testMode="${TEST_MODE}" \
+        --values - \
     < <(gcloud secrets versions access "${SECRET_VERSION}" \
         --secret "infra-values-${ENVIRONMENT}" \
         --project "${PROJECT}" \
     && gcloud secrets versions access "${SECRET_VERSION}" \
         --secret "infra-values-from-files-${ENVIRONMENT}" \
         --project "${PROJECT}" \
-    ) | \
-	kubectl diff -R -f -
+    ) \
+    | sed -n '/---/,$p' \
+    | kubectl diff -R -f -
 }
 
-check_not_empty TAG ENVIRONMENT SECRET_VERSION
+check_not_empty TASK TAG ENVIRONMENT
 eval "$TASK"
diff --git a/scripts/deploy/secrets.sh b/scripts/deploy/secrets.sh
index e377a6d15..a00c7b270 100755
--- a/scripts/deploy/secrets.sh
+++ b/scripts/deploy/secrets.sh
@@ -4,7 +4,7 @@ set -euo pipefail
 
 TASK="$1"
 ENVIRONMENT="$2"
-SECRET_VERSION="$3"
+SECRET_VERSION="${3:-latest}"
 
 PROJECT="stackrox-infra"
 
@@ -96,5 +96,5 @@ revert() {
     upload_secrets
 }
 
-check_not_empty ENVIRONMENT SECRET_VERSION
+check_not_empty TASK ENVIRONMENT
 eval "$TASK"

From ea2ac57b85be630756f459c616957231c1e5b34a Mon Sep 17 00:00:00 2001
From: Tom Martensen <tmartens@redhat.com>
Date: Wed, 11 Oct 2023 11:34:08 +0200
Subject: [PATCH 06/19] fix stupid typo

---
 Makefile | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/Makefile b/Makefile
index 8bdadfa4a..85273e52c 100644
--- a/Makefile
+++ b/Makefile
@@ -226,12 +226,12 @@ endif
 ## Render template
 .PHONY: helm-template
 helm-template: pre-check
-	@./scripts/deploy/helm template $(VERSION) $(ENVIRONMENT) $(SECRET_VERSION)
+	@./scripts/deploy/helm.sh template $(VERSION) $(ENVIRONMENT) $(SECRET_VERSION)
 
 ## Deploy
 .PHONY: helm-deploy
 helm-deploy: pre-check
-	@./scripts/deploy/helm deploy $(VERSION) $(ENVIRONMENT) $(SECRET_VERSION)
+	@./scripts/deploy/helm.sh deploy $(VERSION) $(ENVIRONMENT) $(SECRET_VERSION)
 
 ## Diff
 .PHONY: helm-diff

From cd4eb6629a2ac000c7a1a9e537382776e77df786 Mon Sep 17 00:00:00 2001
From: Tom Martensen <tmartens@redhat.com>
Date: Wed, 11 Oct 2023 11:36:55 +0200
Subject: [PATCH 07/19] fix cap

---
 chart/infra-server/templates/ingress.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/chart/infra-server/templates/ingress.yaml b/chart/infra-server/templates/ingress.yaml
index 9e16b3e4f..11696b295 100644
--- a/chart/infra-server/templates/ingress.yaml
+++ b/chart/infra-server/templates/ingress.yaml
@@ -1,4 +1,4 @@
-{{- if .Capabilities.APIVersions.Has "networking.gke.io/v1/ManagedCertificate" -}}
+{{- if .Capabilities.APIVersions.Has "networking.gke.io/v1/Ingress" -}}
 apiVersion: networking.k8s.io/v1
 kind: Ingress
 

From f3cd6cc2f3b7c5a5d8844ad6e74b528765c4b961 Mon Sep 17 00:00:00 2001
From: Tom Martensen <tmartens@redhat.com>
Date: Wed, 11 Oct 2023 12:22:27 +0200
Subject: [PATCH 08/19] install argo

---
 .github/workflows/PR.yaml | 2 +-
 Makefile                  | 6 +++++-
 2 files changed, 6 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/PR.yaml b/.github/workflows/PR.yaml
index 4c48278c0..ebef9877b 100644
--- a/.github/workflows/PR.yaml
+++ b/.github/workflows/PR.yaml
@@ -97,7 +97,7 @@ jobs:
 
       - name: Deploy infra to dev cluster
         run: |
-          ENVIRONMENT=development TEST_MODE=true make helm-deploy
+          ENVIRONMENT=development TEST_MODE=true make install-argo helm-deploy
           sleep 10 # wait for old pods to disappear so the svc port-forward doesn't connect to them
           kubectl -n infra port-forward svc/infra-server-service 8443:8443 &
           sleep 10
diff --git a/Makefile b/Makefile
index 85273e52c..20d408831 100644
--- a/Makefile
+++ b/Makefile
@@ -261,7 +261,11 @@ secrets-show: pre-check
 
 .PHONY: secrets-edit
 secrets-edit: pre-check
-	./scripts/deploy/secrets.sh edit $(ENVIRONMENT) $(SECRET_VERSION)
+	@./scripts/deploy/secrets.sh edit $(ENVIRONMENT) $(SECRET_VERSION)
+
+.PHONY: secrets-revert
+secrets-revert: pre-check
+	@./scripts/deploy/secrets.sh revert $(ENVIRONMENT) $(SECRET_VERSION)
 
 ##################
 ## Dependencies ##

From ba40223d226e87b7174cc6f05c658d9c3e5b8106 Mon Sep 17 00:00:00 2001
From: Tom Martensen <tmartens@redhat.com>
Date: Wed, 11 Oct 2023 12:44:45 +0200
Subject: [PATCH 09/19] fix bats e2e test target

---
 .github/workflows/PR.yaml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/PR.yaml b/.github/workflows/PR.yaml
index ebef9877b..475871277 100644
--- a/.github/workflows/PR.yaml
+++ b/.github/workflows/PR.yaml
@@ -148,7 +148,7 @@ jobs:
         run: |
           make argo-workflow-lint
 
-      - name: Run e2e tests
+      - name: Run BATS e2e tests
         env:
           INFRA_TOKEN: ${{ secrets.INFRA_TOKEN_DEV }}
         run: |
@@ -158,7 +158,7 @@ jobs:
           $INFRACTL whoami || true
           $INFRACTL version || true
 
-          make e2e-tests || touch FAIL
+          make bats-e2e-tests || touch FAIL
           kubectl -n infra logs -l app=infra-server --tail=-1
           kill %1
 

From 750ed498398123668bd3b4dd027bd0af8dbbfc1c Mon Sep 17 00:00:00 2001
From: Tom Martensen <tmartens@redhat.com>
Date: Wed, 11 Oct 2023 13:07:50 +0200
Subject: [PATCH 10/19] update documentation

---
 CHANGELOG.md  |  2 ++
 DEPLOYMENT.md | 51 ++++++++++++++++-----------------------------------
 2 files changed, 18 insertions(+), 35 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index e76e2ab5f..d6cf5ed35 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -8,6 +8,8 @@ Please avoid adding duplicate information across this changelog and JIRA/doc inp
 
 ## [NEXT RELEASE]
 
+- Deploying infra-server with Helm and GCP Secret Manager
+
 ## [0.8.2]
 
 - Hackathon '23:
diff --git a/DEPLOYMENT.md b/DEPLOYMENT.md
index f6a0ac718..dafe8fc06 100644
--- a/DEPLOYMENT.md
+++ b/DEPLOYMENT.md
@@ -49,27 +49,16 @@ infra-address-production   35.227.207.252  EXTERNAL
 
 ## Configuration
 
-Service configuration is [stored in a GCS bucket](https://console.cloud.google.com/storage/browser/infra-configuration?organizationId=847401270788&project=stackrox-infra).
+Service configuration and secrets are stored in [GCP Secret Manager](https://console.cloud.google.com/security/secret-manager?project=stackrox-infra).
 
-You will need to download this configuration if you plan to make a change to infra. Configuration changes are baked in to the `infra-server` image at build time.
+To view these, run:
 
-To download the configuration locally to `chart/infra-server/configuration`, run:
+`ENVIRONMENT=<development,production> SECRET_VERSION=<latest, 1,2,3,...> make secrets-download`.
 
-`make configuration-download`
+This will download the secrets to `chart/infra-server/configuration/`.
 
-After you change configuration run:
-
-`make create-consolidated-values`
-
-Which will e.g.:
-```
-INFO: Wed Jan 11 06:47:53 PM UTC 2023: Creating a combined values file for chart/infra-server/configuration/development files
-INFO: Wed Jan 11 06:47:53 PM UTC 2023: Creating a combined values file for chart/infra-server/configuration/production files
-```
-
-To upload the local configuration which includes the consolidated values back to the bucket, run:
-
-`make configuration-upload`
+- `<ENVIRONMENT>-values.yaml`: To show or edit a value, do it directly in this file, and use `ENVIRONMENT=<development,production> make secrets-upload` to upload the changes.
+- `<ENVIRONMENT>-values-from-files.yaml`: To show or edit a value, use `ENVIRONMENT=<development,production> SECRET_VERSION=<latest,1,2,3> make secrets-<show, edit>` and follow the instructions. NOTE: This will download a fresh copy of the requested secret version and upload a new version after your changes. That ensures that your local secrets do not go stale.
 
 ## Regenerating the localhost certificates for the gRPC gateway
 
@@ -121,35 +110,27 @@ correct tooling installed with:
 
 Use the `deploy` Github action to update development or production environments with a new release.
 
-### Staging/(dev.infra.rox.systems)
-
-To render a copy of the charts (for inspection), run:
-
-`make render-development`
-
-To then apply that chart to the development cluster, run:
-
-`make install-development-with-rendered`
+### Argo Deployment
 
-To do everything in one command, run:
+To install Argo workflow server, run:
 
-`make install-development`
+`ENVIRONMENT=<development,production> make install-argo`
 
-Note: This will always bounce the infra server pods.
+NOTE: This is a separate step and not a dependant chart for example to avoid too frequent Argo deployments.
 
-### Production
+### Manual deployment
 
 To render a copy of the charts (for inspection), run:
 
-`make render-production`
+`ENVIRONMENT=<development,production> SECRET_VERSION=<latest,1,2,3, ...> make helm-template`
 
-To then apply that chart to the development cluster, run:
+To show the diff between the current Helm release and the charts, run:
 
-`make install-production-with-rendered`
+`ENVIRONMENT=<development,production> SECRET_VERSION=<latest,1,2,3, ...> make helm-diff`
 
-To do everything in one command, run:
+To then apply that chart to the cluster, run:
 
-`make deploy-production`
+`ENVIRONMENT=<development,production> SECRET_VERSION=<latest,1,2,3, ...> make helm-deploy`
 
 ## Verification
 

From 90540ce424a88609253b41192428d219e58ff71d Mon Sep 17 00:00:00 2001
From: Tom Martensen <tmartens@redhat.com>
Date: Wed, 11 Oct 2023 13:09:49 +0200
Subject: [PATCH 11/19] update

---
 DEPLOYMENT.md | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/DEPLOYMENT.md b/DEPLOYMENT.md
index dafe8fc06..06911d29c 100644
--- a/DEPLOYMENT.md
+++ b/DEPLOYMENT.md
@@ -132,6 +132,12 @@ To then apply that chart to the cluster, run:
 
 `ENVIRONMENT=<development,production> SECRET_VERSION=<latest,1,2,3, ...> make helm-deploy`
 
+#### Test Mode
+
+Use the environment variable `TEST_MODE` to disable certain infra service behavior, like:
+
+`TEST_MODE=true ENVIRONMENT=development SECRET_VERSION=latest make helm-deploy`
+
 ## Verification
 
 After deploying the service, browse to the appropriate endpoint to verify that you can login and view the UI.

From 22000e6ca8896c8e65c9253484cdb94c4db57bb8 Mon Sep 17 00:00:00 2001
From: Tom Martensen <tmartens@redhat.com>
Date: Wed, 11 Oct 2023 13:29:15 +0200
Subject: [PATCH 12/19] fix deployment workflow

---
 .github/workflows/deploy.yaml | 4 ++--
 DEPLOYMENT.md                 | 2 ++
 2 files changed, 4 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/deploy.yaml b/.github/workflows/deploy.yaml
index 2eff3204c..3d1c21541 100644
--- a/.github/workflows/deploy.yaml
+++ b/.github/workflows/deploy.yaml
@@ -53,14 +53,14 @@ jobs:
           gcloud container clusters get-credentials infra-${{ inputs.environment }} \
             --project stackrox-infra \
             --region us-west2
-          make install-${{ inputs.environment }}
+          ENVIRONMENT=${{ inputs.environment }} make install-argo helm-deploy
 
       - name: Notify infra channel about new version
         env:
           SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }}
         uses: slackapi/slack-github-action@v1.23.0
         with:
-          channel-id: C01H4DC33K3 #acs-infra
+          channel-id: CVANK5K5W #acs-infra
           payload: >-
             {
                 "blocks": [
diff --git a/DEPLOYMENT.md b/DEPLOYMENT.md
index 06911d29c..624ffa987 100644
--- a/DEPLOYMENT.md
+++ b/DEPLOYMENT.md
@@ -138,6 +138,8 @@ Use the environment variable `TEST_MODE` to disable certain infra service behavi
 
 `TEST_MODE=true ENVIRONMENT=development SECRET_VERSION=latest make helm-deploy`
 
+This is used in the infra PR clusters to set the login referer and disable telemetry.
+
 ## Verification
 
 After deploying the service, browse to the appropriate endpoint to verify that you can login and view the UI.

From 526e1e5c7ae1ced7efa83fe8f320d7eaafcdaf32 Mon Sep 17 00:00:00 2001
From: Tom Martensen <tmartens@redhat.com>
Date: Wed, 11 Oct 2023 13:30:04 +0200
Subject: [PATCH 13/19] fix deployment workflow

---
 .github/workflows/deploy.yaml | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/deploy.yaml b/.github/workflows/deploy.yaml
index 3d1c21541..58dd33166 100644
--- a/.github/workflows/deploy.yaml
+++ b/.github/workflows/deploy.yaml
@@ -60,7 +60,8 @@ jobs:
           SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }}
         uses: slackapi/slack-github-action@v1.23.0
         with:
-          channel-id: CVANK5K5W #acs-infra
+          channel-id: C03KSV3N6N8 #acs-release-automation-testing
+          # channel-id: CVANK5K5W #acs-infra
           payload: >-
             {
                 "blocks": [

From b05e6dd43c99837e34ae95ffbc2d229f2c3f2d1c Mon Sep 17 00:00:00 2001
From: Tom Martensen <tmartens@redhat.com>
Date: Wed, 11 Oct 2023 13:32:34 +0200
Subject: [PATCH 14/19] sequentialize PRs

---
 .github/workflows/PR.yaml | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/.github/workflows/PR.yaml b/.github/workflows/PR.yaml
index 475871277..d40bded07 100644
--- a/.github/workflows/PR.yaml
+++ b/.github/workflows/PR.yaml
@@ -10,6 +10,8 @@ defaults:
   run:
     working-directory: go/src/github.com/stackrox/infra
 
+concurrency: pr-${{ github.ref }}
+
 env:
   CLUSTER_NAME: infra-pr-${{ github.event.pull_request.number }}
   GH_TOKEN: ${{ secrets.ROBOT_ROX_GITHUB_TOKEN }}

From 58cb41d370b108e285ae5f6912321176ad16551d Mon Sep 17 00:00:00 2001
From: Tom Martensen <tmartens@redhat.com>
Date: Wed, 11 Oct 2023 13:50:10 +0200
Subject: [PATCH 15/19] send it

---
 .github/workflows/deploy.yaml | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/.github/workflows/deploy.yaml b/.github/workflows/deploy.yaml
index 58dd33166..3d1c21541 100644
--- a/.github/workflows/deploy.yaml
+++ b/.github/workflows/deploy.yaml
@@ -60,8 +60,7 @@ jobs:
           SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }}
         uses: slackapi/slack-github-action@v1.23.0
         with:
-          channel-id: C03KSV3N6N8 #acs-release-automation-testing
-          # channel-id: CVANK5K5W #acs-infra
+          channel-id: CVANK5K5W #acs-infra
           payload: >-
             {
                 "blocks": [

From cb6a8a4f427b413bdc72e42bbcb5a0c97032af19 Mon Sep 17 00:00:00 2001
From: Tom Martensen <tmartens@redhat.com>
Date: Wed, 11 Oct 2023 14:24:47 +0200
Subject: [PATCH 16/19] add name

---
 .github/workflows/deploy.yaml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.github/workflows/deploy.yaml b/.github/workflows/deploy.yaml
index 3d1c21541..1b2b130b0 100644
--- a/.github/workflows/deploy.yaml
+++ b/.github/workflows/deploy.yaml
@@ -1,3 +1,4 @@
+name: Deploy infra
 run-name: >-
   ${{
     format('Deploy infra version {0} to {1}',

From 5081f5d72b6c85da862ee2e5273a42fe154f9b3c Mon Sep 17 00:00:00 2001
From: Tom Martensen <tmartens@redhat.com>
Date: Wed, 11 Oct 2023 15:03:27 +0200
Subject: [PATCH 17/19] fix infractl pull

---
 Makefile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Makefile b/Makefile
index 20d408831..647a804e0 100644
--- a/Makefile
+++ b/Makefile
@@ -194,7 +194,7 @@ pull-infractl-from-dev-server:
 	@mkdir -p bin
 	@rm -f bin/infractl
 	set -o pipefail; \
-	curl --retry 3 --insecure --silent --show-error --fail --location https://localhost:8443/v1/cli/linux/amd64/upgrade \
+	curl --retry 3 --insecure --silent --show-error --fail --location https://localhost:8443/v1/cli/$(shell go env GOOS)/$(shell go env GOARCH)/upgrade \
           | jq -r ".result.fileChunk" \
           | base64 -d \
           > bin/infractl

From 67ab57ee930d3000887e310e5bc87c6c47da7c77 Mon Sep 17 00:00:00 2001
From: Tom Martensen <tmartens@redhat.com>
Date: Thu, 12 Oct 2023 08:28:33 +0200
Subject: [PATCH 18/19] use correct capabilities lookup

---
 chart/infra-server/templates/ingress.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/chart/infra-server/templates/ingress.yaml b/chart/infra-server/templates/ingress.yaml
index 11696b295..9e16b3e4f 100644
--- a/chart/infra-server/templates/ingress.yaml
+++ b/chart/infra-server/templates/ingress.yaml
@@ -1,4 +1,4 @@
-{{- if .Capabilities.APIVersions.Has "networking.gke.io/v1/Ingress" -}}
+{{- if .Capabilities.APIVersions.Has "networking.gke.io/v1/ManagedCertificate" -}}
 apiVersion: networking.k8s.io/v1
 kind: Ingress
 

From ce8e555ae19ce0136a884d69bf213289654fffc0 Mon Sep 17 00:00:00 2001
From: Tom Martensen <tmartens@redhat.com>
Date: Fri, 13 Oct 2023 10:01:57 +0200
Subject: [PATCH 19/19] use testmode instead of capabilities

---
 chart/infra-server/templates/certificate.yaml | 2 +-
 chart/infra-server/templates/deployment.yaml  | 2 +-
 chart/infra-server/templates/ingress.yaml     | 2 +-
 3 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/chart/infra-server/templates/certificate.yaml b/chart/infra-server/templates/certificate.yaml
index ae08599eb..340896502 100644
--- a/chart/infra-server/templates/certificate.yaml
+++ b/chart/infra-server/templates/certificate.yaml
@@ -1,4 +1,4 @@
-{{- if .Capabilities.APIVersions.Has "networking.gke.io/v1/ManagedCertificate" -}}
+{{- if eq .Values.testMode false -}}
 ---
 
 apiVersion: networking.gke.io/v1
diff --git a/chart/infra-server/templates/deployment.yaml b/chart/infra-server/templates/deployment.yaml
index 574f493c1..b6811b323 100644
--- a/chart/infra-server/templates/deployment.yaml
+++ b/chart/infra-server/templates/deployment.yaml
@@ -30,7 +30,7 @@ spec:
             - name: GOOGLE_APPLICATION_CREDENTIALS
               value: /configuration/google-credentials.json
             - name: TEST_MODE
-              value: "{{ eq .Values.testMode true }}"
+              value: "{{ .Values.testMode }}"
           readinessProbe:
             httpGet:
               path: /
diff --git a/chart/infra-server/templates/ingress.yaml b/chart/infra-server/templates/ingress.yaml
index 9e16b3e4f..41361ff54 100644
--- a/chart/infra-server/templates/ingress.yaml
+++ b/chart/infra-server/templates/ingress.yaml
@@ -1,4 +1,4 @@
-{{- if .Capabilities.APIVersions.Has "networking.gke.io/v1/ManagedCertificate" -}}
+{{- if eq .Values.testMode false -}}
 apiVersion: networking.k8s.io/v1
 kind: Ingress