From 2ce068fc139be412275830efcc8dcec8d2542603 Mon Sep 17 00:00:00 2001 From: Vibhu Prashar Date: Thu, 6 Jun 2024 17:52:39 +0530 Subject: [PATCH] fix: make deploy target This PR address the issue #381 by: * Enabling cert-manager that is necessary by webhooks during the deployment of Operator on k8s. * Added a separate config for k8s that contains the manifests that are specific for deploying on k8s. * `config/default` points to OpenShift specific manifests * `config/k8s` points to manifests specific for k8s * Added overlays in manager to isolate k8s and OpenShift Signed-off-by: Vibhu Prashar --- Makefile | 4 +- ...kepler-operator.clusterserviceversion.yaml | 8 +-- ...stem.sustainable.computing.io_keplers.yaml | 2 +- config/default/kustomization.yaml | 36 +--------- config/default/manager_config_patch.yaml | 10 --- config/default/manager_webhook_patch.yaml | 11 --- .../k8s/default/cainjection_in_keplers.yaml | 7 ++ config/k8s/default/kustomization.yaml | 72 +++++++++++++++++++ .../k8s/default/manager_auth_proxy_patch.yaml | 56 +++++++++++++++ config/k8s/default/manager_webhook_patch.yaml | 23 ++++++ .../k8s/default/webhookcainjection_patch.yaml | 29 ++++++++ config/k8s/kustomization.yaml | 24 +++++++ config/manager/{ => base}/kustomization.yaml | 0 config/manager/{ => base}/manager.yaml | 2 - .../manager/overlays/k8s/kustomization.yaml | 14 ++++ .../overlays/openshift/kustomization.yaml | 17 +++++ config/manifests/kustomization.yaml | 20 ------ hack/tools.sh | 2 + 18 files changed, 251 insertions(+), 86 deletions(-) delete mode 100644 config/default/manager_config_patch.yaml create mode 100644 config/k8s/default/cainjection_in_keplers.yaml create mode 100644 config/k8s/default/kustomization.yaml create mode 100644 config/k8s/default/manager_auth_proxy_patch.yaml create mode 100644 config/k8s/default/manager_webhook_patch.yaml create mode 100644 config/k8s/default/webhookcainjection_patch.yaml create mode 100644 config/k8s/kustomization.yaml rename config/manager/{ => base}/kustomization.yaml (100%) rename config/manager/{ => base}/manager.yaml (98%) create mode 100644 config/manager/overlays/k8s/kustomization.yaml create mode 100644 config/manager/overlays/openshift/kustomization.yaml diff --git a/Makefile b/Makefile index 078bc1fb..b8c804a2 100644 --- a/Makefile +++ b/Makefile @@ -246,7 +246,7 @@ uninstall: manifests kustomize ## Uninstall CRDs from the K8s cluster specified .PHONY: deploy deploy: install ## Deploy controller to the K8s cluster specified in ~/.kube/config. - $(KUSTOMIZE) build config/default | \ + $(KUSTOMIZE) build config/k8s | \ sed -e "s||$(OPERATOR_IMG)|g" \ -e "s||$(KEPLER_IMG)|g" \ | tee tmp/deploy.yaml | \ @@ -254,7 +254,7 @@ deploy: install ## Deploy controller to the K8s cluster specified in ~/.kube/con .PHONY: undeploy undeploy: ## Undeploy controller from the K8s cluster specified in ~/.kube/config. Call with ignore-not-found=true to ignore resource not found errors during deletion. - $(KUSTOMIZE) build config/default | \ + $(KUSTOMIZE) build config/k8s | \ kubectl delete --ignore-not-found=$(ignore-not-found) -f - ##@ Build Dependencies diff --git a/bundle/manifests/kepler-operator.clusterserviceversion.yaml b/bundle/manifests/kepler-operator.clusterserviceversion.yaml index f24e15fd..3af1fb92 100644 --- a/bundle/manifests/kepler-operator.clusterserviceversion.yaml +++ b/bundle/manifests/kepler-operator.clusterserviceversion.yaml @@ -28,7 +28,7 @@ metadata: capabilities: Seamless Upgrades categories: Monitoring containerImage: quay.io/sustainable_computing_io/kepler-operator:0.13.0 - createdAt: "2024-05-22T07:06:13Z" + createdAt: "2024-06-11T18:04:59Z" description: 'Deploys and Manages Kepler on Kubernetes ' operators.operatorframework.io/builder: operator-sdk-v1.27.0 operators.operatorframework.io/internal-objects: |- @@ -259,9 +259,9 @@ spec: containers: - args: - --openshift + - --deployment-namespace=kepler-operator - --leader-elect - --kepler.image=$(RELATED_IMAGE_KEPLER) - - --deployment-namespace=kepler-operator - --zap-log-level=5 command: - /manager @@ -302,10 +302,6 @@ spec: capabilities: drop: - ALL - volumeMounts: - - mountPath: /tmp/k8s-webhook-server/serving-certs - name: cert - readOnly: true securityContext: runAsNonRoot: true serviceAccountName: kepler-operator-controller-manager diff --git a/bundle/manifests/kepler.system.sustainable.computing.io_keplers.yaml b/bundle/manifests/kepler.system.sustainable.computing.io_keplers.yaml index dbe4039e..67584ee4 100644 --- a/bundle/manifests/kepler.system.sustainable.computing.io_keplers.yaml +++ b/bundle/manifests/kepler.system.sustainable.computing.io_keplers.yaml @@ -12,7 +12,7 @@ spec: clientConfig: service: name: kepler-operator-webhook-service - namespace: kepler-operator-system + namespace: kepler-operator path: /convert conversionReviewVersions: - v1 diff --git a/config/default/kustomization.yaml b/config/default/kustomization.yaml index fa298e61..15ee2c1b 100644 --- a/config/default/kustomization.yaml +++ b/config/default/kustomization.yaml @@ -1,5 +1,5 @@ # Adds namespace to all resources. -namespace: kepler-operator-system +namespace: kepler-operator # Value of this field is prepended to the # names of all resources, e.g. a deployment named @@ -15,7 +15,7 @@ namePrefix: kepler-operator- bases: - ../crd - ../rbac -- ../manager +- ../manager/overlays/openshift # [WEBHOOK] To enable webhook, uncomment all the sections with [WEBHOOK] prefix including the one in # crd/kustomization.yaml - ../webhook @@ -30,8 +30,6 @@ patchesStrategicMerge: # endpoint w/o any authn/z, please comment the following line. # - manager_auth_proxy_patch.yaml - - # [WEBHOOK] To enable webhook, uncomment all the sections with [WEBHOOK] prefix including the one in # crd/kustomization.yaml - manager_webhook_patch.yaml @@ -40,33 +38,3 @@ patchesStrategicMerge: # Uncomment 'CERTMANAGER' sections in crd/kustomization.yaml to enable the CA injection in the admission webhooks. # 'CERTMANAGER' needs to be enabled to use ca injection #- webhookcainjection_patch.yaml - -# the following config is for teaching kustomize how to do var substitution -vars: -# [CERTMANAGER] To enable cert-manager, uncomment all sections with 'CERTMANAGER' prefix. -#- name: CERTIFICATE_NAMESPACE # namespace of the certificate CR -# objref: -# kind: Certificate -# group: cert-manager.io -# version: v1 -# name: serving-cert # this name should match the one in certificate.yaml -# fieldref: -# fieldpath: metadata.namespace -#- name: CERTIFICATE_NAME -# objref: -# kind: Certificate -# group: cert-manager.io -# version: v1 -# name: serving-cert # this name should match the one in certificate.yaml -#- name: SERVICE_NAMESPACE # namespace of the service -# objref: -# kind: Service -# version: v1 -# name: webhook-service -# fieldref: -# fieldpath: metadata.namespace -#- name: SERVICE_NAME -# objref: -# kind: Service -# version: v1 -# name: webhook-service diff --git a/config/default/manager_config_patch.yaml b/config/default/manager_config_patch.yaml deleted file mode 100644 index f6f58916..00000000 --- a/config/default/manager_config_patch.yaml +++ /dev/null @@ -1,10 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: controller-manager - namespace: system -spec: - template: - spec: - containers: - - name: manager diff --git a/config/default/manager_webhook_patch.yaml b/config/default/manager_webhook_patch.yaml index 1c378047..b6ea9282 100644 --- a/config/default/manager_webhook_patch.yaml +++ b/config/default/manager_webhook_patch.yaml @@ -12,14 +12,3 @@ spec: - containerPort: 9443 name: webhook-server protocol: TCP - volumeMounts: - - mountPath: /tmp/k8s-webhook-server/serving-certs - name: cert - readOnly: true - # NOTE: this will be removed by the manager kustomization.yaml - # since OLM will add the volume - volumes: - - name: cert - secret: - defaultMode: 420 - secretName: webhook-server-cert diff --git a/config/k8s/default/cainjection_in_keplers.yaml b/config/k8s/default/cainjection_in_keplers.yaml new file mode 100644 index 00000000..43a013b5 --- /dev/null +++ b/config/k8s/default/cainjection_in_keplers.yaml @@ -0,0 +1,7 @@ +# The following patch adds a directive for certmanager to inject CA into the CRD +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + cert-manager.io/inject-ca-from: $(CERTIFICATE_NAMESPACE)/$(CERTIFICATE_NAME) + name: keplers.kepler.system.sustainable.computing.io diff --git a/config/k8s/default/kustomization.yaml b/config/k8s/default/kustomization.yaml new file mode 100644 index 00000000..a600b448 --- /dev/null +++ b/config/k8s/default/kustomization.yaml @@ -0,0 +1,72 @@ +# Adds namespace to all resources. +namespace: kepler-operator + +# Value of this field is prepended to the +# names of all resources, e.g. a deployment named +# "wordpress" becomes "alices-wordpress". +# Note that it should also match with the prefix (text before '-') of the namespace +# field above. +namePrefix: kepler-operator- + +# Labels to add to all resources and selectors. +#commonLabels: +# someName: someValue + +bases: +- ../../crd +- ../../rbac +- ../../manager/overlays/k8s +# [WEBHOOK] To enable webhook, uncomment all the sections with [WEBHOOK] prefix including the one in +# crd/kustomization.yaml +- ../../webhook +# [CERTMANAGER] To enable cert-manager, uncomment all sections with 'CERTMANAGER'. 'WEBHOOK' components are required. +- ../../certmanager +# [PROMETHEUS] To enable prometheus monitor, uncomment all sections with 'PROMETHEUS'. +- ../../prometheus + +patchesStrategicMerge: +# Protect the /metrics endpoint by putting it behind auth. +# If you want your controller-manager to expose the /metrics +# endpoint w/o any authn/z, please comment the following line. +# - manager_auth_proxy_patch.yaml + +# [WEBHOOK] To enable webhook, uncomment all the sections with [WEBHOOK] prefix including the one in +# crd/kustomization.yaml +- manager_webhook_patch.yaml + +# [CERTMANAGER] To enable cert-manager, uncomment all sections with 'CERTMANAGER'. +# Uncomment 'CERTMANAGER' sections in crd/kustomization.yaml to enable the CA injection in the admission webhooks. +# 'CERTMANAGER' needs to be enabled to use ca injection +- webhookcainjection_patch.yaml + +- cainjection_in_keplers.yaml + +# the following config is for teaching kustomize how to do var substitution +vars: +# [CERTMANAGER] To enable cert-manager, uncomment all sections with 'CERTMANAGER' prefix. +- name: CERTIFICATE_NAMESPACE # namespace of the certificate CR + objref: + kind: Certificate + group: cert-manager.io + version: v1 + name: serving-cert # this name should match the one in certificate.yaml + fieldref: + fieldpath: metadata.namespace +- name: CERTIFICATE_NAME + objref: + kind: Certificate + group: cert-manager.io + version: v1 + name: serving-cert # this name should match the one in certificate.yaml +- name: SERVICE_NAMESPACE # namespace of the service + objref: + kind: Service + version: v1 + name: webhook-service + fieldref: + fieldpath: metadata.namespace +- name: SERVICE_NAME + objref: + kind: Service + version: v1 + name: webhook-service diff --git a/config/k8s/default/manager_auth_proxy_patch.yaml b/config/k8s/default/manager_auth_proxy_patch.yaml new file mode 100644 index 00000000..f07d69d9 --- /dev/null +++ b/config/k8s/default/manager_auth_proxy_patch.yaml @@ -0,0 +1,56 @@ +# This patch inject a sidecar container which is a HTTP proxy for the +# controller manager, it performs RBAC authorization against the Kubernetes API using SubjectAccessReviews. +apiVersion: apps/v1 +kind: Deployment +metadata: + name: controller-manager + namespace: system +spec: + template: + spec: + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: kubernetes.io/arch + operator: In + values: + - amd64 + - arm64 + - ppc64le + - s390x + - key: kubernetes.io/os + operator: In + values: + - linux + containers: + - name: kube-rbac-proxy + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - "ALL" + image: gcr.io/kubebuilder/kube-rbac-proxy:v0.13.0 + args: + - "--secure-listen-address=0.0.0.0:8443" + - "--upstream=http://127.0.0.1:8080/" + - "--logtostderr=true" + - "--v=0" + ports: + - containerPort: 8443 + protocol: TCP + name: https + resources: + limits: + cpu: 500m + memory: 128Mi + requests: + cpu: 5m + memory: 64Mi + - name: manager + args: + - "--health-probe-bind-address=:8081" + - "--metrics-bind-address=127.0.0.1:8080" + - "--leader-elect" + - "--zap-log-level=3" diff --git a/config/k8s/default/manager_webhook_patch.yaml b/config/k8s/default/manager_webhook_patch.yaml new file mode 100644 index 00000000..48383d96 --- /dev/null +++ b/config/k8s/default/manager_webhook_patch.yaml @@ -0,0 +1,23 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: controller + namespace: system +spec: + template: + spec: + containers: + - name: manager + ports: + - containerPort: 9443 + name: webhook-server + protocol: TCP + volumeMounts: + - mountPath: /tmp/k8s-webhook-server/serving-certs + name: cert + readOnly: true + volumes: + - name: cert + secret: + defaultMode: 420 + secretName: webhook-server-cert diff --git a/config/k8s/default/webhookcainjection_patch.yaml b/config/k8s/default/webhookcainjection_patch.yaml new file mode 100644 index 00000000..f992f958 --- /dev/null +++ b/config/k8s/default/webhookcainjection_patch.yaml @@ -0,0 +1,29 @@ +# This patch add annotation to admission webhook config and +# the variables $(CERTIFICATE_NAMESPACE) and $(CERTIFICATE_NAME) will be substituted by kustomize. +apiVersion: admissionregistration.k8s.io/v1 +kind: MutatingWebhookConfiguration +metadata: + labels: + app.kubernetes.io/name: mutatingwebhookconfiguration + app.kubernetes.io/instance: mutating-webhook-configuration + app.kubernetes.io/component: webhook + app.kubernetes.io/created-by: kepler-operator + app.kubernetes.io/part-of: kepler-operator + app.kubernetes.io/managed-by: kustomize + name: mutating-webhook-configuration + annotations: + cert-manager.io/inject-ca-from: $(CERTIFICATE_NAMESPACE)/$(CERTIFICATE_NAME) +--- +apiVersion: admissionregistration.k8s.io/v1 +kind: ValidatingWebhookConfiguration +metadata: + labels: + app.kubernetes.io/name: validatingwebhookconfiguration + app.kubernetes.io/instance: validating-webhook-configuration + app.kubernetes.io/component: webhook + app.kubernetes.io/created-by: kepler-operator + app.kubernetes.io/part-of: kepler-operator + app.kubernetes.io/managed-by: kustomize + name: validating-webhook-configuration + annotations: + cert-manager.io/inject-ca-from: $(CERTIFICATE_NAMESPACE)/$(CERTIFICATE_NAME) diff --git a/config/k8s/kustomization.yaml b/config/k8s/kustomization.yaml new file mode 100644 index 00000000..48514eb1 --- /dev/null +++ b/config/k8s/kustomization.yaml @@ -0,0 +1,24 @@ +# These resources constitute the fully configured set of manifests +# used to generate the 'manifests/' directory in a bundle. +resources: +- default + +# [WEBHOOK] To enable webhooks, uncomment all the sections with [WEBHOOK] prefix. +# Do NOT uncomment sections with prefix [CERTMANAGER], as OLM does not support cert-manager. +# These patches remove the unnecessary "cert" volume and its manager container volumeMount. +# patchesJson6902: +# - target: +# group: apps +# version: v1 +# kind: Deployment +# name: controller-manager +# namespace: system +# patch: |- +# # Remove the manager container's "cert" volumeMount, since OLM will create and mount a set of certs. +# # Update the indices in this path if adding or removing containers/volumeMounts in the manager's Deployment. +# - op: remove +# path: /spec/template/spec/containers/0/volumeMounts/0 +# # Remove the "cert" volume, since OLM will create and mount a set of certs. +# # Update the indices in this path if adding or removing volumes in the manager's Deployment. +# - op: remove +# path: /spec/template/spec/volumes/0 diff --git a/config/manager/kustomization.yaml b/config/manager/base/kustomization.yaml similarity index 100% rename from config/manager/kustomization.yaml rename to config/manager/base/kustomization.yaml diff --git a/config/manager/manager.yaml b/config/manager/base/manager.yaml similarity index 98% rename from config/manager/manager.yaml rename to config/manager/base/manager.yaml index 19117049..1df2db5d 100644 --- a/config/manager/manager.yaml +++ b/config/manager/base/manager.yaml @@ -73,10 +73,8 @@ spec: value: '' args: # TODO: move --openshift and deployment-namespace to openshift specific kustomize directory - - --openshift - --leader-elect - --kepler.image=$(RELATED_IMAGE_KEPLER) - - --deployment-namespace=kepler-operator - --zap-log-level=5 image: '' imagePullPolicy: IfNotPresent diff --git a/config/manager/overlays/k8s/kustomization.yaml b/config/manager/overlays/k8s/kustomization.yaml new file mode 100644 index 00000000..691cc803 --- /dev/null +++ b/config/manager/overlays/k8s/kustomization.yaml @@ -0,0 +1,14 @@ +resources: +- ../../base + +patchesJson6902: + - target: + group: apps + version: v1 + kind: Deployment + name: controller + namespace: system + patch: |- + - op: add + path: /spec/template/spec/containers/0/args/0 + value: --deployment-namespace=kepler diff --git a/config/manager/overlays/openshift/kustomization.yaml b/config/manager/overlays/openshift/kustomization.yaml new file mode 100644 index 00000000..f79c5731 --- /dev/null +++ b/config/manager/overlays/openshift/kustomization.yaml @@ -0,0 +1,17 @@ +resources: +- ../../base + +patchesJson6902: + - target: + group: apps + version: v1 + kind: Deployment + name: controller + namespace: system + patch: |- + - op: add + path: /spec/template/spec/containers/0/args/0 + value: --openshift + - op: add + path: /spec/template/spec/containers/0/args/1 + value: --deployment-namespace=kepler-operator diff --git a/config/manifests/kustomization.yaml b/config/manifests/kustomization.yaml index 596609d1..c45ee810 100644 --- a/config/manifests/kustomization.yaml +++ b/config/manifests/kustomization.yaml @@ -5,23 +5,3 @@ resources: - ../default - ../samples - ../scorecard - -# [WEBHOOK] To enable webhooks, uncomment all the sections with [WEBHOOK] prefix. -# Do NOT uncomment sections with prefix [CERTMANAGER], as OLM does not support cert-manager. -# These patches remove the unnecessary "cert" volume and its manager container volumeMount. -patchesJson6902: -- target: - group: apps - version: v1 - kind: Deployment - name: controller-manager - namespace: system - patch: |- - # Remove the manager container's "cert" volumeMount, since OLM will create and mount a set of certs. - # Update the indices in this path if adding or removing containers/volumeMounts in the manager's Deployment. - - op: remove - path: /spec/template/spec/containers/0/volumeMounts/0 - # Remove the "cert" volume, since OLM will create and mount a set of certs. - # Update the indices in this path if adding or removing volumes in the manager's Deployment. - - op: remove - path: /spec/template/spec/volumes/0 diff --git a/hack/tools.sh b/hack/tools.sh index 5b5ccbc8..6529e4b8 100755 --- a/hack/tools.sh +++ b/hack/tools.sh @@ -225,9 +225,11 @@ install_oc() { ok "oc was installed successfully" } + version_jq() { jq --version } + install_jq() { validate_version jq --version "$JQ_VERSION" && { return 0