From 4ca2d23404c64017a413e0ee408f926b51b6898e Mon Sep 17 00:00:00 2001 From: yy <56745951+lingdie@users.noreply.github.com> Date: Wed, 26 Jun 2024 19:47:20 +0800 Subject: [PATCH] feat: helm charts support installing gateway in daemonset mod. (#1054) --- helm/core/templates/daemonset.yaml | 329 ++++++++++++++++++++++++++++ helm/core/templates/deployment.yaml | 4 +- helm/core/values.yaml | 3 + 3 files changed, 335 insertions(+), 1 deletion(-) create mode 100644 helm/core/templates/daemonset.yaml diff --git a/helm/core/templates/daemonset.yaml b/helm/core/templates/daemonset.yaml new file mode 100644 index 0000000000..1e144ea146 --- /dev/null +++ b/helm/core/templates/daemonset.yaml @@ -0,0 +1,329 @@ +{{- if eq .Values.gateway.kind "DaemonSet" -}} +{{- $o11y := .Values.global.o11y }} +{{- $unprivilegedPortSupported := true }} +{{- range $index, $node := (lookup "v1" "Node" "default" "").items }} + {{- $kernelVersion := $node.status.nodeInfo.kernelVersion }} + {{- if $kernelVersion }} + {{- $kernelVersion = regexFind "^(\\d+\\.\\d+\\.\\d+)" $kernelVersion }} + {{- if and $kernelVersion (semverCompare "<4.11.0" $kernelVersion) }} + {{- $unprivilegedPortSupported = false }} + {{- end }} + {{- end }} +{{- end -}} +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: {{ include "gateway.name" . }} + namespace: {{ .Release.Namespace }} + labels: + {{- include "gateway.labels" . | nindent 4}} + annotations: + {{- .Values.gateway.annotations | toYaml | nindent 4 }} +spec: + selector: + matchLabels: + {{- include "gateway.selectorLabels" . | nindent 6 }} + template: + metadata: + annotations: + {{- if .Values.global.enableHigressIstio }} + "enableHigressIstio": "true" + {{- end }} + {{- if .Values.gateway.podAnnotations }} + {{- toYaml .Values.gateway.podAnnotations | nindent 8 }} + {{- end }} + labels: + sidecar.istio.io/inject: "false" + {{- with .Values.gateway.revision }} + istio.io/rev: {{ . }} + {{- end }} + {{- include "gateway.selectorLabels" . | nindent 8 }} + spec: + {{- with .Values.gateway.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} + serviceAccountName: {{ include "gateway.serviceAccountName" . }} + securityContext: + {{- if .Values.gateway.securityContext }} + {{- toYaml .Values.gateway.securityContext | nindent 8 }} + {{- else if and $unprivilegedPortSupported (and (not .Values.gateway.hostNetwork) (semverCompare ">=1.22-0" .Capabilities.KubeVersion.GitVersion)) }} + # Safe since 1.22: https://github.com/kubernetes/kubernetes/pull/103326 + sysctls: + - name: net.ipv4.ip_unprivileged_port_start + value: "0" + {{- end }} + containers: + {{- if $o11y.enabled }} + {{- $config := $o11y.promtail }} + - name: promtail + image: {{ $config.image.repository }}:{{ $config.image.tag }} + imagePullPolicy: IfNotPresent + args: + - -config.file=/etc/promtail/promtail.yaml + env: + - name: 'HOSTNAME' + valueFrom: + fieldRef: + fieldPath: 'spec.nodeName' + ports: + - containerPort: {{ $config.port }} + name: http-metrics + protocol: TCP + readinessProbe: + failureThreshold: 3 + httpGet: + path: /ready + port: {{ $config.port }} + scheme: HTTP + initialDelaySeconds: 10 + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 1 + volumeMounts: + - name: promtail-config + mountPath: "/etc/promtail" + - name: log + mountPath: /var/log/proxy + - name: tmp + mountPath: /tmp + {{- end }} + - name: higress-gateway + image: "{{ .Values.gateway.hub | default .Values.global.hub }}/{{ .Values.gateway.image | default "gateway" }}:{{ .Values.gateway.tag | default .Chart.AppVersion }}" + args: + - proxy + - router + - --domain + - $(POD_NAMESPACE).svc.cluster.local + - --proxyLogLevel=warning + - --proxyComponentLogLevel=misc:error + - --log_output_level=all:info + - --serviceCluster=higress-gateway + securityContext: + {{- if .Values.gateway.containerSecurityContext }} + {{- toYaml .Values.gateway.containerSecurityContext | nindent 12 }} + {{- else if and $unprivilegedPortSupported (and (not .Values.gateway.hostNetwork) (semverCompare ">=1.22-0" .Capabilities.KubeVersion.GitVersion)) }} + # Safe since 1.22: https://github.com/kubernetes/kubernetes/pull/103326 + capabilities: + drop: + - ALL + allowPrivilegeEscalation: false + privileged: false + # When enabling lite metrics, the configuration template files need to be replaced. + {{- if not .Values.global.liteMetrics }} + readOnlyRootFilesystem: true + {{- end }} + runAsUser: 1337 + runAsGroup: 1337 + runAsNonRoot: true + {{- else }} + capabilities: + drop: + - ALL + add: + - NET_BIND_SERVICE + runAsUser: 0 + runAsGroup: 1337 + runAsNonRoot: false + allowPrivilegeEscalation: true + {{- end }} + env: + - name: NODE_NAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: spec.nodeName + - name: POD_NAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.name + - name: POD_NAMESPACE + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.namespace + - name: INSTANCE_IP + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: status.podIP + - name: HOST_IP + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: status.hostIP + - name: SERVICE_ACCOUNT + valueFrom: + fieldRef: + fieldPath: spec.serviceAccountName + - name: PILOT_XDS_SEND_TIMEOUT + value: 60s + - name: PROXY_XDS_VIA_AGENT + value: "true" + - name: ENABLE_INGRESS_GATEWAY_SDS + value: "false" + - name: JWT_POLICY + value: {{ include "controller.jwtPolicy" . }} + - name: ISTIO_META_HTTP10 + value: "1" + - name: ISTIO_META_CLUSTER_ID + value: "{{ $.Values.clusterName | default `Kubernetes` }}" + - name: INSTANCE_NAME + value: "higress-gateway" + {{- if .Values.global.liteMetrics }} + - name: LITE_METRICS + value: "on" + {{- end }} + {{- if include "skywalking.enabled" . }} + - name: ISTIO_BOOTSTRAP_OVERRIDE + value: /etc/istio/custom-bootstrap/custom_bootstrap.json + {{- end }} + {{- with .Values.gateway.networkGateway }} + - name: ISTIO_META_REQUESTED_NETWORK_VIEW + value: "{{.}}" + {{- end }} + {{- range $key, $val := .Values.env }} + - name: {{ $key }} + value: {{ $val | quote }} + {{- end }} + ports: + - containerPort: 15090 + protocol: TCP + name: http-envoy-prom + {{- if or .Values.global.local .Values.global.kind }} + - containerPort: {{ .Values.gateway.httpPort }} + hostPort: {{ .Values.gateway.httpPort }} + name: http + protocol: TCP + - containerPort: {{ .Values.gateway.httpsPort }} + hostPort: {{ .Values.gateway.httpsPort }} + name: https + protocol: TCP + {{- end }} + readinessProbe: + failureThreshold: {{ .Values.gateway.readinessFailureThreshold }} + httpGet: + path: /healthz/ready + port: 15021 + scheme: HTTP + initialDelaySeconds: {{ .Values.gateway.readinessInitialDelaySeconds }} + periodSeconds: {{ .Values.gateway.readinessPeriodSeconds }} + successThreshold: {{ .Values.gateway.readinessSuccessThreshold }} + timeoutSeconds: {{ .Values.gateway.readinessTimeoutSeconds }} + {{- if not (or .Values.global.local .Values.global.kind) }} + resources: + {{- toYaml .Values.gateway.resources | nindent 12 }} + {{- end }} + volumeMounts: + {{- if eq (include "controller.jwtPolicy" .) "third-party-jwt" }} + - name: istio-token + mountPath: /var/run/secrets/tokens + readOnly: true + {{- end }} + - name: config + mountPath: /etc/istio/config + - name: istio-ca-root-cert + mountPath: /var/run/secrets/istio + - name: istio-data + mountPath: /var/lib/istio/data + - name: podinfo + mountPath: /etc/istio/pod + - name: proxy-socket + mountPath: /etc/istio/proxy + {{- if include "skywalking.enabled" . }} + - mountPath: /etc/istio/custom-bootstrap + name: custom-bootstrap-volume + {{- end }} + {{- if .Values.global.volumeWasmPlugins }} + - mountPath: /opt/plugins + name: local-wasmplugins-volume + {{- end }} + {{- if $o11y.enabled }} + - mountPath: /var/log/proxy + name: log + {{- end }} + {{- if .Values.gateway.hostNetwork }} + hostNetwork: {{ .Values.gateway.hostNetwork }} + dnsPolicy: ClusterFirstWithHostNet + {{- end }} + {{- with .Values.gateway.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.gateway.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.gateway.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} + volumes: + {{- if eq (include "controller.jwtPolicy" .) "third-party-jwt" }} + - name: istio-token + projected: + sources: + - serviceAccountToken: + audience: istio-ca + expirationSeconds: 43200 + path: istio-token + {{- end }} + - name: istio-ca-root-cert + configMap: + {{- if .Values.global.enableHigressIstio }} + name: istio-ca-root-cert + {{- else }} + name: higress-ca-root-cert + {{- end }} + - name: config + configMap: + name: higress-config + {{- if include "skywalking.enabled" . }} + - configMap: + defaultMode: 420 + name: higress-custom-bootstrap + name: custom-bootstrap-volume + {{- end }} + - name: istio-data + emptyDir: {} + - name: proxy-socket + emptyDir: {} + {{- if $o11y.enabled }} + - name: log + emptyDir: {} + - name: tmp + emptyDir: {} + - name: promtail-config + configMap: + name: higress-promtail + {{- end }} + - name: podinfo + downwardAPI: + defaultMode: 420 + items: + - fieldRef: + apiVersion: v1 + fieldPath: metadata.labels + path: labels + - fieldRef: + apiVersion: v1 + fieldPath: metadata.annotations + path: annotations + - path: cpu-request + resourceFieldRef: + containerName: higress-gateway + divisor: 1m + resource: requests.cpu + - path: cpu-limit + resourceFieldRef: + containerName: higress-gateway + divisor: 1m + resource: limits.cpu + {{- if .Values.global.volumeWasmPlugins }} + - name: local-wasmplugins-volume + hostPath: + path: /opt/plugins + type: Directory + {{- end }} +{{- end }} diff --git a/helm/core/templates/deployment.yaml b/helm/core/templates/deployment.yaml index cf6f5cbf21..7a472e293f 100644 --- a/helm/core/templates/deployment.yaml +++ b/helm/core/templates/deployment.yaml @@ -1,3 +1,4 @@ +{{- if eq .Values.gateway.kind "Deployment" -}} {{- $o11y := .Values.global.o11y }} {{- $unprivilegedPortSupported := true }} {{- range $index, $node := (lookup "v1" "Node" "default" "").items }} @@ -241,7 +242,7 @@ spec: mountPath: /var/run/secrets/istio - name: istio-data mountPath: /var/lib/istio/data - - name: podinfo + - name: podinfo mountPath: /etc/istio/pod - name: proxy-socket mountPath: /etc/istio/proxy @@ -340,3 +341,4 @@ spec: path: /opt/plugins type: Directory {{- end }} +{{- end }} diff --git a/helm/core/values.yaml b/helm/core/values.yaml index 36176fbd5f..d9e58dd947 100644 --- a/helm/core/values.yaml +++ b/helm/core/values.yaml @@ -396,6 +396,9 @@ gateway: replicas: 2 image: gateway + # -- Use a `DaemonSet` or `Deployment` + kind: Deployment + # The number of successive failed probes before indicating readiness failure. readinessFailureThreshold: 30