feat: helm charts support installing gateway in daemonset mod. (#1054)

alibaba · Jun 26, 2024 · 4ca2d23 · 4ca2d23
1 parent 0ce52de
commit 4ca2d23
Show file tree

Hide file tree

Showing 3 changed files with 335 additions and 1 deletion.
diff --git a/helm/core/templates/daemonset.yaml b/helm/core/templates/daemonset.yaml
@@ -0,0 +1,329 @@
+{{- if eq .Values.gateway.kind "DaemonSet" -}}
+{{- $o11y := .Values.global.o11y  }}
+{{- $unprivilegedPortSupported := true }}
+{{- range $index, $node := (lookup "v1" "Node" "default" "").items }}
+    {{- $kernelVersion := $node.status.nodeInfo.kernelVersion }}
+    {{- if $kernelVersion }}
+      {{- $kernelVersion = regexFind "^(\\d+\\.\\d+\\.\\d+)" $kernelVersion }}
+      {{- if and $kernelVersion (semverCompare "<4.11.0" $kernelVersion) }}
+      {{-   $unprivilegedPortSupported = false }}
+      {{- end }}
+    {{- end }}
+{{- end -}}
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  name: {{ include "gateway.name" . }}
+  namespace: {{ .Release.Namespace }}
+  labels:
+    {{- include "gateway.labels" . | nindent 4}}
+  annotations:
+    {{- .Values.gateway.annotations | toYaml | nindent 4 }}
+spec:
+  selector:
+    matchLabels:
+      {{- include "gateway.selectorLabels" . | nindent 6 }}
+  template:
+    metadata:
+      annotations:
+      {{- if .Values.global.enableHigressIstio }}
+        "enableHigressIstio": "true"
+      {{- end }}
+      {{- if .Values.gateway.podAnnotations }}
+        {{- toYaml .Values.gateway.podAnnotations | nindent 8 }}
+      {{- end }}
+      labels:
+        sidecar.istio.io/inject: "false"
+        {{- with .Values.gateway.revision }}
+        istio.io/rev: {{ . }}
+        {{- end }}
+        {{- include "gateway.selectorLabels" . | nindent 8 }}
+    spec:
+      {{- with .Values.gateway.imagePullSecrets }}
+      imagePullSecrets:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      serviceAccountName: {{ include "gateway.serviceAccountName" . }}
+      securityContext:
+      {{- if .Values.gateway.securityContext }}
+        {{- toYaml .Values.gateway.securityContext | nindent 8 }}
+      {{- else if and $unprivilegedPortSupported (and (not .Values.gateway.hostNetwork) (semverCompare ">=1.22-0" .Capabilities.KubeVersion.GitVersion)) }}
+        # Safe since 1.22: https://github.com/kubernetes/kubernetes/pull/103326
+        sysctls:
+        - name: net.ipv4.ip_unprivileged_port_start
+          value: "0"
+      {{- end }}
+      containers:
+        {{- if $o11y.enabled }}
+          {{- $config := $o11y.promtail }}
+        - name: promtail
+          image: {{ $config.image.repository }}:{{ $config.image.tag }}
+          imagePullPolicy: IfNotPresent
+          args:
+            - -config.file=/etc/promtail/promtail.yaml
+          env:
+            - name: 'HOSTNAME'
+              valueFrom:
+                fieldRef:
+                  fieldPath: 'spec.nodeName'
+          ports:
+            - containerPort: {{ $config.port }}
+              name: http-metrics
+              protocol: TCP
+          readinessProbe:
+            failureThreshold: 3
+            httpGet:
+              path: /ready
+              port: {{ $config.port }}
+              scheme: HTTP
+            initialDelaySeconds: 10
+            periodSeconds: 10
+            successThreshold: 1
+            timeoutSeconds: 1
+          volumeMounts:
+            - name: promtail-config
+              mountPath: "/etc/promtail"
+            - name: log
+              mountPath: /var/log/proxy
+            - name: tmp
+              mountPath: /tmp
+        {{- end }}
+        - name: higress-gateway
+          image: "{{ .Values.gateway.hub | default .Values.global.hub }}/{{ .Values.gateway.image | default "gateway" }}:{{ .Values.gateway.tag | default .Chart.AppVersion }}"
+          args:
+            - proxy
+            - router
+            - --domain
+            - $(POD_NAMESPACE).svc.cluster.local
+            - --proxyLogLevel=warning
+            - --proxyComponentLogLevel=misc:error
+            - --log_output_level=all:info
+            - --serviceCluster=higress-gateway
+          securityContext:
+          {{- if .Values.gateway.containerSecurityContext }}
+            {{- toYaml .Values.gateway.containerSecurityContext | nindent 12 }}
+          {{- else if and $unprivilegedPortSupported (and (not .Values.gateway.hostNetwork) (semverCompare ">=1.22-0" .Capabilities.KubeVersion.GitVersion)) }}
+            # Safe since 1.22: https://github.com/kubernetes/kubernetes/pull/103326
+            capabilities:
+              drop:
+              - ALL
+            allowPrivilegeEscalation: false
+            privileged: false
+          # When enabling lite metrics, the configuration template files need to be replaced.
+          {{- if not .Values.global.liteMetrics }}
+            readOnlyRootFilesystem: true
+          {{- end }}
+            runAsUser: 1337
+            runAsGroup: 1337
+            runAsNonRoot: true
+          {{- else }}
+            capabilities:
+              drop:
+              - ALL
+              add:
+              - NET_BIND_SERVICE
+            runAsUser: 0
+            runAsGroup: 1337
+            runAsNonRoot: false
+            allowPrivilegeEscalation: true
+          {{- end }}
+          env:
+          - name: NODE_NAME
+            valueFrom:
+              fieldRef:
+                apiVersion: v1
+                fieldPath: spec.nodeName
+          - name: POD_NAME
+            valueFrom:
+              fieldRef:
+                apiVersion: v1
+                fieldPath: metadata.name
+          - name: POD_NAMESPACE
+            valueFrom:
+              fieldRef:
+                apiVersion: v1
+                fieldPath: metadata.namespace
+          - name: INSTANCE_IP
+            valueFrom:
+              fieldRef:
+                apiVersion: v1
+                fieldPath: status.podIP
+          - name: HOST_IP
+            valueFrom:
+              fieldRef:
+                apiVersion: v1
+                fieldPath: status.hostIP
+          - name: SERVICE_ACCOUNT
+            valueFrom:
+              fieldRef:
+                fieldPath: spec.serviceAccountName
+          - name: PILOT_XDS_SEND_TIMEOUT
+            value: 60s
+          - name: PROXY_XDS_VIA_AGENT
+            value: "true"
+          - name: ENABLE_INGRESS_GATEWAY_SDS
+            value: "false"
+          - name: JWT_POLICY
+            value: {{ include "controller.jwtPolicy" . }}
+          - name: ISTIO_META_HTTP10
+            value: "1"
+          - name: ISTIO_META_CLUSTER_ID
+            value: "{{ $.Values.clusterName | default `Kubernetes` }}"
+          - name: INSTANCE_NAME
+            value: "higress-gateway"
+          {{- if .Values.global.liteMetrics }}
+          - name: LITE_METRICS
+            value: "on"
+          {{- end }}
+          {{- if include "skywalking.enabled" . }}
+          - name: ISTIO_BOOTSTRAP_OVERRIDE
+            value: /etc/istio/custom-bootstrap/custom_bootstrap.json
+          {{- end }}
+          {{- with .Values.gateway.networkGateway }}
+          - name: ISTIO_META_REQUESTED_NETWORK_VIEW
+            value: "{{.}}"
+          {{- end }}
+          {{- range $key, $val := .Values.env }}
+          - name: {{ $key }}
+            value: {{ $val | quote }}
+          {{- end }}
+          ports:
+          - containerPort: 15090
+            protocol: TCP
+            name: http-envoy-prom
+          {{- if or .Values.global.local .Values.global.kind }}
+          - containerPort: {{ .Values.gateway.httpPort }}
+            hostPort:  {{ .Values.gateway.httpPort }}
+            name: http
+            protocol: TCP
+          - containerPort:  {{ .Values.gateway.httpsPort }}
+            hostPort:  {{ .Values.gateway.httpsPort }}
+            name: https
+            protocol: TCP
+          {{- end }}
+          readinessProbe:
+            failureThreshold: {{ .Values.gateway.readinessFailureThreshold }}
+            httpGet:
+              path: /healthz/ready
+              port: 15021
+              scheme: HTTP
+            initialDelaySeconds: {{ .Values.gateway.readinessInitialDelaySeconds }}
+            periodSeconds: {{ .Values.gateway.readinessPeriodSeconds }}
+            successThreshold: {{ .Values.gateway.readinessSuccessThreshold }}
+            timeoutSeconds: {{ .Values.gateway.readinessTimeoutSeconds }}
+          {{- if not (or .Values.global.local .Values.global.kind) }}
+          resources:
+            {{- toYaml .Values.gateway.resources | nindent 12 }}
+          {{- end }}
+          volumeMounts:
+          {{- if eq (include "controller.jwtPolicy" .) "third-party-jwt" }}
+          - name: istio-token
+            mountPath: /var/run/secrets/tokens
+            readOnly: true
+          {{- end }}
+          - name: config
+            mountPath: /etc/istio/config
+          - name: istio-ca-root-cert
+            mountPath: /var/run/secrets/istio
+          - name: istio-data
+            mountPath: /var/lib/istio/data
+          - name: podinfo
+            mountPath: /etc/istio/pod
+          - name: proxy-socket
+            mountPath: /etc/istio/proxy
+          {{- if include "skywalking.enabled" . }}
+          - mountPath: /etc/istio/custom-bootstrap
+            name: custom-bootstrap-volume
+          {{- end }}
+          {{- if .Values.global.volumeWasmPlugins }}
+          - mountPath: /opt/plugins
+            name: local-wasmplugins-volume
+          {{- end }}
+          {{- if $o11y.enabled }}
+          - mountPath: /var/log/proxy
+            name: log
+          {{- end }}
+      {{- if .Values.gateway.hostNetwork }}
+      hostNetwork: {{ .Values.gateway.hostNetwork }}
+      dnsPolicy: ClusterFirstWithHostNet
+      {{- end }}
+      {{- with .Values.gateway.nodeSelector }}
+      nodeSelector:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      {{- with .Values.gateway.affinity }}
+      affinity:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      {{- with .Values.gateway.tolerations }}
+      tolerations:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      volumes:
+      {{- if eq (include "controller.jwtPolicy" .) "third-party-jwt" }}
+      - name: istio-token
+        projected:
+          sources:
+            - serviceAccountToken:
+                audience: istio-ca
+                expirationSeconds: 43200
+                path: istio-token
+      {{- end }}
+      - name: istio-ca-root-cert
+        configMap:
+      {{- if .Values.global.enableHigressIstio }}
+          name: istio-ca-root-cert
+      {{- else }}
+          name: higress-ca-root-cert
+      {{- end }}
+      - name: config
+        configMap:
+          name: higress-config
+      {{- if include "skywalking.enabled" . }}
+      - configMap:
+          defaultMode: 420
+          name: higress-custom-bootstrap
+        name: custom-bootstrap-volume
+      {{- end }}
+      - name: istio-data
+        emptyDir: {}
+      - name: proxy-socket
+        emptyDir: {}
+      {{- if $o11y.enabled }}
+      - name: log
+        emptyDir: {}
+      - name: tmp
+        emptyDir: {}
+      - name: promtail-config
+        configMap:
+          name: higress-promtail
+      {{- end }}
+      - name: podinfo
+        downwardAPI:
+          defaultMode: 420
+          items:
+          - fieldRef:
+              apiVersion: v1
+              fieldPath: metadata.labels
+            path: labels
+          - fieldRef:
+              apiVersion: v1
+              fieldPath: metadata.annotations
+            path: annotations
+          - path: cpu-request
+            resourceFieldRef:
+              containerName: higress-gateway
+              divisor: 1m
+              resource: requests.cpu
+          - path: cpu-limit
+            resourceFieldRef:
+              containerName: higress-gateway
+              divisor: 1m
+              resource: limits.cpu
+      {{- if .Values.global.volumeWasmPlugins }}
+      - name: local-wasmplugins-volume
+        hostPath:
+          path: /opt/plugins
+          type: Directory
+      {{- end }}
+{{- end }}
diff --git a/helm/core/templates/deployment.yaml b/helm/core/templates/deployment.yaml
@@ -1,3 +1,4 @@
+{{- if eq .Values.gateway.kind "Deployment" -}}
 {{- $o11y := .Values.global.o11y  }}
 {{- $unprivilegedPortSupported := true }}
 {{- range $index, $node := (lookup "v1" "Node" "default" "").items }}
@@ -241,7 +242,7 @@ spec:
             mountPath: /var/run/secrets/istio
           - name: istio-data
             mountPath: /var/lib/istio/data
-          - name: podinfo            
+          - name: podinfo
             mountPath: /etc/istio/pod
           - name: proxy-socket
             mountPath: /etc/istio/proxy
@@ -340,3 +341,4 @@ spec:
           path: /opt/plugins
           type: Directory
       {{- end }}
+{{- end }}
diff --git a/helm/core/values.yaml b/helm/core/values.yaml
@@ -396,6 +396,9 @@ gateway:
   replicas: 2
   image: gateway
 
+  # -- Use a `DaemonSet` or `Deployment`
+  kind: Deployment
+
   # The number of successive failed probes before indicating readiness failure.
   readinessFailureThreshold: 30