From 55abd63a1787b4f18be463571f9910edf6596ae7 Mon Sep 17 00:00:00 2001 From: Petr Beklemishev Date: Thu, 26 Dec 2024 12:25:26 +0700 Subject: [PATCH] Bump citylens to 1.16 (#559) --- charts/citylens/Chart.yaml | 2 +- charts/citylens/README.md | 42 +++++++--- charts/citylens/templates/helpers.tpl | 14 ++++ charts/citylens/templates/web/configmap.yaml | 16 +++- .../web/workers/deployment-map-matcher.yaml | 81 +++++++++++++++++++ .../citylens/templates/workers/configmap.yaml | 1 + charts/citylens/values.yaml | 60 ++++++++++++-- 7 files changed, 199 insertions(+), 17 deletions(-) create mode 100644 charts/citylens/templates/web/workers/deployment-map-matcher.yaml diff --git a/charts/citylens/Chart.yaml b/charts/citylens/Chart.yaml index 0e3927b90..d6fbf1f2a 100644 --- a/charts/citylens/Chart.yaml +++ b/charts/citylens/Chart.yaml @@ -4,7 +4,7 @@ type: application description: A Helm chart for Kubernetes to deploy Citylens service version: 1.32.0 -appVersion: 1.15.0 +appVersion: 1.16.4 maintainers: - name: 2gis diff --git a/charts/citylens/README.md b/charts/citylens/README.md index 490d8feed..a81abc68f 100644 --- a/charts/citylens/README.md +++ b/charts/citylens/README.md @@ -47,7 +47,7 @@ See the [documentation]() to learn about: | Name | Description | Value | | ---------------------- | ------------ | ------------------------------ | | `api.image.repository` | Repository. | `2gis-on-premise/citylens-api` | -| `api.image.tag` | Tag. | `1.15.0` | +| `api.image.tag` | Tag. | `1.16.4` | | `api.image.pullPolicy` | Pull Policy. | `IfNotPresent` | ### Resources settings @@ -133,7 +133,7 @@ See the [documentation]() to learn about: | Name | Description | Value | | ---------------------- | ------------ | ------------------------------ | | `web.image.repository` | Repository. | `2gis-on-premise/citylens-web` | -| `web.image.tag` | Tag. | `1.15.0` | +| `web.image.tag` | Tag. | `1.16.4` | | `web.image.pullPolicy` | Pull Policy. | `IfNotPresent` | ### Resources settings @@ -185,11 +185,12 @@ See the [documentation]() to learn about: ### Custom settings -| Name | Description | Value | -| -------------------- | ------------------------------------------------------------------------ | -------------- | -| `web.logLevel` | Log level. | `WARNING` | -| `web.metricsAppName` | Value for service prometheus metrics label "app_name". | `citylens-web` | -| `web.pgPoolSize` | Postgres: maximum number of connections in connections pool to maintain. | `5` | +| Name | Description | Value | +| ----------------------- | ------------------------------------------------------------------------------------------- | -------------- | +| `web.logLevel` | Log level. | `WARNING` | +| `web.metricsAppName` | Value for service prometheus metrics label "app_name". | `citylens-web` | +| `web.pgPoolSize` | Postgres: maximum number of connections in connections pool to maintain. | `5` | +| `web.pgPoolMaxOverflow` | Postgres: maximum number of extra connections in connections pool (relative of pgPoolSize). | `10` | ### Metadata settings @@ -341,6 +342,26 @@ See the [documentation]() to learn about: | `worker.tracksUploader.tolerations` | Kubernetes pod [tolerations](https://kubernetes.io/docs/concepts/scheduling-eviction/taint-and-toleration/) settings. | `{}` | | `worker.tracksUploader.affinity` | Kubernetes pod [affinity](https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#node-affinity) settings. | `{}` | +### Citylens Map Matcher worker's settings + +| Name | Description | Value | +| ---------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------ | ------- | +| `worker.mapMatcher.enabled` | If Map Matcher worker is enabled for the service. | `false` | +| `worker.mapMatcher.replicas` | A replica count for the pod. | `1` | +| `worker.mapMatcher.revisionHistoryLimit` | Revision history limit (used for [rolling back](https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/) a deployment). | `3` | +| `worker.mapMatcher.baseUrl` | Map Matching API address. Ex.: http://navi-front.svc **Required** | `""` | +| `worker.mapMatcher.key` | Map Matching API key. **Required** | `""` | +| `worker.mapMatcher.interpolation` | Set to `true` for compatibility with Map Matching API which requires gps points timestamps in seconds as integers (w/o millisecond precision support). | `true` | +| `worker.mapMatcher.retries` | Total number of retries in case of Map Matching API unavailability/errors. | `3` | +| `worker.mapMatcher.timeoutSeconds` | Map Matching API timeout, seconds. | `30` | +| `worker.mapMatcher.annotations` | Kubernetes [annotations](https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations/). | `{}` | +| `worker.mapMatcher.labels` | Kubernetes [labels](https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/). | `{}` | +| `worker.mapMatcher.podAnnotations` | Kubernetes [annotations](https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations/). | `{}` | +| `worker.mapMatcher.podLabels` | Kubernetes [labels](https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/). | `{}` | +| `worker.mapMatcher.nodeSelector` | Kubernetes pod [node selectors](https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#nodeselector). | `{}` | +| `worker.mapMatcher.tolerations` | Kubernetes pod [tolerations](https://kubernetes.io/docs/concepts/scheduling-eviction/taint-and-toleration/) settings. | `{}` | +| `worker.mapMatcher.affinity` | Kubernetes pod [affinity](https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#node-affinity) settings. | `{}` | + ### Citylens Dashboard batch events worker's settings | Name | Description | Value | @@ -352,7 +373,7 @@ See the [documentation]() to learn about: | Name | Description | Value | | -------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------- | ---------------------------------- | | `worker.dashboardBatchEvents.image.repository` | Repository. | `2gis-on-premise/citylens-workers` | -| `worker.dashboardBatchEvents.image.tag` | Tag. | `1.15.0` | +| `worker.dashboardBatchEvents.image.tag` | Tag. | `1.16.4` | | `worker.dashboardBatchEvents.image.pullPolicy` | Pull Policy. | `IfNotPresent` | | `worker.dashboardBatchEvents.logLevel` | Worker's log level. | `INFO` | | `worker.dashboardBatchEvents.revisionHistoryLimit` | Revision history limit (used for [rolling back](https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/) a deployment). | `3` | @@ -370,7 +391,7 @@ See the [documentation]() to learn about: | -------------------------------------- | ----------------------------------------------------------------------------------------------------------------------- | ----------------------------------- | | `migrations.enabled` | If migrations needed. | `true` | | `migrations.image.repository` | Repository. | `2gis-on-premise/citylens-database` | -| `migrations.image.tag` | Tag. | `1.15.0` | +| `migrations.image.tag` | Tag. | `1.16.1` | | `migrations.image.pullPolicy` | Pull Policy | `IfNotPresent` | | `migrations.resources.requests.cpu` | A CPU request. | `100m` | | `migrations.resources.requests.memory` | A memory request. | `1Gi` | @@ -385,6 +406,7 @@ See the [documentation]() to learn about: | `kafka.bootstrapServer` | A Kafka broker endpoint. **Required** | `""` | | `kafka.username` | A Kafka username for connection. **Required** | `""` | | `kafka.password` | A Kafka password for connection. **Required** | `""` | +| `kafka.produceTimeoutSeconds` | Timeout for producer. | `5` | | `kafka.topics.frames` | List of topics for Frames saver worker. **Required** | `""` | | `kafka.topics.tracks` | List of topics for Tracks metadata worker. **Required** | `""` | | `kafka.topics.pro` | Topic for frames synchronization with Pro (used by Reporter pro worker). **Required** | `""` | @@ -445,7 +467,7 @@ See the [documentation]() to learn about: | Name | Description | Value | | ------------------- | ----------------------------------------------------------------------------------------------------- | ------ | -| `pro.baseUrl` | PRO API URL (used for filters actualization). Ex: http(s)://pro-api.svc/your_asset_name/filters | `""` | +| `pro.baseUrl` | PRO API URL. Ex: http(s)://pro-api.svc | `""` | | `pro.key` | PRO API auth token | `""` | | `pro.verifySsl` | Set to `false` if pro.baseUrl must be accessed via https without certificate validation. **Required** | `true` | | `pro.framesAssetId` | PRO frames asset id (used for filters actualization). Ex: your_asset_name | `""` | diff --git a/charts/citylens/templates/helpers.tpl b/charts/citylens/templates/helpers.tpl index 6dcf34cbd..12f863f15 100644 --- a/charts/citylens/templates/helpers.tpl +++ b/charts/citylens/templates/helpers.tpl @@ -25,6 +25,10 @@ Expand the name of the chart. {{ include "citylens.name" . }}-logs-saver {{- end }} +{{- define "citylens.map-matcher.name" -}} +{{ include "citylens.name" . }}-map-matcher +{{- end }} + {{- define "citylens.predictions-saver.name" -}} {{ include "citylens.name" . }}-predictions-saver {{- end }} @@ -112,6 +116,16 @@ app.kubernetes.io/instance: {{ include "citylens.logs-saver.name" . }} app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} {{- end }} +{{- define "citylens.map-matcher.selectorLabels" -}} +app.kubernetes.io/name: {{ .Release.Name }} +app.kubernetes.io/instance: {{ include "citylens.map-matcher.name" . }} +{{- end }} + +{{- define "citylens.map-matcher.labels" -}} +{{ include "citylens.map-matcher.selectorLabels" . }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} + {{- define "citylens.predictions-saver.selectorLabels" -}} app.kubernetes.io/name: {{ .Release.Name }} app.kubernetes.io/instance: {{ include "citylens.predictions-saver.name" . }} diff --git a/charts/citylens/templates/web/configmap.yaml b/charts/citylens/templates/web/configmap.yaml index f0d9fe10c..aaa575571 100644 --- a/charts/citylens/templates/web/configmap.yaml +++ b/charts/citylens/templates/web/configmap.yaml @@ -30,9 +30,11 @@ data: db_connections: postgres: {{ include "citylens.pgDSN" . | squote }} postgres_pool_size: {{ .Values.web.pgPoolSize }} + postgres_pool_max_overflow: {{ .Values.web.pgPoolMaxOverflow }} kafka: {{- with .Values.kafka }} - bootstrap_servers: {{ required "A valid .Values.kafka.bootstrap.servers entry required" .bootstrapServer | squote }} + bootstrap_servers: {{ required "A valid .Values.kafka.bootstrap.servers entry required" .bootstrapServer | squote }} + producer_timeout_seconds: {{ .produceTimeoutSeconds }} security: security_protocol: 'SASL_PLAINTEXT' sasl_mechanism: 'SCRAM-SHA-512' @@ -107,6 +109,18 @@ data: coords: [{{ join "," .coords }}] {{- end }} initial_project: {{ .Values.map.initialProject | squote }} + {{- with .Values.worker.mapMatcher }} + {{- if .enabled }} + matching: + enabled: {{ .enabled }} + host: {{ required "A valid .Values.worker.mapMatcher.baseUrl entry required" .baseUrl }} + key: {{ required "A valid .Values.worker.mapMatcher.key entry required" .key }} + interpolation: {{ .interpolation }} + retries_total: {{ .retries }} + retries_backoff_factor: 0.8 + timeout: {{ .timeoutSeconds }} + {{- end }} + {{- end }} register_predictor_ttl_seconds: 600 enabled_modules: header_links: diff --git a/charts/citylens/templates/web/workers/deployment-map-matcher.yaml b/charts/citylens/templates/web/workers/deployment-map-matcher.yaml new file mode 100644 index 000000000..231e0fc1a --- /dev/null +++ b/charts/citylens/templates/web/workers/deployment-map-matcher.yaml @@ -0,0 +1,81 @@ +{{- if .Values.worker.mapMatcher.enabled }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "citylens.map-matcher.name" . }} + {{- if .Values.worker.mapMatcher.annotations }} + annotations: + {{- toYaml .Values.worker.mapMatcher.annotations | nindent 4 }} + {{- end }} + labels: + {{- include "citylens.map-matcher.labels" . | nindent 4 }} + {{- if .Values.worker.mapMatcher.labels }} + {{- toYaml .Values.worker.mapMatcher.labels | nindent 4 }} + {{- end }} +spec: + replicas: {{ .Values.worker.mapMatcher.replicas }} + revisionHistoryLimit: {{ .Values.worker.mapMatcher.revisionHistoryLimit }} + selector: + matchLabels: + {{- include "citylens.map-matcher.selectorLabels" . | nindent 6 }} + strategy: + type: RollingUpdate + rollingUpdate: + maxSurge: 1 + maxUnavailable: 0 + template: + metadata: + annotations: + checksum/config: {{ include (print $.Template.BasePath "/web/configmap.yaml") . | sha256sum }} + checksum/custom-ca: {{ include "citylens.checksum" (merge (dict "path" "/custom-ca.configmap.yaml") $) }} + {{- with .Values.worker.mapMatcher.podAnnotations }} + {{- toYaml . | nindent 8 }} + {{- end }} + labels: + {{- include "citylens.map-matcher.labels" . | nindent 8 }} + spec: + containers: + - name: {{ include "citylens.map-matcher.name" . }} + image: {{ required "A valid .Values.dgctlDockerRegistry entry required" .Values.dgctlDockerRegistry }}/{{ .Values.web.image.repository }}:{{ .Values.web.image.tag }} + imagePullPolicy: {{ .Values.web.image.pullPolicy }} + command: [ "/bin/sh", "-c" ] + args: [ "flask map_matching match" ] + env: + - name: CONFIG_PATH + value: /opt/worker/config/dashboard_config.yaml + {{- if $.Values.customCAs.bundle }} + - name: SSL_CERT_DIR + value: {{ include "citylens.customCA.mountPath" $ }} + {{- end }} + resources: + {{- toYaml .Values.worker.resources | nindent 12 }} + volumeMounts: + - name: config-volume + mountPath: /opt/worker/config + {{- if $.Values.customCAs.bundle }} + - mountPath: {{ include "citylens.customCA.mountPath" $ }} + name: custom-ca + readOnly: true + {{- end }} + volumes: + - name: config-volume + configMap: + name: {{ include "citylens.web.name" . }}-configmap + {{- if $.Values.customCAs.bundle }} + - name: custom-ca + configMap: + name: {{ include "citylens.name" $ }}-custom-ca-configmap + {{- end }} + {{- with .Values.worker.mapMatcher.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.worker.mapMatcher.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.worker.mapMatcher.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} +{{- end }} diff --git a/charts/citylens/templates/workers/configmap.yaml b/charts/citylens/templates/workers/configmap.yaml index 7531b1767..2c3db26a8 100644 --- a/charts/citylens/templates/workers/configmap.yaml +++ b/charts/citylens/templates/workers/configmap.yaml @@ -11,6 +11,7 @@ data: s3_endpoint: {{ required "A valid .Values.s3.endpoint entry required" .endpoint | squote }} bucket_prefix: {{ required "A valid .Values.s3.bucketPrefix entry required" .bucketPrefix | squote }} frame_key_template: {{ include "citylens.s3_constants.frame_key_template" . | squote }} + crop_frame_key_template: {{ include "citylens.s3_constants.crop_frame_key_template" . | squote }} {{- end }} topics: {{- with .Values.kafka.topics }} diff --git a/charts/citylens/values.yaml b/charts/citylens/values.yaml index 66d4c90de..7a20cccdf 100644 --- a/charts/citylens/values.yaml +++ b/charts/citylens/values.yaml @@ -99,7 +99,7 @@ api: image: repository: 2gis-on-premise/citylens-api pullPolicy: IfNotPresent - tag: 1.15.0 + tag: 1.16.4 replicas: 4 @@ -215,6 +215,7 @@ api: # @param web.logLevel Log level. # @param web.metricsAppName Value for service prometheus metrics label "app_name". # @param web.pgPoolSize Postgres: maximum number of connections in connections pool to maintain. +# @param web.pgPoolMaxOverflow Postgres: maximum number of extra connections in connections pool (relative of pgPoolSize). # @section Metadata settings @@ -231,7 +232,7 @@ web: image: repository: 2gis-on-premise/citylens-web pullPolicy: IfNotPresent - tag: 1.15.0 + tag: 1.16.4 replicas: 1 @@ -279,6 +280,7 @@ web: pkce: false pgPoolSize: 5 + pgPoolMaxOverflow: 10 logLevel: WARNING metricsAppName: citylens-web @@ -584,6 +586,52 @@ worker: tolerations: {} +# @section Citylens Map Matcher worker's settings + +# @param worker.mapMatcher.enabled If Map Matcher worker is enabled for the service. + +# @param worker.mapMatcher.replicas A replica count for the pod. + +# @param worker.mapMatcher.revisionHistoryLimit Revision history limit (used for [rolling back](https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/) a deployment). + +# @param worker.mapMatcher.baseUrl Map Matching API address. Ex.: http://navi-front.svc **Required** +# @param worker.mapMatcher.key Map Matching API key. **Required** +# @param worker.mapMatcher.interpolation Set to `true` for compatibility with Map Matching API which requires gps points timestamps in seconds as integers (w/o millisecond precision support). +# @param worker.mapMatcher.retries Total number of retries in case of Map Matching API unavailability/errors. +# @param worker.mapMatcher.timeoutSeconds Map Matching API timeout, seconds. + +# @param worker.mapMatcher.annotations Kubernetes [annotations](https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations/). +# @param worker.mapMatcher.labels Kubernetes [labels](https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/). +# @param worker.mapMatcher.podAnnotations Kubernetes [annotations](https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations/). +# @param worker.mapMatcher.podLabels Kubernetes [labels](https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/). + +# @param worker.mapMatcher.nodeSelector Kubernetes pod [node selectors](https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#nodeselector). +# @param worker.mapMatcher.tolerations Kubernetes pod [tolerations](https://kubernetes.io/docs/concepts/scheduling-eviction/taint-and-toleration/) settings. +# @param worker.mapMatcher.affinity Kubernetes pod [affinity](https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#node-affinity) settings. + + mapMatcher: + + enabled: false + + replicas: 1 + + baseUrl: '' + key: '' + interpolation: true + retries: 3 + timeoutSeconds: 30 + + revisionHistoryLimit: 3 + + annotations: {} + labels: {} + podAnnotations: {} + podLabels: {} + nodeSelector: {} + affinity: {} + tolerations: {} + + # @section Citylens Dashboard batch events worker's settings # @param worker.dashboardBatchEvents.replicas A replica count for the pod. @@ -614,7 +662,7 @@ worker: image: repository: 2gis-on-premise/citylens-workers pullPolicy: IfNotPresent - tag: 1.15.0 + tag: 1.16.4 revisionHistoryLimit: 3 @@ -650,7 +698,7 @@ migrations: image: repository: 2gis-on-premise/citylens-database pullPolicy: IfNotPresent - tag: 1.15.0 + tag: 1.16.1 resources: requests: @@ -667,6 +715,7 @@ migrations: # @param kafka.bootstrapServer A Kafka broker endpoint. **Required** # @param kafka.username A Kafka username for connection. **Required** # @param kafka.password A Kafka password for connection. **Required** +# @param kafka.produceTimeoutSeconds Timeout for producer. # @param kafka.topics.frames List of topics for Frames saver worker. **Required** # @param kafka.topics.tracks List of topics for Tracks metadata worker. **Required** # @param kafka.topics.pro Topic for frames synchronization with Pro (used by Reporter pro worker). **Required** @@ -683,6 +732,7 @@ kafka: bootstrapServer: '' username: '' password: '' + produceTimeoutSeconds: 5 topics: frames: '' tracks: '' @@ -776,7 +826,7 @@ reporters: # @section PRO integration (only when Pro reporter enabled) -# @param pro.baseUrl PRO API URL (used for filters actualization). Ex: http(s)://pro-api.svc/your_asset_name/filters +# @param pro.baseUrl PRO API URL. Ex: http(s)://pro-api.svc # @param pro.key PRO API auth token # @param pro.verifySsl Set to `false` if pro.baseUrl must be accessed via https without certificate validation. **Required** # @param pro.framesAssetId PRO frames asset id (used for filters actualization). Ex: your_asset_name