Merge branch 'main' into refactor/ci/improve-backport-pr-quality

.github/actions/docker-build/action.yaml

@@ -68,7 +68,9 @@ runs:
       id: determine_platforms
       run: |
         if [ "${TARGET_PLATFORMS}" = "" ]; then
-          if [ "${{ github.event_name }}" = "pull_request" ]; then
+          if [[ "$GITHUB_REF" =~ ^refs/heads/main$ ]] || [[ "$GITHUB_REF" =~ ^refs/heads/master$ ]] || [[ "$GITHUB_REF" =~ ^refs/heads/release.* ]] || [[ "${PRIMARY_TAG}" == "nightly" ]]; then
+            platforms=`make docker/platforms`
+          elif [ "${{ github.event_name }}" = "pull_request" ]; then
             platforms="linux/amd64"
           elif [ "${{ github.event_name }}" = "pull_request_target" ]; then
             platforms="linux/amd64"
@@ -91,6 +93,11 @@ runs:
           latest_tags="-t ${IMAGE_NAME}:latest -t ${ALTER_IMAGE_NAME}:latest"
           extra_tags="${extra_tags} ${latest_tags}"
         fi
+        if [[ "$GITHUB_REF" =~ ^refs/heads/main$ ]] || [[ "$GITHUB_REF" =~ ^refs/heads/master$ ]] || [[ "$GITHUB_REF" =~ ^refs/heads/release.* ]] || [[ "${PRIMARY_TAG}" == "nightly" ]]; then
+          commit_hash=${GITHUB_SHA::8}
+          hash_tags="-t ${IMAGE_NAME}:${commit_hash} -t ${ALTER_IMAGE_NAME}:${commit_hash}"
+          extra_tags="${extra_tags} ${hash_tags}"
+        fi
         echo "EXTRA_TAGS is determined: ${extra_tags}"
         echo "EXTRA_TAGS=${extra_tags}" >> $GITHUB_OUTPUT
       env:

.github/workflows/dockers-agent-ngt-image.yml

@@ -80,5 +80,4 @@ jobs:
     uses: ./.github/workflows/_docker-image.yaml
     with:
       target: agent-ngt
-      platforms: linux/amd64,linux/arm64
     secrets: inherit

.github/workflows/dockers-benchmark-job-image.yml

@@ -78,5 +78,4 @@ jobs:
     uses: ./.github/workflows/_docker-image.yaml
     with:
       target: benchmark-job
-      platforms: linux/amd64,linux/arm64
     secrets: inherit

.github/workflows/dockers-helm-operator-image.yml

@@ -68,5 +68,4 @@ jobs:
     uses: ./.github/workflows/_docker-image.yaml
     with:
       target: operator/helm
-      platforms: linux/amd64,linux/arm64
     secrets: inherit

.github/workflows/format.yml

@@ -105,7 +105,7 @@ jobs:
         run: |
           make deps/install
           make format
-          git checkout go.mod go.sum ./rust/Cargo.lock
+          git checkout go.mod go.sum ./example/client/go.mod ./example/client/go.sum ./rust/Cargo.lock
       - name: Check format and deps difference
         run: |
           if git diff --quiet --exit-code; then

charts/vald-helm-operator/crds/valdrelease.yaml

@@ -124,6 +124,20 @@ spec:
                     annotations:
                       type: object
                       x-kubernetes-preserve-unknown-fields: true
+                    clusterRole:
+                      type: object
+                      properties:
+                        enabled:
+                          type: boolean
+                        name:
+                          type: string
+                    clusterRoleBinding:
+                      type: object
+                      properties:
+                        enabled:
+                          type: boolean
+                        name:
+                          type: string
                     enabled:
                       type: boolean
                     env:
@@ -251,13 +265,17 @@ spec:
                             - normalizedcosine
                         enable_copy_on_write:
                           type: boolean
+                        enable_export_index_info_to_k8s:
+                          type: boolean
                         enable_in_memory_mode:
                           type: boolean
                         enable_proactive_gc:
                           type: boolean
                         error_buffer_limit:
                           type: integer
                           minimum: 1
+                        export_index_info_duration:
+                          type: string
                         index_path:
                           type: string
                         initial_delay_max_duration:
@@ -273,12 +291,16 @@ spec:
                           type: string
                         min_load_index_timeout:
                           type: string
+                        namespace:
+                          type: string
                         object_type:
                           type: string
                           enum:
                             - float
                             - float16
                             - uint8
+                        pod_name:
+                          type: string
                         search_edge_size:
                           type: integer
                         vqueue:
@@ -959,6 +981,13 @@ spec:
                         labels:
                           type: object
                           x-kubernetes-preserve-unknown-fields: true
+                    serviceAccount:
+                      type: object
+                      properties:
+                        enabled:
+                          type: boolean
+                        name:
+                          type: string
                     serviceType:
                       type: string
                       enum:

charts/vald/templates/agent/clusterrole.yaml

@@ -0,0 +1,37 @@
+#
+# Copyright (C) 2019-2024 vdaas.org vald team <vald@vdaas.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# You may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+{{- $agent := .Values.agent -}}
+{{- if and $agent.enabled $agent.clusterRole.enabled $agent.ngt.enable_export_index_info_to_k8s }}
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: {{ $agent.clusterRole.name }}
+  labels:
+    app.kubernetes.io/name: {{ include "vald.name" . }}
+    helm.sh/chart: {{ include "vald.chart" . }}
+    app.kubernetes.io/managed-by: {{ .Release.Service }}
+    app.kubernetes.io/instance: {{ .Release.Name }}
+    app.kubernetes.io/version: {{ .Chart.Version }}
+    app.kubernetes.io/component: agent
+rules:
+  - apiGroups:
+      - ""
+    resources:
+      - pods
+    verbs:
+      - list
+      - patch
+{{- end }}

charts/vald/templates/agent/clusterrolebinding.yaml

@@ -0,0 +1,37 @@
+#
+# Copyright (C) 2019-2024 vdaas.org vald team <vald@vdaas.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# You may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+{{- $agent := .Values.agent -}}
+{{- if and $agent.enabled $agent.clusterRoleBinding.enabled $agent.ngt.enable_export_index_info_to_k8s }}
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: {{ $agent.clusterRoleBinding.name }}
+  labels:
+    app.kubernetes.io/name: {{ include "vald.name" . }}
+    helm.sh/chart: {{ include "vald.chart" . }}
+    app.kubernetes.io/managed-by: {{ .Release.Service }}
+    app.kubernetes.io/instance: {{ .Release.Name }}
+    app.kubernetes.io/version: {{ .Chart.Version }}
+    app.kubernetes.io/component: agent
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: {{ $agent.clusterRole.name }}
+subjects:
+  - kind: ServiceAccount
+    name: {{ $agent.serviceAccount.name }}
+    namespace: {{ .Release.Namespace }}
+{{- end }}

charts/vald/templates/agent/daemonset.yaml

@@ -167,6 +167,9 @@ spec:
         {{- toYaml $agent.podSecurityContext | nindent 8 }}
       {{- end }}
       terminationGracePeriodSeconds: {{ $agent.terminationGracePeriodSeconds }}
+      {{- if and $agent.serviceAccount.enabled $agent.ngt.enable_export_index_info_to_k8s }}
+      serviceAccountName: {{ $agent.serviceAccount.name }}
+      {{- end }}
       volumes:
         - name: {{ $agent.name }}-config
           configMap:

charts/vald/templates/agent/deployment.yaml

@@ -171,6 +171,9 @@ spec:
         {{- toYaml $agent.podSecurityContext | nindent 8 }}
       {{- end }}
       terminationGracePeriodSeconds: {{ $agent.terminationGracePeriodSeconds }}
+      {{- if and $agent.serviceAccount.enabled $agent.ngt.enable_export_index_info_to_k8s }}
+      serviceAccountName: {{ $agent.serviceAccount.name }}
+      {{- end }}
       volumes:
         - name: {{ $agent.name }}-config
           configMap:

charts/vald/templates/agent/serviceaccount.yaml

@@ -0,0 +1,29 @@
+#
+# Copyright (C) 2019-2024 vdaas.org vald team <vald@vdaas.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# You may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+{{- $agent := .Values.agent -}}
+{{- if and $agent.enabled $agent.serviceAccount.enabled $agent.ngt.enable_export_index_info_to_k8s }}
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: {{ $agent.serviceAccount.name }}
+  labels:
+    app.kubernetes.io/name: {{ include "vald.name" . }}
+    helm.sh/chart: {{ include "vald.chart" . }}
+    app.kubernetes.io/managed-by: {{ .Release.Service }}
+    app.kubernetes.io/instance: {{ .Release.Name }}
+    app.kubernetes.io/version: {{ .Chart.Version }}
+    app.kubernetes.io/component: agent
+{{- end }}

charts/vald/templates/agent/statefulset.yaml

@@ -209,6 +209,9 @@ spec:
         {{- toYaml $agent.podSecurityContext | nindent 8 }}
       {{- end }}
       terminationGracePeriodSeconds: {{ $agent.terminationGracePeriodSeconds }}
+      {{- if and $agent.serviceAccount.enabled $agent.ngt.enable_export_index_info_to_k8s }}
+      serviceAccountName: {{ $agent.serviceAccount.name }}
+      {{- end }}
       volumes:
         - name: {{ $agent.name }}-config
           configMap:

charts/vald/values.schema.json

@@ -70,6 +70,29 @@
           "type": "object",
           "description": "deployment annotations"
         },
+        "clusterRole": {
+          "type": "object",
+          "properties": {
+            "enabled": {
+              "type": "boolean",
+              "description": "creates clusterRole resource"
+            },
+            "name": { "type": "string", "description": "name of clusterRole" }
+          }
+        },
+        "clusterRoleBinding": {
+          "type": "object",
+          "properties": {
+            "enabled": {
+              "type": "boolean",
+              "description": "creates clusterRoleBinding resource"
+            },
+            "name": {
+              "type": "string",
+              "description": "name of clusterRoleBinding"
+            }
+          }
+        },
         "enabled": { "type": "boolean", "description": "agent enabled" },
         "env": {
           "type": "array",
@@ -238,6 +261,10 @@
               "type": "boolean",
               "description": "enable copy on write saving for more stable backup"
             },
+            "enable_export_index_info_to_k8s": {
+              "type": "boolean",
+              "description": "enable export index info to k8s"
+            },
             "enable_in_memory_mode": {
               "type": "boolean",
               "description": "in-memory mode enabled"
@@ -251,6 +278,10 @@
               "description": "maximum number of core ngt error buffer pool size limit",
               "minimum": 1
             },
+            "export_index_info_duration": {
+              "type": "string",
+              "description": "duration of exporting index info"
+            },
             "index_path": {
               "type": "string",
               "description": "path to index data"
@@ -280,11 +311,19 @@
               "type": "string",
               "description": "minimum duration of load index timeout"
             },
+            "namespace": {
+              "type": "string",
+              "description": "namespace of myself"
+            },
             "object_type": {
               "type": "string",
               "description": "object type. it should be `float` or `uint8` or `float16`. for further details: https://github.com/yahoojapan/NGT/wiki/Command-Quick-Reference",
               "enum": ["float", "float16", "uint8"]
             },
+            "pod_name": {
+              "type": "string",
+              "description": "pod name of myself"
+            },
             "search_edge_size": {
               "type": "integer",
               "description": "search edge size"
@@ -1459,6 +1498,19 @@
             "labels": { "type": "object", "description": "service labels" }
           }
         },
+        "serviceAccount": {
+          "type": "object",
+          "properties": {
+            "enabled": {
+              "type": "boolean",
+              "description": "creates service account"
+            },
+            "name": {
+              "type": "string",
+              "description": "name of service account"
+            }
+          }
+        },
         "serviceType": {
           "type": "string",
           "description": "service type: ClusterIP, LoadBalancer or NodePort",