refactor index manager service add index service API to expose index …

…informations

style: format code with Gofumpt and Prettier

Signed-off-by: kpango <kpango@vdaas.org>

.gitfiles

@@ -161,8 +161,6 @@ apis/grpc/v1/filter/egress/egress_filter.pb.go
 apis/grpc/v1/filter/egress/egress_filter_vtproto.pb.go
 apis/grpc/v1/filter/ingress/ingress_filter.pb.go
 apis/grpc/v1/filter/ingress/ingress_filter_vtproto.pb.go
-apis/grpc/v1/manager/index/index_manager.pb.go
-apis/grpc/v1/manager/index/index_manager_vtproto.pb.go
 apis/grpc/v1/mirror/mirror.go
 apis/grpc/v1/mirror/mirror.pb.go
 apis/grpc/v1/mirror/mirror_vtproto.pb.go
@@ -177,6 +175,8 @@ apis/grpc/v1/vald/filter.pb.go
 apis/grpc/v1/vald/filter_vtproto.pb.go
 apis/grpc/v1/vald/flush.pb.go
 apis/grpc/v1/vald/flush_vtproto.pb.go
+apis/grpc/v1/vald/index.pb.go
+apis/grpc/v1/vald/index_vtproto.pb.go
 apis/grpc/v1/vald/insert.pb.go
 apis/grpc/v1/vald/insert_vtproto.pb.go
 apis/grpc/v1/vald/object.pb.go
@@ -197,12 +197,12 @@ apis/proto/v1/agent/sidecar/sidecar.proto
 apis/proto/v1/discoverer/discoverer.proto
 apis/proto/v1/filter/egress/egress_filter.proto
 apis/proto/v1/filter/ingress/ingress_filter.proto
-apis/proto/v1/manager/index/index_manager.proto
 apis/proto/v1/mirror/mirror.proto
 apis/proto/v1/payload/payload.proto
 apis/proto/v1/rpc/errdetails/error_details.proto
 apis/proto/v1/vald/filter.proto
 apis/proto/v1/vald/flush.proto
+apis/proto/v1/vald/index.proto
 apis/proto/v1/vald/insert.proto
 apis/proto/v1/vald/object.proto
 apis/proto/v1/vald/remove.proto
@@ -214,12 +214,12 @@ apis/swagger/v1/agent/sidecar/sidecar.swagger.json
 apis/swagger/v1/discoverer/discoverer.swagger.json
 apis/swagger/v1/filter/egress/egress_filter.swagger.json
 apis/swagger/v1/filter/ingress/ingress_filter.swagger.json
-apis/swagger/v1/manager/index/index_manager.swagger.json
 apis/swagger/v1/mirror/mirror.swagger.json
 apis/swagger/v1/payload/payload.swagger.json
 apis/swagger/v1/rpc/errdetails/error_details.swagger.json
 apis/swagger/v1/vald/filter.swagger.json
 apis/swagger/v1/vald/flush.swagger.json
+apis/swagger/v1/vald/index.swagger.json
 apis/swagger/v1/vald/insert.swagger.json
 apis/swagger/v1/vald/object.swagger.json
 apis/swagger/v1/vald/remove.swagger.json
@@ -228,6 +228,7 @@ apis/swagger/v1/vald/update.swagger.json
 apis/swagger/v1/vald/upsert.swagger.json
 assets/docs/guides/observability-configuration/architecture.png
 assets/docs/guides/operations/grafana-example.png
+assets/docs/guides/read-replica-and-rotator/architecture.png
 assets/docs/overview/component/agent/agent.drawio
 assets/docs/overview/component/agent/ngt.png
 assets/docs/overview/component/agent/sidecar_backup.png
@@ -556,6 +557,7 @@ docs/api/upsert.md
 docs/contributing/coding-style.md
 docs/contributing/contributing-guide.md
 docs/contributing/development.md
+docs/contributing/reviewer-guideline.md
 docs/contributing/unit-test-guideline.md
 docs/overview/about-vald.md
 docs/overview/architecture.md
@@ -571,8 +573,8 @@ docs/performance/benchmark.md
 docs/performance/continuous-benchmark.md
 docs/performance/loadtest.md
 docs/performance/tuning-search-performance.md
+docs/support/FAQ.md
 docs/support/contacts.md
-docs/support/faq.md
 docs/troubleshooting/README.md
 docs/troubleshooting/client-side.md
 docs/troubleshooting/mirror-gateway.md
@@ -595,6 +597,7 @@ docs/user-guides/mirroring-configuration.md
 docs/user-guides/network-policy.md
 docs/user-guides/observability-configuration.md
 docs/user-guides/operations.md
+docs/user-guides/read-replica-and-rotator.md
 docs/user-guides/sdks.md
 docs/user-guides/upgrade-cluster.md
 example/client/agent/main.go
@@ -1820,6 +1823,14 @@ renovate.json
 rust/Cargo.lock
 rust/Cargo.toml
 rust/bin/agent/Cargo.toml
+rust/bin/agent/src/handler.rs
+rust/bin/agent/src/handler/common.rs
+rust/bin/agent/src/handler/index.rs
+rust/bin/agent/src/handler/insert.rs
+rust/bin/agent/src/handler/remove.rs
+rust/bin/agent/src/handler/search.rs
+rust/bin/agent/src/handler/update.rs
+rust/bin/agent/src/handler/upsert.rs
 rust/bin/agent/src/main.rs
 rust/libs/ngt-rs/Cargo.toml
 rust/libs/ngt-rs/build.rs
@@ -1838,8 +1849,6 @@ rust/libs/proto/src/filter.egress.v1.tonic.rs
 rust/libs/proto/src/filter.ingress.v1.rs
 rust/libs/proto/src/filter.ingress.v1.tonic.rs
 rust/libs/proto/src/lib.rs
-rust/libs/proto/src/manager.index.v1.rs
-rust/libs/proto/src/manager.index.v1.tonic.rs
 rust/libs/proto/src/mirror.v1.rs
 rust/libs/proto/src/mirror.v1.tonic.rs
 rust/libs/proto/src/payload.v1.rs

.github/ISSUE_TEMPLATE/bug_report.md

@@ -22,8 +22,8 @@ assignees: ""
 
 <!--- Please change the versions below along with your environment -->
 
-- Go Version: 1.22.3
+- Go Version: 1.22.5
 - Rust Version: 1.77.2
 - Docker Version: 20.10.8
-- Kubernetes Version: v1.30.0
-- NGT Version: 2.2.1
+- Kubernetes Version: v1.30.2
+- NGT Version: 2.2.2

.github/ISSUE_TEMPLATE/security_issue_report.md

@@ -16,8 +16,8 @@ assignees: ""
 
 <!--- Please change the versions below along with your environment -->
 
-- Go Version: 1.22.3
+- Go Version: 1.22.5
 - Rust Version: 1.77.2
 - Docker Version: 20.10.8
-- Kubernetes Version: v1.30.0
-- NGT Version: 2.2.1
+- Kubernetes Version: v1.30.2
+- NGT Version: 2.2.2

.github/PULL_REQUEST_TEMPLATE.md

@@ -16,11 +16,11 @@
 
 <!--- Please change the versions below along with your environment -->
 
-- Go Version: 1.22.3
+- Go Version: 1.22.5
 - Rust Version: 1.77.2
 - Docker Version: 20.10.8
-- Kubernetes Version: v1.30.0
-- NGT Version: 2.2.1
+- Kubernetes Version: v1.30.2
+- NGT Version: 2.2.2
 
 ### Checklist
 

.github/actions/docker-build/action.yaml

@@ -113,7 +113,11 @@ runs:
           REMOTE="true" \
           DOCKER="docker" \
           BUILDKIT_INLINE_CACHE=0 \
-          DOCKER_OPTS="--platform ${PLATFORMS} --builder ${BUILDER} ${LABEL_OPTS}  --label org.opencontainers.image.version=${PRIMARY_TAG} --label org.opencontainers.image.title=${TARGET}" \
+          DOCKER_OPTS="--platform ${PLATFORMS} --builder ${BUILDER} ${LABEL_OPTS} \
+          --label org.opencontainers.image.version=${PRIMARY_TAG} \
+          --label org.opencontainers.image.title=${TARGET} \
+          --label org.opencontainers.image.created=\"$(date --rfc-3339=ns)\" \
+          --label org.opencontainers.image.licenses=\"Apache 2.0\"" \
           EXTRA_ARGS="${EXTRA_TAGS}" \
           TAG="${PRIMARY_TAG}" \
           docker/build/${TARGET}

.github/workflows/release.yml

@@ -61,7 +61,7 @@ jobs:
           git push origin ${VERSION}
           echo "VERSION=${VERSION}" >> $GITHUB_OUTPUT
       - name: Create release
-        uses: softprops/action-gh-release@v1
+        uses: softprops/action-gh-release@v2
         env:
           GITHUB_TOKEN: ${{ secrets.DISPATCH_TOKEN }}
         with:

Makefile

@@ -454,6 +454,7 @@ format: \
 .PHONY: format/go
 ## run golines, gofumpt, goimports for all go files
 format/go: \
+	crlfmt/install \
 	golines/install \
 	gofumpt/install \
 	strictgoimports/install \
@@ -462,10 +463,12 @@ format/go: \
 	find $(ROOTDIR)/ -type d -name .git -prune -o -type f -regex '.*[^\.pb]\.go' -print | xargs -P$(CORES) $(GOBIN)/gofumpt -w
 	find $(ROOTDIR)/ -type d -name .git -prune -o -type f -regex '.*[^\.pb]\.go' -print | xargs -P$(CORES) $(GOBIN)/strictgoimports -w
 	find $(ROOTDIR)/ -type d -name .git -prune -o -type f -regex '.*\.go' -print | xargs -P$(CORES) $(GOBIN)/goimports -w
+	find $(ROOTDIR)/ -type d -name .git -prune -o -type f -regex '.*\.go' -print | xargs -P$(CORES) $(GOBIN)/crlfmt -w
 
 .PHONY: format/go/test
 ## run golines, gofumpt, goimports for go test files
 format/go/test: \
+	crlfmt/install \
 	golines/install \
 	gofumpt/install \
 	strictgoimports/install \
@@ -474,6 +477,7 @@ format/go/test: \
 	find $(ROOTDIR) -name '*_test.go' | xargs -P$(CORES) $(GOBIN)/gofumpt -w
 	find $(ROOTDIR) -name '*_test.go' | xargs -P$(CORES) $(GOBIN)/strictgoimports -w
 	find $(ROOTDIR) -name '*_test.go' | xargs -P$(CORES) $(GOBIN)/goimports -w
+	find $(ROOTDIR) -name '*_test.go' | xargs -P$(CORES) $(GOBIN)/crlfmt -w
 
 .PHONY: format/yaml
 format/yaml: \
@@ -516,6 +520,7 @@ deps: \
 .PHONY: deps/install
 ## install dependencies
 deps/install: \
+	crlfmt/install \
 	golines/install \
 	gofumpt/install \
 	strictgoimports/install \
@@ -592,8 +597,8 @@ ngt/install: /usr/local/include/NGT/Capi.h
 ## install Faiss
 faiss/install: /usr/local/lib/libfaiss.so
 /usr/local/lib/libfaiss.so:
-	curl -fsSLO https://github.com/facebookresearch/faiss/archive/v$(FAISS_VERSION).tar.gz
-	tar zxf v$(FAISS_VERSION).tar.gz -C $(TEMP_DIR)/
+	curl -fsSL https://github.com/facebookresearch/faiss/archive/v$(FAISS_VERSION).tar.gz -o $(TEMP_DIR)/v$(FAISS_VERSION).tar.gz
+	tar zxf $(TEMP_DIR)/v$(FAISS_VERSION).tar.gz -C $(TEMP_DIR)/
 	cd $(TEMP_DIR)/faiss-$(FAISS_VERSION) && \
 		cmake -DFAISS_ENABLE_GPU=OFF -DFAISS_ENABLE_PYTHON=OFF -DBUILD_TESTING=OFF -DCMAKE_BUILD_TYPE=Release -DBUILD_SHARED_LIBS=ON -B build . && \
 		make -C build -j faiss && \
@@ -607,7 +612,11 @@ faiss/install: /usr/local/lib/libfaiss.so
 lint: \
 	docs/lint \
 	files/lint \
-	vet
+	vet \
+	go/lint
+
+.PHONY: go/lint
+go/lint:
 	$(call go-lint)
 
 .PHONY: vet

Makefile.d/build.mk

@@ -17,8 +17,6 @@
 .PHONY: binary/build
 ## build all binaries
 binary/build: \
-	cmd/agent/core/faiss/faiss \
-	cmd/agent/core/ngt/ngt \
 	cmd/agent/sidecar/sidecar \
 	cmd/discoverer/k8s/discoverer \
 	cmd/gateway/filter/filter \
@@ -32,12 +30,14 @@ binary/build: \
 	cmd/tools/benchmark/job/job \
 	cmd/tools/benchmark/operator/operator \
 	cmd/index/operator/index-operator
+	# cmd/agent/core/ngt/ngt \
+	# cmd/agent/core/faiss/faiss \
 
 
 cmd/agent/core/ngt/ngt: \
 	ngt/install
 	$(eval CGO_ENABLED = 1)
-	$(call go-build,agent/core/ngt,-linkmode 'external',-static -fPIC -pthread -fopenmp -std=gnu++20 -lstdc++ -lm -z relro -z now $(EXTLDFLAGS), cgo,NGT-$(NGT_VERSION),$@)
+	$(call go-build,agent/core/ngt,-linkmode 'external',-static -fPIC -pthread -fopenmp -std=gnu++20 -lstdc++ -lm -z relro -z now -flto -march=native -fno-plt -Ofast -fvisibility=hidden -ffp-contract=fast $(EXTLDFLAGS), cgo,NGT-$(NGT_VERSION),$@)
 
 cmd/agent/core/faiss/faiss: \
 	faiss/install
@@ -89,6 +89,7 @@ cmd/index/operator/index-operator:
 	$(call go-build,index/operator,,-static,,,$@)
 
 cmd/tools/benchmark/job/job:
+	$(eval CGO_ENABLED = 1)
 	$(call go-build,tools/benchmark/job,-linkmode 'external',-static -fPIC -pthread -fopenmp -std=gnu++20 -lhdf5 -lhdf5_hl -lm -ldl, cgo,$(HDF5_VERSION),$@)
 
 cmd/tools/benchmark/operator/operator:

Makefile.d/dependencies.mk

@@ -47,7 +47,8 @@ go/download:
 
 .PHONY: go/deps
 ## install Go package dependencies
-go/deps:
+go/deps: \
+	update/go
 	sed -i "3s/go [0-9]\+\.[0-9]\+\(\.[0-9]\+\)\?/go $(GO_VERSION)/g" $(ROOTDIR)/hack/go.mod.default
 	if $(GO_CLEAN_DEPS); then \
         	rm -rf $(ROOTDIR)/vendor \

Makefile.d/docker.mk

@@ -46,7 +46,9 @@ ifeq ($(REMOTE),true)
 	@echo "starting remote build for $(IMAGE):$(TAG)"
 	DOCKER_BUILDKIT=1 $(DOCKER) buildx build \
 		$(DOCKER_OPTS) \
+		--cache-to type=gha,scope=$(TAG)-buildcache,mode=max \
 		--cache-to type=registry,ref=$(GHCRORG)/$(IMAGE):$(TAG)-buildcache,mode=max \
+		--cache-from type=gha,scope=$(TAG)-buildcache \
 		--cache-from type=registry,ref=$(GHCRORG)/$(IMAGE):$(TAG)-buildcache \
 		--build-arg BUILDKIT_INLINE_CACHE=$(BUILDKIT_INLINE_CACHE) \
 		--build-arg GO_VERSION=$(GO_VERSION) \

Makefile.d/functions.mk

@@ -333,14 +333,12 @@ define gen-vald-crd
 endef
 
 define update-github-actions
-	@for ACTION_NAME in $1; do \
+	@set -e; for ACTION_NAME in $1; do \
 		if [ -n "$$ACTION_NAME" ] && [ "$$ACTION_NAME" != "security-and-quality" ]; then \
 			FILE_NAME=`echo $$ACTION_NAME | tr '/' '_' | tr '-' '_' | tr '[:lower:]' '[:upper:]'`; \
 			if [ -n "$$FILE_NAME" ]; then \
 				if [ "$$ACTION_NAME" = "aquasecurity/trivy-action" ] || [ "$$ACTION_NAME" = "machine-learning-apps/actions-chatops" ]; then \
 					VERSION="master"; \
-				elif [ "$$ACTION_NAME" = "softprops/action-gh-release" ]; then \
-					VERSION="1.0.0"; \
 				else \
 					REPO_NAME=`echo $$ACTION_NAME | cut -d'/' -f1-2`; \
 					VERSION=`curl -fsSL https://api.github.com/repos/$$REPO_NAME/releases/latest | grep -Po '"tag_name": "\K.*?(?=")' | sed 's/v//g' | sed -E 's/[^0-9.]+//g'`;\

Makefile.d/proto.mk

@@ -59,3 +59,4 @@ proto/gen: \
 	find $(ROOTDIR)/apis/grpc/* -name '*.go' | xargs -P$(CORES) sed -i -E "s%google.golang.org/grpc/status%github.com/vdaas/vald/internal/net/grpc/status%g"
 	find $(ROOTDIR)/apis/grpc/* -name '*.go' | xargs -P$(CORES) sed -i -E "s%\"io\"%\"github.com/vdaas/vald/internal/io\"%g"
 	find $(ROOTDIR)/apis/grpc/* -name '*.go' | xargs -P$(CORES) sed -i -E "s%\"sync\"%\"github.com/vdaas/vald/internal/sync\"%g"
+	find $(ROOTDIR)/apis/grpc/* -name '*.go' | xargs -P$(CORES) sed -i -E "s%any%any%g"

Makefile.d/tools.mk

@@ -46,6 +46,12 @@ golines/install: $(GOBIN)/golines
 $(GOBIN)/golines:
 	$(call go-install, github.com/segmentio/golines)
 
+.PHONY: crlfmt/install
+crlfmt/install: $(GOBIN)/crlfmt
+
+$(GOBIN)/crlfmt:
+	$(call go-install, github.com/cockroachdb/crlfmt)
+
 .PHONY: prettier/install
 prettier/install: $(BINDIR)/prettier
 $(BINDIR)/prettier: