refactor index manager service add index service API to expose index …

…informations

style: format code with Gofumpt and Prettier

Signed-off-by: kpango <kpango@vdaas.org>

.gitfiles

@@ -80,6 +80,7 @@
 .github/workflows/dockers-agent-sidecar-image.yml
 .github/workflows/dockers-benchmark-job-image.yml
 .github/workflows/dockers-benchmark-operator-image.yaml
+.github/workflows/dockers-buildbase-image.yml
 .github/workflows/dockers-ci-container-image.yml
 .github/workflows/dockers-dev-container-image.yml
 .github/workflows/dockers-discoverer-k8s-image.yml
@@ -161,8 +162,6 @@ apis/grpc/v1/filter/egress/egress_filter.pb.go
 apis/grpc/v1/filter/egress/egress_filter_vtproto.pb.go
 apis/grpc/v1/filter/ingress/ingress_filter.pb.go
 apis/grpc/v1/filter/ingress/ingress_filter_vtproto.pb.go
-apis/grpc/v1/manager/index/index_manager.pb.go
-apis/grpc/v1/manager/index/index_manager_vtproto.pb.go
 apis/grpc/v1/mirror/mirror.go
 apis/grpc/v1/mirror/mirror.pb.go
 apis/grpc/v1/mirror/mirror_vtproto.pb.go
@@ -177,6 +176,8 @@ apis/grpc/v1/vald/filter.pb.go
 apis/grpc/v1/vald/filter_vtproto.pb.go
 apis/grpc/v1/vald/flush.pb.go
 apis/grpc/v1/vald/flush_vtproto.pb.go
+apis/grpc/v1/vald/index.pb.go
+apis/grpc/v1/vald/index_vtproto.pb.go
 apis/grpc/v1/vald/insert.pb.go
 apis/grpc/v1/vald/insert_vtproto.pb.go
 apis/grpc/v1/vald/object.pb.go
@@ -197,12 +198,12 @@ apis/proto/v1/agent/sidecar/sidecar.proto
 apis/proto/v1/discoverer/discoverer.proto
 apis/proto/v1/filter/egress/egress_filter.proto
 apis/proto/v1/filter/ingress/ingress_filter.proto
-apis/proto/v1/manager/index/index_manager.proto
 apis/proto/v1/mirror/mirror.proto
 apis/proto/v1/payload/payload.proto
 apis/proto/v1/rpc/errdetails/error_details.proto
 apis/proto/v1/vald/filter.proto
 apis/proto/v1/vald/flush.proto
+apis/proto/v1/vald/index.proto
 apis/proto/v1/vald/insert.proto
 apis/proto/v1/vald/object.proto
 apis/proto/v1/vald/remove.proto
@@ -214,12 +215,12 @@ apis/swagger/v1/agent/sidecar/sidecar.swagger.json
 apis/swagger/v1/discoverer/discoverer.swagger.json
 apis/swagger/v1/filter/egress/egress_filter.swagger.json
 apis/swagger/v1/filter/ingress/ingress_filter.swagger.json
-apis/swagger/v1/manager/index/index_manager.swagger.json
 apis/swagger/v1/mirror/mirror.swagger.json
 apis/swagger/v1/payload/payload.swagger.json
 apis/swagger/v1/rpc/errdetails/error_details.swagger.json
 apis/swagger/v1/vald/filter.swagger.json
 apis/swagger/v1/vald/flush.swagger.json
+apis/swagger/v1/vald/index.swagger.json
 apis/swagger/v1/vald/insert.swagger.json
 apis/swagger/v1/vald/object.swagger.json
 apis/swagger/v1/vald/remove.swagger.json
@@ -228,6 +229,7 @@ apis/swagger/v1/vald/update.swagger.json
 apis/swagger/v1/vald/upsert.swagger.json
 assets/docs/guides/observability-configuration/architecture.png
 assets/docs/guides/operations/grafana-example.png
+assets/docs/guides/read-replica-and-rotator/architecture.png
 assets/docs/overview/component/agent/agent.drawio
 assets/docs/overview/component/agent/ngt.png
 assets/docs/overview/component/agent/sidecar_backup.png
@@ -516,6 +518,7 @@ dockers/agent/core/ngt/Dockerfile
 dockers/agent/core/ngt/README.md
 dockers/agent/sidecar/Dockerfile
 dockers/agent/sidecar/README.md
+dockers/buildbase/Dockerfile
 dockers/ci/base/Dockerfile
 dockers/ci/base/README.md
 dockers/dev/Dockerfile
@@ -536,6 +539,7 @@ dockers/manager/index/Dockerfile
 dockers/manager/index/README.md
 dockers/operator/helm/Dockerfile
 dockers/operator/helm/README.md
+dockers/readreplica/job/readreplica/rotate/Dockerfile
 dockers/template.md
 dockers/tools/benchmark/job/Dockerfile
 dockers/tools/benchmark/operator/Dockerfile
@@ -556,6 +560,7 @@ docs/api/upsert.md
 docs/contributing/coding-style.md
 docs/contributing/contributing-guide.md
 docs/contributing/development.md
+docs/contributing/reviewer-guideline.md
 docs/contributing/unit-test-guideline.md
 docs/overview/about-vald.md
 docs/overview/architecture.md
@@ -571,8 +576,8 @@ docs/performance/benchmark.md
 docs/performance/continuous-benchmark.md
 docs/performance/loadtest.md
 docs/performance/tuning-search-performance.md
+docs/support/FAQ.md
 docs/support/contacts.md
-docs/support/faq.md
 docs/troubleshooting/README.md
 docs/troubleshooting/client-side.md
 docs/troubleshooting/mirror-gateway.md
@@ -595,6 +600,7 @@ docs/user-guides/mirroring-configuration.md
 docs/user-guides/network-policy.md
 docs/user-guides/observability-configuration.md
 docs/user-guides/operations.md
+docs/user-guides/read-replica-and-rotator.md
 docs/user-guides/sdks.md
 docs/user-guides/upgrade-cluster.md
 example/client/agent/main.go
@@ -686,6 +692,7 @@ hack/benchmark/internal/starter/gateway/vald/vald_test.go
 hack/benchmark/internal/starter/starter.go
 hack/benchmark/metrics/metrics.go
 hack/benchmark/src/singleflight/singleflight_bench_test.go
+hack/docker/gen/main.go
 hack/git/hooks/pre-commit
 hack/go.mod.default
 hack/gorules/rules.go
@@ -1016,7 +1023,6 @@ internal/hash/hash.go
 internal/hash/hash_test.go
 internal/info/info.go
 internal/info/info_bench_test.go
-internal/info/info_test.go
 internal/info/option.go
 internal/info/option_test.go
 internal/io/copy.go
@@ -1820,6 +1826,14 @@ renovate.json
 rust/Cargo.lock
 rust/Cargo.toml
 rust/bin/agent/Cargo.toml
+rust/bin/agent/src/handler.rs
+rust/bin/agent/src/handler/common.rs
+rust/bin/agent/src/handler/index.rs
+rust/bin/agent/src/handler/insert.rs
+rust/bin/agent/src/handler/remove.rs
+rust/bin/agent/src/handler/search.rs
+rust/bin/agent/src/handler/update.rs
+rust/bin/agent/src/handler/upsert.rs
 rust/bin/agent/src/main.rs
 rust/libs/ngt-rs/Cargo.toml
 rust/libs/ngt-rs/build.rs
@@ -1838,8 +1852,6 @@ rust/libs/proto/src/filter.egress.v1.tonic.rs
 rust/libs/proto/src/filter.ingress.v1.rs
 rust/libs/proto/src/filter.ingress.v1.tonic.rs
 rust/libs/proto/src/lib.rs
-rust/libs/proto/src/manager.index.v1.rs
-rust/libs/proto/src/manager.index.v1.tonic.rs
 rust/libs/proto/src/mirror.v1.rs
 rust/libs/proto/src/mirror.v1.tonic.rs
 rust/libs/proto/src/payload.v1.rs

.github/ISSUE_TEMPLATE/bug_report.md

@@ -22,8 +22,8 @@ assignees: ""
 
 <!--- Please change the versions below along with your environment -->
 
-- Go Version: 1.22.3
+- Go Version: 1.22.5
 - Rust Version: 1.77.2
 - Docker Version: 20.10.8
-- Kubernetes Version: v1.30.0
-- NGT Version: 2.2.1
+- Kubernetes Version: v1.30.3
+- NGT Version: 2.2.2

.github/ISSUE_TEMPLATE/security_issue_report.md

@@ -16,8 +16,8 @@ assignees: ""
 
 <!--- Please change the versions below along with your environment -->
 
-- Go Version: 1.22.3
+- Go Version: 1.22.5
 - Rust Version: 1.77.2
 - Docker Version: 20.10.8
-- Kubernetes Version: v1.30.0
-- NGT Version: 2.2.1
+- Kubernetes Version: v1.30.3
+- NGT Version: 2.2.2

.github/PULL_REQUEST_TEMPLATE.md

@@ -16,11 +16,11 @@
 
 <!--- Please change the versions below along with your environment -->
 
-- Go Version: 1.22.3
+- Go Version: 1.22.5
 - Rust Version: 1.77.2
 - Docker Version: 20.10.8
-- Kubernetes Version: v1.30.0
-- NGT Version: 2.2.1
+- Kubernetes Version: v1.30.3
+- NGT Version: 2.2.2
 
 ### Checklist
 

.github/actions/docker-build/action.yaml

@@ -113,7 +113,11 @@ runs:
           REMOTE="true" \
           DOCKER="docker" \
           BUILDKIT_INLINE_CACHE=0 \
-          DOCKER_OPTS="--platform ${PLATFORMS} --builder ${BUILDER} ${LABEL_OPTS}  --label org.opencontainers.image.version=${PRIMARY_TAG} --label org.opencontainers.image.title=${TARGET}" \
+          DOCKER_OPTS="--platform ${PLATFORMS} --builder ${BUILDER} ${LABEL_OPTS} \
+          --label org.opencontainers.image.version=${PRIMARY_TAG} \
+          --label org.opencontainers.image.title=${TARGET} \
+          --label org.opencontainers.image.created=\"$(date --rfc-3339=ns)\" \
+          --label org.opencontainers.image.licenses=\"Apache 2.0\"" \
           EXTRA_ARGS="${EXTRA_TAGS}" \
           TAG="${PRIMARY_TAG}" \
           docker/build/${TARGET}

.github/workflows/release.yml

@@ -61,7 +61,7 @@ jobs:
           git push origin ${VERSION}
           echo "VERSION=${VERSION}" >> $GITHUB_OUTPUT
       - name: Create release
-        uses: softprops/action-gh-release@v1
+        uses: softprops/action-gh-release@v2
         env:
           GITHUB_TOKEN: ${{ secrets.DISPATCH_TOKEN }}
         with:

Makefile

@@ -48,6 +48,17 @@ VERSION ?= $(eval VERSION := $(shell cat versions/VALD_VERSION))$(VERSION)
 
 NGT_REPO = github.com/yahoojapan/NGT
 
+NPM_GLOBAL_PREFIX := $(eval NPM_GLOBAL_PREFIX := $(shell npm prefix --location=global))$(NPM_GLOBAL_PREFIX)
+
+TEST_NOT_IMPL_PLACEHOLDER = NOT IMPLEMENTED BELOW
+
+TEMP_DIR := $(eval TEMP_DIR := $(shell mktemp -d))$(TEMP_DIR)
+USR_LOCAL = /usr/local
+BINDIR = $(USR_LOCAL)/bin
+LIB_PATH = $(USR_LOCAL)/lib
+$(LIB_PATH):
+	mkdir -p $(LIB_PATH)
+
 GOPRIVATE = $(GOPKG),$(GOPKG)/apis,$(GOPKG)-client-go
 GOPROXY = "https://proxy.golang.org,direct"
 GOPATH := $(eval GOPATH := $(shell go env GOPATH))$(GOPATH)
@@ -60,17 +71,11 @@ GO_CLEAN_DEPS := true
 GOTEST_TIMEOUT = 30m
 CGO_ENABLED = 1
 
-RUST_HOME ?= /usr/local/lib/rust
+RUST_HOME ?= $(LIB_PATH)/rust
 RUSTUP_HOME ?= $(RUST_HOME)/rustup
 CARGO_HOME ?= $(RUST_HOME)/cargo
 RUST_VERSION := $(eval RUST_VERSION := $(shell cat versions/RUST_VERSION))$(RUST_VERSION)
 
-NPM_GLOBAL_PREFIX := $(eval NPM_GLOBAL_PREFIX := $(shell npm prefix --location=global))$(NPM_GLOBAL_PREFIX)
-
-TEST_NOT_IMPL_PLACEHOLDER = NOT IMPLEMENTED BELOW
-
-TEMP_DIR := $(eval TEMP_DIR := $(shell mktemp -d))$(TEMP_DIR)
-
 BUF_VERSION               := $(eval BUF_VERSION := $(shell cat versions/BUF_VERSION))$(BUF_VERSION)
 NGT_VERSION 		  := $(eval NGT_VERSION := $(shell cat versions/NGT_VERSION))$(NGT_VERSION)
 FAISS_VERSION             := $(eval FAISS_VERSION := $(shell cat versions/FAISS_VERSION))$(FAISS_VERSION)
@@ -101,8 +106,6 @@ SWAP_DEPLOYMENT_TYPE ?= deployment
 SWAP_IMAGE           ?= ""
 SWAP_TAG             ?= latest
 
-BINDIR ?= /usr/local/bin
-
 UNAME := $(eval UNAME := $(shell uname -s))$(UNAME)
 OS := $(eval OS := $(shell echo $(UNAME) | tr '[:upper:]' '[:lower:]'))$(OS)
 ARCH := $(eval ARCH := $(shell uname -m))$(ARCH)
@@ -239,6 +242,7 @@ GO_SOURCES = $(eval GO_SOURCES := $(shell find \
 		-not -path '$(ROOTDIR)/hack/benchmark/internal/starter/gateway/*' \
 		-not -path '$(ROOTDIR)/hack/gorules/*' \
 		-not -path '$(ROOTDIR)/hack/license/*' \
+		-not -path '$(ROOTDIR)/hack/docker/*' \
 		-not -path '$(ROOTDIR)/hack/swagger/*' \
 		-not -path '$(ROOTDIR)/hack/tools/*' \
 		-not -path '$(ROOTDIR)/tests/*' \
@@ -270,6 +274,7 @@ GO_OPTION_SOURCES = $(eval GO_OPTION_SOURCES := $(shell find \
 		-not -path '$(ROOTDIR)/hack/benchmark/internal/starter/gateway/*' \
 		-not -path '$(ROOTDIR)/hack/gorules/*' \
 		-not -path '$(ROOTDIR)/hack/license/*' \
+		-not -path '$(ROOTDIR)/hack/docker/*' \
 		-not -path '$(ROOTDIR)/hack/swagger/*' \
 		-not -path '$(ROOTDIR)/hack/tools/*' \
 		-not -path '$(ROOTDIR)/tests/*' \
@@ -412,6 +417,11 @@ files:
 license:
 	$(call gen-license,$(ROOTDIR),$(MAINTAINER))
 
+.PHONY: dockerfile
+## generate dockerfiles
+dockerfile:
+	$(call gen-dockerfile,$(ROOTDIR),$(MAINTAINER))
+
 .PHONY: init
 ## initialize development environment
 init: \
@@ -445,6 +455,7 @@ update: \
 .PHONY: format
 ## format go codes
 format: \
+	dockerfile \
 	license \
 	format/proto \
 	format/go \
@@ -455,6 +466,7 @@ format: \
 .PHONY: format/go
 ## run golines, gofumpt, goimports for all go files
 format/go: \
+	crlfmt/install \
 	golines/install \
 	gofumpt/install \
 	strictgoimports/install \
@@ -463,10 +475,12 @@ format/go: \
 	find $(ROOTDIR)/ -type d -name .git -prune -o -type f -regex '.*[^\.pb]\.go' -print | xargs -P$(CORES) $(GOBIN)/gofumpt -w
 	find $(ROOTDIR)/ -type d -name .git -prune -o -type f -regex '.*[^\.pb]\.go' -print | xargs -P$(CORES) $(GOBIN)/strictgoimports -w
 	find $(ROOTDIR)/ -type d -name .git -prune -o -type f -regex '.*\.go' -print | xargs -P$(CORES) $(GOBIN)/goimports -w
+	find $(ROOTDIR)/ -type d -name .git -prune -o -type f -regex '.*\.go' -print | xargs -P$(CORES) $(GOBIN)/crlfmt -w
 
 .PHONY: format/go/test
 ## run golines, gofumpt, goimports for go test files
 format/go/test: \
+	crlfmt/install \
 	golines/install \
 	gofumpt/install \
 	strictgoimports/install \
@@ -475,6 +489,7 @@ format/go/test: \
 	find $(ROOTDIR) -name '*_test.go' | xargs -P$(CORES) $(GOBIN)/gofumpt -w
 	find $(ROOTDIR) -name '*_test.go' | xargs -P$(CORES) $(GOBIN)/strictgoimports -w
 	find $(ROOTDIR) -name '*_test.go' | xargs -P$(CORES) $(GOBIN)/goimports -w
+	find $(ROOTDIR) -name '*_test.go' | xargs -P$(CORES) $(GOBIN)/crlfmt -w
 
 .PHONY: format/yaml
 format/yaml: \
@@ -517,6 +532,7 @@ deps: \
 .PHONY: deps/install
 ## install dependencies
 deps/install: \
+	crlfmt/install \
 	golines/install \
 	gofumpt/install \
 	strictgoimports/install \
@@ -578,23 +594,23 @@ version/telepresence:
 
 .PHONY: ngt/install
 ## install NGT
-ngt/install: /usr/local/include/NGT/Capi.h
-/usr/local/include/NGT/Capi.h:
+ngt/install: $(USR_LOCAL)/include/NGT/Capi.h
+$(USR_LOCAL)/include/NGT/Capi.h:
 	git clone --depth 1 --branch v$(NGT_VERSION) https://github.com/yahoojapan/NGT $(TEMP_DIR)/NGT-$(NGT_VERSION)
 	cd $(TEMP_DIR)/NGT-$(NGT_VERSION) && \
 		cmake -DCMAKE_C_FLAGS="$(CFLAGS)" -DCMAKE_CXX_FLAGS="$(CXXFLAGS)" "$(NGT_EXTRA_FLAGS)" .
-	make -j -C $(TEMP_DIR)/NGT-$(NGT_VERSION)
+	make -j$(CORES) -C $(TEMP_DIR)/NGT-$(NGT_VERSION)
 	make install -C $(TEMP_DIR)/NGT-$(NGT_VERSION)
 	cd $(ROOTDIR)
 	rm -rf $(TEMP_DIR)/NGT-$(NGT_VERSION)
 	ldconfig
 
 .PHONY: faiss/install
 ## install Faiss
-faiss/install: /usr/local/lib/libfaiss.so
-/usr/local/lib/libfaiss.so:
-	curl -fsSLO https://github.com/facebookresearch/faiss/archive/v$(FAISS_VERSION).tar.gz
-	tar zxf v$(FAISS_VERSION).tar.gz -C $(TEMP_DIR)/
+faiss/install: $(LIB_PATH)/libfaiss.so
+$(LIB_PATH)/libfaiss.so:
+	curl -fsSL https://github.com/facebookresearch/faiss/archive/v$(FAISS_VERSION).tar.gz -o $(TEMP_DIR)/v$(FAISS_VERSION).tar.gz
+	tar zxf $(TEMP_DIR)/v$(FAISS_VERSION).tar.gz -C $(TEMP_DIR)/
 	cd $(TEMP_DIR)/faiss-$(FAISS_VERSION) && \
 		cmake -DFAISS_ENABLE_GPU=OFF -DFAISS_ENABLE_PYTHON=OFF -DBUILD_TESTING=OFF -DCMAKE_BUILD_TYPE=Release -DBUILD_SHARED_LIBS=ON -B build . && \
 		make -C build -j faiss && \
@@ -608,7 +624,11 @@ faiss/install: /usr/local/lib/libfaiss.so
 lint: \
 	docs/lint \
 	files/lint \
-	vet
+	vet \
+	go/lint
+
+.PHONY: go/lint
+go/lint:
 	$(call go-lint)
 
 .PHONY: vet