add Faiss BinaryIndex (#2576)

* add Faiss BinaryIndex

* style: format code with Gofumpt and Prettier

This commit fixes the style issues introduced in 43c35c2 according to the output
from Gofumpt and Prettier.

Details: #2576

* add nprobe option

* fix memory leak by deleting tmpD

* ensure memory cleanup for tmpD in case of exceptions

---------

Co-authored-by: deepsource-autofix[bot] <62050782+deepsource-autofix[bot]@users.noreply.github.com>

apis/docs/v1/docs.md

@@ -894,6 +894,7 @@ Represent search configuration.
 | min_num               | [uint32](#uint32)                                                      |       | Minimum number of result to be returned.     |
 | aggregation_algorithm | [Search.AggregationAlgorithm](#payload-v1-Search-AggregationAlgorithm) |       | Aggregation Algorithm                        |
 | ratio                 | [google.protobuf.FloatValue](#google-protobuf-FloatValue)              |       | Search ratio for agent return result number. |
+| nprobe                | [uint32](#uint32)                                                      |       | Search nprobe.                               |
 
 <a name="payload-v1-Search-IDRequest"></a>
 

apis/grpc/v1/payload/payload_vtproto.pb.go

@@ -184,6 +184,7 @@ func (m *Search_Config) CloneVT() *Search_Config {
 	r.MinNum = m.MinNum
 	r.AggregationAlgorithm = m.AggregationAlgorithm
 	r.Ratio = (*wrapperspb.FloatValue)((*wrapperspb1.FloatValue)(m.Ratio).CloneVT())
+	r.Nprobe = m.Nprobe
 	if len(m.unknownFields) > 0 {
 		r.unknownFields = make([]byte, len(m.unknownFields))
 		copy(r.unknownFields, m.unknownFields)
@@ -2223,6 +2224,9 @@ func (this *Search_Config) EqualVT(that *Search_Config) bool {
 	if !(*wrapperspb1.FloatValue)(this.Ratio).EqualVT((*wrapperspb1.FloatValue)(that.Ratio)) {
 		return false
 	}
+	if this.Nprobe != that.Nprobe {
+		return false
+	}
 	return string(this.unknownFields) == string(that.unknownFields)
 }
 
@@ -5166,6 +5170,11 @@ func (m *Search_Config) MarshalToSizedBufferVT(dAtA []byte) (int, error) {
 		i -= len(m.unknownFields)
 		copy(dAtA[i:], m.unknownFields)
 	}
+	if m.Nprobe != 0 {
+		i = protohelpers.EncodeVarint(dAtA, i, uint64(m.Nprobe))
+		i--
+		dAtA[i] = 0x58
+	}
 	if m.Ratio != nil {
 		size, err := (*wrapperspb1.FloatValue)(m.Ratio).MarshalToSizedBufferVT(dAtA[:i])
 		if err != nil {
@@ -9783,6 +9792,9 @@ func (m *Search_Config) SizeVT() (n int) {
 		l = (*wrapperspb1.FloatValue)(m.Ratio).SizeVT()
 		n += 1 + l + protohelpers.SizeOfVarint(uint64(l))
 	}
+	if m.Nprobe != 0 {
+		n += 1 + protohelpers.SizeOfVarint(uint64(m.Nprobe))
+	}
 	n += len(m.unknownFields)
 	return n
 }
@@ -12382,6 +12394,25 @@ func (m *Search_Config) UnmarshalVT(dAtA []byte) error {
 				return err
 			}
 			iNdEx = postIndex
+		case 11:
+			if wireType != 0 {
+				return fmt.Errorf("proto: wrong wireType = %d for field Nprobe", wireType)
+			}
+			m.Nprobe = 0
+			for shift := uint(0); ; shift += 7 {
+				if shift >= 64 {
+					return protohelpers.ErrIntOverflow
+				}
+				if iNdEx >= l {
+					return io.ErrUnexpectedEOF
+				}
+				b := dAtA[iNdEx]
+				iNdEx++
+				m.Nprobe |= uint32(b&0x7F) << shift
+				if b < 0x80 {
+					break
+				}
+			}
 		default:
 			iNdEx = preIndex
 			skippy, err := protohelpers.Skip(dAtA[iNdEx:])

apis/proto/v1/payload/payload.proto

@@ -96,6 +96,8 @@ message Search {
     AggregationAlgorithm aggregation_algorithm = 9;
     // Search ratio for agent return result number.
     google.protobuf.FloatValue ratio = 10;
+    // Search nprobe.
+    uint32 nprobe = 11;
   }
 
   // AggregationAlgorithm is enum of each aggregation algorithms

apis/swagger/v1/vald/filter.swagger.json

@@ -550,6 +550,11 @@
           "type": "number",
           "format": "float",
           "description": "Search ratio for agent return result number."
+        },
+        "nprobe": {
+          "type": "integer",
+          "format": "int64",
+          "description": "Search nprobe."
         }
       },
       "description": "Represent search configuration."

apis/swagger/v1/vald/search.swagger.json

@@ -464,6 +464,11 @@
           "type": "number",
           "format": "float",
           "description": "Search ratio for agent return result number."
+        },
+        "nprobe": {
+          "type": "integer",
+          "format": "int64",
+          "description": "Search nprobe."
         }
       },
       "description": "Represent search configuration."

charts/vald-helm-operator/crds/valdrelease.yaml

@@ -182,6 +182,11 @@ spec:
                           type: integer
                         max_load_index_timeout:
                           type: string
+                        method_type:
+                          type: string
+                          enum:
+                            - ivfpq
+                            - binaryindex
                         metric_type:
                           type: string
                           enum:

charts/vald/values.yaml

@@ -2295,18 +2295,22 @@ agent:
     # @schema {"name": "agent.faiss.dimension", "type": "integer", "minimum": 1}
     # agent.faiss.dimension -- vector dimension
     dimension: 4096
+    # @schema {"name": "agent.faiss.method_type", "type": "string", "enum": ["ivfpq", "binaryindex"]}
+    # agent.faiss.method_type -- method type
+    # it should be `ivfpq` or `binaryindex`
+    method_type: ivfpq
     # @schema {"name": "agent.faiss.metric_type", "type": "string", "enum": ["innerproduct", "l2"]}
-    # agent.faiss.metric_type-- metric type
+    # agent.faiss.metric_type -- metric type
     # it should be `innerproduct` or `l2`
     metric_type: l2
     # @schema {"name": "agent.faiss.nlist", "type": "integer"}
-    # agent.faiss.nlist-- nlist
+    # agent.faiss.nlist -- nlist
     nlist: 100
     # @schema {"name": "agent.faiss.m", "type": "integer"}
-    # agent.faiss.m-- m
+    # agent.faiss.m -- m
     m: 8
     # @schema {"name": "agent.faiss.nbits_per_idx", "type": "integer"}
-    # agent.faiss.nbits_per_idx-- nbits_per_idx
+    # agent.faiss.nbits_per_idx -- nbits_per_idx
     nbits_per_idx: 8
     # @schema {"name": "agent.faiss.enable_in_memory_mode", "type": "boolean"}
     # agent.faiss.enable_in_memory_mode -- in-memory mode enabled

cmd/agent/core/faiss/sample.yaml

@@ -115,6 +115,7 @@ faiss:
   load_index_timeout_factor: 1ms
   m: 8 # dimension % m == 0, train size >= 2^m(or nlist) * minPointsPerCentroid
   max_load_index_timeout: 10m
+  method_type: "ivfpq"
   metric_type: "innerproduct"
   min_load_index_timeout: 3m
   nbits_per_idx: 8

docs/tutorial/get-started-with-faiss-agent.md

@@ -60,6 +60,7 @@ In this tutorial, you will deploy the basic configuration of Vald that is consis
        load_index_timeout_factor: 1ms
        m: 8  # dimension % m == 0, train size >= 2^m(or nlist) * minPointsPerCentroid
        max_load_index_timeout: 10m
+       method_type: "ivfpq"
        metric_type: "innerproduct"
        min_load_index_timeout: 3m
        nbits_per_idx: 8

internal/config/faiss.go

@@ -37,6 +37,9 @@ type Faiss struct {
 	// ref: https://github.com/facebookresearch/faiss/wiki/FAQ#can-i-ignore-warning-clustering-xxx-points-to-yyy-centroids
 	NbitsPerIdx int `info:"nbits_per_idx" json:"nbits_per_idx,omitempty" yaml:"nbits_per_idx"`
 
+	// MethodType represents the method type
+	MethodType string `info:"method_type" json:"method_type,omitempty" yaml:"method_type"`
+
 	// MetricType represents the metric type
 	MetricType string `info:"metric_type" json:"metric_type,omitempty" yaml:"metric_type"`