diff --git a/.golangci.yml b/.golangci.yml index f4ce6fe..e4a3fd2 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -76,20 +76,18 @@ run: # skip-files: # - sample -# issues: -# exclude-rules: -# - path: info/as_parser_test\.go -# linters: -# - lll # Test code is allowed to have long lines -# - path: asconfig/generate_test\.go -# linters: -# - dupl # Test code is allowed to have duplicate code -# - path: asconfig/asconfig_test\.go -# linters: -# - dupl # Test code is allowed to have duplicate code -# - path: '(.+)test\.go' -# linters: -# - govet # Test code field alignment for sake of space is not a concern -# - linters: -# - lll -# source: "// " +issues: + exclude-rules: + - path: info/as_parser_test\.go + linters: + - lll # Test code is allowed to have long lines + - path: asconfig/generate_test\.go + linters: + - dupl # Test code is allowed to have duplicate code + - path: asconfig/asconfig_test\.go + linters: + - dupl # Test code is allowed to have duplicate code + - path: '(.+)test\.go' + linters: + - govet # Test code field alignment for sake of space is not a concern + - wsl # Auto generated tests cuddle assignments diff --git a/cmd/flags/hnsw.go b/cmd/flags/hnsw.go index cf6b5dc..ee57cdb 100644 --- a/cmd/flags/hnsw.go +++ b/cmd/flags/hnsw.go @@ -57,7 +57,7 @@ func NewHnswIndexCachingFlags() *IndexCachingFlags { func (cf *IndexCachingFlags) NewFlagSet() *pflag.FlagSet { flagSet := &pflag.FlagSet{} flagSet.Var(&cf.MaxEntries, HnswIndexCacheMaxEntries, "Maximum number of entries to cache.") - flagSet.Var(&cf.Expiry, HnswIndexCacheExpiry, "A cache entry will expire after this amount of time has passed since the entry was added to cache, or 'inf' to never expire.") + flagSet.Var(&cf.Expiry, HnswIndexCacheExpiry, "A cache entry will expire after this amount of time has passed since the entry was added to cache, or -1 to never expire.") return flagSet } @@ -85,7 +85,7 @@ func NewHnswRecordCachingFlags() *RecordCachingFlags { func (cf *RecordCachingFlags) NewFlagSet() *pflag.FlagSet { flagSet := &pflag.FlagSet{} flagSet.Var(&cf.MaxEntries, HnswRecordCacheMaxEntries, "Maximum number of entries to cache.") - flagSet.Var(&cf.Expiry, HnswRecordCacheExpiry, "A cache entry will expire after this amount of time has passed since the entry was added to cache, or 'inf' to never expire.") + flagSet.Var(&cf.Expiry, HnswRecordCacheExpiry, "A cache entry will expire after this amount of time has passed since the entry was added to cache, or -1 to never expire.") return flagSet } diff --git a/cmd/flags/optionals.go b/cmd/flags/optionals.go index b69ceac..203ed0d 100644 --- a/cmd/flags/optionals.go +++ b/cmd/flags/optionals.go @@ -209,8 +209,8 @@ func (f *DurationOptionalFlag) Int64() *int64 { return &milli } -// InfDurationOptionalFlag is a flag that can be either a time.duration or infinity. -// It is used for flags like --hnsw-index-cache-expiry which can be set to "infinity" +// InfDurationOptionalFlag is a flag that can be either a time.duration or -1 (never expire). +// It is used for flags like --hnsw-index-cache-expiry which can be set to never expire (-1) type InfDurationOptionalFlag struct { duration DurationOptionalFlag isInfinite bool @@ -224,7 +224,7 @@ func (f *InfDurationOptionalFlag) Set(val string) error { val = strings.ToLower(val) - if val == "inf" || val == "infinity" || val == "-1" { + if val == strconv.Itoa(Infinity) { f.isInfinite = true } else { return fmt.Errorf("invalid duration %s", val) @@ -239,7 +239,7 @@ func (f *InfDurationOptionalFlag) Type() string { func (f *InfDurationOptionalFlag) String() string { if f.isInfinite { - return "infinity" + return "-1" } if f.duration.Val != nil { diff --git a/cmd/flags/optionals_test.go b/cmd/flags/optionals_test.go index 08c9f1c..6659d8f 100644 --- a/cmd/flags/optionals_test.go +++ b/cmd/flags/optionals_test.go @@ -114,30 +114,12 @@ func (suite *OptionalFlagSuite) TestDurationOptionalFlag() { func (suite *OptionalFlagSuite) TestInfDurationOptionalFlag() { f := &InfDurationOptionalFlag{} - err := f.Set("inf") + err := f.Set("-1") if err != nil { suite.T().Errorf("Unexpected error: %v", err) } - suite.Equal("infinity", f.String()) - suite.Equal(int64(-1), *f.Int64()) - f = &InfDurationOptionalFlag{} - - err = f.Set("infinity") - if err != nil { - suite.T().Errorf("Unexpected error: %v", err) - } - - suite.Equal("infinity", f.String()) - suite.Equal(int64(-1), *f.Int64()) - f = &InfDurationOptionalFlag{} - - err = f.Set("-1") - if err != nil { - suite.T().Errorf("Unexpected error: %v", err) - } - - suite.Equal("infinity", f.String()) + suite.Equal("-1", f.String()) suite.Equal(int64(-1), *f.Int64()) f = &InfDurationOptionalFlag{} diff --git a/cmd/writers/indexList.go b/cmd/writers/indexList.go index 043fcf8..819025e 100644 --- a/cmd/writers/indexList.go +++ b/cmd/writers/indexList.go @@ -30,11 +30,13 @@ func NewIndexTableWriter(writer io.Writer, verbose bool, logger *slog.Logger) *I "Dimensions", "Distance Metric", "Unmerged", + "Vector Records", + "Size", + "Unmerged %", } verboseHeadings := append(table.Row{}, headings...) verboseHeadings = append( verboseHeadings, - "Vector Records", "Vertices", "Labels*", "Storage", @@ -79,11 +81,13 @@ func (itw *IndexTableWriter) AppendIndexRow( index.Dimensions, index.VectorDistanceMetric, status.GetUnmergedRecordCount(), + status.GetIndexHealerVectorRecordsIndexed(), + formatBytes(calculateIndexSize(index, status)), + getPercentUnmerged(status), } if itw.verbose { row = append(row, - status.GetIndexHealerVectorRecordsIndexed(), status.GetIndexHealerVerticesValid(), index.Labels, ) @@ -145,3 +149,51 @@ func convertMillisecondToDuration[T int64 | uint64 | uint32](m T) time.Duration func convertFloatToPercentStr(f float32) string { return fmt.Sprintf("%.2f%%", f) } + +// calculateIndexSize calculates the size of the index in bytes +func calculateIndexSize(index *protos.IndexDefinition, status *protos.IndexStatusResponse) int64 { + // Each dimension is a float32 + vectorSize := int64(index.Dimensions) * 4 + // Each index record has ~500 bytes of overhead + the vector size + indexRecSize := 500 + vectorSize + // The total size is the number of records times the size of each record + return indexRecSize * status.GetIndexHealerVerticesValid() +} + +// formatBytes converts bytes to human readable string format +func formatBytes(bytes int64) string { + const ( + B = 1 + KB = 1024 * B + MB = 1024 * KB + GB = 1024 * MB + TB = 1024 * GB + PB = 1024 * TB + ) + + switch { + case bytes >= PB: + return fmt.Sprintf("%.2f PB", float64(bytes)/float64(PB)) + case bytes >= TB: + return fmt.Sprintf("%.2f TB", float64(bytes)/float64(TB)) + case bytes >= GB: + return fmt.Sprintf("%.2f GB", float64(bytes)/float64(GB)) + case bytes >= MB: + return fmt.Sprintf("%.2f MB", float64(bytes)/float64(MB)) + case bytes >= KB: + return fmt.Sprintf("%.2f KB", float64(bytes)/float64(KB)) + default: + return fmt.Sprintf("%d B", bytes) + } +} + +func getPercentUnmerged(status *protos.IndexStatusResponse) string { + unmergedCount := status.GetUnmergedRecordCount() + + verticies := status.GetIndexHealerVerticesValid() + if verticies == 0 { + return "0%" + } + + return fmt.Sprintf("%.2f%%", float64(unmergedCount)/float64(verticies)*100) +} diff --git a/cmd/writers/indexList_test.go b/cmd/writers/indexList_test.go new file mode 100644 index 0000000..2a3e23f --- /dev/null +++ b/cmd/writers/indexList_test.go @@ -0,0 +1,175 @@ +package writers + +import ( + "testing" + + "github.com/aerospike/avs-client-go/protos" +) + +func Test_calculateIndexSize(t *testing.T) { + type args struct { + index *protos.IndexDefinition + status *protos.IndexStatusResponse + } + tests := []struct { + name string + args args + want int64 + }{ + { + name: "positive simple", + args: args{ + index: &protos.IndexDefinition{ + Dimensions: 100, + }, + status: &protos.IndexStatusResponse{ + IndexHealerVerticesValid: 10, + }, + }, + want: 9000, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := calculateIndexSize(tt.args.index, tt.args.status); got != tt.want { + t.Errorf("calculateIndexSize() = %v, want %v", got, tt.want) + } + }) + } +} + +func Test_formatBytes(t *testing.T) { + type args struct { + bytes int64 + } + tests := []struct { + name string + args args + want string + }{ + { + name: "petabytes", + args: args{ + bytes: 1024 * 1024 * 1024 * 1024 * 1024, + }, + want: "1.00 PB", + }, + { + name: "terabytes", + args: args{ + bytes: 1024 * 1024 * 1024 * 1024, + }, + want: "1.00 TB", + }, + { + name: "gigabytes", + args: args{ + bytes: 1024 * 1024 * 1024, + }, + want: "1.00 GB", + }, + { + name: "megabytes", + args: args{ + bytes: 1024 * 1024, + }, + want: "1.00 MB", + }, + { + name: "kilobytes", + args: args{ + bytes: 1024, + }, + want: "1.00 KB", + }, + { + name: "bytes", + args: args{ + bytes: 512, + }, + want: "512 B", + }, + { + name: "zero bytes", + args: args{ + bytes: 0, + }, + want: "0 B", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := formatBytes(tt.args.bytes); got != tt.want { + t.Errorf("formatBytes() = %v, want %v", got, tt.want) + } + }) + } +} +func Test_getPercentUnmerged(t *testing.T) { + type args struct { + status *protos.IndexStatusResponse + } + tests := []struct { + name string + args args + want string + }{ + { + name: "zero vertices", + args: args{ + status: &protos.IndexStatusResponse{ + IndexHealerVerticesValid: 0, + UnmergedRecordCount: 10, + }, + }, + want: "0%", + }, + { + name: "zero unmerged records", + args: args{ + status: &protos.IndexStatusResponse{ + IndexHealerVerticesValid: 100, + UnmergedRecordCount: 0, + }, + }, + want: "0.00%", + }, + { + name: "50 percent unmerged", + args: args{ + status: &protos.IndexStatusResponse{ + IndexHealerVerticesValid: 100, + UnmergedRecordCount: 50, + }, + }, + want: "50.00%", + }, + { + name: "100 percent unmerged", + args: args{ + status: &protos.IndexStatusResponse{ + IndexHealerVerticesValid: 100, + UnmergedRecordCount: 100, + }, + }, + want: "100.00%", + }, + { + name: "33.33 percent unmerged", + args: args{ + status: &protos.IndexStatusResponse{ + IndexHealerVerticesValid: 300, + UnmergedRecordCount: 100, + }, + }, + want: "33.33%", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := getPercentUnmerged(tt.args.status); got != tt.want { + t.Errorf("getPercentUnmerged() = %v, want %v", got, tt.want) + } + }) + } +} diff --git a/e2e_test.go b/e2e_test.go index 9e935b9..218db75 100644 --- a/e2e_test.go +++ b/e2e_test.go @@ -306,7 +306,7 @@ func (suite *CmdTestSuite) TestSuccessfulCreateIndexCmd() { name: "test with infinite record cache expiry", indexName: "recinfidx", indexNamespace: "test", - cmd: "index create -y -n test -i recinfidx -d 256 -m SQUARED_EUCLIDEAN --vector-field vector0 --hnsw-record-cache-expiry inf", + cmd: "index create -y -n test -i recinfidx -d 256 -m SQUARED_EUCLIDEAN --vector-field vector0 --hnsw-record-cache-expiry -1", expectedIndex: tests.NewIndexDefinitionBuilder(false, "recinfidx", "test", 256, protos.VectorDistanceMetric_SQUARED_EUCLIDEAN, "vector0"). WithHnswRecordCacheExpiry(-1). Build(), @@ -315,7 +315,7 @@ func (suite *CmdTestSuite) TestSuccessfulCreateIndexCmd() { name: "test with infinite index cache expiry", indexName: "idxinfidx", indexNamespace: "test", - cmd: "index create -y -n test -i idxinfidx -d 256 -m SQUARED_EUCLIDEAN --vector-field vector0 --hnsw-index-cache-expiry inf", + cmd: "index create -y -n test -i idxinfidx -d 256 -m SQUARED_EUCLIDEAN --vector-field vector0 --hnsw-index-cache-expiry -1", expectedIndex: tests.NewIndexDefinitionBuilder(false, "idxinfidx", "test", 256, protos.VectorDistanceMetric_SQUARED_EUCLIDEAN, "vector0"). WithHnswIndexCacheExpiry(-1). Build(), @@ -751,8 +751,8 @@ func (suite *CmdTestSuite) TestSuccessfulListIndexCmd() { }, cmd: "index list --no-color --format 1", expectedTable: `Indexes -,Name,Namespace,Field,Dimensions,Distance Metric,Unmerged -1,list,test,vector,256,COSINE,0 +,Name,Namespace,Field,Dimensions,Distance Metric,Unmerged,Vector Records,Size,Unmerged % +1,list,test,vector,256,COSINE,0,0,0 B,0% `, }, { @@ -767,9 +767,9 @@ func (suite *CmdTestSuite) TestSuccessfulListIndexCmd() { }, cmd: "index list --no-color --format 1", expectedTable: `Indexes -,Name,Namespace,Set,Field,Dimensions,Distance Metric,Unmerged -1,list2,bar,barset,vector,256,HAMMING,0 -2,list1,test,,vector,256,COSINE,0 +,Name,Namespace,Set,Field,Dimensions,Distance Metric,Unmerged,Vector Records,Size,Unmerged % +1,list2,bar,barset,vector,256,HAMMING,0,0,0 B,0% +2,list1,test,,vector,256,COSINE,0,0,0 B,0% `, }, { @@ -795,8 +795,8 @@ func (suite *CmdTestSuite) TestSuccessfulListIndexCmd() { }, cmd: "index list --verbose --no-color --format 1", expectedTable: `Indexes -,Name,Namespace,Set,Field,Dimensions,Distance Metric,Unmerged,Vector Records,Vertices,Labels*,Storage,Index Parameters -1,list2,bar,barset,vector,256,HAMMING,0,0,0,map[],"Namespace\,bar +,Name,Namespace,Set,Field,Dimensions,Distance Metric,Unmerged,Vector Records,Size,Unmerged %,Vertices,Labels*,Storage,Index Parameters +1,list2,bar,barset,vector,256,HAMMING,0,0,0 B,0%,0,map[],"Namespace\,bar Set\,list2","HNSW Max Edges\,16 Ef\,100 @@ -818,7 +818,7 @@ Healer Parallelism*\,1 Merge Index Parallelism*\,80 Merge Re-Index Parallelism*\,26 Enable Vector Integrity Check\,true" -2,list1,test,,vector,256,COSINE,0,0,0,map[foo:bar],"Namespace\,test +2,list1,test,,vector,256,COSINE,0,0,0 B,0%,0,map[foo:bar],"Namespace\,test Set\,list1","HNSW Max Edges\,16 Ef\,100