Merge branch 'main' into alpha

server/metrics/auth.go

@@ -0,0 +1,66 @@
+// Copyright 2022 Tigris Data, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package metrics
+
+import (
+	"context"
+
+	"github.com/uber-go/tally"
+)
+
+var (
+	AuthOkRequests    tally.Scope
+	AuthErrorRequests tally.Scope
+	AuthRespTime      tally.Scope
+)
+
+func getAuthOkTagKeys() []string {
+	return []string{
+		"grpc_method",
+		"env",
+		"service",
+		"version",
+	}
+}
+
+func getAuthTimerTagKeys() []string {
+	return []string{
+		"grpc_method",
+		"env",
+		"service",
+		"version",
+	}
+}
+
+func getAuthErrorTagKeys() []string {
+	return []string{
+		"grpc_method",
+		"env",
+		"service",
+		"version",
+		"error_source",
+		"error_code",
+	}
+}
+
+func GetAuthBaseTags(ctx context.Context) map[string]string {
+	return getGrpcTagsFromContext(ctx)
+}
+
+func initializeAuthScopes() {
+	AuthOkRequests = AuthMetrics.SubScope("count")
+	AuthErrorRequests = AuthMetrics.SubScope("count")
+	AuthRespTime = AuthMetrics.SubScope("response")
+}

server/metrics/fdb.go

@@ -83,6 +83,10 @@ func getFdbReqErrorTags(reqMethodName string, code string) map[string]string {
 	}
 }
 
+func GetFdbBaseTags(reqMthodName string) map[string]string {
+	return getFdbReqOkTags(reqMthodName)
+}
+
 func GetFdbOkTags(ctx context.Context, reqMethodName string) map[string]string {
 	return addTigrisTenantToTags(ctx, getFdbReqOkTags(reqMethodName))
 }

server/metrics/metrics.go

@@ -35,17 +35,23 @@ var (
 	SessionMetrics tally.Scope
 	SizeMetrics    tally.Scope
 	NetworkMetrics tally.Scope
+	AuthMetrics    tally.Scope
 )
 
+func getVersion() string {
+	if util.Version != "" {
+		return util.Version
+	} else {
+		return "dev"
+	}
+}
+
 func GetGlobalTags() map[string]string {
-	res := map[string]string{
+	return map[string]string{
 		"service": util.Service,
 		"env":     config.GetEnvironment(),
+		"version": getVersion(),
 	}
-	if res["version"] = util.Version; res["version"] == "" {
-		res["version"] = "dev"
-	}
-	return res
 }
 
 func InitializeMetrics() io.Closer {
@@ -78,6 +84,8 @@ func InitializeMetrics() io.Closer {
 		// Network netrics
 		NetworkMetrics = root.SubScope("net")
 		initializeNetworkScopes()
+		AuthMetrics = root.SubScope("auth")
+		initializeAuthScopes()
 	}
 	return closer
 }

server/metrics/requests.go

@@ -17,9 +17,9 @@ package metrics
 import (
 	"context"
 	"fmt"
+	"github.com/tigrisdata/tigris/server/config"
 	"strings"
 
-	"github.com/tigrisdata/tigris/server/config"
 	"github.com/tigrisdata/tigris/server/request"
 	"github.com/uber-go/tally"
 	"google.golang.org/grpc"

server/metrics/tags.go

@@ -15,8 +15,15 @@
 package metrics
 
 import (
+	"context"
 	"errors"
 	"strconv"
+	"strings"
+
+	"google.golang.org/grpc"
+
+	"github.com/tigrisdata/tigris/server/config"
+	"github.com/tigrisdata/tigris/util"
 
 	"github.com/apple/foundationdb/bindings/go/src/fdb"
 	api "github.com/tigrisdata/tigris/api/server/v1"
@@ -54,7 +61,8 @@ func getTigrisError(err error) (string, bool) {
 	return "", false
 }
 
-func getTagsForError(err error) map[string]string {
+func getTagsForError(err error, source string) map[string]string {
+	// The source parameter is only considered when the source cannot be determined from the error itself
 	value, isFdbError := getFdbError(err)
 	if isFdbError {
 		return map[string]string{
@@ -70,10 +78,17 @@ func getTagsForError(err error) map[string]string {
 			"error_value":  value,
 		}
 	}
-	// TODO: handle search errors
+	// Generic errors
+	if err == nil {
+		// Should only happen in test cases, if this is seen in production non-errors are counted as errors
+		return map[string]string{
+			"error_source": source,
+			"error_value":  "none",
+		}
+	}
 	return map[string]string{
-		"error_source": "unknown",
-		"error_value":  "unknown",
+		"error_source": source,
+		"error_value":  err.Error(),
 	}
 }
 
@@ -100,15 +115,28 @@ func GetDbCollTagsForReq(req interface{}) map[string]string {
 	return map[string]string{}
 }
 
+func getDefaultValue(tagKey string) string {
+	switch tagKey {
+	case "env":
+		return config.GetEnvironment()
+	case "service":
+		return util.Service
+	case "version":
+		return getVersion()
+	default:
+		return UnknownValue
+	}
+}
+
 func standardizeTags(tags map[string]string, stdKeys []string) map[string]string {
 	res := tags
 	for _, tagKey := range stdKeys {
 		if _, ok := tags[tagKey]; !ok {
 			// tag is missing, need to add it
-			res[tagKey] = UnknownValue
+			res[tagKey] = getDefaultValue(tagKey)
 		} else {
 			if res[tagKey] == "" {
-				res[tagKey] = UnknownValue
+				res[tagKey] = getDefaultValue(tagKey)
 			}
 		}
 	}
@@ -126,3 +154,16 @@ func standardizeTags(tags map[string]string, stdKeys []string) map[string]string
 	}
 	return res
 }
+
+func getGrpcTagsFromContext(ctx context.Context) map[string]string {
+	fullMethodName, fullMethodNameFound := grpc.Method(ctx)
+	if fullMethodNameFound {
+		return map[string]string{
+			"grpc_method": strings.Split(fullMethodName, "/")[2],
+		}
+	} else {
+		return map[string]string{
+			"grpc_method": UnknownValue,
+		}
+	}
+}

server/metrics/tags_test.go

@@ -47,15 +47,17 @@ func TestTagsHelpers(t *testing.T) {
 
 	t.Run("Test getTagsForError", func(t *testing.T) {
 		assert.Equal(t, map[string]string{
-			"error_source": "unknown",
-			"error_value":  "unknown",
-		}, getTagsForError(nil))
+			"error_source": "test_source",
+			"error_value":  "none",
+		}, getTagsForError(nil, "test_source"))
 
-		fdbErrTags := getTagsForError(fdb.Error{Code: 1})
+		// For specific errors, the source is ignored
+		fdbErrTags := getTagsForError(fdb.Error{Code: 1}, "ignored_source")
 		assert.Equal(t, "fdb", fdbErrTags["error_source"])
 		assert.Equal(t, "1", fdbErrTags["error_value"])
 
-		tigrisErrTags := getTagsForError(&api.TigrisError{Code: api.Code_NOT_FOUND})
+		// For specific errors, the source is ignored
+		tigrisErrTags := getTagsForError(&api.TigrisError{Code: api.Code_NOT_FOUND}, "ignored_source")
 		assert.Equal(t, "tigris_server", tigrisErrTags["error_source"])
 		assert.Equal(t, "NOT_FOUND", tigrisErrTags["error_value"])
 	})

server/metrics/tracing.go

@@ -33,6 +33,7 @@ const (
 	FdbSpanType               string = "fdb"
 	SearchSpanType            string = "search"
 	SessionSpanType           string = "session"
+	AuthSpanType              string = "auth"
 )
 
 type SpanMeta struct {
@@ -81,7 +82,7 @@ func (s *SpanMeta) GetRequestTimerTags() map[string]string {
 }
 
 func (s *SpanMeta) GetRequestErrorTags(err error) map[string]string {
-	return standardizeTags(mergeTags(s.tags, getTagsForError(err)), getRequestErrorTagKeys())
+	return standardizeTags(mergeTags(s.tags, getTagsForError(err, "request")), getRequestErrorTagKeys())
 }
 
 func (s *SpanMeta) GetFdbOkTags() map[string]string {
@@ -93,7 +94,7 @@ func (s *SpanMeta) GetFdbTimerTags() map[string]string {
 }
 
 func (s *SpanMeta) GetFdbErrorTags(err error) map[string]string {
-	return standardizeTags(mergeTags(s.tags, getTagsForError(err)), getFdbErrorTagKeys())
+	return standardizeTags(mergeTags(s.tags, getTagsForError(err, "fdb")), getFdbErrorTagKeys())
 }
 
 func (s *SpanMeta) GetSearchOkTags() map[string]string {
@@ -105,7 +106,7 @@ func (s *SpanMeta) GetSearchTimerTags() map[string]string {
 }
 
 func (s *SpanMeta) GetSearchErrorTags(err error) map[string]string {
-	return standardizeTags(mergeTags(s.tags, getTagsForError(err)), getSearchErrorTagKeys())
+	return standardizeTags(mergeTags(s.tags, getTagsForError(err, "search")), getSearchErrorTagKeys())
 }
 
 func (s *SpanMeta) GetSessionOkTags() map[string]string {
@@ -117,7 +118,7 @@ func (s *SpanMeta) GetSessionTimerTags() map[string]string {
 }
 
 func (s *SpanMeta) GetSessionErrorTags(err error) map[string]string {
-	return standardizeTags(mergeTags(s.tags, getTagsForError(err)), getSessionErrorTagKeys())
+	return standardizeTags(mergeTags(s.tags, getTagsForError(err, "session")), getSessionErrorTagKeys())
 }
 
 func (s *SpanMeta) GetNamespaceSizeTags() map[string]string {
@@ -136,9 +137,21 @@ func (s *SpanMeta) GetNetworkTags() map[string]string {
 	return standardizeTags(s.tags, getNetworkTagKeys())
 }
 
+func (s *SpanMeta) GetAuthOkTags() map[string]string {
+	return standardizeTags(s.tags, getAuthOkTagKeys())
+}
+
+func (s *SpanMeta) GetAuthTimerTags() map[string]string {
+	return standardizeTags(s.tags, getAuthTimerTagKeys())
+}
+
+func (s *SpanMeta) GetAuthErrorTags(err error) map[string]string {
+	return standardizeTags(mergeTags(s.tags, getTagsForError(err, "auth")), getAuthErrorTagKeys())
+}
+
 func (s *SpanMeta) SaveSpanMetaToContext(ctx context.Context) (context.Context, error) {
 	if s.span == nil {
-		return nil, fmt.Errorf("Parent span was not created")
+		return nil, fmt.Errorf("parent span was not created")
 	}
 	ctx = context.WithValue(ctx, SpanMetaCtxKey{}, s)
 	return ctx, nil
@@ -171,6 +184,8 @@ func (s *SpanMeta) StartTracing(ctx context.Context, childOnly bool) context.Con
 		// This is a child span, parents need to be marked
 		spanOpts = append(spanOpts, tracer.ChildOf(parentSpanMeta.span.Context()))
 		s.parent = parentSpanMeta
+		// Copy the tags from the parent span
+		s.AddTags(parentSpanMeta.GetTags())
 	}
 	if childOnly && !parentExists {
 		// There is no parent span, no need to start tracing here
@@ -204,18 +219,19 @@ func (s *SpanMeta) FinishTracing(ctx context.Context) context.Context {
 	return ctx
 }
 
-func (s *SpanMeta) FinishWithError(ctx context.Context, err error) {
+func (s *SpanMeta) FinishWithError(ctx context.Context, source string, err error) context.Context {
 	if s.span == nil {
-		return
+		return nil
 	}
 	errCode := status.Code(err)
 	s.span.SetTag("grpc.code", errCode.String())
-	errTags := getTagsForError(err)
+	errTags := getTagsForError(err, source)
 	for k, v := range errTags {
 		s.span.SetTag(k, v)
 	}
 	finishOptions := []tracer.FinishOption{tracer.WithError(err)}
 	s.span.Finish(finishOptions...)
 	s.span = nil
 	ClearSpanMetaContext(ctx)
+	return ctx
 }

server/middleware/auth.go

@@ -20,10 +20,13 @@ import (
 	"strings"
 	"time"
 
+	"github.com/tigrisdata/tigris/server/metrics"
+
 	"github.com/auth0/go-jwt-middleware/v2/jwks"
 	"github.com/auth0/go-jwt-middleware/v2/validator"
 	"github.com/rs/zerolog/log"
 	api "github.com/tigrisdata/tigris/api/server/v1"
+
 	"github.com/tigrisdata/tigris/lib/set"
 	"github.com/tigrisdata/tigris/server/config"
 	"github.com/tigrisdata/tigris/server/request"
@@ -98,6 +101,23 @@ func GetJWTValidator(config *config.Config) *validator.Validator {
 	}
 	return jwtValidator
 }
+
+func MeasuredAuthFunction(ctx context.Context, jwtValidator *validator.Validator, config *config.Config) (ctxResult context.Context, err error) {
+	spanMeta := metrics.NewSpanMeta("auth", "auth", metrics.AuthSpanType, metrics.GetAuthBaseTags(ctx))
+	ctxResult = spanMeta.StartTracing(ctx, false)
+	timer := metrics.AuthRespTime.Tagged(spanMeta.GetAuthTimerTags()).Timer("time").Start()
+	ctxResult, err = AuthFunction(ctxResult, jwtValidator, config)
+	timer.Stop()
+	if err != nil {
+		metrics.AuthErrorRequests.Tagged(spanMeta.GetAuthErrorTags(err)).Counter("error").Inc(1)
+		ctxResult = spanMeta.FinishWithError(ctxResult, "auth", err)
+		return
+	}
+	metrics.AuthOkRequests.Tagged(spanMeta.GetAuthOkTags()).Counter("ok").Inc(1)
+	ctxResult = spanMeta.FinishTracing(ctxResult)
+	return
+}
+
 func AuthFunction(ctx context.Context, jwtValidator *validator.Validator, config *config.Config) (ctxResult context.Context, err error) {
 	defer func() {
 		if err != nil {

server/middleware/middleware.go

@@ -37,8 +37,15 @@ import (
 func Get(config *config.Config, tenantMgr *metadata.TenantManager, txMgr *transaction.Manager) (grpc.UnaryServerInterceptor, grpc.StreamServerInterceptor) {
 	jwtValidator := GetJWTValidator(config)
 	// inline closure to access the state of jwtValidator
-	authFunction := func(ctx context.Context) (context.Context, error) {
-		return AuthFunction(ctx, jwtValidator, config)
+	var authFunction func(ctx context.Context) (context.Context, error)
+	if config.Tracing.Enabled {
+		authFunction = func(ctx context.Context) (context.Context, error) {
+			return MeasuredAuthFunction(ctx, jwtValidator, config)
+		}
+	} else {
+		authFunction = func(ctx context.Context) (context.Context, error) {
+			return AuthFunction(ctx, jwtValidator, config)
+		}
 	}
 
 	excludedMethods := set.New()