diff --git a/server/v2/api/grpcgateway/doc.go b/server/v2/api/grpcgateway/doc.go new file mode 100644 index 000000000000..cbdce577f2fc --- /dev/null +++ b/server/v2/api/grpcgateway/doc.go @@ -0,0 +1,11 @@ +// Package grpcgateway provides a custom http mux that utilizes the global gogoproto registry to match +// grpc gateway requests to query handlers. POST requests with JSON bodies and GET requests with query params are supported. +// Wildcard endpoints (i.e. foo/bar/{baz}), as well as catch-all endpoints (i.e. foo/bar/{baz=**} are supported. Using +// header `x-cosmos-block-height` allows you to specify a height for the query. +// +// The URL matching logic is achieved by building regular expressions from the gateway HTTP annotations. These regular expressions +// are then used to match against incoming requests to the HTTP server. +// +// In cases where the custom http mux is unable to handle the query (i.e. no match found), the request will fall back to the +// ServeMux from github.com/grpc-ecosystem/grpc-gateway/runtime. +package grpcgateway diff --git a/server/v2/api/grpcgateway/interceptor.go b/server/v2/api/grpcgateway/interceptor.go index 7119d452a9b9..0b8086b4d034 100644 --- a/server/v2/api/grpcgateway/interceptor.go +++ b/server/v2/api/grpcgateway/interceptor.go @@ -2,6 +2,7 @@ package grpcgateway import ( "net/http" + "regexp" "strconv" "strings" @@ -20,16 +21,28 @@ import ( var _ http.Handler = &gatewayInterceptor[transaction.Tx]{} +// queryMetadata holds information related to handling gateway queries. +type queryMetadata struct { + // queryInputProtoName is the proto name of the query's input type. + queryInputProtoName string + // wildcardKeyNames are the wildcard key names from the query's HTTP annotation. + // for example /foo/bar/{baz}/{qux} would produce []string{"baz", "qux"} + // this is used for building the query's parameter map. + wildcardKeyNames []string +} + // gatewayInterceptor handles routing grpc-gateway queries to the app manager's query router. type gatewayInterceptor[T transaction.Tx] struct { logger log.Logger // gateway is the fallback grpc gateway mux handler. gateway *runtime.ServeMux - // customEndpointMapping is a mapping of custom GET options on proto RPC handlers, to the fully qualified method name. + // regexpToQueryMetadata is a mapping of regular expressions of HTTP annotations to metadata for the query. + // it is built from parsing the HTTP annotations obtained from the gogoproto global registry.' // - // example: /cosmos/bank/v1beta1/denoms_metadata -> cosmos.bank.v1beta1.Query.DenomsMetadata - customEndpointMapping map[string]string + // TODO: it might be interesting to make this a 'most frequently used' data structure, so frequently used regexp's are + // iterated over first. + regexpToQueryMetadata map[*regexp.Regexp]queryMetadata // appManager is used to route queries to the application. appManager appmanager.AppManager[T] @@ -41,20 +54,20 @@ func newGatewayInterceptor[T transaction.Tx](logger log.Logger, gateway *runtime if err != nil { return nil, err } + regexQueryMD := createRegexMapping(getMapping) return &gatewayInterceptor[T]{ logger: logger, gateway: gateway, - customEndpointMapping: getMapping, + regexpToQueryMetadata: regexQueryMD, appManager: am, }, nil } -// ServeHTTP implements the http.Handler interface. This function will attempt to match http requests to the -// interceptors internal mapping of http annotations to query request type names. -// If no match can be made, it falls back to the runtime gateway server mux. +// ServeHTTP implements the http.Handler interface. This method will attempt to match request URIs to its internal mapping +// of gateway HTTP annotations. If no match can be made, it falls back to the runtime gateway server mux. func (g *gatewayInterceptor[T]) ServeHTTP(writer http.ResponseWriter, request *http.Request) { g.logger.Debug("received grpc-gateway request", "request_uri", request.RequestURI) - match := matchURL(request.URL, g.customEndpointMapping) + match := matchURL(request.URL, g.regexpToQueryMetadata) if match == nil { // no match cases fall back to gateway mux. g.gateway.ServeHTTP(writer, request) @@ -143,3 +156,18 @@ func getHTTPGetAnnotationMapping() (map[string]string, error) { return httpGets, nil } + +// createRegexMapping converts the annotationMapping (HTTP annotation -> query input type name) to a +// map of regular expressions for that HTTP annotation pattern, to queryMetadata. +func createRegexMapping(annotationMapping map[string]string) map[*regexp.Regexp]queryMetadata { + regexQueryMD := make(map[*regexp.Regexp]queryMetadata) + for annotation, queryInputName := range annotationMapping { + pattern, wildcardNames := patternToRegex(annotation) + reg := regexp.MustCompile(pattern) + regexQueryMD[reg] = queryMetadata{ + queryInputProtoName: queryInputName, + wildcardKeyNames: wildcardNames, + } + } + return regexQueryMD +} diff --git a/server/v2/api/grpcgateway/uri.go b/server/v2/api/grpcgateway/uri.go index 6531447cf889..a3d1d5f43665 100644 --- a/server/v2/api/grpcgateway/uri.go +++ b/server/v2/api/grpcgateway/uri.go @@ -25,9 +25,10 @@ type uriMatch struct { // Params are any wildcard/query params found in the request. // // example: - // - foo/bar/{baz} - foo/bar/qux -> {baz: qux} - // - foo/bar?baz=qux - foo/bar -> {baz: qux} - Params map[string]string + // - foo/bar/{baz} - foo/bar/qux -> {baz: {"qux"}} + // - foo/bar?baz=qux - foo/bar -> {baz: {"qux"} + // - foo/bar?denom=atom&denom=stake -> {denom: {"atom", "stake"}} + Params map[string][]string } // HasParams reports whether the uriMatch has any params. @@ -37,50 +38,36 @@ func (uri uriMatch) HasParams() bool { // matchURL attempts to find a match for the given URL. // NOTE: if no match is found, nil is returned. -func matchURL(u *url.URL, getPatternToQueryInputName map[string]string) *uriMatch { +func matchURL(u *url.URL, regexpToQueryMetadata map[*regexp.Regexp]queryMetadata) *uriMatch { uriPath := strings.TrimRight(u.Path, "/") queryParams := u.Query() - params := make(map[string]string) + params := make(map[string][]string) for key, vals := range queryParams { if len(vals) > 0 { - // url.Values contains a slice for the values as you are able to specify a key multiple times in URL. - // example: https://localhost:9090/do/something?color=red&color=blue&color=green - // We will just take the first value in the slice. - params[key] = vals[0] + params[key] = vals } } - - // for simple cases where there are no wildcards, we can just do a map lookup. - if inputName, ok := getPatternToQueryInputName[uriPath]; ok { - return &uriMatch{ - QueryInputName: inputName, - Params: params, - } - } - - // attempt to find a match in the pattern map. - for getPattern, queryInputName := range getPatternToQueryInputName { - getPattern = strings.TrimRight(getPattern, "/") - - regexPattern, wildcardNames := patternToRegex(getPattern) - - regex := regexp.MustCompile(regexPattern) - matches := regex.FindStringSubmatch(uriPath) - - if len(matches) > 1 { - // first match is the full string, subsequent matches are capture groups - for i, name := range wildcardNames { - params[name] = matches[i+1] + for reg, qmd := range regexpToQueryMetadata { + matches := reg.FindStringSubmatch(uriPath) + switch { + case len(matches) == 1: + return &uriMatch{ + QueryInputName: qmd.queryInputProtoName, + Params: params, + } + case len(matches) > 1: + // first match is the URI, subsequent matches are the wild card values. + for i, name := range qmd.wildcardKeyNames { + params[name] = []string{matches[i+1]} } return &uriMatch{ - QueryInputName: queryInputName, + QueryInputName: qmd.queryInputProtoName, Params: params, } } } - return nil } @@ -148,17 +135,20 @@ func createMessage(match *uriMatch) (gogoproto.Message, error) { // if the uri match has params, we need to populate the message with the values of those params. if match.HasParams() { - // convert flat params map to nested structure nestedParams := make(map[string]any) - for key, value := range match.Params { + for key, values := range match.Params { parts := strings.Split(key, ".") current := nestedParams // step through nested levels for i, part := range parts { if i == len(parts)-1 { - // Last part - set the value - current[part] = value + // Last part - set the value(s) + if len(values) == 1 { + current[part] = values[0] // single value + } else { + current[part] = values // slice of values + } } else { // continue nestedness if _, exists := current[part]; !exists { diff --git a/server/v2/api/grpcgateway/uri_test.go b/server/v2/api/grpcgateway/uri_test.go index 4bb74c39e40e..beffdfbcbd61 100644 --- a/server/v2/api/grpcgateway/uri_test.go +++ b/server/v2/api/grpcgateway/uri_test.go @@ -6,6 +6,7 @@ import ( "io" "net/http" "net/url" + "regexp" "testing" gogoproto "github.com/cosmos/gogoproto/proto" @@ -23,19 +24,19 @@ func TestMatchURI(t *testing.T) { name: "simple match, no wildcards", uri: "https://localhost:8080/foo/bar", mapping: map[string]string{"/foo/bar": "query.Bank"}, - expected: &uriMatch{QueryInputName: "query.Bank", Params: map[string]string{}}, + expected: &uriMatch{QueryInputName: "query.Bank", Params: map[string][]string{}}, }, { name: "match with query parameters", uri: "https://localhost:8080/foo/bar?baz=qux", mapping: map[string]string{"/foo/bar": "query.Bank"}, - expected: &uriMatch{QueryInputName: "query.Bank", Params: map[string]string{"baz": "qux"}}, + expected: &uriMatch{QueryInputName: "query.Bank", Params: map[string][]string{"baz": {"qux"}}}, }, { name: "match with multiple query parameters", uri: "https://localhost:8080/foo/bar?baz=qux&foo=/msg.type.bank.send", mapping: map[string]string{"/foo/bar": "query.Bank"}, - expected: &uriMatch{QueryInputName: "query.Bank", Params: map[string]string{"baz": "qux", "foo": "/msg.type.bank.send"}}, + expected: &uriMatch{QueryInputName: "query.Bank", Params: map[string][]string{"baz": {"qux"}, "foo": {"/msg.type.bank.send"}}}, }, { name: "wildcard match at the end", @@ -43,7 +44,7 @@ func TestMatchURI(t *testing.T) { mapping: map[string]string{"/foo/bar/{baz}": "bar"}, expected: &uriMatch{ QueryInputName: "bar", - Params: map[string]string{"baz": "buzz"}, + Params: map[string][]string{"baz": {"buzz"}}, }, }, { @@ -52,7 +53,7 @@ func TestMatchURI(t *testing.T) { mapping: map[string]string{"/foo/{baz}/bar": "bar"}, expected: &uriMatch{ QueryInputName: "bar", - Params: map[string]string{"baz": "buzz"}, + Params: map[string][]string{"baz": {"buzz"}}, }, }, { @@ -61,7 +62,16 @@ func TestMatchURI(t *testing.T) { mapping: map[string]string{"/foo/bar/{q1}/{q2}": "bar"}, expected: &uriMatch{ QueryInputName: "bar", - Params: map[string]string{"q1": "baz", "q2": "buzz"}, + Params: map[string][]string{"q1": {"baz"}, "q2": {"buzz"}}, + }, + }, + { + name: "match with multiple query parameters", + uri: "https://localhost:8080/bank/supply/by_denom?denom=foo&denom=bar", + mapping: map[string]string{"/bank/supply/by_denom": "queryDenom"}, + expected: &uriMatch{ + QueryInputName: "queryDenom", + Params: map[string][]string{"denom": {"foo", "bar"}}, }, }, { @@ -70,7 +80,7 @@ func TestMatchURI(t *testing.T) { mapping: map[string]string{"/foo/bar/{ibc_token=**}": "bar"}, expected: &uriMatch{ QueryInputName: "bar", - Params: map[string]string{"ibc_token": "ibc/token/stuff"}, + Params: map[string][]string{"ibc_token": {"ibc/token/stuff"}}, }, }, { @@ -85,14 +95,15 @@ func TestMatchURI(t *testing.T) { t.Run(tc.name, func(t *testing.T) { u, err := url.Parse(tc.uri) require.NoError(t, err) - actual := matchURL(u, tc.mapping) + regexpMapping := createRegexMapping(tc.mapping) + actual := matchURL(u, regexpMapping) require.Equal(t, tc.expected, actual) }) } } func TestURIMatch_HasParams(t *testing.T) { - u := uriMatch{Params: map[string]string{"foo": "bar"}} + u := uriMatch{Params: map[string][]string{"foo": {"bar"}}} require.True(t, u.HasParams()) u = uriMatch{} @@ -111,10 +122,11 @@ type Pagination struct { const dummyProtoName = "dummy" type DummyProto struct { - Foo string `protobuf:"bytes,1,opt,name=foo,proto3" json:"foo,omitempty"` - Bar bool `protobuf:"varint,2,opt,name=bar,proto3" json:"bar,omitempty"` - Baz int `protobuf:"varint,3,opt,name=baz,proto3" json:"baz,omitempty"` - Page *Pagination `protobuf:"bytes,4,opt,name=page,proto3" json:"page,omitempty"` + Foo string `protobuf:"bytes,1,opt,name=foo,proto3" json:"foo,omitempty"` + Bar bool `protobuf:"varint,2,opt,name=bar,proto3" json:"bar,omitempty"` + Baz int `protobuf:"varint,3,opt,name=baz,proto3" json:"baz,omitempty"` + Denoms []string `protobuf:"bytes,4,rep,name=denoms,proto3" json:"denoms,omitempty"` + Page *Pagination `protobuf:"bytes,4,opt,name=page,proto3" json:"page,omitempty"` } func (d DummyProto) Reset() {} @@ -141,7 +153,11 @@ func TestCreateMessage(t *testing.T) { name: "message with params", uri: uriMatch{ QueryInputName: dummyProtoName, - Params: map[string]string{"foo": "blah", "bar": "true", "baz": "1352"}, + Params: map[string][]string{ + "foo": {"blah"}, + "bar": {"true"}, + "baz": {"1352"}, + }, }, expected: &DummyProto{ Foo: "blah", @@ -149,11 +165,47 @@ func TestCreateMessage(t *testing.T) { Baz: 1352, }, }, + { + name: "message with slice param", + uri: uriMatch{ + QueryInputName: dummyProtoName, + Params: map[string][]string{ + "foo": {"blah"}, + "bar": {"true"}, + "baz": {"1352"}, + "denoms": {"atom", "stake"}, + }, + }, + expected: &DummyProto{ + Foo: "blah", + Bar: true, + Baz: 1352, + Denoms: []string{"atom", "stake"}, + }, + }, + { + name: "message with multiple param for single value field should fail", + uri: uriMatch{ + QueryInputName: dummyProtoName, + Params: map[string][]string{ + "foo": {"blah", "blahhh"}, // foo is a single value field. + "bar": {"true"}, + "baz": {"1352"}, + "denoms": {"atom", "stake"}, + }, + }, + expErr: true, + }, { name: "message with nested params", uri: uriMatch{ QueryInputName: dummyProtoName, - Params: map[string]string{"foo": "blah", "bar": "true", "baz": "1352", "page.limit": "3"}, + Params: map[string][]string{ + "foo": {"blah"}, + "bar": {"true"}, + "baz": {"1352"}, + "page.limit": {"3"}, + }, }, expected: &DummyProto{ Foo: "blah", @@ -166,7 +218,13 @@ func TestCreateMessage(t *testing.T) { name: "message with multi nested params", uri: uriMatch{ QueryInputName: dummyProtoName, - Params: map[string]string{"foo": "blah", "bar": "true", "baz": "1352", "page.limit": "3", "page.nest.foo": "5"}, + Params: map[string][]string{ + "foo": {"blah"}, + "bar": {"true"}, + "baz": {"1352"}, + "page.limit": {"3"}, + "page.nest.foo": {"5"}, + }, }, expected: &DummyProto{ Foo: "blah", @@ -179,7 +237,11 @@ func TestCreateMessage(t *testing.T) { name: "invalid params should error out", uri: uriMatch{ QueryInputName: dummyProtoName, - Params: map[string]string{"foo": "blah", "bar": "235235", "baz": "true"}, + Params: map[string][]string{ + "foo": {"blah"}, + "bar": {"235235"}, + "baz": {"true"}, + }, }, expErr: true, }, @@ -262,3 +324,70 @@ func TestCreateMessageFromJson(t *testing.T) { }) } } + +func Test_patternToRegex(t *testing.T) { + tests := []struct { + name string + pattern string + wildcards []string + wildcardValues []string + shouldMatch string + shouldNotMatch []string + }{ + { + name: "simple match, no wildcards", + pattern: "/foo/bar/baz", + shouldMatch: "/foo/bar/baz", + shouldNotMatch: []string{"/foo/bar", "/foo", "/foo/bar/baz/boo"}, + }, + { + name: "match with wildcard", + pattern: "/foo/bar/{baz}", + wildcards: []string{"baz"}, + shouldMatch: "/foo/bar/hello", + wildcardValues: []string{"hello"}, + shouldNotMatch: []string{"/foo/bar", "/foo/bar/baz/boo"}, + }, + { + name: "match with multiple wildcards", + pattern: "/foo/{bar}/{baz}/meow", + wildcards: []string{"bar", "baz"}, + shouldMatch: "/foo/hello/world/meow", + wildcardValues: []string{"hello", "world"}, + shouldNotMatch: []string{"/foo/bar/baz/boo", "/foo/bar/baz"}, + }, + { + name: "match catch-all wildcard", + pattern: `/foo/bar/{baz=**}`, + wildcards: []string{"baz"}, + shouldMatch: `/foo/bar/this/is/a/long/wildcard`, + wildcardValues: []string{"this/is/a/long/wildcard"}, + shouldNotMatch: []string{"/foo/bar", "/foo", "/foo/baz/bar/long/wild/card"}, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + regString, wildcards := patternToRegex(tt.pattern) + // should produce the same wildcard keys + require.Equal(t, tt.wildcards, wildcards) + reg := regexp.MustCompile(regString) + + // handle the "should match" case. + matches := reg.FindStringSubmatch(tt.shouldMatch) + require.True(t, len(matches) > 0) // there should always be a match. + // when matches > 1, this means we got wildcard values to handle. the test should have wildcard values. + if len(matches) > 1 { + require.Greater(t, len(tt.wildcardValues), 0) + } + // matches[0] is the URL, everything else should be those wildcard values. + if len(tt.wildcardValues) > 0 { + require.Equal(t, matches[1:], tt.wildcardValues) + } + + // should never match these. + for _, notMatch := range tt.shouldNotMatch { + require.Len(t, reg.FindStringSubmatch(notMatch), 0) + } + }) + } +}