Hello gzip middleware

.golangci.yml

@@ -60,6 +60,7 @@ linters-settings:
           - github.com/slok/go-http-metrics
           - github.com/purini-to/zapmw
           - github.com/caitlinelfring/go-env-default
+          - github.com/go-http-utils/headers
 
 issues:
   max-same-issues: 0 # unlimited

go.mod

@@ -6,6 +6,7 @@ require (
 	github.com/Shopify/toxiproxy v2.1.4+incompatible
 	github.com/caitlinelfring/go-env-default v1.1.0
 	github.com/ethereum/go-ethereum v1.13.5
+	github.com/go-http-utils/headers v0.0.0-20181008091004-fed159eddc2a
 	github.com/gorilla/mux v1.8.1
 	github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f
 	github.com/pkg/errors v0.9.1

go.sum

@@ -15,6 +15,8 @@ github.com/deckarep/golang-set/v2 v2.1.0/go.mod h1:VAky9rY/yGXJOLEDv3OMci+7wtDpO
 github.com/ethereum/go-ethereum v1.13.5 h1:U6TCRciCqZRe4FPXmy1sMGxTfuk8P7u2UoinF3VbaFk=
 github.com/ethereum/go-ethereum v1.13.5/go.mod h1:yMTu38GSuyxaYzQMViqNmQ1s3cE84abZexQmTgenWk0=
 github.com/go-chi/chi v4.0.2+incompatible/go.mod h1:eB3wogJHnLi3x/kFX2A+IbTBlXxmMeXJVKy9tTv1XzQ=
+github.com/go-http-utils/headers v0.0.0-20181008091004-fed159eddc2a h1:v6zMvHuY9yue4+QkG/HQ/W67wvtQmWJ4SDo9aK/GIno=
+github.com/go-http-utils/headers v0.0.0-20181008091004-fed159eddc2a/go.mod h1:I79BieaU4fxrw4LMXby6q5OS9XnoR9UIKLOzDFjUmuw=
 github.com/go-ole/go-ole v1.2.6 h1:/Fpf6oFPoeFik9ty7siob0G6Ke8QvQEuVcuChpwXzpY=
 github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0=
 github.com/go-stack/stack v1.8.1 h1:ntEHSVwIt7PNXNpgPmVfMrNhLtgjlmnZha2kOpuRiDw=

internal/middleware/gunzip.go

@@ -0,0 +1,42 @@
+package middleware
+
+import (
+	"bytes"
+	"compress/gzip"
+	"io"
+	"net/http"
+	"strings"
+
+	"github.com/go-http-utils/headers"
+)
+
+func Gunzip(next http.Handler) http.Handler {
+	fn := func(w http.ResponseWriter, req *http.Request) {
+		// Skip if not gzip.
+		//
+		if !strings.Contains(req.Header.Get(headers.ContentEncoding), "gzip") {
+			next.ServeHTTP(w, req)
+
+			return
+		}
+
+		body := &bytes.Buffer{}
+
+		r, err := gzip.NewReader(req.Body)
+		if err != nil {
+			http.Error(w, http.StatusText(http.StatusInternalServerError), http.StatusInternalServerError)
+		}
+
+		if _, err := io.Copy(body, r); err != nil { // nolint:gosec
+			http.Error(w, http.StatusText(http.StatusInternalServerError), http.StatusInternalServerError)
+		}
+
+		req.Header.Del(headers.ContentEncoding)
+		req.Body = io.NopCloser(body)
+		req.ContentLength = int64(body.Len())
+
+		next.ServeHTTP(w, req)
+	}
+
+	return http.HandlerFunc(fn)
+}

internal/middleware/gunzip_test.go

@@ -0,0 +1,67 @@
+package middleware
+
+import (
+	"bytes"
+	"compress/gzip"
+	"io"
+	"net/http"
+	"net/http/httptest"
+	"strings"
+	"testing"
+
+	"github.com/go-http-utils/headers"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestGunzip(t *testing.T) {
+	t.Parallel()
+
+	eth_chainId := `{"jsonrpc":"2.0","id":1,"method":"eth_chainId","params":[]}`
+
+	t.Run("compressed request", func(t *testing.T) {
+		tests := http.HandlerFunc(func(res http.ResponseWriter, req *http.Request) {
+			body := &strings.Builder{}
+
+			nbytes, err := io.Copy(body, req.Body)
+			assert.True(t, nbytes > 0)
+			assert.Nil(t, err)
+
+			assert.Equal(t, eth_chainId, body.String())
+			assert.Equal(t, int64(len(eth_chainId)), req.ContentLength)
+			assert.NotContains(t, req.Header.Get(headers.ContentEncoding), "gzip")
+		})
+
+		body := &bytes.Buffer{}
+		w := gzip.NewWriter(body)
+
+		nbytes, err := io.Copy(w, bytes.NewBufferString(eth_chainId))
+
+		assert.Nil(t, err)
+		assert.True(t, nbytes > 0)
+		assert.Nil(t, w.Close())
+
+		request := httptest.NewRequest(http.MethodPost, "http://localhost", body)
+		request.Header.Set(headers.ContentEncoding, "gzip")
+
+		Gunzip(tests).
+			ServeHTTP(httptest.NewRecorder(), request)
+	})
+
+	t.Run("uncompressed HTTP request", func(t *testing.T) {
+		tests := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+			body := &strings.Builder{}
+
+			nbytes, err := io.Copy(body, r.Body)
+			assert.True(t, nbytes > 0)
+			assert.Nil(t, err)
+
+			assert.Equal(t, eth_chainId, body.String())
+			assert.Equal(t, int64(len(eth_chainId)), r.ContentLength)
+			assert.NotContains(t, r.Header.Get(headers.ContentEncoding), "gzip")
+		})
+
+		Gunzip(tests).
+			ServeHTTP(httptest.NewRecorder(),
+				httptest.NewRequest(http.MethodPost, "http://localhost", bytes.NewBufferString(eth_chainId)))
+	})
+}

internal/middleware/gzip.go

@@ -0,0 +1,43 @@
+package middleware
+
+import (
+	"bytes"
+	"compress/gzip"
+	"io"
+	"net/http"
+	"strings"
+
+	"github.com/go-http-utils/headers"
+)
+
+func Gzip(next http.Handler) http.Handler {
+	fn := func(res http.ResponseWriter, req *http.Request) {
+		// Skip if compressed.
+		if strings.Contains(req.Header.Get(headers.ContentEncoding), "gzip") {
+			next.ServeHTTP(res, req)
+
+			return
+		}
+
+		body := &bytes.Buffer{}
+		w := gzip.NewWriter(body)
+
+		if _, err := io.Copy(w, req.Body); err != nil {
+			http.Error(res,
+				http.StatusText(http.StatusInternalServerError), http.StatusInternalServerError)
+		}
+
+		if err := w.Close(); err != nil {
+			http.Error(res,
+				http.StatusText(http.StatusInternalServerError), http.StatusInternalServerError)
+		}
+
+		req.Header.Set(headers.ContentEncoding, "gzip")
+		req.Body = io.NopCloser(body)
+		req.ContentLength = int64(body.Len())
+
+		next.ServeHTTP(res, req)
+	}
+
+	return http.HandlerFunc(fn)
+}

internal/middleware/gzip_test.go

@@ -0,0 +1,62 @@
+package middleware
+
+import (
+	"bytes"
+	"compress/gzip"
+	"io"
+	"net/http"
+	"net/http/httptest"
+	"testing"
+
+	"github.com/go-http-utils/headers"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestGzip(t *testing.T) {
+	t.Parallel()
+
+	eth_chainId := `{"jsonrpc":"2.0","id":1,"method":"eth_chainId","params":[]}`
+	t.Run("compressed HTTP request", func(t *testing.T) {
+		tests := http.HandlerFunc(func(res http.ResponseWriter, req *http.Request) {
+			body := &bytes.Buffer{}
+
+			w := gzip.NewWriter(body)
+			nbytes, err := io.Copy(w, bytes.NewBufferString(eth_chainId))
+			assert.Nil(t, err)
+			assert.True(t, nbytes > 0)
+			assert.Nil(t, w.Close())
+
+			assert.Equal(t, int64(body.Len()), req.ContentLength)
+			assert.Equal(t, io.NopCloser(body), req.Body)
+			assert.Contains(t, req.Header.Get(headers.ContentEncoding), "gzip")
+		})
+
+		Gzip(tests).
+			ServeHTTP(httptest.NewRecorder(),
+				httptest.NewRequest(http.MethodPost, "http://localhost", bytes.NewBufferString(eth_chainId)),
+			)
+	})
+
+	t.Run("uncompressed HTTP request", func(t *testing.T) {
+		tests := http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
+
+			body := &bytes.Buffer{}
+
+			r, err := gzip.NewReader(req.Body)
+			assert.Nil(t, err)
+
+			nbytes, err := io.Copy(body, r)
+			assert.Nil(t, err)
+			assert.True(t, nbytes > 0)
+			assert.Nil(t, r.Close())
+
+			assert.Equal(t, eth_chainId, body.String())
+			assert.Contains(t, req.Header.Get(headers.ContentEncoding), "gzip")
+		})
+
+		Gzip(tests).
+			ServeHTTP(httptest.NewRecorder(),
+				httptest.NewRequest(http.MethodPost, "http://localhost", bytes.NewBufferString(eth_chainId)),
+			)
+	})
+}

internal/proxy/proxy.go

@@ -7,6 +7,7 @@ import (
 	"net/http/httputil"
 	"time"
 
+	"github.com/0xProject/rpc-gateway/internal/middleware"
 	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/client_golang/prometheus/promauto"
 )
@@ -112,7 +113,11 @@ func (h *Proxy) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 		pw := NewResponseWriter()
 		r.Body = io.NopCloser(bytes.NewBuffer(body.Bytes()))
 
-		target.Proxy.ServeHTTP(pw, r)
+		if target.Config.Connection.HTTP.Compression {
+			middleware.Gzip(target.Proxy).ServeHTTP(pw, r)
+		} else {
+			middleware.Gunzip(target.Proxy).ServeHTTP(pw, r)
+		}
 
 		if h.HasNodeProviderFailed(pw.statusCode) {
 			h.metricResponseTime.WithLabelValues(target.Config.Name, r.Method).Observe(time.Since(start).Seconds())

internal/proxy/reverse_proxy.go

@@ -1,48 +1,16 @@
 package proxy
 
 import (
-	"bytes"
-	"compress/gzip"
-	"io"
 	"net"
 	"net/http"
 	"net/http/httputil"
 	"net/url"
-	"strings"
 	"time"
 
 	"github.com/mwitkow/go-conntrack"
 	"github.com/pkg/errors"
-
-	"go.uber.org/zap"
 )
 
-func doProcessRequest(r *http.Request, config TargetConfig) error {
-	if strings.Contains(r.Header.Get("Content-Encoding"), "gzip") && !config.Connection.HTTP.Compression {
-		return errors.Wrap(doGunzip(r), "gunzip failed")
-	}
-
-	return nil
-}
-
-func doGunzip(r *http.Request) error {
-	uncompressed, err := gzip.NewReader(r.Body)
-	if err != nil {
-		return errors.Wrap(err, "cannot decompress the data")
-	}
-
-	body := &bytes.Buffer{}
-	if _, err := io.Copy(body, uncompressed); err != nil { // nolint:gosec
-		return errors.Wrap(err, "cannot read uncompressed data")
-	}
-
-	r.Header.Del("Content-Encoding")
-	r.Body = io.NopCloser(body)
-	r.ContentLength = int64(body.Len())
-
-	return nil
-}
-
 func NewReverseProxy(targetConfig TargetConfig, config Config) (*httputil.ReverseProxy, error) {
 	target, err := url.Parse(targetConfig.Connection.HTTP.URL)
 	if err != nil {
@@ -55,12 +23,6 @@ func NewReverseProxy(targetConfig TargetConfig, config Config) (*httputil.Revers
 		r.URL.Scheme = target.Scheme
 		r.URL.Host = target.Host
 		r.URL.Path = target.Path
-
-		if err := doProcessRequest(r, targetConfig); err != nil {
-			zap.L().Error("cannot process request", zap.Error(err))
-		}
-
-		zap.L().Debug("request forward", zap.String("URL", r.URL.String()))
 	}
 
 	conntrackDialer := conntrack.NewDialContextFunc(