From 9711f991e978db820ddb917c498283029b5f360e Mon Sep 17 00:00:00 2001 From: Germano Eichenberg Date: Fri, 9 Dec 2022 15:04:52 -0300 Subject: [PATCH] Add DISABLE_GLOBAL_RATELIMIT_DETECTION flag, return a 429 not a 500 when /users/@me or /gateway/bot calls 429 --- CONFIG.md | 10 ++++++++++ README.md | 35 ++++++++++++++++++----------------- lib/discord.go | 25 ++++++++++++++++--------- lib/queue.go | 1 + lib/queue_manager.go | 13 +++++++++---- main.go | 4 +++- 6 files changed, 57 insertions(+), 31 deletions(-) diff --git a/CONFIG.md b/CONFIG.md index 6ac18d4..6ad38c8 100644 --- a/CONFIG.md +++ b/CONFIG.md @@ -65,6 +65,16 @@ Allows you to define custom global request limits for one or multiple bots. The Format: Command separated list of **user id** and limit combo, separated by `:` and with no spaces at all. Don't use application ids. Example: `392827169497284619:100,227115752396685313:80` + +##### DISABLE_GLOBAL_RATELIMIT_DETECTION +Disables the optimistic global rest limit detection. This detection uses the /gateway/bot endpoint, which has a low ratelimit and can cause issues with requests being dropped/delayed as cluster size grows. + +You probably want to set BOT_RATELIMIT_OVERRIDES if you set this to true. + +Default: false + +In the future, this will be the only possible behavior. + ## Unstable env vars Collection of env vars that may be removed at any time, mainly used for Discord introducing new behaviour on their edge api versions diff --git a/README.md b/README.md index 919bb7d..f7d8ec9 100644 --- a/README.md +++ b/README.md @@ -24,23 +24,24 @@ The proxy sits between the client and discord. Instead of pointing to discord.co Configuration options are -| Variable | Value | Default | -|-----------------|-----------------------------------------------|-------------------------| -| LOG_LEVEL | panic, fatal, error, warn, info, debug, trace | info | -| PORT | number | 8080 | -| METRICS_PORT | number | 9000 | -| ENABLE_METRICS | boolean | true | -| ENABLE_PPROF | boolean | false | -| BUFFER_SIZE | number | 50 | -| OUTBOUND_IP | string | "" | -| BIND_IP | string | 0.0.0.0 | -| REQUEST_TIMEOUT | number (milliseconds) | 5000 | -| CLUSTER_PORT | number | 7946 | -| CLUSTER_MEMBERS | string list (comma separated) | "" | -| CLUSTER_DNS | string | "" | -| MAX_BEARER_COUNT| number | 1024 | -| DISABLE_HTTP_2 | bool | true | -| BOT_RATELIMIT_OVERRIDES | string list (comma separated) | "" | +| Variable | Value | Default | +|-----------------|---------------------------------------------|---------| +| LOG_LEVEL | panic, fatal, error, warn, info, debug, trace | info | +| PORT | number | 8080 | +| METRICS_PORT | number | 9000 | +| ENABLE_METRICS | boolean | true | +| ENABLE_PPROF | boolean | false | +| BUFFER_SIZE | number | 50 | +| OUTBOUND_IP | string | "" | +| BIND_IP | string | 0.0.0.0 | +| REQUEST_TIMEOUT | number (milliseconds) | 5000 | +| CLUSTER_PORT | number | 7946 | +| CLUSTER_MEMBERS | string list (comma separated) | "" | +| CLUSTER_DNS | string | "" | +| MAX_BEARER_COUNT| number | 1024 | +| DISABLE_HTTP_2 | bool | true | +| BOT_RATELIMIT_OVERRIDES | string list (comma separated) | "" | +| DISABLE_GLOBAL_RATELIMIT_DETECTION | boolean | false | Information on each config var can be found [here](https://github.com/germanoeich/nirn-proxy/blob/main/CONFIG.md) diff --git a/lib/discord.go b/lib/discord.go index 28db651..cf99f86 100644 --- a/lib/discord.go +++ b/lib/discord.go @@ -22,14 +22,16 @@ var contextTimeout time.Duration var globalOverrideMap = make(map[string]uint) +var disableRestLimitDetection = false + type BotGatewayResponse struct { SessionStartLimit map[string]int `json:"session_start_limit"` } type BotUserResponse struct { - Id string `json:"id"` + Id string `json:"id"` Username string `json:"username"` - Discrim string `json:"discriminator"` + Discrim string `json:"discriminator"` } func createTransport(ip string, disableHttp2 bool) http.RoundTripper { @@ -56,9 +58,9 @@ func createTransport(ip string, disableHttp2 bool) http.RoundTripper { } dialer := &net.Dialer{ - LocalAddr: addr, - Timeout: 30 * time.Second, - KeepAlive: 30 * time.Second, + LocalAddr: addr, + Timeout: 30 * time.Second, + KeepAlive: 30 * time.Second, } dialContext := func(ctx context.Context, network, addr string) (net.Conn, error) { @@ -109,15 +111,17 @@ func parseGlobalOverrides(overrides string) { } } -func ConfigureDiscordHTTPClient(ip string, timeout time.Duration, disableHttp2 bool, globalOverrides string) { +func ConfigureDiscordHTTPClient(ip string, timeout time.Duration, disableHttp2 bool, globalOverrides string, disableRestDetection bool) { transport := createTransport(ip, disableHttp2) client = &http.Client{ Transport: transport, - Timeout: 90 * time.Second, + Timeout: 90 * time.Second, } contextTimeout = timeout + disableRestLimitDetection = disableRestDetection + parseGlobalOverrides(globalOverrides) } @@ -137,6 +141,10 @@ func GetBotGlobalLimit(token string, user *BotUserResponse) (uint, error) { return 50, nil } + if disableRestLimitDetection { + return 50, nil + } + bot, err := doDiscordReq(context.Background(), "/api/v9/gateway/bot", "GET", nil, map[string][]string{"Authorization": {token}}, "") if err != nil { @@ -205,7 +213,7 @@ func GetBotUser(token string) (*BotUserResponse, error) { } func doDiscordReq(ctx context.Context, path string, method string, body io.ReadCloser, header http.Header, query string) (*http.Response, error) { - discordReq, err := http.NewRequestWithContext(ctx, method, "https://discord.com" + path + "?" + query, body) + discordReq, err := http.NewRequestWithContext(ctx, method, "https://discord.com"+path+"?"+query, body) if err != nil { return nil, err } @@ -231,7 +239,6 @@ func doDiscordReq(ctx context.Context, path string, method string, body io.ReadC status = "429 Shared" } } - RequestHistogram.With(map[string]string{"route": route, "status": status, "method": method, "clientId": identifier.(string)}).Observe(elapsed) } diff --git a/lib/queue.go b/lib/queue.go index a55a936..203383a 100644 --- a/lib/queue.go +++ b/lib/queue.go @@ -75,6 +75,7 @@ func NewRequestQueue(processor func(ctx context.Context, item *QueueItem) (*http botLimit: limit, }, nil } + return nil, err } diff --git a/lib/queue_manager.go b/lib/queue_manager.go index b366d91..f0f1ce4 100644 --- a/lib/queue_manager.go +++ b/lib/queue_manager.go @@ -297,10 +297,15 @@ func (m *QueueManager) fulfillRequest(resp *http.ResponseWriter, req *http.Reque } if err != nil { - (*resp).WriteHeader(500) - (*resp).Write([]byte(err.Error())) - ErrorCounter.Inc() - logEntry.WithFields(logrus.Fields{"function": "getOrCreateQueue", "queueType": queueType}).Error(err) + if strings.HasPrefix(err.Error(), "429") { + Generate429(resp) + logEntry.WithFields(logrus.Fields{"function": "getOrCreateQueue", "queueType": queueType}).Warn(err) + } else { + (*resp).WriteHeader(500) + (*resp).Write([]byte(err.Error())) + ErrorCounter.Inc() + logEntry.WithFields(logrus.Fields{"function": "getOrCreateQueue", "queueType": queueType}).Error(err) + } return } diff --git a/main.go b/main.go index 2447c60..2288d8b 100644 --- a/main.go +++ b/main.go @@ -74,7 +74,9 @@ func main() { globalOverrides := lib.EnvGet("BOT_RATELIMIT_OVERRIDES", "") - lib.ConfigureDiscordHTTPClient(outboundIp, time.Duration(timeout)*time.Millisecond, disableHttp2, globalOverrides) + disableGlobalRatelimitDetection := lib.EnvGetBool("DISABLE_GLOBAL_RATELIMIT_DETECTION", false) + + lib.ConfigureDiscordHTTPClient(outboundIp, time.Duration(timeout)*time.Millisecond, disableHttp2, globalOverrides, disableGlobalRatelimitDetection) port := lib.EnvGet("PORT", "8080") bindIp := lib.EnvGet("BIND_IP", "0.0.0.0")