Skip to content

Commit

Permalink
perf(middleware): optimize rotation for cookie and proxy
Browse files Browse the repository at this point in the history
  • Loading branch information
jaxron committed Oct 8, 2024
1 parent 1d8a6a4 commit 4557518
Show file tree
Hide file tree
Showing 2 changed files with 46 additions and 30 deletions.
38 changes: 23 additions & 15 deletions middleware/cookie/cookie.go
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ import (
"context"
"net/http"
"sync"
"sync/atomic"

"github.com/jaxron/axonet/pkg/client/logger"
"github.com/jaxron/axonet/pkg/client/middleware"
Expand All @@ -17,18 +18,24 @@ const (

// CookieMiddleware manages cookie rotation for HTTP requests.
type CookieMiddleware struct {
mu sync.RWMutex
cookies [][]*http.Cookie
logger logger.Logger
cookies [][]*http.Cookie
cookieCount int
current atomic.Uint64
mu sync.RWMutex
logger logger.Logger
}

// New creates a new CookieMiddleware instance.
func New(cookies [][]*http.Cookie) *CookieMiddleware {
return &CookieMiddleware{
mu: sync.RWMutex{},
cookies: cookies,
logger: &logger.NoOpLogger{},
m := &CookieMiddleware{
cookies: cookies,
cookieCount: len(cookies),
current: atomic.Uint64{},
mu: sync.RWMutex{},
logger: &logger.NoOpLogger{},
}
m.current.Store(0)
return m
}

// Process applies cookie logic before passing the request to the next middleware.
Expand Down Expand Up @@ -61,17 +68,16 @@ func (m *CookieMiddleware) Process(ctx context.Context, httpClient *http.Client,

// selectCookieSet chooses the next cookie set to use.
func (m *CookieMiddleware) selectCookieSet() []*http.Cookie {
m.mu.Lock()
defer m.mu.Unlock()
m.mu.RLock()
defer m.mu.RUnlock()

if len(m.cookies) == 0 {
if m.cookieCount == 0 {
return nil
}

cookieSet := m.cookies[0]
m.cookies = append(m.cookies[1:], cookieSet)

return cookieSet
current := m.current.Add(1) - 1
index := current % uint64(m.cookieCount) // #nosec G115
return m.cookies[index]
}

// UpdateCookies updates the list of cookies at runtime.
Expand All @@ -80,6 +86,8 @@ func (m *CookieMiddleware) UpdateCookies(cookies [][]*http.Cookie) {
defer m.mu.Unlock()

m.cookies = cookies
m.cookieCount = len(cookies)
m.current.Store(0)

m.logger.WithFields(logger.Int("cookie_sets", len(cookies))).Debug("Cookies updated")
}
Expand All @@ -89,7 +97,7 @@ func (m *CookieMiddleware) GetCookieCount() int {
m.mu.RLock()
defer m.mu.RUnlock()

return len(m.cookies)
return m.cookieCount
}

// SetLogger sets the logger for the middleware.
Expand Down
38 changes: 23 additions & 15 deletions middleware/proxy/proxy.go
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ import (
"net/http"
"net/url"
"sync"
"sync/atomic"

"github.com/jaxron/axonet/pkg/client/logger"
"github.com/jaxron/axonet/pkg/client/middleware"
Expand All @@ -21,18 +22,24 @@ const (

// ProxyMiddleware manages proxy rotation for HTTP requests.
type ProxyMiddleware struct {
mu sync.RWMutex
proxies []*url.URL
logger logger.Logger
proxies []*url.URL
proxyCount int
current atomic.Uint64
mu sync.RWMutex
logger logger.Logger
}

// New creates a new ProxyMiddleware instance.
func New(proxies []*url.URL) *ProxyMiddleware {
return &ProxyMiddleware{
mu: sync.RWMutex{},
proxies: proxies,
logger: &logger.NoOpLogger{},
m := &ProxyMiddleware{
proxies: proxies,
proxyCount: len(proxies),
current: atomic.Uint64{},
mu: sync.RWMutex{},
logger: &logger.NoOpLogger{},
}
m.current.Store(0)
return m
}

// Process applies proxy logic before passing the request to the next middleware.
Expand Down Expand Up @@ -64,17 +71,16 @@ func (m *ProxyMiddleware) Process(ctx context.Context, httpClient *http.Client,

// selectProxy chooses the next proxy to use.
func (m *ProxyMiddleware) selectProxy() *url.URL {
m.mu.Lock()
defer m.mu.Unlock()
m.mu.RLock()
defer m.mu.RUnlock()

if len(m.proxies) == 0 {
if m.proxyCount == 0 {
return nil
}

proxy := m.proxies[0]
m.proxies = append(m.proxies[1:], proxy)

return proxy
current := m.current.Add(1) - 1
index := current % uint64(m.proxyCount) // #nosec G115
return m.proxies[index]
}

// applyProxyToClient applies the proxy to the given http.Client.
Expand Down Expand Up @@ -120,6 +126,8 @@ func (m *ProxyMiddleware) UpdateProxies(newProxies []*url.URL) {
defer m.mu.Unlock()

m.proxies = newProxies
m.proxyCount = len(newProxies)
m.current.Store(0)

m.logger.WithFields(logger.Int("proxy_count", len(newProxies))).Debug("Proxies updated")
}
Expand All @@ -129,7 +137,7 @@ func (m *ProxyMiddleware) GetProxyCount() int {
m.mu.RLock()
defer m.mu.RUnlock()

return len(m.proxies)
return m.proxyCount
}

// SetLogger sets the logger for the middleware.
Expand Down

0 comments on commit 4557518

Please sign in to comment.