From edddd95c84bdde713c36da9b002c0c63b43bc83b Mon Sep 17 00:00:00 2001 From: Zhima-Mochi Date: Tue, 23 Jan 2024 17:46:59 +0800 Subject: [PATCH] Test wsl2 redis cluster connection --- .dockerignore | 3 ++ docker/config.linux.yaml | 5 +- docker/config.windows.yaml | 5 +- docker/mongodb/init.sh | 2 + docker/mongodb/scripts/mongo-init.js | 4 +- docker/redis/docker-compose-test.yml | 77 ++++++++++++++++++++++++++++ docker/redis/docker-compose.yml | 67 ++---------------------- docker/redis/init.sh | 9 +--- docker/redis/redis/redis.conf | 6 +++ docs/docs.go | 2 +- docs/swagger.json | 2 +- docs/swagger.yaml | 2 +- handlers/redirection/impl.go | 2 +- init.sh | 3 +- main.go | 6 ++- models/url.go | 10 ++-- pkg/cache/redis/impl.go | 12 ++++- pkg/database/mongodb/impl.go | 5 +- tests/api_stress_test.py | 52 +++++++++++++++++++ 19 files changed, 176 insertions(+), 98 deletions(-) create mode 100644 docker/redis/docker-compose-test.yml create mode 100644 docker/redis/redis/redis.conf create mode 100644 tests/api_stress_test.py diff --git a/.dockerignore b/.dockerignore index 69c24d7..6343ecd 100644 --- a/.dockerignore +++ b/.dockerignore @@ -3,3 +3,6 @@ README.md docker/ tests/ docs/design/ +init.sh +Dockerfile +docker-compose* \ No newline at end of file diff --git a/docker/config.linux.yaml b/docker/config.linux.yaml index de05da7..01b9677 100644 --- a/docker/config.linux.yaml +++ b/docker/config.linux.yaml @@ -7,8 +7,5 @@ redis: - 172.17.0.1:7000 - 172.17.0.1:7001 - 172.17.0.1:7002 - - 172.17.0.1:7003 - - 172.17.0.1:7004 - - 172.17.0.1:7005 - password: "" + password: password db: 0 \ No newline at end of file diff --git a/docker/config.windows.yaml b/docker/config.windows.yaml index 43650af..ab68a17 100644 --- a/docker/config.windows.yaml +++ b/docker/config.windows.yaml @@ -7,8 +7,5 @@ redis: - host.docker.internal:7000 - host.docker.internal:7001 - host.docker.internal:7002 - - host.docker.internal:7003 - - host.docker.internal:7004 - - host.docker.internal:7005 - password: "" + password: password db: 0 \ No newline at end of file diff --git a/docker/mongodb/init.sh b/docker/mongodb/init.sh index 9130a80..ef0ca0a 100755 --- a/docker/mongodb/init.sh +++ b/docker/mongodb/init.sh @@ -44,3 +44,5 @@ sleep 10 # Execute the final initialization script docker-compose exec router01 sh -c "mongosh -u root --authenticationDatabase admin -p password < /scripts/mongo-init.js" + +echo "MongoDB Sharded Cluster is ready!" \ No newline at end of file diff --git a/docker/mongodb/scripts/mongo-init.js b/docker/mongodb/scripts/mongo-init.js index 3e9af19..0914b39 100644 --- a/docker/mongodb/scripts/mongo-init.js +++ b/docker/mongodb/scripts/mongo-init.js @@ -2,10 +2,10 @@ sh.enableSharding("linkZapURL") // Setup shardingKey for collection `linkZapURL.url` -db.adminCommand({ shardCollection: "linkZapURL.url", key: { seq: 1 } }) +db.adminCommand({ shardCollection: "linkZapURL.url", key: { shardID: 1 } }) db = db.getSiblingDB("linkZapURL"); -db.url.createIndex({ seq: 1, ID: 1 }, { unique: true }); +db.url.createIndex({ shardID: 1, ID: 1 }, { unique: true }); diff --git a/docker/redis/docker-compose-test.yml b/docker/redis/docker-compose-test.yml new file mode 100644 index 0000000..502f9ae --- /dev/null +++ b/docker/redis/docker-compose-test.yml @@ -0,0 +1,77 @@ +version: '3.9' + +services: + + redis-node-1: + image: redis:7.2.4 + container_name: redis-node-1 + links: + - redis-node-2 + - redis-node-3 + - redis-node-4 + - redis-node-5 + - redis-node-6 + command: redis-server --port 6379 --bind 0.0.0.0 --cluster-enabled yes --cluster-node-timeout 5000 --cluster-config-file nodes.conf --requirepass password --masterauth password --protected-mode no + restart: always + volumes: + - redis-node-1-data:/data + ports: + - "7000:6379" + + redis-node-2: + image: redis:7.2.4 + container_name: redis-node-2 + command: redis-server --port 6379 --bind 0.0.0.0 --cluster-enabled yes --cluster-node-timeout 5000 --cluster-config-file nodes.conf --requirepass password --masterauth password --protected-mode no + restart: always + volumes: + - redis-node-2-data:/data + ports: + - "7001:6379" + + redis-node-3: + image: redis:7.2.4 + container_name: redis-node-3 + command: redis-server --port 6379 --bind 0.0.0.0 --cluster-enabled yes --cluster-node-timeout 5000 --cluster-config-file nodes.conf --requirepass password --masterauth password --protected-mode no + restart: always + volumes: + - redis-node-3-data:/data + ports: + - "7002:6379" + + redis-node-4: + image: redis:7.2.4 + container_name: redis-node-4 + command: redis-server --port 6379 --bind 0.0.0.0 --cluster-enabled yes --cluster-node-timeout 5000 --cluster-config-file nodes.conf --requirepass password --masterauth password --protected-mode no + restart: always + volumes: + - redis-node-4-data:/data + ports: + - "7003:6379" + + redis-node-5: + image: redis:7.2.4 + container_name: redis-node-5 + command: redis-server --port 6379 --bind 0.0.0.0 --cluster-enabled yes --cluster-node-timeout 5000 --cluster-config-file nodes.conf --requirepass password --masterauth password --protected-mode no + restart: always + volumes: + - redis-node-5-data:/data + ports: + - "7004:6379" + + redis-node-6: + image: redis:7.2.4 + container_name: redis-node-6 + command: redis-server --port 6379 --bind 0.0.0.0 --cluster-enabled yes --cluster-node-timeout 5000 --cluster-config-file nodes.conf --requirepass password --masterauth password --protected-mode no + restart: always + volumes: + - redis-node-6-data:/data + ports: + - "7005:6379" + +volumes: + redis-node-1-data: + redis-node-2-data: + redis-node-3-data: + redis-node-4-data: + redis-node-5-data: + redis-node-6-data: diff --git a/docker/redis/docker-compose.yml b/docker/redis/docker-compose.yml index 5a454c1..c7a8f8a 100644 --- a/docker/redis/docker-compose.yml +++ b/docker/redis/docker-compose.yml @@ -5,73 +5,12 @@ services: redis-node-1: image: redis:7.2.4 container_name: redis-node-1 - links: - - redis-node-2 - - redis-node-3 - - redis-node-4 - - redis-node-5 - - redis-node-6 - command: redis-server --port 6379 --bind 0.0.0.0 --cluster-enabled yes --cluster-node-timeout 5000 --cluster-config-file nodes.conf + command: redis-server --port 6379 --bind 0.0.0.0 --requirepass password --masterauth password --protected-mode no restart: always volumes: - - redis-cluster-node-1-data:/data + - redis-node-1-data:/data ports: - "7000:6379" - redis-node-2: - image: redis:7.2.4 - container_name: redis-node-2 - command: redis-server --port 6379 --bind 0.0.0.0 --cluster-enabled yes --cluster-node-timeout 5000 --cluster-config-file nodes.conf - restart: always - volumes: - - redis-cluster-node-2-data:/data - ports: - - "7001:6379" - - redis-node-3: - image: redis:7.2.4 - container_name: redis-node-3 - command: redis-server --port 6379 --bind 0.0.0.0 --cluster-enabled yes --cluster-node-timeout 5000 --cluster-config-file nodes.conf - restart: always - volumes: - - redis-cluster-node-3-data:/data - ports: - - "7002:6379" - - redis-node-4: - image: redis:7.2.4 - container_name: redis-node-4 - command: redis-server --port 6379 --bind 0.0.0.0 --cluster-enabled yes --cluster-node-timeout 5000 --cluster-config-file nodes.conf - restart: always - volumes: - - redis-cluster-node-4-data:/data - ports: - - "7003:6379" - - redis-node-5: - image: redis:7.2.4 - container_name: redis-node-5 - command: redis-server --port 6379 --bind 0.0.0.0 --cluster-enabled yes --cluster-node-timeout 5000 --cluster-config-file nodes.conf - restart: always - volumes: - - redis-cluster-node-5-data:/data - ports: - - "7004:6379" - - redis-node-6: - image: redis:7.2.4 - container_name: redis-node-6 - command: redis-server --port 6379 --bind 0.0.0.0 --cluster-enabled yes --cluster-node-timeout 5000 --cluster-config-file nodes.conf - restart: always - volumes: - - redis-cluster-node-6-data:/data - ports: - - "7005:6379" - volumes: - redis-cluster-node-1-data: - redis-cluster-node-2-data: - redis-cluster-node-3-data: - redis-cluster-node-4-data: - redis-cluster-node-5-data: - redis-cluster-node-6-data: \ No newline at end of file + redis-node-1-data: diff --git a/docker/redis/init.sh b/docker/redis/init.sh index 8345bef..742d32a 100755 --- a/docker/redis/init.sh +++ b/docker/redis/init.sh @@ -7,11 +7,6 @@ docker-compose build docker-compose down -v # Start all services -docker-compose up -d --remove-orphans +docker-compose up -d - -# Wait for the services to start. -echo "Waiting for services to start..." -sleep 3 - -docker-compose exec redis-node-1 sh -c "echo 'yes' | redis-cli --cluster create redis-node-1:6379 redis-node-2:6379 redis-node-3:6379 redis-node-4:6379 redis-node-5:6379 redis-node-6:6379 --cluster-replicas 1" \ No newline at end of file +echo "Redis Cluster is ready!" diff --git a/docker/redis/redis/redis.conf b/docker/redis/redis/redis.conf new file mode 100644 index 0000000..11bb4a1 --- /dev/null +++ b/docker/redis/redis/redis.conf @@ -0,0 +1,6 @@ +cluster-enabled yes +cluster-config-file nodes.conf +cluster-node-timeout 5000 +appendonly no +protected-mode no +bind 0.0.0.0 \ No newline at end of file diff --git a/docs/docs.go b/docs/docs.go index e660a3f..e752053 100644 --- a/docs/docs.go +++ b/docs/docs.go @@ -114,7 +114,7 @@ const docTemplate = `{ "properties": { "expireAt": { "type": "string", - "example": "2021-02-08T09:20:41Z" + "example": "2025-02-08T09:20:41Z" }, "url": { "type": "string", diff --git a/docs/swagger.json b/docs/swagger.json index 62ef807..2bf6c74 100644 --- a/docs/swagger.json +++ b/docs/swagger.json @@ -103,7 +103,7 @@ "properties": { "expireAt": { "type": "string", - "example": "2021-02-08T09:20:41Z" + "example": "2025-02-08T09:20:41Z" }, "url": { "type": "string", diff --git a/docs/swagger.yaml b/docs/swagger.yaml index b5d3546..9739694 100644 --- a/docs/swagger.yaml +++ b/docs/swagger.yaml @@ -2,7 +2,7 @@ definitions: main.ShortenRequest: properties: expireAt: - example: "2021-02-08T09:20:41Z" + example: "2025-02-08T09:20:41Z" type: string url: example: https://example.com diff --git a/handlers/redirection/impl.go b/handlers/redirection/impl.go index dcf2101..6764310 100644 --- a/handlers/redirection/impl.go +++ b/handlers/redirection/impl.go @@ -66,7 +66,7 @@ func (im *impl) Redirect(ctx context.Context, code string) (*models.URL, error) Code: code, } - err := u.ToBSON() + err := u.ToModel() if err != nil { return nil, err } diff --git a/init.sh b/init.sh index 54e2b9f..b70a304 100755 --- a/init.sh +++ b/init.sh @@ -1,5 +1,4 @@ #!/bin/bash - # Stop and remove containers docker-compose -f docker-compose.windows.yml down -v @@ -7,4 +6,4 @@ docker-compose -f docker-compose.windows.yml down -v docker-compose -f docker-compose.windows.yml build # Start all services -docker-compose -f docker-compose.windows.yml up -d \ No newline at end of file +docker-compose -f docker-compose.windows.yml up \ No newline at end of file diff --git a/main.go b/main.go index ec5baf7..61b9a2b 100644 --- a/main.go +++ b/main.go @@ -111,7 +111,7 @@ func main() { type ShortenRequest struct { URL string `json:"url" binding:"required" example:"https://example.com"` - ExpireAt string `json:"expireAt" binding:"required" example:"2021-02-08T09:20:41Z"` + ExpireAt string `json:"expireAt" binding:"required" example:"2025-02-08T09:20:41Z"` ExpireAtInt64 int64 `json:"-" binding:"-"` } @@ -172,7 +172,7 @@ func (h *Handler) Shorten(g *gin.Context) { // @Description Redirects to the original URL. // @Tags URL Redirection // @Param code path string true "Shortened URL Code" -// @Success 301 string string "Moved Permanently" +// @Success 301 {string} string "Moved Permanently" // @Failure 404 {object} map[string]any // @Failure 500 {object} map[string]any // @Router /{code} [get] @@ -190,5 +190,7 @@ func (h *Handler) Redirect(g *gin.Context) { return } + log.Printf("Redirecting to %s", url.URL) + g.Redirect(http.StatusMovedPermanently, url.URL) } diff --git a/models/url.go b/models/url.go index 7f1cdcd..24f6ec6 100644 --- a/models/url.go +++ b/models/url.go @@ -41,7 +41,7 @@ var ( var num int64 - for i := 0; i < len(code); i++ { + for i := len(code) - 1; i >= 0; i-- { if n, ok := base58alphabetMap[code[i]]; !ok { return 0, ErrInvalidCode } else { @@ -61,14 +61,14 @@ func init() { type URL struct { ID int64 `json:"-" bson:"ID"` - Seq uint8 `json:"-" bson:"seq"` + ShardID int64 `json:"-" bson:"shardID"` Code string `json:"code" bson:"-"` URL string `json:"url" bson:"url"` ExpireAt int64 `json:"expireAt" bson:"expireAt"` } func (u *URL) ToBSON() error { - u.Seq = getSeq(u) + u.ShardID = getShardID(u) return nil } @@ -104,8 +104,8 @@ func getCode(u *URL) (string, error) { return code, nil } -func getSeq(u *URL) uint8 { - return uint8(u.ID & 0xFF) +func getShardID(u *URL) int64 { + return (u.ID >> 24) % 1000 } func getID(u *URL) (int64, error) { diff --git a/pkg/cache/redis/impl.go b/pkg/cache/redis/impl.go index 5767b64..7361f83 100644 --- a/pkg/cache/redis/impl.go +++ b/pkg/cache/redis/impl.go @@ -13,13 +13,23 @@ import ( ) type impl struct { + // client *redis.ClusterClient client *redis.Client config *config.Redis } func NewRedis(config *config.Redis) (cache.Cache, error) { + // client := redis.NewClusterClient(&redis.ClusterOptions{ + // Addrs: config.Addrs, + // Password: config.Password, + + // RouteByLatency: true, + // RouteRandomly: true, + // }) + client := redis.NewClient(&redis.Options{ - Addr: config.Addrs[0], + Addr: config.Addrs[0], + Password: config.Password, }) // Ping the primary diff --git a/pkg/database/mongodb/impl.go b/pkg/database/mongodb/impl.go index 28172c6..290a12b 100644 --- a/pkg/database/mongodb/impl.go +++ b/pkg/database/mongodb/impl.go @@ -42,10 +42,9 @@ func (im *impl) getCollection(collectionName string) *mongo.Collection { func (im *impl) Get(ctx context.Context, table string, key int64, result interface{}) error { collection := im.getCollection(table) - seq := key & 0xFF filter := bson.M{ - "seq": seq, - "ID": key, + "shardID": (key >> 24) % 1000, + "ID": key, } err := collection.FindOne(ctx, filter).Decode(result) diff --git a/tests/api_stress_test.py b/tests/api_stress_test.py new file mode 100644 index 0000000..411062e --- /dev/null +++ b/tests/api_stress_test.py @@ -0,0 +1,52 @@ +import requests +import time +from concurrent.futures import ThreadPoolExecutor + + +def send_request(url, data): + try: + headers = {'Content-Type': 'application/json'} + response = requests.post( + url=url, + json=data, + headers=headers, + ) + if response.status_code == 201: + return 0 + else: + return response.json() + except requests.exceptions.RequestException as e: + return str(e) + + +def main(): + url = "http://localhost:9000/" + number_of_requests = 100 + data = { + "expireAt": "2025-02-08T09:20:41Z", + "url": "https://google.com" + } + + start_time = time.time() + + with ThreadPoolExecutor(max_workers=10) as executor: + responses = list(executor.map( + lambda x: send_request(url, data), range(number_of_requests)) + ) + + print(responses) + + end_time = time.time() + duration = end_time - start_time + + success_count = responses.count(0) + failure_count = number_of_requests - success_count + + print(f"Total requests: {number_of_requests}") + print(f"Successful requests: {success_count}") + print(f"Failed requests: {failure_count}") + print(f"Total time taken: {duration} seconds") + + +if __name__ == "__main__": + main()