change back to 500 requests since 1k caused error

tests/system-test-kind.sh

@@ -47,15 +47,17 @@ if [ "$replicas" -eq 1 ]; then
 fi
 
 
-# Send 1000 requests in parallel to stapi backend using openai python client and threading
-SCRIPT_DIR=$(dirname "$0")
-python3 $SCRIPT_DIR/test_openai_embedding.py --requests 1000 --model text-embedding-ada-002
+requests=500
+echo "Send $requests requests in parallel to stapi backend using openai python client and threading"
+python3 $SCRIPT_DIR/test_openai_embedding.py \
+  --requests $requests \
+  --model text-embedding-ada-002 \
+  --client-per-thread False
 
-# Ensure replicas has been scaled up to more than 1 after sending 1000 parallel requests
 replicas=$(kubectl get deployment stapi-minilm-l6-v2 -o jsonpath='{.spec.replicas}')
 if [ "$replicas" -ge 2 ]; then
-  echo "Test passed: Expected 2 or more replicas after sending more than 1000 requests, got $replicas"
+  echo "Test passed: Expected 2 or more replicas after sending more than $requests requests, got $replicas"
   else
-  echo "Test failed: Expected 2 or more replicas after sending more than 1000 requests, got $replicas"
+  echo "Test failed: Expected 2 or more replicas after sending more than $requests requests, got $replicas"
   exit 1
 fi

tests/test_openai_embedding.py

@@ -7,15 +7,21 @@
 parser.add_argument("--requests", type=int, default=60)
 parser.add_argument("--model", type=str, default="text-embedding-ada-002")
 parser.add_argument("--text", type=str, default="Generate an embedding for me")
+parser.add_argument("--client-per-thread", type=bool, default=False)
 args = parser.parse_args()
 
-client = OpenAI(
-    api_key="this won't be used",
-    base_url=args.base_url,
-)
+def create_client():
+    return OpenAI(
+        api_key="this won't be used",
+        base_url=args.base_url,
+    )
+
+client = create_client()
 
 def embedding_request(index: int):
     print (f"Request {index} of {args.requests}")
+    if args.client_per_thread:
+        client = create_client()
     embedding = client.embeddings.create(model=args.model, input=args.text)
     print (f"Finished {index} of {args.requests}")
     return embedding