add manifests

substratusai · Nov 2, 2024 · da2f09c · da2f09c
1 parent ecccafa
commit da2f09c
Showing 1 changed file with 24 additions and 0 deletions.
diff --git a/manifests/models/llama-3.1-70b-instruct-fp8-gh200.yaml b/manifests/models/llama-3.1-70b-instruct-fp8-gh200.yaml
@@ -0,0 +1,24 @@
+# Source: models/templates/models.yaml
+apiVersion: kubeai.org/v1
+kind: Model
+metadata:
+  name: llama-3.1-70b-instruct-fp8-gh200
+spec:
+  features: [TextGeneration]
+  owner:
+  url: hf://neuralmagic/Meta-Llama-3.1-70B-Instruct-FP8
+  engine: VLLM
+  args:
+    - --max-model-len=32768
+    - --max-num-batched-token=32768
+    - --max-num-seqs=512
+    - --gpu-memory-utilization=0.9
+    - --enable-prefix-caching
+    - --enable-chunked-prefill
+    - --disable-log-requests
+    - --kv-cache-dtype=fp8
+    - --enforce-eager
+  env:
+    VLLM_ATTENTION_BACKEND: FLASHINFER
+  targetRequests: 512
+  resourceProfile: nvidia-gpu-gh200:1