comment out env variables and set max seq=1

substratusai · Nov 17, 2024 · facdf4e · facdf4e
1 parent 11fd736
commit facdf4e
Showing 1 changed file with 4 additions and 3 deletions.
diff --git a/manifests/models/opt-125m-cpu.yaml b/manifests/models/opt-125m-cpu.yaml
@@ -5,11 +5,12 @@ metadata:
   name: opt-125m-cpu
 spec:
   features: [TextGeneration]
-  env:
-    VLLM_RPC_TIMEOUT: "10000"
-    VLLM_CPU_KVCACHE_SPACE: "1"
+  # env:
+  #   VLLM_RPC_TIMEOUT: "10000"
+  #   VLLM_CPU_KVCACHE_SPACE: "1"
   args:
   - --disable-frontend-multiprocessing
+  - --max-num-seqs=1
   owner: facebook
   url: hf://facebook/opt-125m
   engine: VLLM