chore: Update GPU mapping for model launches by commenting out unused configurations and adjust GPU memory utilization to 0.3. Add additional dependencies to requirements.txt for enhanced functionality.

app.py

@@ -20,13 +20,20 @@ MODELS = dict()
 
 # Launch models via vLLM
 model_gpu_mapping = [
-    (0, 1000), (0, 1500),
-    (1, 2000), (1, 2500),
-    (2, 3000), (2, 3500),
-    (3, 4000), (3, 4500),
-    (4, 5000), (4, 5500),
-    (5, 6000), (5, 6500),
-    (6, 7000), (6, 7500),
+    # (0, 1000), 
+    # (0, 1500),
+    # (1, 2000), 
+    # (1, 2500),
+    # (2, 3000), 
+    # (2, 3500),
+    (2, 4000), 
+    # (3, 4500),
+    (2, 5000), 
+    # (4, 5500),
+    (3, 6000), 
+    # (5, 6500),
+    (3, 7000), 
+    # (6, 7500),
 ]
 
 for index, (gpu_id, iter_num) in enumerate(model_gpu_mapping):
@@ -44,7 +51,7 @@ for index, (gpu_id, iter_num) in enumerate(model_gpu_mapping):
             "--model", model_name,
             "--port", str(port),
             "--quantization", "bitsandbytes",
-            "--gpu-memory-utilization", "0.4",
+            "--gpu-memory-utilization", "0.3",
             "--trust-remote-code",
         ],
         env={**os.environ, "CUDA_VISIBLE_DEVICES": str(gpu_id)},

requirements.txt

@@ -1,2 +1,8 @@
 gradio
-gradio[oauth]
+gradio[oauth]
+vllm
+bitsandbytes
+transformers
+datasets
+ninja
+flash-attn