fix(half-precision): use half precision for T4

main.py

@@ -52,7 +52,13 @@ engine_llama_3_2: LLM = LLM(
     # https://github.com/vllm-project/vllm/blob/v0.6.4/vllm/config.py#L98-L102
     # max_model_len=32768,
     enforce_eager=True,            # Disable CUDA graph
-    dtype='auto',                  # Use 'half' if you want half precision
+
+    # File "/home/user/.local/lib/python3.12/site-packages/vllm/worker/worker.py",
+    # line 479, in _check_if_gpu_supports_dtype
+    # Bfloat16 is only supported on GPUs with compute capability of at least 8.0.
+    # Your Tesla T4 GPU has compute capability 7.5.
+    # You can use float16 instead by explicitly setting the`dtype` flag in CLI, for example: --dtype=half.
+    dtype='half',                  # Use 'half' for T4
 )
 
 # ValueError: max_num_batched_tokens (512) is smaller than max_model_len (32768).
@@ -67,7 +73,7 @@ engine_sailor_chat: LLM = LLM(
     tensor_parallel_size=1,
     # max_model_len=32768,
     enforce_eager=True,            # Disable CUDA graph
-    dtype='auto',                  # Use 'half' if you want half precision
+    dtype='half',                  # Use 'half' for T4
 )