SongGeneration

Runtime error

root commited on Feb 14

Commit

239c833

1 Parent(s): 69ad7bf

distribute gpu

Files changed (1) hide show

levo_inference.py CHANGED Viewed

@@ -47,7 +47,7 @@ class LeVoInference(torch.nn.Module):
             tensor_parallel_size=self.cfg.vllm.device_num,
             enforce_eager=True,
             dtype="bfloat16",
-            gpu_memory_utilization=self.cfg.vllm.gpu_memory_utilization,
             max_num_seqs=4,
             tokenizer=None,
             skip_tokenizer_init=True,

             tensor_parallel_size=self.cfg.vllm.device_num,
             enforce_eager=True,
             dtype="bfloat16",
+            gpu_memory_utilization=0.7,
             max_num_seqs=4,
             tokenizer=None,
             skip_tokenizer_init=True,