init

app.py

@@ -1,30 +1,31 @@
 import spaces
 import gradio as gr
-import io
-from urllib.request import urlopen
 
 import soundfile as sf
 import torch
-from transformers import AutoModelForCausalLM, AutoProcessor, GenerationConfig
+from transformers import AutoModelForCausalLM, AutoProcessor, infer_device
 
 
 
 
-MODEL_ID = "microsoft/Phi-4-multimodal-instruct"
+model_path = "microsoft/Phi-4-multimodal-instruct"
+device = f"{infer_device()}:0"
 
-processor = AutoProcessor.from_pretrained(MODEL_ID, trust_remote_code=True)
+processor = AutoProcessor.from_pretrained(model_path, trust_remote_code=True)
 model = AutoModelForCausalLM.from_pretrained(
-    MODEL_ID,
-    device_map="cuda" if torch.cuda.is_available() else "cpu",
-    torch_dtype="auto",
-    _attn_implementation="flash_attention_2",
+    model_path,
+    device_map=device,
+    dtype=torch.float16,
 )
 
-model.load_adapter(MODEL_ID, adapter_name="speech", device_map="cuda" if torch.cuda.is_available() else "cpu", adapter_kwargs={"subfolder": 'speech-lora'})
+model.load_adapter(
+    model_path,
+    adapter_name="speech",
+    device_map=device,
+    adapter_kwargs={"subfolder": 'speech-lora'}
+)
 model.set_adapter("speech")
 
-generation_config = GenerationConfig.from_pretrained(MODEL_ID)
-
 @spaces.GPU
 def run_phi4(audio_path: str, instruction: str) -> str:
     if not audio_path:
@@ -32,21 +33,38 @@ def run_phi4(audio_path: str, instruction: str) -> str:
 
     audio, samplerate = sf.read(audio_path)
 
-    user_prompt = "<|user|>"
-    assistant_prompt = "<|assistant|>"
-    prompt_suffix = "<|end|>"
-
-    prompt = f"{user_prompt}<|audio_1|>{instruction}{prompt_suffix}{assistant_prompt}"
+    messages = [
+        {
+            "role": "user",
+            "content": [
+                {"type": "audio", "url": audio_path},
+                {"type": "text", "text": instruction},
+            ],
+        }
+    ]
+
+    chat_text = processor.apply_chat_template(
+        messages,
+        add_generation_prompt=True,
+        tokenize=False,
+        return_dict=False,
+    )
 
-    inputs = processor(text=prompt, audios=[(audio, samplerate)], return_tensors="pt").to(model.device)
+    inputs = processor(
+        text=chat_text,
+        audios=[(audio, samplerate)],
+        return_tensors="pt",
+    ).to(model.device)
 
-    output_ids = model.generate(
+    generate_ids = model.generate(
         **inputs,
-        max_new_tokens=4096,
-        generation_config=generation_config,
+        max_new_tokens=1000,
+        do_sample=False,
     )
-    output_ids = output_ids[:, inputs["input_ids"].shape[1]:]
-    response = processor.batch_decode(output_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0]
+    generate_ids = generate_ids[:, inputs['input_ids'].shape[1]:]
+    response = processor.batch_decode(
+        generate_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False
+    )[0]
     return response