try again

app.py

@@ -1,30 +1,29 @@
 import spaces
 import gradio as gr
+import io
+from urllib.request import urlopen
 
 import soundfile as sf
-from transformers import AutoModelForCausalLM, AutoProcessor
+import torch
+from transformers import AutoModelForCausalLM, AutoProcessor, GenerationConfig
 
 
 
 
-model_path = "microsoft/Phi-4-multimodal-instruct"
+MODEL_ID = "microsoft/Phi-4-multimodal-instruct"
 
-processor = AutoProcessor.from_pretrained(model_path, trust_remote_code=True)
+processor = AutoProcessor.from_pretrained(MODEL_ID, trust_remote_code=True)
 model = AutoModelForCausalLM.from_pretrained(
-    model_path,
-    device_map="auto",
-    trust_remote_code=True,
-    _attn_implementation="flash_attention_2",
+    MODEL_ID,
+    device_map="cuda" if torch.cuda.is_available() else "cpu",
+    torch_dtype="auto",
 )
 
-model.load_adapter(
-    model_path,
-    adapter_name="speech",
-    device_map="auto",
-    adapter_kwargs={"subfolder": 'speech-lora'}
-)
+model.load_adapter(MODEL_ID, adapter_name="speech", device_map="cuda" if torch.cuda.is_available() else "cpu", adapter_kwargs={"subfolder": 'speech-lora'})
 model.set_adapter("speech")
 
+generation_config = GenerationConfig.from_pretrained(MODEL_ID)
+
 @spaces.GPU
 def run_phi4(audio_path: str, instruction: str) -> str:
     if not audio_path:
@@ -32,38 +31,21 @@ def run_phi4(audio_path: str, instruction: str) -> str:
 
     audio, samplerate = sf.read(audio_path)
 
-    messages = [
-        {
-            "role": "user",
-            "content": [
-                {"type": "audio", "url": audio_path},
-                {"type": "text", "text": instruction},
-            ],
-        }
-    ]
-
-    chat_text = processor.apply_chat_template(
-        messages,
-        add_generation_prompt=True,
-        tokenize=False,
-        return_dict=False,
-    )
+    user_prompt = "<|user|>"
+    assistant_prompt = "<|assistant|>"
+    prompt_suffix = "<|end|>"
+
+    prompt = f"{user_prompt}<|audio_1|>{instruction}{prompt_suffix}{assistant_prompt}"
 
-    inputs = processor(
-        text=chat_text,
-        audios=[(audio, samplerate)],
-        return_tensors="pt",
-    ).to(model.device)
+    inputs = processor(text=prompt, audios=[(audio, samplerate)], return_tensors="pt").to(model.device)
 
-    generate_ids = model.generate(
+    output_ids = model.generate(
         **inputs,
-        max_new_tokens=1000,
-        do_sample=False,
+        max_new_tokens=4096,
+        generation_config=generation_config,
     )
-    generate_ids = generate_ids[:, inputs['input_ids'].shape[1]:]
-    response = processor.batch_decode(
-        generate_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False
-    )[0]
+    output_ids = output_ids[:, inputs["input_ids"].shape[1]:]
+    response = processor.batch_decode(output_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0]
     return response