Update app.py

app.py

@@ -1,16 +1,13 @@
 import spaces
 import torch
 import gradio as gr
-import yt_dlp as youtube_dl
-from transformers import pipeline, AutoTokenizer, AutoModelForCausalLM, TextIteratorStreamer
-from threading import Thread
-import tempfile
+from transformers import AutoTokenizer, LlamaForCausalLM
+import bitsandbytes, flash_attn
 import os
 
 MODEL_NAME = "openai/whisper-large-v3-turbo"
 BATCH_SIZE = 8
 FILE_LIMIT_MB = 1000
-YT_LENGTH_LIMIT_S = 3600  # limit to 1 hour YouTube files
 
 device = 0 if torch.cuda.is_available() else "cpu"
 
@@ -22,12 +19,16 @@ pipe = pipeline(
     device=device,
 )
 
-# Hugging Face Token for the LLM model
-HF_TOKEN = os.getenv("HF_TOKEN")  # Make sure to set this in the environment variables
-
 # Load tokenizer and model for SOAP note generation
-tokenizer = AutoTokenizer.from_pretrained("NousResearch/Hermes-3-Llama-3.1-8B")
-model = AutoModelForCausalLM.from_pretrained("NousResearch/Hermes-3-Llama-3.1-8B", device_map="auto")
+tokenizer = AutoTokenizer.from_pretrained("NousResearch/Hermes-3-Llama-3.1-8B", trust_remote_code=True)
+model = LlamaForCausalLM.from_pretrained(
+    "NousResearch/Hermes-3-Llama-3.1-8B",
+    torch_dtype=torch.float16,
+    device_map="auto",
+    load_in_8bit=False,
+    load_in_4bit=True,
+    use_flash_attention_2=True
+)
 
 # Prompt for SOAP note generation
 sys_prompt = "You are a world class clinical assistant."
@@ -50,50 +51,23 @@ def transcribe(inputs, task):
     text = pipe(inputs, batch_size=BATCH_SIZE, generate_kwargs={"task": task}, return_timestamps=True)["text"]
     return text
 
-# Function to download audio from YouTube
-def download_yt_audio(yt_url, filename):
-    info_loader = youtube_dl.YoutubeDL()
-    try:
-        info = info_loader.extract_info(yt_url, download=False)
-    except youtube_dl.utils.DownloadError as err:
-        raise gr.Error(str(err))
-
-    file_length_s = sum(x * int(t) for x, t in zip([3600, 60, 1], info["duration_string"].split(":")) if t.isdigit())
-    if file_length_s > YT_LENGTH_LIMIT_S:
-        raise gr.Error(f"Video too long. Maximum allowed duration is {YT_LENGTH_LIMIT_S / 60} minutes.")
-
-    ydl_opts = {"outtmpl": filename, "format": "worstvideo[ext=mp4]+bestaudio[ext=m4a]/best[ext=mp4]/best"}
-    with youtube_dl.YoutubeDL(ydl_opts) as ydl:
-        ydl.download([yt_url])
-
-# Function to transcribe YouTube audio
-@spaces.GPU
-def yt_transcribe(yt_url, task):
-    with tempfile.TemporaryDirectory() as tmpdirname:
-        filepath = os.path.join(tmpdirname, "video.mp4")
-        download_yt_audio(yt_url, filepath)
-        with open(filepath, "rb") as f:
-            inputs = f.read()
-    inputs = pipe.feature_extractor.ffmpeg_read(inputs, pipe.feature_extractor.sampling_rate)
-    inputs = {"array": inputs, "sampling_rate": pipe.feature_extractor.sampling_rate}
-    text = pipe(inputs, batch_size=BATCH_SIZE, generate_kwargs={"task": task}, return_timestamps=True)["text"]
-    return f'<iframe width="500" height="320" src="https://www.youtube.com/embed/{yt_url.split("?v=")[-1]}"> </iframe>', text
-
 # Function to generate SOAP notes using LLM
 def generate_soap(transcribed_text):
-    prompt = f"{sys_prompt}\n\n{task_prompt}\n{transcribed_text}"
-    inputs = tokenizer(prompt, return_tensors="pt").input_ids.to(model.device)
-    outputs = model.generate(inputs, max_new_tokens=512)
-    return tokenizer.decode(outputs[0], skip_special_tokens=True)
+    prompt = f"<|im_start|>system\n{sys_prompt}<|im_end|>\n<|im_start|>user\n{task_prompt}\n{transcribed_text}<|im_end|>\n<|im_start|>assistant"
+    input_ids = tokenizer(prompt, return_tensors="pt").input_ids.to("cuda")
+    generated_ids = model.generate(input_ids, max_new_tokens=2048, temperature=0.8, repetition_penalty=1.1, do_sample=True, eos_token_id=tokenizer.eos_token_id)
+    response = tokenizer.decode(generated_ids[0][input_ids.shape[-1]:], skip_special_tokens=True, clean_up_tokenization_space=True)
+    return response
 
 # Gradio Interfaces for different inputs
 demo = gr.Blocks(theme=gr.themes.Ocean())
 
+# Interface for microphone or file transcription
 mf_transcribe = gr.Interface(
     fn=transcribe,
     inputs=[gr.Audio(sources="microphone", type="filepath"), gr.Radio(["transcribe", "translate"], label="Task", value="transcribe")],
     outputs="text",
-    title="Whisper Large V3 Turbo: Transcribe Audio",
+    title="Audio Transcribe",
     description="Transcribe long-form microphone or audio inputs."
 )
 
@@ -101,16 +75,10 @@ file_transcribe = gr.Interface(
     fn=transcribe,
     inputs=[gr.Audio(sources="upload", type="filepath", label="Audio file"), gr.Radio(["transcribe", "translate"], label="Task", value="transcribe")],
     outputs="text",
-    title="Whisper Large V3: Transcribe Audio"
-)
-
-yt_transcribe = gr.Interface(
-    fn=yt_transcribe,
-    inputs=[gr.Textbox(lines=1, placeholder="Paste the URL to a YouTube video here", label="YouTube URL"), gr.Radio(["transcribe", "translate"], label="Task", value="transcribe")],
-    outputs=["html", "text"],
-    title="Whisper Large V3: Transcribe YouTube"
+    title="Audio Transcribe"
 )
 
+# SOAP Note generation interface
 soap_note = gr.Interface(
     fn=generate_soap,
     inputs="text",
@@ -119,7 +87,10 @@ soap_note = gr.Interface(
     description="Convert transcribed conversation to a clinical SOAP note with structured sections (Subjective, Objective, Assessment, Plan)."
 )
 
+# Tabbed interface integrating SOAP note below transcription
 with demo:
-    gr.TabbedInterface([mf_transcribe, file_transcribe, yt_transcribe, soap_note], ["Microphone", "Audio file", "YouTube", "SOAP Note"])
+    with gr.TabbedInterface([mf_transcribe, file_transcribe], ["Microphone", "Audio file"]) as transcribe_tab:
+        transcribe_tab.outputs[0]  # Output from transcription feeds directly to SOAP note
+        soap_note  # SOAP note interface placed directly below transcription output
 
 demo.queue().launch(ssr_mode=False)