Added English audio feature

app.py

@@ -1,31 +1,35 @@
 import torch
-from transformers import AutoModelForSeq2SeqLM, AutoTokenizer
+from transformers import AutoModelForSeq2SeqLM, AutoTokenizer, pipeline
 from IndicTransToolkit.processor import IndicProcessor
 import gradio as gr
 import requests
 from datetime import datetime
+import tempfile
 
 # Supabase configuration
 SUPABASE_URL = "https://gptmdbhzblfybdnohqnh.supabase.co" 
-SUPABASE_API_KEY = "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJpc3MiOiJzdXBhYmFzZSIsInJlZiI6ImdwdG1kYmh6YmxmeWJkbm9ocW5oIiwicm9sZSI6ImFub24iLCJpYXQiOjE3NDc0NjY1NDgsImV4cCI6MjA2MzA0MjU0OH0.CfWArts6Kd_x7Wj0a_nAyGJfrFt8F7Wdy_MdYDj9e7U"                 # ← Replace with your anon/public API key
+SUPABASE_API_KEY = "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJpc3MiOiJzdXBhYmFzZSIsInJlZiI6ImdwdG1kYmh6YmxmeWJkbm9ocW5oIiwicm9sZSI6ImFub24iLCJpYXQiOjE3NDc0NjY1NDgsImV4cCI6MjA2MzA0MjU0OH0.CfWArts6Kd_x7Wj0a_nAyGJfrFt8F7Wdy_MdYDj9e7U"
 SUPABASE_TABLE = "translations"
 
 # Device configuration
 DEVICE = "cuda" if torch.cuda.is_available() else "cpu"
 
-# Load both models ahead of time
+# Load translation models
 model_en_to_indic = AutoModelForSeq2SeqLM.from_pretrained("ai4bharat/indictrans2-en-indic-1B", trust_remote_code=True).to(DEVICE)
 tokenizer_en_to_indic = AutoTokenizer.from_pretrained("ai4bharat/indictrans2-en-indic-1B", trust_remote_code=True)
 model_indic_to_en = AutoModelForSeq2SeqLM.from_pretrained("ai4bharat/indictrans2-indic-en-1B", trust_remote_code=True).to(DEVICE)
 tokenizer_indic_to_en = AutoTokenizer.from_pretrained("ai4bharat/indictrans2-indic-en-1B", trust_remote_code=True)
 ip = IndicProcessor(inference=True)
 
-# Separate save function (only called if user clicks Save button)
+# Whisper STT and English TTS pipelines
+asr = pipeline("automatic-speech-recognition", model="openai/whisper-small")
+tts_en = pipeline("text-to-speech", model="espnet/kan-bayashi_ljspeech_vits")
+
+# Save to Supabase
 def save_to_supabase(input_text, output_text, direction):
     if not input_text.strip() or not output_text.strip():
         return "Nothing to save."
 
-    # Choose table name based on direction
     table_name = "translations" if direction == "en_to_ks" else "ks_to_en_translations"
 
     payload = {
@@ -47,19 +51,12 @@ def save_to_supabase(input_text, output_text, direction):
             json=payload,
             timeout=10
         )
-
-        if response.status_code == 201:
-            return "✅ Saved successfully!"
-        else:
-            print("SAVE ERROR:", response.status_code, response.text)
-            return "❌ Failed to save."
+        return "✅ Saved successfully!" if response.status_code == 201 else "❌ Failed to save."
     except Exception as e:
         print("SAVE EXCEPTION:", e)
         return "❌ Save request error."
 
-
-
-# Function to retrieve recent translation history from Supabase
+# Fetch translation history
 def get_translation_history(direction="en_to_ks"):
     table_name = "translations" if direction == "en_to_ks" else "ks_to_en_translations"
 
@@ -74,19 +71,16 @@ def get_translation_history(direction="en_to_ks"):
             headers=headers,
             timeout=10
         )
-
         if response.status_code == 200:
             records = response.json()
             return "\n\n".join(
                 [f"Input: {r['input_text']} → Output: {r['output_text']}" for r in records]
             )
-        else:
-            return "Failed to load history."
+        return "Failed to load history."
     except Exception as e:
         print("HISTORY FETCH ERROR:", e)
         return "Error loading history."
 
-
 # Translation function
 def translate(text, direction):
     if not text.strip():
@@ -123,34 +117,47 @@ def translate(text, direction):
         print("Translation Error:", e)
         return "⚠️ Translation failed.", gr.update(), gr.update()
 
-
-
-# Toggle function to switch direction and update labels
+# Transcribe English audio
+def transcribe_audio(audio_path):
+    try:
+        result = asr(audio_path)
+        return result["text"]
+    except Exception as e:
+        print("STT Error:", e)
+        return "⚠️ Transcription failed."
+
+# Synthesize English audio if direction is ks_to_en
+def synthesize_tts(text, direction):
+    if direction == "ks_to_en":
+        try:
+            result = tts_en(text)
+            return (result["sampling_rate"], result["audio"])
+        except Exception as e:
+            print("TTS Error:", e)
+    return None
+
+# Direction switch
 def switch_direction(direction, input_text_val, output_text_val):
     new_direction = "ks_to_en" if direction == "en_to_ks" else "en_to_ks"
     input_label = "Kashmiri Text" if new_direction == "ks_to_en" else "English Text"
     output_label = "English Translation" if new_direction == "ks_to_en" else "Kashmiri Translation"
-
-    # Swap input/output text too
     return (
         new_direction,
         gr.update(value=output_text_val, label=input_label),
         gr.update(value=input_text_val, label=output_label)
     )
 
-
-# Update your Gradio interface block
+# Gradio interface
 with gr.Blocks() as interface:
     gr.HTML("""
-<div style="display: flex; justify-content: space-between; align-items: center; padding: 10px;">
-    <img src="https://raw.githubusercontent.com/BurhaanRasheedZargar/Images/211321a234613a9c3dd944fe9367cf13d1386239/assets/left_logo.png" style="height:150px; width:auto;">
-    <h2 style="margin: 0; text-align: center;">English ↔ Kashmiri Translator</h2>
-    <img src="https://raw.githubusercontent.com/BurhaanRasheedZargar/Images/77797f7f7cbee328fa0f9d31cf3e290441e04cd3/assets/right_logo.png">
-</div>
-""")
+    <div style="display: flex; justify-content: space-between; align-items: center; padding: 10px;">
+        <img src="https://raw.githubusercontent.com/BurhaanRasheedZargar/Images/211321a234613a9c3dd944fe9367cf13d1386239/assets/left_logo.png" style="height:150px; width:auto;">
+        <h2 style="margin: 0; text-align: center;">English ↔ Kashmiri Translator</h2>
+        <img src="https://raw.githubusercontent.com/BurhaanRasheedZargar/Images/77797f7f7cbee328fa0f9d31cf3e290441e04cd3/assets/right_logo.png">
+    </div>
+    """)
 
-
-    translation_direction = gr.State(value="en_to_ks")  
+    translation_direction = gr.State(value="en_to_ks")
 
     with gr.Row():
         input_text = gr.Textbox(lines=2, label="English Text", placeholder="Enter text....")
@@ -159,12 +166,17 @@ with gr.Blocks() as interface:
     with gr.Row():
         translate_button = gr.Button("Translate")
         save_button = gr.Button("Save Translation")
-        switch_button = gr.Button("Switch")  # ← New button
+        switch_button = gr.Button("Switch")
 
     save_status = gr.Textbox(label="Save Status", interactive=False)
     history_box = gr.Textbox(lines=10, label="Translation History", interactive=False)
 
-    # Actions
+    with gr.Row():
+        audio_input = gr.Audio(source="microphone", type="filepath", label="🎙️ Speak in English")
+        audio_output = gr.Audio(label="🔊 English Output Audio")
+    stt_translate_button = gr.Button("🎤 Transcribe & Translate")
+
+    # Click events
     translate_button.click(
         fn=translate,
         inputs=[input_text, translation_direction],
@@ -172,23 +184,34 @@ with gr.Blocks() as interface:
     )
 
     save_button.click(
-    fn=save_to_supabase,
-    inputs=[input_text, output_text, translation_direction],
-    outputs=save_status
-).then(
-    fn=get_translation_history,
-    inputs=translation_direction,
-    outputs=history_box
-)
-
-
+        fn=save_to_supabase,
+        inputs=[input_text, output_text, translation_direction],
+        outputs=save_status
+    ).then(
+        fn=get_translation_history,
+        inputs=translation_direction,
+        outputs=history_box
+    )
 
     switch_button.click(
-    fn=switch_direction,
-    inputs=[translation_direction, input_text, output_text],
-    outputs=[translation_direction, input_text, output_text]
-)
+        fn=switch_direction,
+        inputs=[translation_direction, input_text, output_text],
+        outputs=[translation_direction, input_text, output_text]
+    )
 
+    stt_translate_button.click(
+        fn=transcribe_audio,
+        inputs=audio_input,
+        outputs=input_text
+    ).then(
+        fn=translate,
+        inputs=[input_text, translation_direction],
+        outputs=[output_text, input_text, output_text]
+    ).then(
+        fn=synthesize_tts,
+        inputs=[output_text, translation_direction],
+        outputs=audio_output
+    )
 
 if __name__ == "__main__":
-    interface.launch(share=True, inbrowser=True)
+    interface.queue().launch()

postBuild

@@ -1 +1,7 @@
-python -c "from transformers import AutoModelForSeq2SeqLM, AutoTokenizer; AutoModelForSeq2SeqLM.from_pretrained('ai4bharat/indictrans2-en-indic-1B'); AutoTokenizer.from_pretrained('ai4bharat/indictrans2-en-indic-1B'); AutoModelForSeq2SeqLM.from_pretrained('ai4bharat/indictrans2-indic-en-1B'); AutoTokenizer.from_pretrained('ai4bharat/indictrans2-indic-en-1B')"
+python -c "from transformers import AutoModelForSeq2SeqLM, AutoTokenizer, pipeline; \
+AutoModelForSeq2SeqLM.from_pretrained('ai4bharat/indictrans2-en-indic-1B'); \
+AutoTokenizer.from_pretrained('ai4bharat/indictrans2-en-indic-1B'); \
+AutoModelForSeq2SeqLM.from_pretrained('ai4bharat/indictrans2-indic-en-1B'); \
+AutoTokenizer.from_pretrained('ai4bharat/indictrans2-indic-en-1B'); \
+pipeline('automatic-speech-recognition', model='openai/whisper-small'); \
+pipeline('text-to-speech', model='espnet/kan-bayashi_ljspeech_vits')"

requirements.txt

@@ -1,6 +1,7 @@
 torch>=1.12
 transformers>=4.30.0
+sentencepiece             # Required for tokenizer in IndicTrans2
+torchaudio                # Required by Whisper and ESPnet TTS
 gradio
 requests
-sentencepiece  # Optional, may still be needed by transformers
 git+https://github.com/VarunGumma/IndicTransToolkit.git