Added Audio Support

app.py

@@ -7,7 +7,8 @@ from utils import (
     retrieve_context_approx,
     build_prompt,
     ask_gemini,
-    load_documents_gradio,  # Import the new function
+    load_documents_gradio,
+    transcribe
 )
 
 client = authenticate()
@@ -37,20 +38,50 @@ def handle_question(query):
     answer = ask_gemini(prompt, client)
     return f"### My Insights :\n\n{answer.strip()}"
 
+def route_question(text_input, audio_input):
+  if text_input.strip():
+    return handle_question(text_input)
+  elif audio_input is not None:
+    transcribed = transcribe(audio_input)
+    return handle_question(transcribed)
+  else:
+    return "Please provide a question by typing or speaking."
 
-with gr.Blocks(theme='NoCrypt/miku') as demo:
-    gr.Markdown("## Ask Questions from Your Uploaded Documents")
-    #gr.Image(value="bg.JPG", visible=True)
+def show_audio():
+  return gr.update(visible=True)
 
-    file_input = gr.File(label="Upload Your File", file_types=['.pdf', '.txt', '.docx', '.csv', '.json', '.pptx', '.xml', '.xlsx'], file_count='multiple')
+css="""
+#micbttn {
+  background-color: #FFCCCB;
+  font-size: 30px;
+  height: 59px;
+}
 
-    process_btn = gr.Button("Process Document")
-    status = gr.Textbox(label="Processing Status")
+#micINP {
+  background-color: #FFCCCB;
+}
+"""
 
-    question = gr.Textbox(label="Ask a Question")
-    answer = gr.Markdown()
+with gr.Blocks(css=css, theme='NoCrypt/miku') as demo:
+  gr.Markdown("## Ask Questions from Your Uploaded Documents")
+  file_input = gr.File(label="Upload Your File", file_types=['.pdf', '.txt', '.docx', '.csv', '.json', '.pptx', '.xml', '.xlsx'], file_count='multiple')
 
-    process_btn.click(upload_and_process, inputs=file_input, outputs=status)
-    question.submit(handle_question, inputs=question, outputs=answer)
+  process_btn = gr.Button("Process Document")
+  status = gr.Textbox(label="Processing Status")
+
+  gr.Markdown("### Ask your question (type or speak):")
+
+  with gr.Row():
+    text_question = gr.Textbox(placeholder="Type your question...", scale=9, show_label=False)
+    mic_btn = gr.Button("🎤", scale=1, elem_id="micbttn")
+  
+  audio_input = gr.Audio(sources=["microphone"], type="numpy", visible=False, label=None, elem_id="micINP")
+  
+  submit_btn = gr.Button("Submit")
+  answer = gr.Markdown()
+
+  process_btn.click(upload_and_process, inputs=file_input, outputs=status)
+  mic_btn.click(show_audio, outputs=audio_input)
+  submit_btn.click(route_question, inputs=[text_question, audio_input], outputs=answer)
 
 demo.launch(share=True)  # Or demo.deploy(hf_space="your-username/your-space-name")

requirements.txt

@@ -11,4 +11,7 @@ unstructured[pdf]
 unstructured[docx]
 unstructured[ppt]
 unstructured[excel]
-unstructured[xml]
+unstructured[xml]
+torch
+torchaudio
+transformers

utils.py

@@ -12,6 +12,7 @@ warnings.filterwarnings("ignore")
 from google import genai
 from google.genai import types
 from sentence_transformers import SentenceTransformer
+from transformers import pipeline
 from langchain_community.document_loaders import(
     UnstructuredPDFLoader,
     TextLoader,
@@ -138,4 +139,21 @@ def ask_gemini(prompt, client):
     contents=[prompt],
     config=types.GenerateContentConfig(max_output_tokens=2048, temperature=0.5, seed=42),
   )
-  return response.text
+  return response.text
+
+# Speech2Text:
+def transcribe(audio, model="openai/whisper-base.en"):
+  if audio is None:
+    raise ValueError("No audio detected!")
+  
+  transcriber = pipeline("automatic-speech-recognition", model=model)
+  sr, y = audio # Sampling rate (KHz) and y= amplitude array
+
+  if y.ndim > 1: # Convert to Mono (CH=1) if Stereo (CH=2; L & R)
+    y = y.mean(1)
+
+  y = y.astype(np.float32)
+  y /= np.max(np.abs(y)) # Normalizing the amplitude values in range [-1,1]
+
+  result = transcriber({"sampling_rate" : sr, "raw" : y})
+  return result["text"]