fix: Integration speech to text and text to speech with UI

.env.example

@@ -1,2 +1,4 @@
 GEMINI_API_KEY=your_api_key_here
 GROQ_API_KEY=your_api_key_here
+TEXT_TO_AUDIO=your_hugging_face_api_key_here
+STREAMLIT_WATCHER_TYPE=none

.gitignore

@@ -1,4 +1,7 @@
 ### Python ###
+audio/
+models/
+
 # Byte-compiled / optimized / DLL files
 __pycache__/
 *.py[cod]

.streamlit/config.toml

@@ -0,0 +1,4 @@
+[theme]
+base="dark"
+primaryColor = "#4B9EFF"
+

app.py

@@ -1,3 +1,7 @@
+import os
+os.environ["STREAMLIT_WATCHER_TYPE"] = "none"
+os.environ["PYTORCH_DISABLE_WIN_FIX"] = "1"
+
 import streamlit as st
 import json
 from datetime import datetime
@@ -8,7 +12,7 @@ from core.answering_competitor import Answering_competitor
 from core.response_evaluator import scorer
 from core.summary_utils import custom_css, generate_text_summary, clean_json_response
 from core.generate_summary import generate_summary_content
-
+from core.speech_converter import text_to_audio, load_model
 
 # Page configuration
 st.set_page_config(
@@ -17,6 +21,8 @@ st.set_page_config(
     layout="wide"
 )
 
+whisper_model = load_model()
+
 if 'questions_generated' not in st.session_state:
     st.session_state.questions_generated = []
 
@@ -69,16 +75,6 @@ if st.session_state.page_stack[-1] == "Loading":
 
 if st.session_state.page_stack[-1] == "Summary":    
     st.markdown(custom_css, unsafe_allow_html=True)
-
-
-    # with st.container(border=True):
-
-    #     if "improvement_summary" not in st.session_state or not st.session_state["improvement_summary"]:
-    #         st.session_state["improvement_summary"] = improvement_summary(least_scores(st.session_state.track_score))
-            
-    #     for i in st.session_state["improvement_summary"].split("\n"):
-    #         st.markdown(i)
-    
     st.session_state.user_answers = [st.session_state[f"user_answer_{ques}"] for ques in range(len(st.session_state.questions))]
     st.session_state.ai_answers = [st.session_state[f"llm_answer_{ques}"] for ques in range(len(st.session_state.questions))]
     st.session_state.scores = [
@@ -301,6 +297,7 @@ if st.session_state.page_stack[-1].startswith("Ques_"):
     with user_ans:
         st.markdown("**👤 Your Answer:**")
         if f"submitted_ans_{ques}" in st.session_state and st.session_state[f"submitted_ans_{ques}"]:
+            st.audio_input(label="Record Audio",disabled=True, label_visibility="collapsed")
             st.text_area(
                     "Type your answer here",
                     value=st.session_state[f"user_answer_{ques}"],
@@ -308,19 +305,40 @@ if st.session_state.page_stack[-1].startswith("Ques_"):
                     label_visibility="collapsed",
                     disabled=True
                     )
+            
+            
         else:
+            audio_file = st.audio_input(label="Record Audio",key=f"audio_ip_{ques}" ,label_visibility="collapsed")
+            transcribed_text = ""
+            if audio_file:
+                file_path = os.path.join("audio",f"user_answer_{ques}.wav")
+                try:
+                    with open(file_path, "wb") as f:
+                        audio_file.seek(0)
+                        f.write(audio_file.read())
+
+                    transcribed_text = whisper_model.transcribe(file_path)["text"]
+                    # print(st.session_state[f"transcribed_{ques}"])
+                    # st.rerun()
+                except Exception as e:
+                    st.error(f"Error occured while transcribing {e}")
+            
             user_answer = st.text_area(
                     "Type your answer here",
-                    value="",
+                    value= transcribed_text,
                     height=150,
                     label_visibility="collapsed",
                     )
 
+                    
+
 
     with llm_ans:
         st.markdown("**🤖 Rival's Answer:**")
         if f"submitted_ans_{ques}" in st.session_state and st.session_state[f"submitted_ans_{ques}"]:
             st.text_area("Competitor Response",st.session_state[f"llm_answer_{ques}"], height=150, label_visibility="collapsed")
+            st.audio(data=os.path.join("audio",f"llm_answer_{ques}.wav"))
+
         else:
             # st.text_area("Competitor Response",value="Hidden values", height=150, label_visibility="collapsed")
             st.markdown(f"""
@@ -346,12 +364,20 @@ if st.session_state.page_stack[-1].startswith("Ques_"):
                     {st.session_state[f"llm_answer_{ques}"]}
                 </div>
                 """, unsafe_allow_html=True)
+            try:
+                os.makedirs("audio",exist_ok=True)
+                text_to_audio(st.session_state[f"llm_answer_{ques}"], os.path.join("audio",f"llm_answer_{ques}.wav"))
+            except Exception as e:
+                print(e)
+                st.error(f"An error occurred {e}")
 
-    _,submit_area,_ = st.columns([1,1,1])
+    left_area,_ = st.columns([1,1])
+    _, submit_area, _  = left_area.columns([1,3,1])
     with submit_area:
-        if st.button("Submit Answer", use_container_width=True):
+        if st.button("🚀 Submit & Compare Answers", use_container_width=True, type="primary"):
             st.session_state[f"submitted_ans_{ques}"] = True
             st.session_state[f"user_answer_{ques}"] = user_answer
+            print("User answer\n",st.session_state[f"user_answer_{ques}"])
             st.rerun()
     
     
@@ -399,7 +425,7 @@ if st.session_state.page_stack[-1].startswith("Ques_"):
 
     with next:
         if ques < len(st.session_state.questions)-1:
-            if st.button("➡️ Next Question"):
+            if st.button("Next Question ➡️"):
                 st.session_state.page_stack.append(f"Ques_{ques+1}")
                 st.rerun()
         if ques == len(st.session_state.questions) -1:

core/input_comp_gen.py

@@ -1,12 +1,10 @@
 import streamlit as st
-import json  
-from typing import Dict, Any, List, Tuple
+import json
 from model import generate_response 
 from utils import FileProcessor 
-from speech_converter import audio_to_text, text_to_audio, load_model
+from speech_converter import text_to_audio, load_model
 import tempfile
 import os
-import torch
 
 # Initialize the model through speech_converter
 whisper_model = load_model()

core/response_evaluator.py

@@ -1,7 +1,6 @@
 from core.model import generate_response
 import json
 from typing import List
-from pydantic import BaseModel
 from core.utils import trim_backticks, Collect_score
 
 

core/speech_converter.py

@@ -1,12 +1,7 @@
 import whisper
 import streamlit as st
 import pyttsx3
-import os
-import torch
 
-# Set PyTorch settings to avoid thread/loop errors
-torch.set_num_threads(1)
-torch.set_num_interop_threads(1)
 
 @st.cache_resource(show_spinner="Loading speech recognition model...")
 def load_model():
@@ -39,14 +34,31 @@ def audio_to_text(audio_file_path=None):
         print(f"Audio processing error: {e}")
         return None
 
-def text_to_audio(text):
+def text_to_audio(text, filepath):
     """Converts text to speech using pyttsx3"""
     engine = pyttsx3.init()
     voices = engine.getProperty("voices")
-    engine.setProperty("rate", 125)
-    engine.setProperty("voice", voices[1].id)
-    engine.say(text)
+    engine.setProperty("rate", 125) 
+    engine.setProperty("voice", voices[0].id)
+    # engine.say(text)
+    engine.save_to_file(text, filepath)
     engine.runAndWait()
 
+# def text_to_audio(text=None, filepath=None):
+#     client = InferenceClient(
+#         provider="fal-ai",
+#         api_key=os.getenv("TEXT_TO_AUDIO"),
+#     )
+#     # audio is returned as bytes
+#     audio_bytes=client.text_to_speech(
+#         text,
+#         model="hexgrad/Kokoro-82M",
+#     )
+
+#     with open(filepath, "wb") as f:
+#         f.write(audio_bytes)
+
+
+
 if __name__ == "__main__":
     text_to_audio("Test speech conversion")

requirements.txt

@@ -1,6 +1,7 @@
+openai-whisper
+streamlit
+PyPDF2
 openai>=1.0.0
 python-dotenv>=1.0.0
-SpeechRecognition==3.14.3
 pyttsx3==2.98
-wavio==0.0.9
 groq==0.26.0