Add update to buzz user on being idle

.gitignore

@@ -3,3 +3,4 @@ __pycache__
 .chroma
 initialize.sh
 conversations.log
+custom_gpt_voice assistant_demo.mp4

app.py

@@ -1,100 +1,45 @@
-
 import os
-import boto3
+import nltk 
 import openai
-import whisper
-import logging
-import base64
+import time
 import gradio as gr
-from io import BytesIO
-
-from langchain import OpenAI
-from langchain.chains import RetrievalQA
-from langchain.vectorstores import Chroma
-from langchain.document_loaders import DirectoryLoader
-from langchain.embeddings.openai import OpenAIEmbeddings
-from langchain.text_splitter import CharacterTextSplitter
-from assets.char_poses_base64 import idle_html_base_64, thinking_html_base_64, talking_html_base64
-
-logging.basicConfig(level="INFO",
-                    filename='conversations.log',
-                    filemode='a',
-                    format='%(asctime)s %(message)s',
-                    datefmt='%H:%M:%S')
+from threading import Thread
 
-logger = logging.getLogger('voice_agent')
+from assets.char_poses_base64 import (
+    CHAR_IDLE_HTML, CHAR_THINKING_HTML, CHAR_TALKING_HTML)
 
+from app_utils import (
+    get_chat_history, initialize_knowledge_base, 
+    text_to_speech_gen, logging, buzz_user)
 
 global FUNC_CALL
 FUNC_CALL = 0
 
-OPENAI_API_KEY = os.getenv('OPENAI_API_KEY')
-AWS_ACCESS_KEY_ID = os.getenv('AWS_ACCESS_KEY_ID')
-AWS_SECRET_ACCESS_KEY = os.getenv('AWS_SECRET_ACCESS_KEY')
-AWS_REGION_NAME = 'ap-south-1'
+global BUZZ_TIMEOUT
+BUZZ_TIMEOUT = 60
 
 GENERAL_RSPONSE_TRIGGERS = ["I don't understand the question.", "I don't know", "Hello, my name is", "mentioned in the context provided"]
 MESSAGES = [{"role": "system", "content": "You are a helpful assistant.."}]
 
-CHAR_IDLE = f'<img src="{idle_html_base_64}"></img>'
-CHAR_TALKING = f'<img src="{talking_html_base64}"></img>'
-CHAR_THINKING = f'<img src="{thinking_html_base_64}"></img>'
+LOGGER = logging.getLogger('voice_agent')
 AUDIO_HTML = ''
 
 # Uncomment If this is your first Run: 
-import nltk 
 nltk.download('averaged_perceptron_tagger')
+conv_model, voice_model = initialize_knowledge_base()
 
 
-def initialize_knowledge_base():
-    
-    loader = DirectoryLoader('profiles', glob='**/*.txt')
-    docs = loader.load()
-
-    char_text_splitter = CharacterTextSplitter(chunk_size=1000, chunk_overlap=0)
-    doc_texts = char_text_splitter.split_documents(docs)
-
-    openAI_embeddings = OpenAIEmbeddings()
-    vStore = Chroma.from_documents(doc_texts, openAI_embeddings)
-
-    conv_model = RetrievalQA.from_chain_type(
-        llm=OpenAI(), 
-        chain_type="stuff", 
-        retriever=vStore.as_retriever(
-            search_kwargs={"k": 1}
-            )
-        )
-    voice_model = whisper.load_model("tiny")
-
-    return conv_model, voice_model
-
-
-def text_to_speech_gen(answer):
-
-    polly = boto3.client('polly',
-                aws_access_key_id=AWS_ACCESS_KEY_ID,
-                aws_secret_access_key=AWS_SECRET_ACCESS_KEY,
-                region_name=AWS_REGION_NAME)
-
-    response = polly.synthesize_speech(
-        Text=answer, 
-        VoiceId='Matthew', 
-        OutputFormat='mp3', 
-        Engine = "neural")
-    
-    audio_stream = response['AudioStream'].read()
-    audio_html = audio_to_html(audio_stream)
-
-    return audio_html
-    
+def idle_timer():
+    global BUZZ_TIMEOUT
 
-def audio_to_html(audio_bytes):
-    audio_io = BytesIO(audio_bytes)
-    audio_io.seek(0)
-    audio_base64 = base64.b64encode(audio_io.read()).decode("utf-8")
-    audio_html = f'<audio src="data:audio/mpeg;base64,{audio_base64}" controls autoplay></audio>'
+    while True:
+        print('started countdown')
+        time.sleep(BUZZ_TIMEOUT)
+        buzz_user()
 
-    return audio_html
+        if BUZZ_TIMEOUT == 80:
+            time.sleep(BUZZ_TIMEOUT)
+            BUZZ_TIMEOUT = 60
 
 
 def update_img():
@@ -102,17 +47,9 @@ def update_img():
     FUNC_CALL += 1
 
     if FUNC_CALL % 2== 0:
-        CHARACTER_STATE = CHAR_TALKING
+        return CHAR_TALKING_HTML
     else:
-        CHARACTER_STATE = CHAR_THINKING
-        
-    return CHARACTER_STATE
-
-
-def user(user_message, history):
-    return "", history + [[user_message, None]]
-
-conv_model, voice_model = initialize_knowledge_base()
+        return CHAR_THINKING_HTML
 
 
 def get_response(history, audio_input):
@@ -120,6 +57,9 @@ def get_response(history, audio_input):
     query_type = 'text'
     question =history[-1][0]
 
+    global BUZZ_TIMEOUT
+    BUZZ_TIMEOUT = 80
+
     if not question:
         if audio_input:
             query_type = 'audio'
@@ -130,8 +70,8 @@ def get_response(history, audio_input):
         else:
             return None, None
 
-    logger.info("\nquery_type: %s", query_type)
-    logger.info("query_text: %s", question)
+    LOGGER.info("\nquery_type: %s", query_type)
+    LOGGER.info("query_text: %s", question)
     print('\nquery_type:', query_type)
     print('\nquery_text:', question)
 
@@ -139,7 +79,7 @@ def get_response(history, audio_input):
         question = 'hello'
     
     answer = conv_model.run(question)
-    logger.info("\ndocument_response: %s", answer)
+    LOGGER.info("\ndocument_response: %s", answer)
     print('\ndocument_response:', answer)
 
     for trigger in GENERAL_RSPONSE_TRIGGERS:
@@ -154,7 +94,7 @@ def get_response(history, audio_input):
                     )
             answer = chat.choices[0].message.content
             MESSAGES.append({"role": "assistant", "content": answer})
-            logger.info("general_response: %s", answer)
+            LOGGER.info("general_response: %s", answer)
             print('\ngeneral_response:', answer)
 
     AUDIO_HTML = text_to_speech_gen(answer)
@@ -162,12 +102,14 @@ def get_response(history, audio_input):
 
     return history, AUDIO_HTML
 
+buzz_usr_proc = Thread(target=idle_timer)
 
 with gr.Blocks(title="Your Assistance Pal!") as demo: 
+
     with gr.Row():
         output_html = gr.HTML(label="Felix's Voice", value=AUDIO_HTML)
         output_html.visible = False
-        assistant_character = gr.HTML(label=None, value=CHAR_IDLE, show_label=False)
+        assistant_character = gr.HTML(label=None, value=CHAR_IDLE_HTML, show_label=False)
         with gr.Column(scale=0.1):
             chatbot = gr.Chatbot(label='Send a text or a voice input').style(height=285)
             with gr.Row():
@@ -176,14 +118,16 @@ with gr.Blocks(title="Your Assistance Pal!") as demo:
                     audio_input = gr.Audio(source="microphone", type='filepath', show_label=False).style(container=False)
                     button = gr.Button(value="Send")
 
-    msg.submit(user, [msg, chatbot], [msg, chatbot]
+    msg.submit(get_chat_history, [msg, chatbot], [msg, chatbot]
                 ).then(update_img, outputs=[assistant_character]
                 ).then(get_response, [chatbot, audio_input], [chatbot, output_html]
                 ).then(update_img, outputs=[assistant_character])
 
-    button.click(user, [msg, chatbot], [msg, chatbot]
+    button.click(get_chat_history, [msg, chatbot], [msg, chatbot]
                 ).then(update_img, outputs=[assistant_character]
                 ).then(get_response, [chatbot, audio_input], [chatbot, output_html]
                 ).then(update_img, outputs=[assistant_character])
     
-demo.launch(debug=False, favicon_path='assets/favicon.png', show_api=False, share=False)
+    buzz_usr_proc.start()
+    
+demo.launch(debug=False, favicon_path='assets/favicon.png', show_api=False, share=True)

app_utils.py

@@ -0,0 +1,90 @@
+import os
+import whisper
+from io import BytesIO
+import base64
+import boto3
+from pydub import AudioSegment
+from pydub.playback import play
+import logging
+
+from langchain import OpenAI
+from langchain.chains import RetrievalQA
+from langchain.vectorstores import Chroma
+from langchain.document_loaders import DirectoryLoader
+from langchain.embeddings.openai import OpenAIEmbeddings
+from langchain.text_splitter import CharacterTextSplitter
+
+
+OPENAI_API_KEY = os.getenv('OPENAI_API_KEY')
+AWS_ACCESS_KEY_ID = os.getenv('AWS_ACCESS_KEY_ID')
+AWS_SECRET_ACCESS_KEY = os.getenv('AWS_SECRET_ACCESS_KEY')
+AWS_REGION_NAME = 'ap-south-1'
+
+
+logging.basicConfig(level="INFO",
+                    filename='conversations.log',
+                    filemode='a',
+                    format='%(asctime)s %(message)s',
+                    datefmt='%H:%M:%S')
+
+
+def buzz_user():
+    input_prompt = AudioSegment.from_mp3('assets/timeout_audio.mp3')
+    play(input_prompt)
+    
+
+def initialize_knowledge_base():
+    
+    loader = DirectoryLoader('profiles', glob='**/*.txt')
+    docs = loader.load()
+
+    char_text_splitter = CharacterTextSplitter(chunk_size=1000, chunk_overlap=0)
+    doc_texts = char_text_splitter.split_documents(docs)
+
+    openAI_embeddings = OpenAIEmbeddings()
+    vStore = Chroma.from_documents(doc_texts, openAI_embeddings)
+
+    conv_model = RetrievalQA.from_chain_type(
+        llm=OpenAI(), 
+        chain_type="stuff", 
+        retriever=vStore.as_retriever(
+            search_kwargs={"k": 1}
+            )
+        )
+    voice_model = whisper.load_model("tiny")
+
+    return conv_model, voice_model
+
+
+def text_to_speech_gen(answer):
+
+    polly = boto3.client('polly',
+                aws_access_key_id=AWS_ACCESS_KEY_ID,
+                aws_secret_access_key=AWS_SECRET_ACCESS_KEY,
+                region_name=AWS_REGION_NAME)
+
+    response = polly.synthesize_speech(
+        Text=answer, 
+        VoiceId='Matthew', 
+        OutputFormat='mp3', 
+        Engine = "neural")
+    
+    audio_stream = response['AudioStream'].read()
+    audio_html = audio_to_html(audio_stream)
+
+    return audio_html
+    
+
+def audio_to_html(audio_bytes):
+    audio_io = BytesIO(audio_bytes)
+    audio_io.seek(0)
+    audio_base64 = base64.b64encode(audio_io.read()).decode("utf-8")
+    audio_html = f'<audio src="data:audio/mpeg;base64,{audio_base64}" controls autoplay></audio>'
+
+    return audio_html
+
+
+def get_chat_history(user_message, history):
+    return "", history + [[user_message, None]]
+
+