init

.gitignore

@@ -0,0 +1,162 @@
+apps
+
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py,cover
+.hypothesis/
+.pytest_cache/
+cover/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+.pybuilder/
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+#   For a library or package, you might want to ignore these files since the code is
+#   intended to run in multiple environments; otherwise, check them in:
+# .python-version
+
+# pipenv
+#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+#   However, in case of collaboration, if having platform-specific dependencies or dependencies
+#   having no cross-platform support, pipenv may install dependencies that don't work, or not
+#   install all needed dependencies.
+#Pipfile.lock
+
+# poetry
+#   Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
+#   This is especially recommended for binary packages to ensure reproducibility, and is more
+#   commonly ignored for libraries.
+#   https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
+#poetry.lock
+
+# pdm
+#   Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
+#pdm.lock
+#   pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it
+#   in version control.
+#   https://pdm.fming.dev/#use-with-ide
+.pdm.toml
+
+# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
+__pypackages__/
+
+# Celery stuff
+celerybeat-schedule
+celerybeat.pid
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+
+# pytype static type analyzer
+.pytype/
+
+# Cython debug symbols
+cython_debug/
+
+# PyCharm
+#  JetBrains specific template is maintained in a separate JetBrains.gitignore that can
+#  be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
+#  and can be added to the global gitignore or merged into this file.  For a more nuclear
+#  option (not recommended) you can uncomment the following to ignore the entire idea folder.
+#.idea/

app.py

@@ -0,0 +1,88 @@
+import gradio as gr
+import time
+from utils import *
+import os
+
+grammar_prompt="""
+I want you to act as a grammar mistake checker and make the sentence more fluent. You take all the user input and auto correct it. Just reply to user input with correct grammar and reasons, DO NOT reply the context of the question of the user input. If the user input is grammatically correct and fluent, just ignore it. Sample of the conversation will show below: 
+
+Correct: today is a good day.
+
+Original: today is a good day.
+Corrected: Today is a good day. 
+Reason: Capitalize the first letter of the sentence.
+
+###
+
+"""
+        
+chat_history = [
+    {"role": "system", "content": os.environ.get("SECRET_PROMPT","You are a chat bot. Talk to me!")},
+]
+def convert_chatbox(chat_history):
+    return [f"{i['role']}: {i['content']}" for i in chat_history]
+
+with gr.Blocks() as demo:
+    chatbot = gr.Chatbot()
+    with gr.Row():
+        msg = gr.Textbox()
+        audio = gr.Audio(source="microphone", type="filepath", streaming=False)
+    player = gr.Audio( type="filepath", label="Speaker",interactive=False)
+    summary = gr.Button("Summary")
+    summary_box = gr.Textbox(label="Summary")
+    # functions
+    def respond(message):
+        # TODO: replace this with real GPT model
+        chat_history.append({'role': 'user', 'content': message})
+        result = generate_response(chat_history)
+        mesg=result['choices'][0]['message']
+        print("recv: ", mesg)
+        
+        response = mesg['content']
+        chat_history.append(mesg)
+        
+        # write to file
+        result = tts(response)
+        with open("temp.wav", "wb") as audio_file:
+            audio_file.write(result.audio_data)
+
+        print("write to temp.wav")
+        
+        chatbot.value.append((message,response))
+        print("chat_history: ", chatbot.value)
+        
+        return None, "temp.wav", chatbot.value
+
+    msg.submit(respond, [msg], [msg, player,chatbot])
+    
+    
+    def transcribe(audio_file):
+        print("start transcribe, ", audio_file)
+        
+        start = time.time()
+        text = recognize_from_file(audio_file)
+        print("use ", time.time()-start)
+        
+        print("transcribe done, ", text)
+        return respond(text)
+
+    audio.change(transcribe, [audio], [audio, player, chatbot])
+    
+    def summary_response():
+        messages = [
+        ]
+        sentences = []
+        for user,assistant in chatbot.value:
+            sentences.append("Correct: " + user)
+        messages.append({'role': 'user', 'content': grammar_prompt + "\n".join(sentences)})
+        
+        result = generate_response(messages)
+        mesg=result['choices'][0]['message']
+        corrected = mesg['content']
+        print("recv: ", mesg)
+        
+        return corrected
+    
+    summary.click(summary_response, None, summary_box, queue=False)
+
+demo.launch()

requirements.txt

@@ -0,0 +1,3 @@
+openai>=0.27.7
+azure-cognitiveservices-speech==1.28.0
+gradio==3.23.0

utils/__init__.py

@@ -0,0 +1,3 @@
+from .asr import *
+from .tts import *
+from .llm import *

utils/asr.py

@@ -0,0 +1,29 @@
+import azure.cognitiveservices.speech as speechsdk
+import os
+import time
+# Replace with your own subscription key and service region
+
+def get_recoginizer(**kwargs):
+    speech_config = speechsdk.SpeechConfig(subscription=os.environ.get('SPEECH_KEY'), region=os.environ.get('SPEECH_REGION'))
+    speech_config.speech_recognition_language="en-US"
+    audio_config = speechsdk.audio.AudioConfig(**kwargs)
+    return speechsdk.SpeechRecognizer(speech_config=speech_config,audio_config=audio_config)
+
+def recognize_from_file(file=None):
+    # This example requires environment variables named "SPEECH_KEY" and "SPEECH_REGION"
+    speech_config = speechsdk.SpeechConfig(subscription=os.environ.get('SPEECH_KEY'), region=os.environ.get('SPEECH_REGION'))
+    speech_config.speech_recognition_language="en-US"
+    # speech_config.set_proxy("127.0.0.1", 2080)
+    audio_config = speechsdk.audio.AudioConfig(filename=file)
+    
+    speech_recognizer = speechsdk.SpeechRecognizer(speech_config=speech_config, audio_config=audio_config)
+    
+    result = speech_recognizer.recognize_once_async().get()
+    return result.text
+        
+
+if __name__ == '__main__':
+    for audio_file in os.listdir("audio_samples"):
+        start = time.time()
+        print(recognize_from_file(f"audio_samples/{audio_file}"), " in ", time.time()-start)
+    

utils/bark_example.py

@@ -0,0 +1,23 @@
+import os
+os.environ["SUNO_USE_SMALL_MODELS"] = "True"
+os.environ["SUNO_OFFLOAD_CPU"] = "True"
+from bark import SAMPLE_RATE, generate_audio, preload_models
+from scipy.io.wavfile import write as write_wav
+from IPython.display import Audio
+
+# download and load all models
+preload_models()
+
+# generate audio from text
+text_prompt = """
+    Of course! Here is our menu. We have a wide variety of authentic Chinese dishes to choose from. 
+"""
+import torch
+with torch.no_grad():
+    audio_array = generate_audio(text_prompt, history_prompt="v2/en_speaker_1")
+
+    # save audio to disk
+    write_wav("bark_generation.wav", SAMPLE_RATE, audio_array)
+    
+    # play text in notebook
+    Audio(audio_array, rate=SAMPLE_RATE)

utils/llm.py

@@ -0,0 +1,39 @@
+# Import necessary libraries
+import gradio as gr
+import openai
+import os
+
+# Set up OpenAI API key
+openai.api_key = os.environ['OPENAI_API_KEY']
+openai.proxy =  os.getenv('HTTP_PROXY', "")
+
+# Define function to generate bot response
+# messages=[
+#     {"role": "system", "content": "You are a helpful assistant."},
+#     {"role": "user", "content": "Who won the world series in 2020?"},
+#     {"role": "assistant", "content": "The Los Angeles Dodgers won the World Series in 2020."},
+#     {"role": "user", "content": "Where was it played?"}
+# ]
+
+# Call OpenAI GPT-3 API to generate a response
+def generate_response(messages, model = "gpt-3.5-turbo"):
+        
+    response = openai.ChatCompletion.create(
+        model=model,
+        messages = messages
+        )
+
+    # Extract and return the generated response
+    return response
+
+if __name__ == '__main__':
+    messages=[
+        {"role": "system", "content": "You are a helpful assistant."},
+        {"role": "user", "content": "Who won the world series in 2020?"},
+        {"role": "assistant", "content": "The Los Angeles Dodgers won the World Series in 2020."},
+        {"role": "user", "content": "Where was it played?"}
+    ]
+    response = generate_response(messages)
+    print(response)
+    print(response['choices'][0]['message']['content'])    
+  

utils/tts.py

@@ -0,0 +1,40 @@
+import azure.cognitiveservices.speech as speechsdk
+import os
+
+# Create a speech synthesizer object
+speech_config = speechsdk.SpeechConfig(subscription=os.environ.get('SPEECH_KEY'), region=os.environ.get('SPEECH_REGION'))
+speech_config.speech_synthesis_voice_name = "en-GB-ElliotNeural"; # Set the desired voice here
+speech_config.set_proxy("127.0.0.1", 2080)
+speech_synthesizer = speechsdk.SpeechSynthesizer(speech_config=speech_config)
+
+def tts(text)-> speechsdk.SpeechSynthesisResult:
+    # Synthesize the text to audio
+    speech_synthesis_result = speech_synthesizer.speak_text(text)
+    
+    if speech_synthesis_result.reason == speechsdk.ResultReason.SynthesizingAudioCompleted:
+        print("Speech synthesized for text [{}]".format(text))
+    elif speech_synthesis_result.reason == speechsdk.ResultReason.Canceled:
+        cancellation_details = speech_synthesis_result.cancellation_details
+        print("Speech synthesis canceled: {}".format(cancellation_details.reason))
+        if cancellation_details.reason == speechsdk.CancellationReason.Error:
+            if cancellation_details.error_details:
+                print("Error details: {}".format(cancellation_details.error_details))
+                print("Did you set the speech resource key and region values?")
+    return speech_synthesis_result
+
+if __name__ == '__main__':
+
+    # Save the audio to file
+    import time
+    
+    for i,text in enumerate([
+        "Welcome to Jade Palace. My name is Jack, I'll be your server tonight. How can I help you?",
+        "We have a variety of traditional Chinese dishes. Some of our most popular items are Kung Pao Chicken, Sweet and Sour Pork, Beef with Broccoli, and Egg Foo Young. We also offer Dim Sum, fresh seafood, noodle soups, and of course classic dishes like Spring Rolls, Dumplings, and Won Ton Soup. Would you like to see our full menu?",
+        "Absolutely, my apologies. Please, right this way. Here we are, I have a nice quiet table for you in the corner. Please have a seat. Would you like to start with some tea or a drink while you look over the menu? We have jasmine tea, oolong tea, Tsingtao beer or hot sake if you prefer. Just let me know when you are ready to order. Thank you."
+    ]):
+        start = time.time()
+        result = tts(text)
+        with open(f"audio_samples/audo_{i}.wav", "wb") as audio_file:
+            audio_file.write(result.audio_data)
+        print(result, " in ", time.time()-start)
+