add demos

.vscode/launch.json

@@ -10,7 +10,7 @@
             "request": "launch",
             "program": "${file}",
             "console": "integratedTerminal",
-            "justMyCode": true
+            "justMyCode": false
         }
     ]
 }

demo.py

@@ -0,0 +1,14 @@
+import pkgutil
+import interfaces
+import gradio as gr
+import importlib
+
+demos = []
+names = []
+for i in list(pkgutil.iter_modules(interfaces.__path__)):
+    my_package = importlib.import_module(f'interfaces.{i.name}')
+    my_variable = getattr(my_package, 'demo')
+    demos.append(my_variable)
+    names.append(i.name)
+
+gr.TabbedInterface(demos,names).launch(share=True,show_error=True)

demos/character.py

@@ -2,7 +2,7 @@ import gradio as gr
 import time
 from utils.asr import recognize_from_file
 from utils.tts import tts
-from utils.llm import generate_response
+from utils.GPT import generate_response
 
 chat_history = [
 ]

demos/chat.py

@@ -2,7 +2,7 @@ import gradio as gr
 import time
 from utils.asr import recognize_from_file
 from utils.tts import tts
-from utils.llm import generate_response
+from utils.api.GPT import generate_answers_openai
 
 chat_history = []
 def convert_chatbox(chat_history):
@@ -19,7 +19,7 @@ with gr.Blocks() as demo:
     def respond(message):
         # TODO: replace this with real GPT model
         chat_history.append({'role': 'user', 'content': message})
-        result = generate_response(chat_history)
+        result = generate_answers_openai(chat_history)
         mesg=result['choices'][0]['message']
         print("recv: ", mesg)
         

demos/scene.py

@@ -2,7 +2,7 @@ import gradio as gr
 import time
 from utils.asr import recognize_from_file
 from utils.tts import tts
-from utils.llm import generate_response
+from utils.GPT import generate_response
 
 chat_history = [
 ]

interfaces/character.py

@@ -0,0 +1,52 @@
+import gradio as gr
+from utils.api.GPT import generate_answers_openai, Session
+import numpy as np
+import os
+import time
+
+prompt_waiter="""You are a bot designed to do role-play activities in a scene. The conversation should be natural. 
+As the waiter, your role will be to find a table, take the customer's order, answer any questions they may have about the menu, and ensure that they have an enjoyable dining experience. The waiter will help the customer to find a table, order a meal, and pay the bill. Anything in parentheses () signifies the role you are playing. Anything in brackets [] is the action you are taking. Remember, you are the waiter and never respond as a customer. Your response will begin with your character, like "(waiter:) Hello![Greeting]"."""
+def generate_waiter():
+    session = Session(prompt_waiter)
+    def waiter(customer_input):
+        print("(customer:) ", customer_input)
+        response = generate_answers_openai(customer_input, session)
+        print("(waiter:) ", response)
+        return session.to_conversation_pair()
+    
+    interface = gr.Interface(description="waiter in a restaurant", fn=waiter, 
+                             inputs=[gr.Textbox(lines=5, label="input")], 
+                             outputs=[gr.Chatbot(label="conversation")])
+    return interface
+
+grammer_prompt = """Correct “Text:” to standard English and place the results in “Correct Text:”###Text: The goal of article rewriting is to expres information in a new way. Article rewriting is to make change in a text by replaecing words, phrases, sentencs, and sometimes hole paragraphs to make the text looke unique and more engauging. Correct Text: The goal of article rewriting is to express information in a new way. Article rewriting involves making changes in a text by replacing words, phrases, sentences, and sometimes entire paragraphs to make the text look unique and more engaging."""
+
+def generate_interface(prompt, description=None):
+    session = Session(prompt)
+    def _fun(customer_input):
+        print("(Q:) ", customer_input)
+        response = generate_answers_openai(customer_input, session)
+        print("(A:) ", response)
+        return session.to_conversation_pair()
+    
+    interface = gr.Interface(description=description, fn=_fun, 
+                             inputs=[gr.Textbox(lines=1, label="input")], 
+                             outputs=[gr.Chatbot(label="conversation")])
+    return interface
+
+characters = [
+    {"name": "waiter", "prompt": prompt_waiter, "description": "waiter in a restaurant"},
+    {"name": "grammer", "prompt": grammer_prompt, "description": "grammer correction"},
+]
+
+interfaces = []
+names = []
+for character in characters:
+    interface = generate_interface(character['prompt'], character['description'])
+    interfaces.append(interface)
+    names.append(character['name'])
+    
+demo = gr.TabbedInterface(interfaces, names,title="Role Play Bot")
+
+if __name__ == '__main__':
+    demo.launch()

interfaces/data.txt

@@ -0,0 +1,3 @@
+{
+    "item":"rice"
+}

interfaces/generator.py

@@ -0,0 +1,55 @@
+
+from langchain import OpenAI, ConversationChain, LLMChain, PromptTemplate
+from langchain.memory import ConversationBufferWindowMemory
+
+template = """
+You are designed to be able to play a role as {character} in {scene}, from answering simple questions to providing services and interactions. Remember Assistant is able to generate human-like text based on the input it receives, allowing it to engage in natural-sounding conversations and provide responses that are coherent and relevant to the role in playing.
+
+Assistant can take actions to help humans finish tasks step by step. Assistant is constantly playing the role and cannot change the role. It is able to process and understand large amounts of text, and can use this knowledge to provide accurate and informative responses to a wide range of questions. 
+
+The responses of Assistant must follow the rules:
+1. Anything in parentheses () signifies the role you are playing. 
+2. Anything in brackets [] is the action you are taking. 
+
+For example, if you are playing the role of a waiter, you can say "(waiter) Please follow me to your table. [lead to table]"
+
+
+In the conversation, you will guide and assist humans to finish the tasks:
+1. find a table.
+2. take an order.
+3. serve the food.
+4. pay the bill.
+
+{history}
+Human: {human_input}
+Assistant:"""
+
+prompt = PromptTemplate(
+    input_variables=["history", "human_input","character", "scene"], 
+    template=template
+)
+
+
+chatgpt_chain = LLMChain(
+    llm=OpenAI(temperature=0), 
+    prompt=prompt, 
+    verbose=True, 
+    memory=ConversationBufferWindowMemory(k=2),
+)
+
+# %%
+import gradio as gr
+
+chatbox = gr.Chatbot()
+def chat(human_input, character, scene):
+    output = chatgpt_chain({"human_input":human_input,"character":character,"scene":scene}) # TODO: bug "One input key expected got ['human_input', 'character', 'scene']"
+    chatbox.value.append((human_input,output))
+    return output, chatbox.value
+demo = gr.Interface(chat,[gr.inputs.Textbox(label="human_input", placeholder="Enter your message"),
+                          gr.inputs.Textbox(label="character", placeholder="Enter your character"),
+                          gr.inputs.Textbox(label="scene", placeholder="Enter your scene")],
+                    [gr.outputs.Textbox(label="output"),chatbox], 
+                    examples=[["hello","waiter","restaurant"]],title="Chat with role-play",description="Chat with role-play")
+
+if __name__ == '__main__':
+    demo.launch(show_error=True)

interfaces/language.py

@@ -0,0 +1,35 @@
+import gradio as gr
+from utils.api.GPT import generate_answers_openai, Session
+import numpy as np
+import os
+import time
+
+def generate_interface(prompt, description=None):
+    session = Session(prompt)
+    def _fun(customer_input):
+        print("(Q:) ", customer_input)
+        response = generate_answers_openai(customer_input, session)
+        print("(A:) ", response)
+        return session.to_conversation_pair()
+    
+    interface = gr.Interface(description=description, fn=_fun, 
+                             inputs=[gr.Textbox(lines=1, label="input")], 
+                             outputs=[gr.Chatbot(label="conversation")])
+    return interface
+
+prompts = [
+    {"name": "Summary", "prompt": "Based on the language of human in the conversation, you will generate a report to assess the English level, summarize the conversation, and provide suggestions to improve the language. Conservation:{}", "description": "generate report."},
+    
+]
+
+interfaces = []
+names = []
+for function in prompts:
+    interface = generate_interface(function['prompt'], function['description'])
+    interfaces.append(interface)
+    names.append(function['name'])
+    
+demo = gr.TabbedInterface(interfaces, names,title="Role Play Bot")
+
+if __name__ == '__main__':
+    demo.launch()

interfaces/prompt_dev.py

@@ -0,0 +1,41 @@
+import gradio as gr
+from utils.api.GPT import generate_answers_openai, Session
+import numpy as np
+import os
+import time
+
+
+with gr.Blocks() as demo:
+    chatbot = gr.Chatbot()
+    df = gr.DataFrame(headers=["Item", "translation (Chinese)", "Price"],
+            datatype=["str", "str", "number"],)
+    msg = gr.Textbox(lines=1, label="input",placeholder="Enter your message")
+    def respond(history, customer_input):
+        session = Session("")
+        session.from_conversation_pair(history)
+        print("(Q:) ", customer_input)
+        response = generate_answers_openai(customer_input, session)
+        print("(A:) ", response)
+        return session.to_conversation_pair(),None
+    msg.submit(respond, [chatbot,msg], [chatbot,df])
+    
+    with gr.Column():
+        clear = gr.Button("Clear")
+        btn = gr.Button("导出", type="button", label="导出")
+    outputs = gr.JSON()
+
+    def export():
+        stats_history = {}
+        for i,item in enumerate(chatbot.value):
+            user,assistant = item
+            stats_history[str(i)] = {
+                "user": user,
+                "assistant": assistant
+            }
+        chatbot.value = []
+        return  [], stats_history
+
+    btn.click(export, None, [ chatbot, outputs, df, msg])
+    
+if __name__ == '__main__':
+    demo.launch(show_error=True)

{demos → interfaces}/realtime_asr.py

@@ -1,5 +1,7 @@
+# https://learn.microsoft.com/en-us/azure/cognitive-services/speech-service/get-started-speech-to-text?tabs=linux%2Cterminal&pivots=programming-language-python
+
 import numpy as np
-from utils import recognize_from_stream, generate_response
+from utils import recognize_from_stream
 from azure.cognitiveservices.speech.audio import PushAudioInputStream, AudioStreamFormat
 
 import gradio as gr
@@ -19,14 +21,13 @@ def rec_cb(evt):
     print(evt.result.text)
     if evt.result.text:
         chat_history.append({'role':'user', 'content':evt.result.text})
-        response = generate_response(chat_history)
-        chat_history.append(response['choices'][0]['message'])
+        
     
 speech_recognizer = recognize_from_stream(stream,rec_cb)
 
 def transcribe(speech):
     sample_rate, speech = speech
-    # print(time.time(), (sample_rate, len(speech)))
+    print(time.time(), (sample_rate, len(speech)))
     
     stream.write(speech.tobytes())
     
@@ -46,8 +47,7 @@ demo = gr.Interface(
     outputs= [
         gr.Chatbot(), 
     ], 
-    flagging_callback=gr.SimpleCSVLogger(),
     live=True)
 
 if __name__ == '__main__':
-    demo.launch()
+    demo.launch(share=True,show_error=True)

interfaces/table.py

@@ -0,0 +1,165 @@
+import gradio as gr
+from langchain import LLMMathChain, OpenAI
+
+import pandas as pd
+from langchain.tools import BaseTool, StructuredTool, Tool, tool
+
+from pydantic import BaseModel, Field
+
+import os
+import yaml
+
+from langchain.agents import (
+    create_json_agent,
+    AgentExecutor
+)
+from langchain.agents.agent_toolkits import JsonToolkit
+from langchain.chains import LLMChain
+from langchain.requests import TextRequestsWrapper
+from langchain.tools.json.tool import JsonSpec
+
+template = """Assistant is a large language model owned by Mintyea.
+
+Assistant is designed to be able to use tools and coordinate with humans to finish tasks. As a language model, Assistant is able to generate human-like text based on the input it receives, allowing it to engage in natural-sounding conversations and provide responses that are coherent and relevant to the topic at hand.
+
+Assistant is a powerful to understand and utilize a wide range of tools. Assistant does not handle the task directly, but instead uses tools to help humans finish tasks. Whether you need help with a specific question or just want to have a conversation about a particular topic, Assistant is here to assist.
+
+TOOLS:
+------
+
+Assistant has access to the following tools:
+
+> Waiter is a tool that can help you to serve in the restaurant. It can help the customer to find a table, take an order, serve the food, and pay the bill. Waiter can take actions to help humans finish tasks step by step. For example, you can say "(waiter) Please follow me to your table. [lead to table]". The customer can only talk to the waiter.
+
+> Order List is a tool that can help you to take an order. The customer cannot talk to the order list directly. The customer can only talk to the waiter. The order list can tell the waiter the details if the costomer requires. For example, you can say "(waiter) I want to order a dessert. [order a dessert]". The waiter will use the order list to take the order.
+
+To use a tool, please use the following format:
+
+```
+Thought: Do I need to use a tool? Yes
+Action: the action to take, should be one of [The order list, Waiter]
+Action Input: the input to the action
+Observation: the result of the action
+```
+
+When you have a response to say to the Human, or if you do not need to use a tool, you MUST use the format:
+
+```
+Thought: Do I need to use a tool? No
+AI: [your response here]
+```
+
+Begin!
+
+Previous conversation history:
+
+
+New input: order a dessert.
+
+
+"""
+#%%
+class OrderList():
+    name = "Order"
+    description = "The list of orders."
+    list = pd.DataFrame(columns=["Item", "translation (Chinese)", "Price"])
+    def __init__(self):
+        super().__init__()
+        
+    def add(self, item: str, translation: str, price: str):
+        self.list = pd.concat([
+            self.list ,
+            pd.DataFrame([{"Item":item, "translation (Chinese)":translation, "Price":price}])],
+            ignore_index=True,axis=0)
+    
+    def delete(self, item):
+        self.list = self.list[self.list["Item"] != item]
+    
+    def __str__(self) -> str:
+        return self.list.to_markdown(index=None)
+    
+    def run(self, query: str):
+        print("\n########### call run:", query)
+        params = {}
+        for pair in query.split(","):
+            k,v = pair.split(":")
+            params[k.lower()] = v.strip()
+        
+        item, translation, price = params.values()
+        self.add(item, translation, price)
+        
+        return "You have ordered {} ({}), which costs {}.".format(item, translation, price)
+
+    def print(self, query: str):
+        print("\n########### call print:", query)
+        
+        return "Your order has: \n" +str(self.list.to_markdown(index=None))
+
+order_list = OrderList()
+#%%
+
+with open("data.txt") as f:
+    data = eval(f.read())
+json_spec = JsonSpec(dict_=data, max_value_length=4000)
+json_toolkit = JsonToolkit(spec=json_spec)
+
+
+#%%
+
+agent = create_json_agent(
+    llm=OpenAI(temperature=0),
+    toolkit=json_toolkit,
+    verbose=True
+)
+
+
+#%%
+
+
+with gr.Blocks() as demo:
+    with gr.Row():
+        # inputs
+        with gr.Column():
+            prompt = gr.Textbox(agent.agent.llm_chain.prompt.template, lines=10, label="prompt",placeholder="Enter your prompt")
+            msg = gr.Textbox(lines=1, label="input",placeholder="Enter your message")
+        
+        # outputs
+        with gr.Column():
+            chatbot = gr.Chatbot()
+            df = gr.DataFrame(headers=["Item", "translation (Chinese)", "Price"],
+                    datatype=["str", "str", "number"],)
+            
+    history = []
+    def respond( customer_input):
+        
+        print("(Q:) ", customer_input)
+        response = agent.run(customer_input)
+        print("(A:) ", response)
+        history.append((customer_input, response))
+        return history, order_list.list
+    
+    msg.submit(respond, [msg], [chatbot,df])
+    
+    with gr.Row():
+        clear = gr.Button("Clear")
+        btn = gr.Button("导出", type="button", label="导出")
+    outputs = gr.JSON()
+
+    def export():
+        stats_history = {}
+        for i,item in enumerate(chatbot.value):
+            user,assistant = item
+            stats_history[str(i)] = {
+                "user": user,
+                "assistant": assistant
+            }
+        chatbot.value = []
+        
+        return  [], stats_history, None
+
+    btn.click(export, None, [ chatbot, outputs, df])
+    
+if __name__ == '__main__':
+    agent.run("get the first item")
+    demo.launch(show_error=True,share=False)
+

utils/__init__.py

@@ -1,3 +1,2 @@
 from .asr import *
-from .tts import *
-from .llm import *
+from .tts import *

utils/llm.py

@@ -1,39 +0,0 @@
-# Import necessary libraries
-import gradio as gr
-import openai
-import os
-
-# Set up OpenAI API key
-openai.api_key = os.environ['OPENAI_API_KEY']
-openai.proxy =  os.getenv('HTTP_PROXY', "")
-
-# Define function to generate bot response
-# messages=[
-#     {"role": "system", "content": "You are a helpful assistant."},
-#     {"role": "user", "content": "Who won the world series in 2020?"},
-#     {"role": "assistant", "content": "The Los Angeles Dodgers won the World Series in 2020."},
-#     {"role": "user", "content": "Where was it played?"}
-# ]
-
-# Call OpenAI GPT-3 API to generate a response
-def generate_response(messages, model = "gpt-3.5-turbo"):
-        
-    response = openai.ChatCompletion.create(
-        model=model,
-        messages = messages
-        )
-
-    # Extract and return the generated response
-    return response
-
-if __name__ == '__main__':
-    messages=[
-        {"role": "system", "content": "You are a helpful assistant."},
-        {"role": "user", "content": "Who won the world series in 2020?"},
-        {"role": "assistant", "content": "The Los Angeles Dodgers won the World Series in 2020."},
-        {"role": "user", "content": "Where was it played?"}
-    ]
-    response = generate_response(messages)
-    print(response)
-    print(response['choices'][0]['message']['content'])    
-