Add rough draft of Gradio chatbot

Manual prompting to open ChatGPT interface based on history.

chatbot/app.py

@@ -0,0 +1,172 @@
+import gradio as gr
+from langchain.chat_models import ChatOpenAI
+from langchain.chains import ConversationChain
+from langchain.memory import ConversationBufferMemory
+
+from langchain.prompts.chat import (
+    HumanMessagePromptTemplate,
+    MessagesPlaceholder,
+    ChatPromptTemplate,
+)
+from engineer_prompt import init_prompt
+
+# from transformers import (
+#     BlipProcessor,
+#     BlipForConditionalGeneration,
+#     BlipForQuestionAnswering,
+# )
+# import torch
+# from PIL import Image
+
+# class ImageCaptioning:
+#     def __init__(self, device):
+#         print(f"Initializing ImageCaptioning to {device}")
+#         self.device = device
+#         self.torch_dtype = torch.float16 if "cuda" in device else torch.float32
+#         self.processor = BlipProcessor.from_pretrained(
+#             "Salesforce/blip-image-captioning-base"
+#         )
+#         self.model = BlipForConditionalGeneration.from_pretrained(
+#             "Salesforce/blip-image-captioning-base", torch_dtype=self.torch_dtype
+#         ).to(self.device)
+
+#     def inference(self, image_path):
+#         inputs = self.processor(Image.open(image_path), return_tensors="pt").to(
+#             self.device, self.torch_dtype
+#         )
+#         out = self.model.generate(**inputs)
+#         captions = self.processor.decode(out[0], skip_special_tokens=True)
+#         print(
+#             f"\nProcessed ImageCaptioning, Input Image: {image_path}, Output Text: {captions}"
+#         )
+#         return captions
+
+
+# class VisualQuestionAnswering:
+#     def __init__(self, device):
+#         print(f"Initializing VisualQuestionAnswering to {device}")
+#         self.torch_dtype = torch.float16 if "cuda" in device else torch.float32
+#         self.device = device
+#         self.processor = BlipProcessor.from_pretrained("Salesforce/blip-vqa-base")
+#         self.model = BlipForQuestionAnswering.from_pretrained(
+#             "Salesforce/blip-vqa-base", torch_dtype=self.torch_dtype
+#         ).to(self.device)
+
+#     def inference(self, image_path, question):
+#         raw_image = Image.open(image_path).convert("RGB")
+#         inputs = self.processor(raw_image, question, return_tensors="pt").to(
+#             self.device, self.torch_dtype
+#         )
+#         out = self.model.generate(**inputs)
+#         answer = self.processor.decode(out[0], skip_special_tokens=True)
+#         print(
+#             f"\nProcessed VisualQuestionAnswering, Input Image: {image_path}, Input Question: {question}, "
+#             f"Output Answer: {answer}"
+#         )
+#         return
+
+
+class ConversationBot:
+    def __init__(
+        self,
+    ):
+        self.chat = ChatOpenAI(temperature=1, verbose=True)
+        self.memory = ConversationBufferMemory(return_messages=True)
+        self.init_prompt_msgs = init_prompt.messages
+        self.ai_prompt_questions = {
+            "ingredients": self.init_prompt_msgs[1],
+            "allergies": self.init_prompt_msgs[3],
+            "recipe_open_params": self.init_prompt_msgs[5],
+        }
+
+    def respond(self, user_msg, chat_history):
+        response = self._get_bot_response(user_msg, chat_history)
+        chat_history.append((user_msg, response))
+        return "", chat_history
+
+    def init_conversation(self, formatted_chat_prompt):
+        self.conversation = ConversationChain(
+            llm=self.chat,
+            memory=self.memory,
+            prompt=formatted_chat_prompt,
+            verbose=True,
+        )
+
+    def reset(self):
+        self.memory.clear()
+
+    def _get_bot_response(self, user_msg: str, chat_history) -> str:
+        if len(chat_history) < 2:
+            return self.ai_prompt_questions["allergies"].prompt.template
+
+        if len(chat_history) < 3:
+            return self.ai_prompt_questions["recipe_open_params"].prompt.template
+
+        if len(chat_history) < 4:
+            user = 0
+            ai = 1
+            user_msgs = [msg_pair[user] for msg_pair in chat_history[1:]]
+            f_init_prompt = init_prompt.format_prompt(
+                ingredients=user_msgs[0],
+                allergies=user_msgs[1],
+                recipe_freeform_input=user_msg,
+            )
+            chat_msgs = f_init_prompt.to_messages()
+            results = self.chat.generate([chat_msgs])
+            chat_msgs.extend(
+                [
+                    results.generations[0][0].message,
+                    MessagesPlaceholder(variable_name="history"),
+                    HumanMessagePromptTemplate.from_template("{input}"),
+                ]
+            )
+            open_prompt = ChatPromptTemplate.from_messages(chat_msgs)
+            # prepare the open conversation chain from this point
+            self.init_conversation(open_prompt)
+            return results.generations[0][0].message.content
+
+        response = self.conversation.predict(input=user_msg)
+        return response
+
+    # def run_image(self, image, state, txt, lang):
+    #     image_filename = os.path.join("image", f"{str(uuid.uuid4())[:8]}.png")
+    #     print("======>Auto Resize Image...")
+    #     img = Image.open(image.name)
+    #     width, height = img.size
+    #     ratio = min(512 / width, 512 / height)
+    #     width_new, height_new = (round(width * ratio), round(height * ratio))
+    #     width_new = int(np.round(width_new / 64.0)) * 64
+    #     height_new = int(np.round(height_new / 64.0)) * 64
+    #     img = img.resize((width_new, height_new))
+    #     img = img.convert("RGB")
+    #     img.save(image_filename, "PNG")
+    #     print(f"Resize image form {width}x{height} to {width_new}x{height_new}")
+    #     description = self.models["ImageCaptioning"].inference(image_filename)
+    #     Human_prompt = f'\nHuman: provide a figure named {image_filename}. The description is: {description}. This information helps you to understand this image, but you should use tools to finish following tasks, rather than directly imagine from my description. If you understand, say "Received". \n'
+    #     self.memory.buffer = (
+    #         self.agent.memory.buffer + Human_prompt + "AI: " + AI_prompt
+    #     )
+    #     state = state + [(f"![](file={image_filename})*{image_filename}*", AI_prompt)]
+    #     print(
+    #         f"\nProcessed run_image, Input image: {image_filename}\nCurrent state: {state}\n"
+    #         f"Current Memory: {self.agent.memory.buffer}"
+    #     )
+    #     return state, state, f"{txt} {image_filename} "
+
+
+with gr.Blocks() as demo:
+    bot = ConversationBot()
+    chatbot = gr.Chatbot(
+        value=[(None, bot.ai_prompt_questions["ingredients"].prompt.template)]
+    )
+
+    msg = gr.Textbox()
+    clear = gr.Button("Clear")
+
+    msg.submit(
+        fn=bot.respond, inputs=[msg, chatbot], outputs=[msg, chatbot], queue=False
+    )
+    clear.click(lambda: None, None, chatbot, queue=False).then(bot.reset)
+
+if __name__ == "__main__":
+    demo.launch()