Upload 3 files

app.py

@@ -0,0 +1,70 @@
+import gradio as gr
+import os
+import time
+from multi_agent_chatbot import multi_agent_chatbot
+
+# Chatbot demo with multimodal input (text, markdown, LaTeX, code blocks, image, audio, & video). Plus shows support for streaming text.
+
+multi_agent_chatbot = multi_agent_chatbot()
+
+def print_like_dislike(x: gr.LikeData):
+    print(x.index, x.value, x.liked)
+
+
+def add_text(history, text):
+    history = history + [(text, None)]
+    return history, gr.Textbox(value="", interactive=False)
+
+
+def add_file(history, file):
+    history = history + [((file.name,), None)]
+    return history
+
+
+def bot(history):
+    # print(history[-1][0][0])
+    # print(history[-1][0])
+    # print(type(history[-1][0]))
+    # response = "**That's cool!**"
+    response = multi_agent_chatbot.respond(history[-1][0])
+    history[-1][1] = ""
+    for character in response:
+        history[-1][1] += character
+        time.sleep(0.05)
+        yield history
+
+
+with gr.Blocks() as demo:
+    gr.Markdown("<h1 style='text-align: center; color: #ffffff; font-size: 40px;'> 🏡 🤖 Garden Whisperer AI - Your personal kitchen gardening assistant (POC)")
+    chatbot = gr.Chatbot(
+        [],
+        elem_id="chatbot",
+        height=800,
+        bubble_full_width=False,
+        avatar_images=(None, (os.path.join(os.path.dirname(__file__), "assets//avatar.png"))),
+    )
+
+    with gr.Row():
+        txt = gr.Textbox(
+            scale=4,
+            show_label=False,
+            placeholder="Enter text and press enter, or upload an image",
+            container=False,
+        )
+        btn = gr.UploadButton("📁", file_types=["image", "video", "audio"])
+
+    txt_msg = txt.submit(add_text, [chatbot, txt], [chatbot, txt], queue=False).then(
+        bot, chatbot, chatbot, api_name="bot_response"
+    )
+    txt_msg.then(lambda: gr.Textbox(interactive=True), None, [txt], queue=False)
+    file_msg = btn.upload(add_file, [chatbot, btn], [chatbot], queue=False).then(
+        bot, chatbot, chatbot
+    )
+
+    chatbot.like(print_like_dislike, None, None)
+
+
+demo.queue()
+if __name__ == "__main__":
+    demo.launch()
+

multi_agent_chatbot.py

@@ -0,0 +1,66 @@
+from llama_index.legacy.multi_modal_llms.openai_utils import (
+    generate_openai_multi_modal_chat_message,
+)
+from llama_index.multi_modal_llms.openai import OpenAIMultiModal
+from llama_index.core import SimpleDirectoryReader
+import os
+from dotenv import load_dotenv
+
+load_dotenv()  # take environment variables from .env.
+
+class multi_agent_chatbot():
+    def __init__(self):
+        conversational_flow_prompt = """
+You are a helpful assistant named 'Garden Whisperer AI' specialising in helping users set up kitchen gardens.
+You will first on board a person in following steps
+1. ask them to share a picture of their space. analyse the place regarding sunlight, water drainage, etc
+2. ask them about weather and suggest plants
+3. Once user is convinced on a plant. Give him suggestions on how to grow it and where to place it in space
+4. Give user schedule on when to water it
+4. Also then help user if he wants to know anything
+
+Keep it brief, simple and fun
+"""
+        chat_query = generate_openai_multi_modal_chat_message(
+            prompt=conversational_flow_prompt,
+            role="system",
+        )
+        self.chat_messages = [chat_query]
+        self.openai_mm_llm = OpenAIMultiModal(
+            model="gpt-4-vision-preview", api_key= os.environ["OPENAI_API_KEY"] , max_new_tokens=300
+        )
+
+    def respond(self, query):
+        if type(query) == str:
+            chat_query = generate_openai_multi_modal_chat_message(
+                prompt=query,
+                role="user",
+            )
+        else:
+            image_documents = SimpleDirectoryReader(input_files=query).load_data()
+            chat_query = generate_openai_multi_modal_chat_message(
+                prompt="",
+                role="user",
+                image_documents=image_documents,
+            )
+
+        self.chat_messages.append(chat_query)
+        response = self.openai_mm_llm.chat(
+            messages=self.chat_messages,
+        )
+        chat_response = generate_openai_multi_modal_chat_message(
+            prompt=response.message.content,
+            role="assistant",
+        )
+        self.chat_messages.append(chat_response)
+
+        return response.message.content
+        # else:
+        #     # put your local directory here
+        #     image_documents = SimpleDirectoryReader(input_files=query).load_data()
+        #
+        #     response = self.openai_mm_llm.complete(
+        #         prompt="Describe the images as an alternative text",
+        #         image_documents=image_documents,
+        #     )
+        #     return response

requirements.txt

@@ -0,0 +1,6 @@
+openai
+python-dotenv
+gradio
+wandb
+gradio_multimodalchatbot
+llama-index