Update app.py

app.py

@@ -1,103 +1,75 @@
-import os
-import gradio as gr
-from langchain.agents import  initialize_agent,AgentType
-from langchain.chat_models import AzureChatOpenAI
+from langchain.agents import load_tools
+from langchain.agents import initialize_agent
+from langchain.agents import AgentType
+from langchain.llms import OpenAI
+from langchain.chat_models import ChatOpenAI
 from langchain.chains.conversation.memory import ConversationBufferWindowMemory
+
+import os
+
+OPENAI_API_KEY = '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'
+
+llm = ChatOpenAI(openai_api_key=OPENAI_API_KEY, temperature=0, model_name='gpt-3.5-turbo')
+
 import torch
-from transformers import BlipProcessor,BlipForConditionalGeneration
+from transformers import BlipProcessor, BlipForConditionalGeneration
+
+image_to_text_model = "Salesforce/blip-image-captioning-large"
+device = 'cuda' if torch.cuda.is_available() else 'cpu'
+
+processor = BlipProcessor.from_pretrained(image_to_text_model)
+model = BlipForConditionalGeneration.from_pretrained(image_to_text_model).to(device)
+
+from transformers.models.oneformer.modeling_oneformer import OneFormerModelOutput
 import requests
 from PIL import Image
+
+def describeImage(image_url):
+  image_object = Image.open(image_url).convert('RGB')
+  # image
+  inputs = processor(image_object, return_tensors="pt").to(device)
+  outputs = model.generate(**inputs)
+  return processor.decode(outputs[0], skip_special_tokens=True)
+
+
 from langchain.tools import BaseTool
-from langchain.chains import LLMChain
-from langchain import PromptTemplate, FewShotPromptTemplate
-
-OPENAI_API_KEY='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'
-OPENAI_API_BASE=os.getenv("OPENAI_API_BASE")
-DEP_NAME=os.getenv("deployment_name")
-llm=AzureChatOpenAI(deployment_name=DEP_NAME,openai_api_base=OPENAI_API_BASE,openai_api_key=OPENAI_API_KEY,openai_api_version="2023-03-15-preview",model_name="gpt-3.5-turbo")
-
-image_to_text_model="Salesforce/blip-image-captioning-large"
-device= 'cuda' if torch.cuda.is_available() else 'cpu'
-
-processor=BlipProcessor.from_pretrained(image_to_text_model)
-
-model=BlipForConditionalGeneration.from_pretrained(image_to_text_model).to(device)
-
-def descImage(image_url):
-    image_obj=Image.open(image_url).convert('RGB')
-    inputs=processor(image_obj,return_tensors='pt').to(device)
-    outputs=model.generate(**inputs)
-    return processor.decode(outputs[0],skip_special_tokens=True)
-
-def toChinese(en:str):
-    pp="翻译下面语句到中文\n{en}"
-    prompt = PromptTemplate(
-            input_variables=["en"],
-            template=pp
-        )
-    llchain=LLMChain(llm=llm,prompt=prompt)
-    return llchain.run(en)
-
-class DescTool(BaseTool):
-    name="Describe Image Tool"
-    description="use this tool to describe an image"
-
-    def _run(self,url:str):
-        description=descImage(url)
+
+class DescribeImageTool(BaseTool):
+    name = "Describe Image Tool"
+    description = 'use this tool to describe an image.'
+
+    def _run(self, url: str):
+        description = describeImage(url)
         return description
-    def _arun(
-        self,query:str):
-        raise NotImplementedError('未实现')
-
-tools=[DescTool()]
-memory=ConversationBufferWindowMemory(
-                          memory_key='chat_history',
-                          k=5,
-                          return_messages=True
-                      )
-
-agent=initialize_agent(
-                      agent=AgentType.CHAT_CONVERSATIONAL_REACT_DESCRIPTION,
-                      tools=tools,
-                      llm=llm,
-                      verbose=False,
-                      max_iterations=3,
-                      early_stopping_method='generate',
-                      memory=memory
-                      )
-def reset_user_input():
-    return gr.update(value='')
-def reset_state():
-    return [], []
-
-def predict(file,input, chatbot,history):
-    input1=f""+input+"\n"+file
-    out=agent(input1)
-    anws=toChinese(out['output'])
-    chatbot.append(input)
-    chatbot[-1] = (input, anws)
-    yield chatbot, history
-    return
-
-with gr.Blocks(css=".chat-blocks{height:calc(100vh - 332px);} .mychat{flex:1} .mychat .block{min-height:100%} .mychat .block .wrap{max-height: calc(100vh - 330px);} .myinput{flex:initial !important;min-height:180px}") as demo:
-    title = '图像识别'
-    demo.title=title
-    with gr.Column(elem_classes="chat-blocks"):
-        with gr.Row(elem_classes="mychat"):
-            file = gr.Image(type="filepath")
-            chatbot = gr.Chatbot(label="图像识别", show_label=False)
-        with gr.Column(elem_classes="myinput"):
-            user_input = gr.Textbox(show_label=False, placeholder="请输入...", lines=1).style(
-                container=False)
-            submitBtn = gr.Button("提交", variant="primary", elem_classes="btn1")
-            emptyBtn = gr.Button("清除历史").style(container=False)
-
-    history = gr.State([])
-    submitBtn.click(predict, [file,user_input, chatbot,history], [chatbot, history],
-                        show_progress=True)
-    submitBtn.click(reset_user_input, [], [user_input])
-    emptyBtn.click(reset_state, outputs=[chatbot, history], show_progress=True)
-
-
-
-demo.queue(api_open=False,concurrency_count=20).launch()
+    
+    def _arun(self, query: str):
+        raise NotImplementedError("Async operation not supported yet")
+
+tools = [DescribeImageTool()]
+
+
+agent = initialize_agent(
+    agent='chat-conversational-react-description',
+    tools=tools,
+    llm=llm,
+    verbose=True,
+    max_iterations=3,
+    early_stopping_method='generate',
+    memory=ConversationBufferWindowMemory(
+        memory_key='chat_history',
+        k=5,
+        return_messages=True
+    )
+)
+
+
+import gradio as gr
+def segment(image):
+    #pass  # Implement your image segmentation model here...
+    print(image)
+    image_url = image
+    
+    return agent(f"Describe the following image:\n{image_url}").get('output').replace('The response to your last comment is','')
+
+demo = gr.Interface(segment, gr.Image(type="filepath",shape=(200, 200)), "text")
+demo.launch()