新建app.py,requirements.txt

app.py

@@ -0,0 +1,86 @@
+import os
+import torch
+from langchain.agents import load_tools
+from langchain.agents import initialize_agent
+from langchain.agents import AgentType
+from langchain.llms import OpenAI
+from langchain.chat_models import AzureChatOpenAI
+from langchain.chains.conversation.memory import ConversationBufferWindowMemory
+from transformers import BlipProcessor,BlipForConditionalGeneration
+from transformers.models.oneformer.modeling_oneformer import OneFormerModelOutput
+import requests
+from PIL import Image
+from langchain.tools import BaseTool
+import gradio as gr
+
+OPENAI_API_KEY = os.getenv("OPENAI_API_KEY")
+OPENAI_API_BASE = os.getenv("OPENAI_AI_BASE")
+DEP_NAME = os.getenv("deployment name")
+
+# llm = ChatOpenAI(openai_api_key=OPENAI_API_KEY, temperature=0, model_name='gpt-3.5-turbo')
+llm = AzureChatOpenAI(deployment_name=DEP_NAME,openai_api_base=OPENAI_API_BASE,openai_api_key=OPENAI_API_KEY,openai_api_version="2023-03-15-preview",model_name="gpt-3.5-turbo")
+
+image_to_text_model = "Salesforce/blip-image-captioning-large"
+device = 'cuda' if torch.cuda.is_available() else 'cpu'
+
+processor = BlipProcessor.from_pretrained(image_to_text_model)
+model = BlipForConditionalGeneration.from_pretrained(image_to_text_model).to(device)
+
+def describeImage(image_url):
+  image_object = Image.open(requests.get(image_url, stream=True).raw).convert('RGB')
+  # image
+  inputs = processor(image_object, return_tensors="pt").to(device)
+  outputs = model.generate(**inputs)
+  return processor.decode(outputs[0], skip_special_tokens=True)
+
+img_url = 'https://img1.baidu.com/it/u=1919509102,1927615551&fm=253&fmt=auto&app=120&f=JPEG?w=889&h=500'
+description = describeImage(img_url)
+# print(description)
+
+
+class DescribeImageTool(BaseTool):
+  name = "Describe Image Tool"
+  description = 'use this tool to describe an image.'
+
+  def _run(self, url: str):
+    description = describeImage(url)
+    return description
+
+  def _arun(self, query: str):
+    raise NotImplementedError("Async operation not supported yet")
+
+
+tools = [DescribeImageTool()]
+
+
+agent = initialize_agent(
+    agent='chat-conversational-react-description',
+    tools=tools,
+    llm=llm,
+    verbose=True,
+    max_iterations=3,
+    early_stopping_method='generate',
+    memory=ConversationBufferWindowMemory(
+        memory_key='chat_history',
+        k=5,
+        return_messages=True
+    )
+)
+
+def descImage(req , image_url) :
+    output = agent(f"{req}:\n{image_url}")
+    print( output )
+    desc = output['output']
+    return desc
+
+desc = descImage('描述该图片' , img_url)
+# print(desc)
+
+output = agent(f"将该段翻译为中文：{desc}")
+print(output['output'])
+
+def greet(name):
+    return "Hello " + name + "!!"
+
+if __name__ == "__main__":
+    iface = gr.Interface(fn=descImage,inputs="text",output="text")

requirements.txt

@@ -0,0 +1,6 @@
+langchain
+openai
+transformers
+torch
+pillow
+gradio