update

app.py

@@ -2,32 +2,33 @@ import gradio as gr
 import openai
 import os
 import json
+import numpy as np
+import torch
+
+from transformers import AutoProcessor, AutoModelForCausalLM
+
+checkpoint = "microsoft/git-base"
+processor = AutoProcessor.from_pretrained(checkpoint)
+model = AutoModelForCausalLM.from_pretrained(checkpoint)
 
 openai.organization = os.getenv("API_ORG")
 openai.api_key = os.getenv("API_KEY")
 app_password = os.getenv("APP_PASSWORD")
 app_username = os.getenv("APP_USERNAME")
 
-
-def generate(prompt):
-    response = openai.Image.create(
-      prompt=prompt,
-      n=1,
-      size="256x256"
-    )
-    return response['data'][0]['url']
-
-examples = [
-    ["きのこの山"],
-    ["たけのこの里"],
-]
+def generate(input_image):
+    device = "cuda" if torch.cuda.is_available() else "cpu"
+    inputs = processor(images=input_image, return_tensors="pt").to(device)
+    pixel_values = inputs.pixel_values
+    generated_ids = model.generate(pixel_values=pixel_values, max_length=50)
+    generated_caption = processor.batch_decode(generated_ids, skip_special_tokens=True)[0]
+    return generated_caption
 
 demo = gr.Interface(
     fn=generate,
-    inputs=gr.components.Textbox(lines=5, label="Prompt"),
-    outputs=gr.components.Image(type="filepath", label="Generated Image"),
+    inputs=gr.Image(label="Input", elem_id="input_image", type="pil"),
+    outputs=gr.Text(label="Generated Caption"),
     flagging_options=[],
-    examples=examples
 )
 
 demo.launch(share=False, auth=(app_username, app_password))

requirements.txt

@@ -1,2 +1,3 @@
 gradio
 openai
+transformers