Spaces:
Build error
Build error
Commit
·
4de593d
1
Parent(s):
8698c5e
Update app.py
Browse files
app.py
CHANGED
|
@@ -1,93 +1,39 @@
|
|
| 1 |
import os
|
| 2 |
-
|
| 3 |
-
os.system("pip install -U gradio")
|
| 4 |
-
import sys
|
| 5 |
-
import radio as gr
|
| 6 |
-
cmd22 = "pip install pydantic==1.*"
|
| 7 |
-
|
| 8 |
-
cmd0 = "pip -m pip install 'https://github.com/facebookresearch/detectron2.git@5aeb252b194b93dc2879b4ac34bc51a31b5aee13'"
|
| 9 |
-
# cmd0 = "python -m pip install 'git+https://github.com/facebookresearch/detectron2.git'"
|
| 10 |
-
# cmd0 = "python -m pip install 'https://github.com/facebookresearch/detectron2.git'"
|
| 11 |
-
|
| 12 |
-
os.system(cmd0)
|
| 13 |
-
os.system(cmd22)
|
| 14 |
-
|
| 15 |
-
# clone and install Detic
|
| 16 |
-
os.system(
|
| 17 |
-
"git clone https://github.com/facebookresearch/Detic.git --recurse-submodules"
|
| 18 |
-
)
|
| 19 |
-
os.chdir("Detic")
|
| 20 |
-
|
| 21 |
-
# Install detectron2
|
| 22 |
import torch
|
| 23 |
-
|
| 24 |
-
# Some basic setup:
|
| 25 |
-
# Setup detectron2 logger
|
| 26 |
-
import detectron2
|
| 27 |
-
from detectron2.utils.logger import setup_logger
|
| 28 |
-
|
| 29 |
-
setup_logger()
|
| 30 |
-
|
| 31 |
-
# import some common libraries
|
| 32 |
-
import sys
|
| 33 |
import numpy as np
|
| 34 |
-
|
| 35 |
-
|
| 36 |
-
# import some common detectron2 utilities
|
| 37 |
-
from detectron2 import model_zoo
|
| 38 |
from detectron2.engine import DefaultPredictor
|
| 39 |
from detectron2.config import get_cfg
|
| 40 |
from detectron2.utils.visualizer import Visualizer
|
| 41 |
-
from detectron2.data import MetadataCatalog
|
| 42 |
-
|
| 43 |
-
# Detic libraries
|
| 44 |
-
sys.path.insert(0, "third_party/CenterNet2/projects/CenterNet2/")
|
| 45 |
-
sys.path.insert(0, "third_party/CenterNet2/")
|
| 46 |
from centernet.config import add_centernet_config
|
| 47 |
from detic.config import add_detic_config
|
| 48 |
from detic.modeling.utils import reset_cls_test
|
|
|
|
| 49 |
|
| 50 |
-
|
|
|
|
| 51 |
|
| 52 |
-
|
| 53 |
-
|
| 54 |
-
add_centernet_config(cfg)
|
| 55 |
-
add_detic_config(cfg)
|
| 56 |
-
cfg.MODEL.DEVICE = "cpu"
|
| 57 |
-
cfg.merge_from_file("configs/Detic_LCOCOI21k_CLIP_SwinB_896b32_4x_ft4x_max-size.yaml")
|
| 58 |
-
cfg.MODEL.WEIGHTS = "https://dl.fbaipublicfiles.com/detic/Detic_LCOCOI21k_CLIP_SwinB_896b32_4x_ft4x_max-size.pth"
|
| 59 |
-
cfg.MODEL.ROI_HEADS.SCORE_THRESH_TEST = 0.5 # set threshold for this model
|
| 60 |
-
cfg.MODEL.ROI_BOX_HEAD.ZEROSHOT_WEIGHT_PATH = "rand"
|
| 61 |
-
cfg.MODEL.ROI_HEADS.ONE_CLASS_PER_PROPOSAL = (
|
| 62 |
-
True # For better visualization purpose. Set to False for all classes.
|
| 63 |
-
)
|
| 64 |
-
predictor = DefaultPredictor(cfg)
|
| 65 |
-
|
| 66 |
-
BUILDIN_CLASSIFIER = {
|
| 67 |
-
"lvis": "datasets/metadata/lvis_v1_clip_a+cname.npy",
|
| 68 |
-
"objects365": "datasets/metadata/o365_clip_a+cnamefix.npy",
|
| 69 |
-
"openimages": "datasets/metadata/oid_clip_a+cname.npy",
|
| 70 |
-
"coco": "datasets/metadata/coco_clip_a+cname.npy",
|
| 71 |
-
}
|
| 72 |
-
|
| 73 |
-
BUILDIN_METADATA_PATH = {
|
| 74 |
-
"lvis": "lvis_v1_val",
|
| 75 |
-
"objects365": "objects365_v2_val",
|
| 76 |
-
"openimages": "oid_val_expanded",
|
| 77 |
-
"coco": "coco_2017_val",
|
| 78 |
-
}
|
| 79 |
-
|
| 80 |
-
session_token = os.environ.get("SessionToken")
|
| 81 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 82 |
|
| 83 |
def generate_caption(object_list_str, api_key, temperature):
|
| 84 |
query = f"You are an intelligent image captioner. I will hand you the objects and their position, and you should give me a detailed description that IS BOTH SUPER CONCISE AND SHORT for the photo. In this photo we have the following objects\n{object_list_str}"
|
| 85 |
|
| 86 |
-
# query = f"You are an intelligent image captioner. I will hand you the objects and their position, and you should give me a detailed description for the photo. In this photo we have the following objects\n{object_list_str}"
|
| 87 |
llm = OpenAIChat(
|
| 88 |
model_name="gpt-3.5-turbo", openai_api_key=api_key, temperature=temperature
|
| 89 |
)
|
| 90 |
-
# not gpt-4 yet!
|
| 91 |
|
| 92 |
try:
|
| 93 |
caption = llm(query)
|
|
@@ -97,7 +43,6 @@ def generate_caption(object_list_str, api_key, temperature):
|
|
| 97 |
|
| 98 |
return caption
|
| 99 |
|
| 100 |
-
|
| 101 |
def inference(img, vocabulary, api_key, temperature):
|
| 102 |
metadata = MetadataCatalog.get(BUILDIN_METADATA_PATH[vocabulary])
|
| 103 |
classifier = BUILDIN_CLASSIFIER[vocabulary]
|
|
@@ -144,7 +89,6 @@ def inference(img, vocabulary, api_key, temperature):
|
|
| 144 |
gpt_response,
|
| 145 |
)
|
| 146 |
|
| 147 |
-
|
| 148 |
with gr.Blocks() as demo:
|
| 149 |
with gr.Column():
|
| 150 |
gr.Markdown("# Image Captioning using Detic and ChatGPT with LangChain 🦜️🔗")
|
|
@@ -180,5 +124,4 @@ with gr.Blocks() as demo:
|
|
| 180 |
outputs=[outviz, output_desc],
|
| 181 |
)
|
| 182 |
|
| 183 |
-
|
| 184 |
demo.launch(debug=False)
|
|
|
|
| 1 |
import os
|
| 2 |
+
import gradio as gr
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 3 |
import torch
|
| 4 |
+
import cv2
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 5 |
import numpy as np
|
| 6 |
+
from PIL import Image
|
| 7 |
+
from detectron2.utils.logger import setup_logger
|
|
|
|
|
|
|
| 8 |
from detectron2.engine import DefaultPredictor
|
| 9 |
from detectron2.config import get_cfg
|
| 10 |
from detectron2.utils.visualizer import Visualizer
|
| 11 |
+
from detectron2.data import MetadataCatalog
|
| 12 |
+
from langchain.llms import OpenAI, OpenAIChat
|
|
|
|
|
|
|
|
|
|
| 13 |
from centernet.config import add_centernet_config
|
| 14 |
from detic.config import add_detic_config
|
| 15 |
from detic.modeling.utils import reset_cls_test
|
| 16 |
+
from pydantic import BaseModel, Field, PydanticUserError
|
| 17 |
|
| 18 |
+
class BaseModelWithA(BaseModel):
|
| 19 |
+
a: float
|
| 20 |
|
| 21 |
+
class Foo(BaseModelWithA):
|
| 22 |
+
pass
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 23 |
|
| 24 |
+
try:
|
| 25 |
+
class Bar(Foo):
|
| 26 |
+
x: float = 12.3
|
| 27 |
+
a: float = 123.0 # Add type annotation here
|
| 28 |
+
except PydanticUserError as exc_info:
|
| 29 |
+
assert exc_info.code == 'model-field-overridden'
|
| 30 |
|
| 31 |
def generate_caption(object_list_str, api_key, temperature):
|
| 32 |
query = f"You are an intelligent image captioner. I will hand you the objects and their position, and you should give me a detailed description that IS BOTH SUPER CONCISE AND SHORT for the photo. In this photo we have the following objects\n{object_list_str}"
|
| 33 |
|
|
|
|
| 34 |
llm = OpenAIChat(
|
| 35 |
model_name="gpt-3.5-turbo", openai_api_key=api_key, temperature=temperature
|
| 36 |
)
|
|
|
|
| 37 |
|
| 38 |
try:
|
| 39 |
caption = llm(query)
|
|
|
|
| 43 |
|
| 44 |
return caption
|
| 45 |
|
|
|
|
| 46 |
def inference(img, vocabulary, api_key, temperature):
|
| 47 |
metadata = MetadataCatalog.get(BUILDIN_METADATA_PATH[vocabulary])
|
| 48 |
classifier = BUILDIN_CLASSIFIER[vocabulary]
|
|
|
|
| 89 |
gpt_response,
|
| 90 |
)
|
| 91 |
|
|
|
|
| 92 |
with gr.Blocks() as demo:
|
| 93 |
with gr.Column():
|
| 94 |
gr.Markdown("# Image Captioning using Detic and ChatGPT with LangChain 🦜️🔗")
|
|
|
|
| 124 |
outputs=[outviz, output_desc],
|
| 125 |
)
|
| 126 |
|
|
|
|
| 127 |
demo.launch(debug=False)
|