Duplicate from Daniton/Captioning

Co-authored-by: DanitonAI <Daniton@users.noreply.huggingface.co>

.gitattributes

@@ -0,0 +1,34 @@
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text

README.md

@@ -0,0 +1,13 @@
+---
+title: Captioning
+emoji: 👁
+colorFrom: green
+colorTo: blue
+sdk: gradio
+sdk_version: 3.19.1
+app_file: app.py
+pinned: false
+duplicated_from: Daniton/Captioning
+---
+
+Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference

app.py

@@ -0,0 +1,39 @@
+import pathlib
+
+import gradio as gr
+import open_clip
+import torch
+
+device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+
+model, _, transform = open_clip.create_model_and_transforms(
+    "coca_ViT-B-32",
+    pretrained="laion2b_s13b_b90k"
+)
+model.to(device)
+model.eval()
+
+def output_generate(image):
+    im = transform(image).unsqueeze(0).to(device)
+    with torch.no_grad(), torch.cuda.amp.autocast():
+        generated = model.generate(im, seq_len=20)
+    return open_clip.decode(generated[0].detach()).split("<end_of_text>")[0].replace("<start_of_text>", "")
+
+def inference_caption(image):
+    im = transform(image).unsqueeze(0).to(device)
+    with torch.no_grad(), torch.cuda.amp.autocast():
+        generated = model.generate(
+            im, 
+            generation_type="beam_search",
+            top_p=1.0, 
+            min_seq_len=30, 
+            seq_len=100, 
+            repetition_penalty=1.2
+        )
+    return open_clip.decode(generated[0].detach()).split("<end_of_text>")[0].replace("<start_of_text>", "")
+
+image_input = gr.inputs.Image(type="pil")
+caption_output = gr.outputs.Textbox(label="Caption Output")
+caption_interface = gr.Interface(fn=inference_caption, inputs=image_input, outputs=caption_output, capture_session=True, title="CoCa: Contrastive Captioners", description="An open source implementation of CoCa: Contrastive Captioners are Image-Text Foundation Models.", examples=[path.as_posix() for path in sorted(pathlib.Path("images").glob("*.jpg"))], allow_flagging=False)
+
+caption_interface.launch()

requirements.txt

@@ -0,0 +1,4 @@
+open_clip_torch
+transformers
+--extra-index-url https://download.pytorch.org/whl/cu113
+torch