Initial commit: BLIP I mage captioning demo

.gitignore

@@ -0,0 +1,6 @@
+venv/
+__pycache__/
+*.csv
+*.json
+*.zip
+outputs_captioned/

README.md

@@ -0,0 +1,10 @@
+---
+title: Multimodal Image Captioning with BLIP (Demo)
+sdk: gradio
+python_version: "3.10"
+app_file: app.py
+tags: [image-captioning, blip, gradio, portfolio]
+---
+
+This Space hosts an interactive demo of my fine-tuned BLIP model for image captioning.
+Upload images, generate captions, and download results (CSV/JSON/ZIP).

app.py

@@ -0,0 +1,437 @@
+# ============================================================
+# BLIP Captioning — Pro Demo (CPU, Gradio v5)
+# ------------------------------------------------------------
+# UI POLISH + flexible model source (local or Hub).
+# - Logic unchanged: same presets, generation, rendering, downloads.
+# - Loads model from HF Hub if MODEL_ID_OR_PATH is set; else uses local ./blip_caption_model/final
+# - Optional HF_TOKEN (for private models) is supported but not required for public models.
+# ============================================================
+
+import os
+import re
+import io
+import json
+import zipfile
+from typing import Dict, List, Tuple, Optional
+
+import pandas as pd
+import torch
+from PIL import Image, ImageDraw, ImageFont
+import gradio as gr
+from transformers import (
+    BlipForConditionalGeneration,
+    BlipProcessor,
+    AutoTokenizer,
+    GenerationConfig,
+    __version__ as TF_VER,
+)
+
+# =======================
+# Global Configuration
+# =======================
+
+os.environ["TOKENIZERS_PARALLELISM"] = "false"
+
+HERE = os.path.dirname(__file__)
+# Flexible source:
+# - On Spaces, set MODEL_ID_OR_PATH="your-username/blip-caption-model"
+# - Locally, leave unset to use the fine-tuned weights in ./blip_caption_model/final
+FINAL_DIR = os.getenv(
+    "MODEL_ID_OR_PATH",
+    os.path.join(HERE, "blip_caption_model", "final")
+)
+BASE_ID = "Salesforce/blip-image-captioning-base"
+
+# Optional token (only needed if your model repo is private)
+HF_TOKEN = os.getenv("HF_TOKEN", None)
+
+device = torch.device("cpu")
+torch.set_num_threads(max(1, (os.cpu_count() or 2) - 1))
+
+# =======================
+# Load Model & Processor
+# =======================
+
+print("🔧 torch:", torch.__version__, "| transformers:", TF_VER)
+print("🔄 Loading model from:", FINAL_DIR)
+
+# Load model + (optional) generation config
+model = BlipForConditionalGeneration.from_pretrained(FINAL_DIR, token=HF_TOKEN)
+try:
+    model.generation_config = GenerationConfig.from_pretrained(FINAL_DIR, token=HF_TOKEN)
+except Exception:
+    pass
+
+# Prefer hub processor (fast if torchvision is installed); fall back to saved preprocessor
+try:
+    processor = BlipProcessor.from_pretrained(BASE_ID, use_fast=True)
+    print("ℹ️ Processor: hub:", BASE_ID)
+except Exception as e:
+    print("⚠️ Hub processor failed; using saved preprocessor. Reason:", e)
+    processor = BlipProcessor.from_pretrained(FINAL_DIR, token=HF_TOKEN)
+
+# Kaggle-style preprocessing (224 + center-crop)
+processor.image_processor.size = {"height": 224, "width": 224}
+if hasattr(processor.image_processor, "do_center_crop"):
+    processor.image_processor.do_center_crop = True
+if hasattr(processor.image_processor, "crop_size"):
+    processor.image_processor.crop_size = {"height": 224, "width": 224}
+
+# =======================
+# Tokenizer Alignment
+# =======================
+
+def _lm_head_rows(m: BlipForConditionalGeneration) -> int:
+    try:
+        return m.text_decoder.cls.predictions.decoder.weight.shape[0]
+    except Exception:
+        return m.get_input_embeddings().weight.shape[0]
+
+lm_rows = _lm_head_rows(model)
+
+def load_tokenizer_matching_head(model_head_rows: int) -> Tuple[AutoTokenizer, List[str]]:
+    # Try hub tokenizer first (often matches base), else fall back to saved FINAL_DIR
+    try:
+        tok = AutoTokenizer.from_pretrained(BASE_ID)
+        print("ℹ️ Tokenizer: hub:", BASE_ID)
+        if len(tok) == model_head_rows:
+            return tok, []
+        else:
+            print(f"⚠️ Hub tokenizer size {len(tok)} != LM head {model_head_rows}.")
+    except Exception as e:
+        print("⚠️ Hub tokenizer failed; trying saved. Reason:", e)
+
+    tok = AutoTokenizer.from_pretrained(FINAL_DIR, token=HF_TOKEN)
+    print("ℹ️ Tokenizer: FINAL_DIR")
+    extra: List[str] = []
+    if len(tok) < model_head_rows:
+        need = model_head_rows - len(tok)
+        extra = [f"<extra_tok_{i}>" for i in range(need)]
+        tok.add_tokens(extra)
+        tok.add_special_tokens({"additional_special_tokens": extra})
+        print(f"ℹ️ Added {len(extra)} dummy *special* tokens to match LM head.")
+    return tok, extra
+
+tokenizer, EXTRA_TOKENS = load_tokenizer_matching_head(lm_rows)
+
+# Pad/eos config + left padding (decoder-only-friendly)
+if tokenizer.pad_token is None:
+    tokenizer.pad_token = tokenizer.eos_token
+tokenizer.padding_side = "left"
+model.config.pad_token_id = tokenizer.pad_token_id
+model.generation_config.pad_token_id = tokenizer.pad_token_id
+if tokenizer.eos_token_id is not None:
+    model.generation_config.eos_token_id = tokenizer.eos_token_id
+
+processor.tokenizer = tokenizer
+
+# If we added dummy specials, ban them during generation
+BAD_WORDS_IDS: Optional[List[List[int]]] = None
+if EXTRA_TOKENS:
+    bad = [tokenizer.convert_tokens_to_ids(t) for t in EXTRA_TOKENS]
+    BAD_WORDS_IDS = [[i] for i in bad if i is not None]
+
+def _strip_extra_tokens(text: str) -> str:
+    if not EXTRA_TOKENS:
+        return text
+    return re.sub(r"\s*<extra_tok_\d+>\s*", " ", text).strip()
+
+model.to(device).eval()
+print("✅ Ready. Device:", device)
+print("📏 Inference image size:", getattr(processor.image_processor, "size", None))
+print(f"🧪 vocab check -> lm_head rows: {lm_rows} | tokenizer size: {len(tokenizer)}")
+
+# =======================
+# Decoding Presets
+# =======================
+
+BASE_ARGS = dict(
+    min_length=5,
+    no_repeat_ngram_size=2,
+    early_stopping=True,
+    do_sample=False,
+)
+
+PRESETS: Dict[str, Dict] = {
+    "Quality":   dict(num_beams=5, max_length=35, length_penalty=1.05, **BASE_ARGS),
+    "Balanced":  dict(num_beams=3, max_length=32, length_penalty=1.0,  **BASE_ARGS),
+    "Fast (CPU)":dict(num_beams=1, max_length=28, length_penalty=1.0,  **BASE_ARGS),
+}
+
+# =======================
+# Caption Rendering (Image)
+# =======================
+
+def _get_font(size: int) -> ImageFont.FreeTypeFont:
+    try:
+        return ImageFont.truetype("arial.ttf", size)
+    except Exception:
+        try:
+            import PIL
+            fp = os.path.join(os.path.dirname(PIL.__file__), "fonts", "DejaVuSans.ttf")
+            return ImageFont.truetype(fp, size)
+        except Exception:
+            return ImageFont.load_default()
+
+def _wrap_lines(draw: ImageDraw.ImageDraw, text: str, font: ImageFont.ImageFont, max_width: int) -> List[str]:
+    words = text.split()
+    lines: List[str] = []
+    line = ""
+    for w in words:
+        trial = (line + " " + w).strip()
+        if draw.textlength(trial, font=font) <= max_width:
+            line = trial
+        else:
+            if line:
+                lines.append(line)
+            line = w
+    if line:
+        lines.append(line)
+    return lines
+
+def render_captioned_image(src_path: str, caption: str, out_dir: str, font_size_ratio: float = 0.045) -> str:
+    img = Image.open(src_path).convert("RGB")
+    W, H = img.size
+
+    font_size = max(14, int(W * font_size_ratio))
+    font = _get_font(font_size)
+    padding = int(font_size * 0.6)
+
+    draw_tmp = ImageDraw.Draw(img)
+    lines = _wrap_lines(draw_tmp, caption, font, max_width=W - 2 * padding)
+
+    ascent, descent = font.getmetrics()
+    line_h = ascent + descent
+    text_h = line_h * len(lines)
+    box_h = text_h + 2 * padding
+
+    new_img = Image.new("RGB", (W, H + box_h), "white")
+    new_img.paste(img, (0, 0))
+
+    draw = ImageDraw.Draw(new_img)
+    y = H + padding
+    for l in lines:
+        draw.text((padding, y), l, font=font, fill=(0, 0, 0))
+        y += line_h
+
+    os.makedirs(out_dir, exist_ok=True)
+    base = os.path.basename(src_path)
+    name, _ = os.path.splitext(base)
+    out_path = os.path.join(out_dir, f"{name}_captioned.jpg")
+    new_img.save(out_path, quality=95)
+    return out_path
+
+# =======================
+# Captioning Functions
+# =======================
+
+@torch.no_grad()
+def caption_one(path: str, beams: int, maxlen: int, lenpen: float) -> str:
+    img = Image.open(path).convert("RGB")
+    batch = processor(images=img, return_tensors="pt").to(device)
+
+    gen_kwargs = dict(
+        num_beams=int(beams),
+        max_length=int(maxlen),
+        length_penalty=float(lenpen),
+        **BASE_ARGS,
+    )
+    if BAD_WORDS_IDS is not None:
+        gen_kwargs["bad_words_ids"] = BAD_WORDS_IDS
+
+    ids = model.generate(pixel_values=batch["pixel_values"], **gen_kwargs)
+    text = tokenizer.decode(ids[0], skip_special_tokens=True)
+    return _strip_extra_tokens(text)
+
+def caption_many(paths: List[str],
+                 preset: str,
+                 beams: int,
+                 maxlen: int,
+                 lenpen: float,
+                 export_captioned: bool):
+    p = PRESETS[preset]
+    beams  = int(beams or p["num_beams"])
+    maxlen = int(maxlen or p["max_length"])
+    lenpen = float(lenpen or p["length_penalty"])
+
+    rows, gallery = [], []
+    captioned_paths, captioned_names = [], []
+    cap_map: Dict[str, str] = {}
+
+    out_img_dir = os.path.abspath("outputs_captioned")
+    for pth in (paths or []):
+        try:
+            cap = caption_one(pth, beams, maxlen, lenpen)
+            name = os.path.basename(pth)
+            rows.append({"image": name, "caption": cap})
+            gallery.append((pth, cap))
+            if export_captioned:
+                cpath = render_captioned_image(pth, cap, out_img_dir)
+                captioned_paths.append(cpath)
+                display_name = os.path.basename(cpath)
+                captioned_names.append(display_name)
+                cap_map[display_name] = cpath
+        except Exception as e:
+            rows.append({"image": os.path.basename(pth) if pth else "unknown",
+                         "caption": f"[error] {e}"})
+
+    df = pd.DataFrame(rows)
+    csv_path  = os.path.abspath("demo_captions.csv")
+    json_path = os.path.abspath("demo_captions.json")
+    df.to_csv(csv_path, index=False)
+    with open(json_path, "w", encoding="utf-8") as f:
+        json.dump(rows, f, ensure_ascii=False, indent=2)
+
+    zip_path = None
+    if export_captioned and captioned_paths:
+        zip_path = os.path.abspath("captioned_images.zip")
+        with zipfile.ZipFile(zip_path, "w", compression=zipfile.ZIP_DEFLATED) as zf:
+            for cp in captioned_paths:
+                zf.write(cp, arcname=os.path.basename(cp))
+
+    one_dd_choices = captioned_names
+    multi_cb_choices = captioned_names
+
+    return (
+        gallery, df, csv_path, json_path, zip_path,
+        gr.update(choices=one_dd_choices, value=(one_dd_choices[0] if one_dd_choices else None)),
+        gr.update(choices=multi_cb_choices, value=[]),
+        cap_map
+    )
+
+# =======================
+# Download Helpers
+# =======================
+
+def download_one(selected_name: Optional[str], cap_state: Dict[str, str]) -> Optional[str]:
+    if not selected_name or not cap_state:
+        return None
+    return cap_state.get(selected_name)
+
+def download_multi(selected_names: Optional[List[str]], cap_state: Dict[str, str]) -> Optional[str]:
+    if not selected_names or not cap_state:
+        return None
+    sel_paths = [cap_state[n] for n in selected_names if n in cap_state]
+    if not sel_paths:
+        return None
+    out_zip = os.path.abspath("captioned_selection.zip")
+    with zipfile.ZipFile(out_zip, "w", compression=zipfile.ZIP_DEFLATED) as zf:
+        for p in sel_paths:
+            zf.write(p, arcname=os.path.basename(p))
+    return out_zip
+
+# =======================
+# Gradio UI (Green theme)
+# =======================
+
+from gradio.themes.base import Base
+from gradio.themes.utils import colors
+
+THEME = Base(
+    primary_hue=colors.green,   # ✅ action buttons, toggles, sliders → green
+    secondary_hue=colors.gray,
+)
+
+CUSTOM_CSS = """
+h1, h2, h3, h4, h5, h6 {
+  color: #228B22 !important;   /* forest green headings */
+}
+#gallery .grid-wrap .label {
+  background: rgba(255,255,255,0.9);
+  border-radius: 10px;
+  padding: 6px 10px;
+  font-size: 0.95rem;
+  line-height: 1.25rem;
+}
+footer, .disclaimer {
+  color: #555;
+  font-size: 0.9rem;
+}
+"""
+
+with gr.Blocks(title="Multimodal Image Captioning with BLIP", theme=THEME, css=CUSTOM_CSS) as demo:
+    gr.Markdown(
+        """
+# 🖼️ Multimodal Image Captioning with BLIP (Demo)
+Upload one or many images and generate captions using a fine-tuned BLIP model.
+
+**How it works**
+1. **Upload** JPG/PNG/WebP/BMP images  
+2. Pick a **Preset** (Quality / Balanced / Fast)  
+3. *(Optional)* Tune **Advanced** settings  
+4. Click **Generate Captions** → view **gallery & table**  
+5. **Download** CSV/JSON and **captioned images** (all / single / selected)
+        """.strip()
+    )
+
+    with gr.Row():
+        with gr.Column(scale=6):
+            uploader = gr.File(
+                label="Upload image(s)",
+                file_types=[".jpg", ".jpeg", ".png", ".bmp", ".webp"],
+                file_count="multiple",
+                type="filepath",
+            )
+            gr.Markdown("_Supported formats: JPG, PNG, WebP, BMP. Images resized to 224×224 for inference._")
+        with gr.Column(scale=4):
+            preset = gr.Radio(
+                choices=list(PRESETS.keys()),
+                value="Quality",
+                label="Preset",
+            )
+            with gr.Accordion("Advanced settings (optional)", open=False):
+                beams  = gr.Slider(1, 8,  value=PRESETS["Quality"]["num_beams"], step=1, label="num_beams")
+                maxlen = gr.Slider(16, 64, value=PRESETS["Quality"]["max_length"], step=1, label="max_length")
+                lenpen = gr.Slider(0.8, 1.5, value=PRESETS["Quality"]["length_penalty"], step=0.05, label="length_penalty")
+            export_chk = gr.Checkbox(value=True, label="Also save captioned images")
+            gr.Markdown("_Generates copies with captions rendered for download (ZIP / per-image)._")
+            run = gr.Button("🚀 Generate Captions", variant="primary")
+
+    with gr.Row():
+        gallery = gr.Gallery(label="Results (original image + caption below)", elem_id="gallery", columns=2, height="auto")
+        table = gr.Dataframe(label="Captions Table", interactive=False)
+
+    with gr.Row():
+        csv_out  = gr.File(label="📥 Download CSV")
+        json_out = gr.File(label="📥 Download JSON")
+        zip_all  = gr.File(label="📦 Download ALL captioned (.zip)")
+
+    gr.Markdown("### Download individual / selected captioned images")
+    with gr.Row():
+        with gr.Column():
+            dl_one_dd   = gr.Dropdown(choices=[], value=None, label="Pick a captioned image")
+            dl_one_btn  = gr.Button("⬇️ Download selected image")
+            dl_one_file = gr.File(label="Selected captioned image")
+        with gr.Column():
+            dl_multi_cb  = gr.CheckboxGroup(choices=[], label="Select multiple captioned images")
+            dl_multi_btn = gr.Button("📦 Zip & download selected")
+            dl_multi_zip = gr.File(label="Selected captioned images (.zip)")
+
+    gr.Markdown(
+        """
+<div class="disclaimer">
+<strong>Notes.</strong><br>
+• Captions are deterministic with the same settings (beam search, no sampling).<br>
+• Preprocessing is fixed at 224×224 + center-crop to match training.<br>
+• Extra special tokens (if any) are banned during generation and stripped from outputs.
+</div>
+        """,
+        elem_classes=["disclaimer"]
+    )
+
+    CAP_STATE = gr.State({})
+
+    def _dispatch(files, preset, beams, maxlen, lenpen, export_chk):
+        return caption_many(files, preset, beams, maxlen, lenpen, export_chk)
+
+    run.click(
+        _dispatch,
+        inputs=[uploader, preset, beams, maxlen, lenpen, export_chk],
+        outputs=[gallery, table, csv_out, json_out, zip_all, dl_one_dd, dl_multi_cb, CAP_STATE],
+    )
+
+    dl_one_btn.click(download_one, inputs=[dl_one_dd, CAP_STATE], outputs=[dl_one_file])
+    dl_multi_btn.click(download_multi, inputs=[dl_multi_cb, CAP_STATE], outputs=[dl_multi_zip])
+
+if __name__ == "__main__":
+    demo.launch()

requirements.txt

@@ -0,0 +1,12 @@
+--extra-index-url https://download.pytorch.org/whl/cpu
+torch==2.8.0+cpu
+# torchvision==0.19.0+cpu     # optional if you want the "fast" image processor
+
+transformers==4.56.0
+huggingface-hub>=0.34.0,<1.0
+tokenizers==0.22.0
+safetensors>=0.4.3
+
+gradio==5.27.0
+pillow>=10.0.0
+pandas>=2.3.0