app.py

import os
import sys
import subprocess
subprocess.check_call([sys.executable, "-m", "pip", "uninstall", "-y", "deepspeed"])
import random
import spaces
import numpy as np
import torch
from PIL import Image
import gradio as gr
from diffusers import DiffusionPipeline
from blip3o.conversation import conv_templates
from blip3o.model.builder import load_pretrained_model
from blip3o.utils import disable_torch_init
from blip3o.mm_utils import get_model_name_from_path
from qwen_vl_utils import process_vision_info
from huggingface_hub import snapshot_download
from transformers import AutoModelForCausalLM, AutoTokenizer, AutoProcessor

processor = AutoProcessor.from_pretrained("Qwen/Qwen2.5-VL-7B-Instruct")

# Constants
MAX_SEED = 10000

HUB_MODEL_ID = "BLIP3o/BLIP3o-Model-8B"
model_snapshot_path = snapshot_download(repo_id=HUB_MODEL_ID)
diffusion_path = os.path.join(model_snapshot_path, "diffusion-decoder")

def set_global_seed(seed: int = 42):
    random.seed(seed)
    np.random.seed(seed)
    torch.manual_seed(seed)
    torch.cuda.manual_seed_all(seed)

def add_template(prompt_list: list[str]) -> str:
    conv = conv_templates['qwen'].copy()
    conv.append_message(conv.roles[0], prompt_list[0])
    conv.append_message(conv.roles[1], None)
    return conv.get_prompt()

def make_prompt(text: str) -> list[str]:
    raw = f"Please generate image based on the following caption: {text}"
    return [add_template([raw])]

def randomize_seed_fn(seed: int, randomize: bool) -> int:
    return random.randint(0, MAX_SEED) if randomize else seed

@spaces.GPU
def generate_image(prompt: str, final_seed: int, guidance_scale: float, images_to_generate: int, progress: gr.Progress = gr.Progress(track_tqdm=True)) -> list[Image.Image]:
    set_global_seed(final_seed)
    formatted = make_prompt(prompt)
    images = []
    for _ in range(images_to_generate):
        out = pipe(formatted, guidance_scale=guidance_scale)
        images.append(out.image)
    return images

@spaces.GPU
def process_image(prompt: str, img: Image.Image, progress: gr.Progress = gr.Progress(track_tqdm=True)) -> str:
    messages = [{
        "role": "user",
        "content": [
            {"type": "image", "image": img},
            {"type": "text", "text": prompt},
        ],
    }]
    # print(messages) # Kept original print for debugging if needed
    text_prompt_for_qwen = processor.apply_chat_template(
        messages, tokenize=False, add_generation_prompt=True
    )
    image_inputs, video_inputs = process_vision_info(messages)
    inputs = processor(
        text=[text_prompt_for_qwen],
        images=image_inputs,
        videos=video_inputs,
        padding=True,
        return_tensors="pt",
    ).to('cuda')
    generated_ids = multi_model.generate(**inputs, max_new_tokens=1024)
    input_token_len = inputs.input_ids.shape[1]
    generated_ids_trimmed = generated_ids[:, input_token_len:]
    output_text = processor.batch_decode(
        generated_ids_trimmed, skip_special_tokens=True,
        clean_up_tokenization_spaces=False
    )[0]
    return output_text

print("Diffusion path: ", diffusion_path)
# Initialize model + pipeline
disable_torch_init()

tokenizer, multi_model, _ = load_pretrained_model(
    model_snapshot_path, None, get_model_name_from_path(model_snapshot_path)
)

pipe = DiffusionPipeline.from_pretrained(
    diffusion_path,
    custom_pipeline="pipeline_llava_gen",
    torch_dtype=torch.bfloat16,
    use_safetensors=True,
    variant="bf16",
    multimodal_encoder=multi_model,
    tokenizer=tokenizer,
    safety_checker=None
)
pipe.vae.to('cuda')
pipe.unet.to('cuda')

# Gradio UI
with gr.Blocks(title="BLIP3-o") as demo:
    gr.Markdown('''# BLIP3-o
    A fully open source unified model for both image understanding and generation, check our Github: https://github.com/JiuhaiChen/BLIP3o and Paper: https://arxiv.org/abs/2505.09568
    ''')

    with gr.Row():
        with gr.Column():
            with gr.Tabs():
                with gr.TabItem("Text → Image (Image Generation)"):
                    prompt_gen_input = gr.Textbox(
                        label="Prompt",
                        placeholder="Describe the image you want...",
                        lines=1
                    )
                    seed_slider = gr.Slider(
                        label="Seed",
                        minimum=0, maximum=int(MAX_SEED),
                        step=1, value=42
                    )
                    randomize_checkbox = gr.Checkbox(
                        label="Randomize seed", value=False
                    )
                    guidance_slider = gr.Slider(
                        label="Guidance Scale",
                        minimum=1.0, maximum=30.0,
                        step=0.5, value=3.0
                    )
                    images_to_generate = gr.Slider(
                        label="Number of images",
                        minimum=1, maximum=4,
                        step=1, value=4
                    )
                    run_image_gen_btn = gr.Button("Generate Image")

                    text_gen_examples_data = [
                        ["A cute cat."],
                        ["A young woman with freckles wearing a straw hat, standing in a golden wheat field."],
                        ["A group of friends having a picnic in the park."]
                    ]
                    gr.Examples(
                        examples=text_gen_examples_data,
                        inputs=[prompt_gen_input],
                        cache_examples=False, # As per original
                        label="Image Generation Examples"
                    )

                with gr.TabItem("Image → Text (Image Understanding)"):
                    image_understand_input = gr.Image(label="Input Image", type="pil")
                    prompt_understand_input = gr.Textbox(
                        label="Question about image",
                        placeholder="Describe what you want to know about the image (e.g., What is in this image?)",
                        lines=1
                    )
                    run_image_understand_btn = gr.Button("Understand Image")

                    image_understanding_examples_data = [
                        ["animal-compare.png", "Are these two pictures showing the same kind of animal?"],
                        ["funny_image.jpeg", "Why is this image funny?"],
                        ["animal-compare.png", "Describe this image in detail."],
                    ]
                    gr.Examples(
                        examples=image_understanding_examples_data,
                        inputs=[image_understand_input, prompt_understand_input],
                        cache_examples=False, # As per original
                        label="Image Understanding Examples"
                    )
            
            clean_btn  = gr.Button("Clear All Inputs/Outputs")

        with gr.Column():
            output_gallery = gr.Gallery(label="Generated Images", columns=2, visible=True) # Default to visible, content will control
            output_text    = gr.Textbox(label="Generated Text", visible=False, lines=5, interactive=False)


    @spaces.GPU
    def run_generate_image_tab(prompt, seed, guidance, num_images, progress=gr.Progress(track_tqdm=True)):
        # Seed is already finalized by the randomize_seed_fn in the click chain
        imgs = generate_image(prompt, seed, guidance, num_images, progress=progress)
        return (
            gr.update(value=imgs, visible=True),
            gr.update(value="", visible=False)
        )

    @spaces.GPU
    def run_process_image_tab(img, prompt, progress=gr.Progress(track_tqdm=True)):
        if img is None:
            return (
                gr.update(value=[], visible=False),
                gr.update(value="Please upload an image for understanding.", visible=True)
            )
        txt = process_image(prompt, img, progress=progress)
        return (
            gr.update(value=[], visible=False),
            gr.update(value=txt, visible=True)
        )

    def clean_all_fn():
        return (
            # Tab 1 inputs
            gr.update(value=""),  # prompt_gen_input
            gr.update(value=42),  # seed_slider
            gr.update(value=False), # randomize_checkbox
            gr.update(value=3.0), # guidance_slider
            # Tab 2 inputs
            gr.update(value=None), # image_understand_input
            gr.update(value=""),  # prompt_understand_input
            # Outputs
            gr.update(value=[], visible=True), # output_gallery (reset and keep visible for next gen)
            gr.update(value="", visible=False) # output_text (reset and hide)
        )

    gen_inputs = [prompt_gen_input, seed_slider, guidance_slider, images_to_generate]
    
    run_image_gen_btn.click(
        fn=randomize_seed_fn,
        inputs=[seed_slider, randomize_checkbox],
        outputs=[seed_slider]
    ).then(
        fn=run_generate_image_tab,
        inputs=gen_inputs, # prompt_gen_input, seed_slider (updated), guidance_slider
        outputs=[output_gallery, output_text]
    )

    prompt_gen_input.submit(
        fn=randomize_seed_fn,
        inputs=[seed_slider, randomize_checkbox],
        outputs=[seed_slider]
    ).then(
        fn=run_generate_image_tab,
        inputs=gen_inputs,
        outputs=[output_gallery, output_text]
    )

    # Event listeners for Image -> Text
    understand_inputs = [image_understand_input, prompt_understand_input]

    run_image_understand_btn.click(
        fn=run_process_image_tab,
        inputs=understand_inputs,
        outputs=[output_gallery, output_text]
    )

    prompt_understand_input.submit(
        fn=run_process_image_tab,
        inputs=understand_inputs,
        outputs=[output_gallery, output_text]
    )

    clean_btn.click(
        fn=clean_all_fn,
        inputs=[],
        outputs=[
            prompt_gen_input, seed_slider, randomize_checkbox, guidance_slider,
            image_understand_input, prompt_understand_input,
            output_gallery, output_text
        ]
    )

if __name__ == "__main__":
    demo.launch(share=True)