app.py

import spaces
import gradio as gr
import torch
torch.jit.script = lambda f: f  # Avoid script error in lambda
from t2v_metrics import VQAScore, list_all_vqascore_models


def update_model(model_name):
    return VQAScore(model=model_name, device="cuda")

# Use global variables for model pipe and current model name
global model_pipe, cur_model_name
cur_model_name = "clip-flant5-xl"
model_pipe = update_model(cur_model_name)


# Ensure GPU context manager is imported correctly (assuming spaces is a module you have)
#try:
    #from spaces import GPU # i believe this is wrong, spaces package does not have "GPU"
#except ImportError:
    # GPU = lambda duration: (lambda f: f)  # Dummy decorator if spaces.GPU is not available

if torch.cuda.is_available():
    model_pipe.device = "cuda"
else:
    print("CUDA is not available")
    
@spaces.GPU # a duration lower than 60 does not work, leave as is.
def generate(model_name, image, text):
    global model_pipe, cur_model_name
    
    if model_name != cur_model_name:
        cur_model_name = model_name  # Update the current model name
        model_pipe = update_model(model_name)
    
    print("Image:", image)  # Debug: Print image path
    print("Text:", text)  # Debug: Print text input
    print("Using model:", model_name)
    
    try:
        result = model_pipe(images=[image], texts=[text]).cpu()[0][0].item()  # Perform the model inference
        print("Result:", result)
    except RuntimeError as e:
        print(f"RuntimeError during model inference: {e}")
        raise e
    
    return result


def rank_images(model_name, images, text):
    global model_pipe, cur_model_name
    
    if model_name != cur_model_name:
        cur_model_name = model_name  # Update the current model name
        model_pipe = update_model(model_name)

    images = [image_tuple[0] for image_tuple in images]
    print("Images:", images)  # Debug: Print image paths
    print("Text:", text)  # Debug: Print text input
    print("Using model:", model_name)
    
    try:
        results = model_pipe(images=images, texts=[text]).cpu()[:, 0].tolist()  # Perform the model inference on all images
        print("Initial results: should be imgs x texts", results)
        ranked_results = sorted(zip(images, results), key=lambda x: x[1], reverse=True)  # Rank results
        ranked_images = [(img, f"Rank: {rank + 1} - Score: {score:.2f}") for rank, (img, score) in enumerate(ranked_results)]  # Pair images with their scores and rank
        print("Ranked Results:", ranked_results)
    except RuntimeError as e:
        print(f"RuntimeError during model inference: {e}")
        raise e
    
    return ranked_images


### EXAMPLES ###
example_imgs = ["0_imgs/DALLE3.png", 
                "0_imgs/DeepFloyd.jpg",
                "0_imgs/Midjourney.jpg",
                "0_imgs/SDXL.jpg"]
example_prompt0 = "Two dogs of different breeds playfully chasing around a tree"
example_prompt1 = "Two dogs of the same breed playing on the grass"
###

# # Create the first demo
# demo_vqascore = gr.Interface(
#     fn=generate,  # function to call
#     inputs=[
#         gr.Dropdown(["clip-flant5-xxl", "clip-flant5-xl", ], label="Model Name"), 
#         gr.Image(type="filepath"), 
#         gr.Textbox(label="Prompt")
#     ],  # define the types of inputs
#     examples=[
#         ["clip-flant5-xl", example_imgs[0], example_prompt0],
#         ["clip-flant5-xl", example_imgs[0], example_prompt1],
#         ],
#     outputs="number",  # define the type of output
#     title="VQAScore",  # title of the app
#     description="This model evaluates the similarity between an image and a text prompt."
# )

# # Create the second demo
# demo_vqascore_ranking = gr.Interface(
#     fn=rank_images,  # function to call
#     inputs=[
#         gr.Dropdown(["clip-flant5-xl", "clip-flant5-xxl"], label="Model Name"), 
#         gr.Gallery(label="Generated Images"), 
#         gr.Textbox(label="Prompt")
#     ],  # define the types of inputs
#     outputs=gr.Gallery(label="Ranked Images"),  # define the type of output
#     examples=[
#         ["clip-flant5-xl", [[img, ""] for img in example_imgs], example_prompt0],
#         ["clip-flant5-xl", [[img, ""] for img in example_imgs], example_prompt1]
        
#     ],
#     title="VQAScore Ranking",  # title of the app
#     description="This model ranks a gallery of images based on their similarity to a text prompt.",
#     allow_flagging='never'  
# )

# Custom component for loading examples
def load_example(model_name, images, prompt):
    return model_name, images, prompt


# demo_vqascore = gr.Interface(
#     fn=generate,  # function to call
#     inputs=[
#         gr.Dropdown(["clip-flant5-xxl", "clip-flant5-xl", ], label="Model Name"), 
#         gr.Image(type="filepath"), 
#         gr.Textbox(label="Prompt")
#     ],  # define the types of inputs
#     examples=[
#         ["clip-flant5-xl", example_imgs[0], example_prompt0],
#         ["clip-flant5-xl", example_imgs[0], example_prompt1],
#         ],
#     outputs="number",  # define the type of output
#     title="VQAScore",  # title of the app
#     description="This model evaluates the similarity between an image and a text prompt."
# )

# Create the second demo: VQAScore Ranking
with gr.Blocks() as demo_vqascore_ranking:
    # gr.Markdown("# VQAScore Ranking\nThis model ranks a gallery of images based on their similarity to a text prompt.")
    gr.Markdown("""
    # VQAScore Ranking
    This demo ranks a gallery of images by their VQAScores to an input text prompt. Try examples 1 and 2, or use your own images and prompts.
    If you encounter errors, the model may not have loaded on the GPU properly. Retrying usually resolves this issue.
    """)
    
    with gr.Row():
        with gr.Column():
            model_dropdown = gr.Dropdown(["clip-flant5-xxl", "clip-flant5-xl"], value="clip-flant5-xxl", label="Model Name")
            prompt = gr.Textbox(label="Prompt")
            gallery = gr.Gallery(label="Input Image(s)", elem_id="input-gallery", columns=4, allow_preview=True)
            rank_button = gr.Button("Submit")
            
        with gr.Column():

            ranked_gallery = gr.Gallery(label="Output: Ranked Images with Scores", elem_id="ranked-gallery", columns=4, allow_preview=True)
            rank_button.click(fn=rank_images, inputs=[model_dropdown, gallery, prompt], outputs=ranked_gallery)

            
            example1_button = gr.Button("Load Example 1")
            example2_button = gr.Button("Load Example 2")
            example1_button.click(fn=lambda: load_example("clip-flant5-xxl", example_imgs, example_prompt0), inputs=[], outputs=[model_dropdown, gallery, prompt])
            example2_button.click(fn=lambda: load_example("clip-flant5-xxl", example_imgs, example_prompt1), inputs=[], outputs=[model_dropdown, gallery, prompt])
    
# # Create the second demo
# with gr.Blocks() as demo_vqascore_ranking:
#     gr.Markdown("# VQAScore Ranking\nThis model ranks a gallery of images based on their similarity to a text prompt.")
#     model_dropdown = gr.Dropdown(["clip-flant5-xxl", "clip-flant5-xl"], value="clip-flant5-xxl",  label="Model Name")
#     gallery = gr.Gallery(label="Generated Images", elem_id="input-gallery", columns=4, allow_preview=True)
#     prompt = gr.Textbox(label="Prompt")
#     rank_button = gr.Button("Rank Images")
#     ranked_gallery = gr.Gallery(label="Ranked Images with Scores", elem_id="ranked-gallery", columns=4, allow_preview=True)
    
#     rank_button.click(fn=rank_images, inputs=[model_dropdown, gallery, prompt], outputs=ranked_gallery)
    
#     # Custom example buttons
#     example1_button = gr.Button("Load Example 1")
#     example2_button = gr.Button("Load Example 2")
    
#     example1_button.click(fn=lambda: load_example("clip-flant5-xxl", example_imgs, example_prompt0), inputs=[], outputs=[model_dropdown, gallery, prompt])
#     example2_button.click(fn=lambda: load_example("clip-flant5-xxl", example_imgs, example_prompt1), inputs=[], outputs=[model_dropdown, gallery, prompt])

#     # Layout to allow user to input their own data
#     with gr.Row():
#         gr.Column([model_dropdown, gallery, prompt, rank_button])
#         gr.Column([example1_button, example2_button])

# Launch the interface
demo_vqascore_ranking.queue()
demo_vqascore_ranking.launch(share=False)