Update app.py

app.py

@@ -15,9 +15,18 @@ model = AutoModelForImageTextToText.from_pretrained(
     torch_dtype=torch.bfloat16
 ).to("cuda").eval()
 
-def create_scoring_instruction(prompt, is_editing=True):
-    if is_editing:
-        return f"""You are an expert image editing evaluator. Your task is to evaluate the quality of an edited image based on a source image and a user instruction. Afterwards, you need to suggest how to refine the original user request to produce better image edits (if any).
+TASK_CHOICES = [
+    "Pointwise - Image Editing",
+    "Pointwise - T2I Generation",
+    "Pairwise - Image Editing",
+    "Pairwise - T2I Generation",
+]
+
+# ============================================================
+# Instruction Templates
+# ============================================================
+
+POINTWISE_EDITING_INSTRUCTION = """You are an expert image editing evaluator. Your task is to evaluate the quality of an edited image based on a source image and a user instruction. Afterwards, you need to suggest how to refine the original user request to produce better image edits (if any).
 
 User Instruction: {prompt}
 You are provided with two images:
@@ -83,13 +92,11 @@ Output your evaluation in the following format:
 # User Request Refinement:
 ## Refinement Comments: [Specific suggestions for improving the user request]
 ## Refined Request: [The improved, more specific user request for editing like a standard user instruction]"""
-    else:
-        # Generation Prompt (Simplified, no Image Faithfulness)
-        return f"""You are an expert image generation evaluator. Your task is to evaluate the quality of a generated image based on a user instruction. Afterwards, you need to suggest how to refine the original user request to produce better images (if any).
+
+POINTWISE_T2I_INSTRUCTION = """You are an expert image generation evaluator. Your task is to evaluate the quality of a generated image based on  a user instruction. Afterwards, you need to suggest how to refine the original user request to produce better image generation (if any).
 
 User Instruction: {prompt}
-You are provided with one image:
-1. Generated Image <image>
+Generated Image: <image>
 
 Your task is to evaluate the Generated Image against the User Instruction.
 To do this, you must first assess the image on three critical aspects, provide justifications and absolute scores in 1-4 scale.
@@ -102,7 +109,7 @@ To do this, you must first assess the image on three critical aspects, provide j
 - **1 (Major deviations):** Key elements are completely missing, altered, or contradicted. Instruction is ignored.
 
 **2. Physical and Visual Quality** (Technical errors, composition, realism, and physics)
-- **4 (No noticeable flaws):** The image is physically plausible (correct lighting, shadows, geometry, anatomy). No visible artifacts (seams, blurring, noise).
+- **4 (No noticeable flaws):** The image is physically plausible (correct lighting, shadows, geometry, human anatomy). No visible artifacts (seams, blurring, noise). And all elements work together cohesively.
 - **3 (Minor flaws):** Small inaccuracies that are noticeable but not strongly disruptive (e.g., slight lighting mismatch, minor texture issues).
 - **2 (Some flaws):** Clear physical or visual errors that disrupt the image (e.g., incorrect perspective, "floating" objects, wrong shadow direction, obvious seams).
 - **1 (Severe flaws):** Major physical/visual errors (e.g., impossible geometry, distorted anatomy, garbled objects, severe artifacts).
@@ -114,11 +121,12 @@ To do this, you must first assess the image on three critical aspects, provide j
 - **1 (Major deviations):** Text is unreadable, severely distorted, or missing. (Use N/A if no text generation is required).
 
 ### Scoring Methodology (CRITICAL)
-During assessment for each aspect, recall the initial user request and the scoring rubrics of the aspect, provide scores with detailed justifications for each image and reflect fine-grained preferences.
-1. **Anchor:** Have a global inspection based on the user request and the resulting generation. Determine the rough integer score level (1, 2, 3, or 4) according to the definitions provided.
+During assessment for each aspect, recall the initial user request, source image and the scoring rubrics of the aspect, provide scores with detailed justifications for each image and reflect fine-grained preferences.
+1. **Anchor:** Have a global inspection based on the user request and the resulting generation. Determine the rough integer score level (1, 2, 3, or 4) according to the definitions provided .
 2. **Justify and Adjust:** Do careful visual analysis and identify specific flaws in generation. Justify the score with concrete evidence and scoring logic. Fine-tune this anchor score into a float value. Add small increments for exceptional execution or deduct points for specific flaws.
+   - *Example:* deduct points from 4.0 for slight flaws if the assessed dimension is close to satisfaction. add increments from 1.0 or 2.0 based on severity of flaws.
 
-Afterwards, try to construct a refined user request that helps the visual generation model to produce better images.
+Afterwards, try to construct a refined user request that helps the visual generation model to produce better image edits.
 Think of the weaknesses identified in the judgement, then map them to instruction details and apply specific fixes.
 Provide a final new user request that enrich the initial user request.
 
@@ -138,56 +146,219 @@ Output your evaluation in the following format:
 # Summary: [ Summary of the evaluation ]
 
 # User Request Refinement:
-## Refinement Comments: [Specific suggestions for improving the user request]
-## Refined Request: [The improved, more specific user request for editing like a standard user instruction]"""
+## Refinement Comments: [ Specific suggestions for improving generation quality ]
+## Refined Request: [ The improved, more specific user request ]"""
+
+PAIRWISE_EDITING_INSTRUCTION = """You are an expert image editing evaluator. Your task is to evaluate the quality of an edited image based on a source image and a user instruction.
+
+User Instruction: {prompt}
+You are provided with three images:
+1. Source Image <image>
+2. Edited Image A <image>
+3. Edited Image B <image>
+
+Your task is to compare the two Edited Images according to the User Instruction and source image.
+To do this, you must compare the image on four critical aspects, provide absolute scores for each image and determine who wins. 
+
+### Critical Aspects & Scoring Rubric
+**1. Text Faithfulness** (How accurately does the output follow the instruction?)
+- **4 (Full match):** All key elements (objects, colors, actions) are represented exactly as described. No hallucinations or unrequested changes.
+- **3 (Minor mismatch):** Most key elements are present, but minor details are missing, incorrect, or slightly inaccurate.
+- **2 (Some mismatch):** Some key elements are missing, altered, or interpreted incorrectly.
+- **1 (Major deviations):** Key elements are completely missing, altered, or contradicted. Instruction is ignored.
+
+**2. Image Faithfulness** (How well are the non-edited parts and key input elements preserved?)
+- **4 (Uses input fully):** All relevant elements from the input (background, style, lighting, identity) are accurately preserved or transformed as instructed.
+- **3 (Minor mismatch):** Most relevant elements are preserved, but a few aspects (e.g., background details, lighting consistency) are missing or incorrectly handled.
+- **2 (Partial mismatch):** Some elements are carried over, but key aspects of the original image are lost or distorted.
+- **1 (Fails to use input):** Key elements of the input image are ignored, misinterpreted, or destroyed.
+
+**3. Physical and Visual Quality** (Technical errors, composition, realism, and physics)
+- **4 (No noticeable flaws):** The image is physically plausible (correct lighting, shadows, geometry, anatomy). No visible artifacts (seams, blurring, noise).
+- **3 (Minor flaws):** Small inaccuracies that are noticeable but not strongly disruptive (e.g., slight lighting mismatch, minor texture issues).
+- **2 (Some flaws):** Clear physical or visual errors that disrupt the image (e.g., incorrect perspective, "floating" objects, wrong shadow direction, obvious seams).
+- **1 (Severe flaws):** Major physical/visual errors (e.g., impossible geometry, distorted anatomy, garbled objects, severe artifacts).
+
+**4. Text Rendering** (Only if the instruction involves generating text)
+- **4 (Full match):** Text is correct, legible, and integrated well.
+- **3 (Mostly match):** Minor misspellings or inconsistent capitalization.
+- **2 (Partial match):** Major misspellings or distorted text.
+- **1 (Major deviations):** Text is unreadable, severely distorted, or missing. (Use N/A if no text generation is required).
+
+### Scoring Methodology (CRITICAL)
+During assessment for each aspect, recall the initial user request, source image and the scoring rubrics of the aspect, provide scores with detailed justifications for each image and reflect fine-grained preferences.  
+1. **Anchor:** Have a global inspection. Determine the rough integer score level (1, 2, 3, or 4) according to the definitions provided (you can also refer to the given human preference or rating). 
+2. **Justify and Adjust:** Do careful visual analysis and identify specific flaws in generation. Justify the score with concrete evidence and scoring logic. Fine-tune this anchor score into a float value. Add small increments for exceptional execution or deduct points for specific flaws. 
+   - *Example:* deduct points from 4.0 for slight flaws if the assessed dimension is close to satisfaction. add increments from 1.0 or 2.0 based on severity of flaws.
+3. **Compare:** Ensure the difference between Score A and Score B reflects the correct preference.  
+
+Output your evaluation in the following format:
+# User Request Analysis 
+[ understanding the user request, and what needs to be considered during image editing ]
+# Detailed Judgement
+1. Text Faithfulness: 
+## Justification: [ Comparative Analysis: Given the request, source image and the scoring rubrics, which image is better in this dimension? Provide concrete evidence and scoring logic. e.g., Image A is roughly [X] score level because [reason]. Deduct/Add points for [specific details] to reach final score. ]
+## Score A: [float score for Image A]
+## Score B: [float score for Image B]
+## Winner: [Image A or Image B or It's a tie]
+2. Image Faithfulness: 
+## Justification: [ Similar to above. Comparative analysis with concrete evidence and scoring logic for image faithfulness. ]
+## Score A: [float score for Image A]
+## Score B: [float score for Image B]
+## Winner: [Image A or Image B or It's a tie] 
+3. Physical and Visual Quality: 
+## Justification: [ Similar to above. Comparative analysis with concrete evidence and scoring logic. Since physical/visual quality is often not perfect, give 4.0 sparingly only when it is perfectly realistic. ]
+## Score A: [float score for Image A]
+## Score B: [float score for Image B]
+## Winner: [Image A or Image B or It's a tie]
+4. Text Rendering: 
+## Justification: [ Similar to above. Comparative analysis with concrete evidence and scoring logic. Since text rendering is often challenging, give 4.0 sparingly only if it is perfect. ]
+## Score A: [float score for Image A]
+## Score B: [float score for Image B]
+## Winner: [N/A or Image A or Image B or It's a tie]
+# Summary: [Summary of the evaluation]"""
+
+PAIRWISE_T2I_INSTRUCTION = """You are an expert image evaluator. Your task is to evaluate the quality of two generated images based on a user instruction.
+
+User Instruction: {prompt}
+You are provided with two images:
+1. Generated Image A <image>
+2. Generated Image B <image>
+
+Your task is to compare the two Generated Images according to the User Instruction.
+To do this, you must compare the image on three critical aspects, provide absolute scores for each image and determine who wins. 
+
+### Critical Aspects & Scoring Rubric
+**1. Text Faithfulness** (How accurately does the output follow the instruction?)
+- **4 (Full match):** All key elements (objects, colors, actions) are represented exactly as described. No hallucinations or unrequested changes.
+- **3 (Minor mismatch):** Most key elements are present, but minor details are missing, incorrect, or slightly inaccurate.
+- **2 (Some mismatch):** Some key elements are missing, altered, or interpreted incorrectly.
+- **1 (Major deviations):** Key elements are completely missing, altered, or contradicted. Instruction is ignored.
+
+**2. Physical and Visual Quality** (Technical errors, composition, realism, and physics)
+- **4 (No noticeable flaws):** The image is physically plausible (correct lighting, shadows, geometry, anatomy). No visible artifacts (seams, blurring, noise).
+- **3 (Minor flaws):** Small inaccuracies that are noticeable but not strongly disruptive (e.g., slight lighting mismatch, minor texture issues).
+- **2 (Some flaws):** Clear physical or visual errors that disrupt the image (e.g., incorrect perspective, "floating" objects, wrong shadow direction, obvious seams).
+- **1 (Severe flaws):** Major physical/visual errors (e.g., impossible geometry, distorted anatomy, garbled objects, severe artifacts).
+
+**3. Text Rendering** (Only if the instruction involves generating text)
+- **4 (Full match):** Text is correct, legible, and integrated well.
+- **3 (Mostly match):** Minor misspellings or inconsistent capitalization.
+- **2 (Partial match):** Major misspellings or distorted text.
+- **1 (Major deviations):** Text is unreadable, severely distorted, or missing. (Use N/A if no text generation is required).
+
+### Scoring Methodology (CRITICAL)
+For every aspect, you must first recap the initial user request and the scoring rubrics of the aspect, then follow this "Anchor and Adjust" process to compare and score the two images:
+1. **Anchor:** Determine the rough integer score level (1, 2, 3, or 4) based on the definitions provided.
+2. **Adjust:** Fine-tune this anchor score into a float value. Add small increments for exceptional execution or deduct points for specific flaws. 
+   - *Example:* deduct points from 4.0 for slight flaws if the assessed dimension is close to satisfaction. add increments from 1.0 or 2.0 based on severity of flaws.
+3. **Compare:** Ensure the difference between Score A and Score B reflects the magnitude of the preference. (e.g., A large gap implies one is significantly better; if one is only slightly better, the fine-grained scorings based on identified flaws help explain the preference).
+
+Output your evaluation in the following format:
+# User Request Analysis 
+[ understanding the user request, try to analyze or decompose the user request deeper. Think of what the request might imply or what needs to be inferred to successfully execute the request. ]
+# Detailed Judgement
+1. Text Faithfulness: 
+## Justification: [ Comparative Analysis: Given the request and the scoring rubrics, which image is better in this dimension? Provide concrete evidence and scoring logic. e.g., Image A is roughly [X] score level because [reason]. Deduct/Add points for [specific details] to reach final score. ]
+## Score A: [float score for Image A]
+## Score B: [float score for Image B]
+## Winner: [Image A or Image B or It's a tie]
+2. Physical and Visual Quality: 
+## Justification: [ Similar to above. Comparative analysis with concrete evidence and scoring logic. Since physical/visual quality is often not perfect, give 4.0 sparingly only when it is perfectly realistic. ]
+## Score A: [float score for Image A]
+## Score B: [float score for Image B]
+## Winner: [Image A or Image B or It's a tie]
+3. Text Rendering: 
+## Justification: [ Similar to above. Comparative analysis with concrete evidence and scoring logic. Since text rendering is often challenging, give 4.0 sparingly only if it is perfect. ]
+## Score A: [float score for Image A]
+## Score B: [float score for Image B]
+## Winner: [N/A or Image A or Image B or It's a tie]
+# Summary: [Summary of the evaluation]"""
+
+def create_instruction(prompt, task_type):
+    """Create the appropriate instruction based on the task type."""
+    if task_type == "Pointwise - Image Editing":
+        return POINTWISE_EDITING_INSTRUCTION.format(prompt=prompt)
+    elif task_type == "Pointwise - T2I Generation":
+        return POINTWISE_T2I_INSTRUCTION.format(prompt=prompt)
+    elif task_type == "Pairwise - Image Editing":
+        return PAIRWISE_EDITING_INSTRUCTION.format(prompt=prompt)
+    elif task_type == "Pairwise - T2I Generation":
+        return PAIRWISE_T2I_INSTRUCTION.format(prompt=prompt)
+    else:
+        raise ValueError(f"Unknown task type: {task_type}")
+
+def update_ui_for_task(task_type):
+    """Update image component visibility and labels based on selected task type."""
+    if task_type == "Pointwise - Image Editing":
+        return (
+            gr.update(visible=True, label="Source Image"),
+            gr.update(visible=True, label="Edited Image"),
+            gr.update(visible=False, label="Image B", value=None),
+        )
+    elif task_type == "Pointwise - T2I Generation":
+        return (
+            gr.update(visible=True, label="Generated Image"),
+            gr.update(visible=False, label="(unused)", value=None),
+            gr.update(visible=False, label="(unused)", value=None),
+        )
+    elif task_type == "Pairwise - Image Editing":
+        return (
+            gr.update(visible=True, label="Source Image"),
+            gr.update(visible=True, label="Image A"),
+            gr.update(visible=True, label="Image B"),
+        )
+    elif task_type == "Pairwise - T2I Generation":
+        return (
+            gr.update(visible=True, label="Image A"),
+            gr.update(visible=True, label="Image B"),
+            gr.update(visible=False, label="(unused)", value=None),
+        )
 
 @spaces.GPU
-def model_inference(instruction_text, source_image, edited_image):
-    # Determine files list based on inputs
-    files = []
-    if source_image:
-        files.append(source_image)
-    if edited_image:
-        files.append(edited_image)
-    
-    # Check if files are provided
-    if not files or len(files) == 0:
-        yield "Error: Please upload at least one image (Generated Image or Source Image)."
+def model_inference(task_type, instruction_text, image1, image2, image3):
+    """Run model inference based on the selected task type and uploaded images."""
+    # Validate inputs and collect images based on task
+    if task_type == "Pointwise - Image Editing":
+        if not image1 or not image2:
+            yield "Error: Please upload both Source Image and Edited Image."
+            return
+        files = [image1, image2]
+    elif task_type == "Pointwise - T2I Generation":
+        if not image1:
+            yield "Error: Please upload the Generated Image."
+            return
+        files = [image1]
+    elif task_type == "Pairwise - Image Editing":
+        if not image1 or not image2 or not image3:
+            yield "Error: Please upload Source Image, Image A, and Image B."
+            return
+        files = [image1, image2, image3]
+    elif task_type == "Pairwise - T2I Generation":
+        if not image1 or not image2:
+            yield "Error: Please upload both Image A and Image B."
+            return
+        files = [image1, image2]
+    else:
+        yield "Error: Unknown task type selected."
         return
 
-    # Determine mode based on number of images
-    is_editing = len(files) >= 2
-    
     # Load images
     loaded_images = [load_image(image) for image in files]
-    
-    # Construct content for the model
-    instruction = create_scoring_instruction(instruction_text, is_editing=is_editing)
-    
+
+    # Build instruction with <image> placeholders
+    instruction = create_instruction(instruction_text, task_type)
+
     # Interleave images into the <image> placeholders
     content = []
     parts = instruction.split("<image>")
-    
-    # We expect 2 parts for 1 image (Generation), and 3 parts for 2 images (Editing)
-    if is_editing:
-        # Expect 2 images
-        content.append({"type": "text", "text": parts[0]})
-        content.append({"type": "image", "image": loaded_images[0]}) # Source
-        content.append({"type": "text", "text": parts[1]})
-        content.append({"type": "image", "image": loaded_images[1]}) # Edited
-        if len(parts) > 2:
-            content.append({"type": "text", "text": parts[2]})
-    else:
-        # Expect 1 image
-        content.append({"type": "text", "text": parts[0]})
-        content.append({"type": "image", "image": loaded_images[0]}) # Generated
-        if len(parts) > 1:
-            content.append({"type": "text", "text": parts[1]})
-
-    messages = [
-        {"role": "user", "content": content}
-    ]
-    
+    for i, part in enumerate(parts):
+        content.append({"type": "text", "text": part})
+        if i < len(loaded_images):
+            content.append({"type": "image", "image": loaded_images[i]})
+
+    messages = [{"role": "user", "content": content}]
+
     # Generate and stream text
     prompt = processor.apply_chat_template(messages, tokenize=False, add_generation_prompt=True)
     inputs = processor(
@@ -196,7 +367,7 @@ def model_inference(instruction_text, source_image, edited_image):
         return_tensors="pt",
         padding=True,
     ).to("cuda")
-    
+
     streamer = TextIteratorStreamer(processor, skip_prompt=True, skip_special_tokens=True)
     generation_kwargs = dict(inputs, streamer=streamer, max_new_tokens=2048)
 
@@ -208,37 +379,78 @@ def model_inference(instruction_text, source_image, edited_image):
         buffer += new_text
         yield buffer
 
+# ============================================================
+# Gradio UI
+# ============================================================
+
 with gr.Blocks() as demo:
     gr.HTML(html_header)
-    
+
+    with gr.Row():
+        task_selector = gr.Radio(
+            choices=TASK_CHOICES,
+            value="Pointwise - Image Editing",
+            label="Task Type",
+            info="Select the evaluation task",
+        )
+
     with gr.Row():
         with gr.Column(scale=1):
-            instruction = gr.Textbox(label="User Instruction", lines=5, placeholder="Enter editing instruction here...")
+            instruction = gr.Textbox(
+                label="User Instruction",
+                lines=5,
+                placeholder="Enter the user instruction / prompt here..."
+            )
             submit_btn = gr.Button("Evaluate", variant="primary")
-        
+
+        with gr.Column(scale=1):
+            image1 = gr.Image(
+                label="Source Image",
+                type="filepath",
+                sources=["upload", "clipboard"],
+            )
+
         with gr.Column(scale=1):
-            source_image = gr.Image(label="Source Image", type="filepath", sources=["upload", "clipboard"])
-            
+            image2 = gr.Image(
+                label="Edited Image",
+                type="filepath",
+                sources=["upload", "clipboard"],
+            )
+
         with gr.Column(scale=1):
-            edited_image = gr.Image(label="Edited Image", type="filepath", sources=["upload", "clipboard"])
-            
+            image3 = gr.Image(
+                label="Image B",
+                type="filepath",
+                sources=["upload", "clipboard"],
+                visible=False,
+            )
+
     output = gr.Textbox(label="Evaluation Result", lines=25)
 
+    # Wire task selector to update image visibility/labels
+    task_selector.change(
+        fn=update_ui_for_task,
+        inputs=[task_selector],
+        outputs=[image1, image2, image3],
+    )
+
+    # Wire evaluate button
     submit_btn.click(
         fn=model_inference,
-        inputs=[instruction, source_image, edited_image],
-        outputs=output
+        inputs=[task_selector, instruction, image1, image2, image3],
+        outputs=output,
     )
 
+    # Examples for different tasks
     gr.Examples(
         examples=[
-            ["Remove the arrows from the blue sign and add the text of Detour ahead, no right turns.", "example_images/0016cb70b187efe39969766dc4b3f9ed_b63ed6db519f685c33b860b511879cfe2fa7351059a17ebe5eafa83213e222fb_13_source.png", "example_images/0016cb70b187efe39969766dc4b3f9ed_b63ed6db519f685c33b860b511879cfe2fa7351059a17ebe5eafa83213e222fb_13_ovis_u1_Image A.png"]
+            ["Pointwise - Image Editing", "Remove the arrows from the blue sign and add the text of Detour ahead, no right turns.", "example_images/0016cb70b187efe39969766dc4b3f9ed_b63ed6db519f685c33b860b511879cfe2fa7351059a17ebe5eafa83213e222fb_13_source.png", "example_images/0016cb70b187efe39969766dc4b3f9ed_b63ed6db519f685c33b860b511879cfe2fa7351059a17ebe5eafa83213e222fb_13_ovis_u1_Image A.png", None],
         ],
-        inputs=[instruction, source_image, edited_image]
+        inputs=[task_selector, instruction, image1, image2, image3],
     )
-    
+
     gr.Markdown(tos_markdown)
     gr.Markdown(learn_more_markdown)
     gr.Markdown(bibtext)
 
-demo.launch(debug=True)
+demo.launch(debug=True)