Update app.py

app.py

@@ -12,6 +12,7 @@ import torch
 import numpy as np
 from PIL import Image, ImageOps
 import cv2
+import pymupdf
 
 from transformers import (
     Qwen2VLForConditionalGeneration,
@@ -35,32 +36,40 @@ MAX_INPUT_TOKEN_LENGTH = int(os.getenv("MAX_INPUT_TOKEN_LENGTH", "4096"))
 
 device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
 
-# Load olmOCR-7B-0225-preview
-MODEL_ID_M = "allenai/olmOCR-7B-0225-preview"
-processor_m = AutoProcessor.from_pretrained(MODEL_ID_M, trust_remote_code=True)
-model_m = Qwen2VLForConditionalGeneration.from_pretrained(
-    MODEL_ID_M,
-    trust_remote_code=True,
-    torch_dtype=torch.float16
-).to(device).eval()
-
-# Load ByteDance's Dolphin
-MODEL_ID_K = "ByteDance/Dolphin"
-processor_k = AutoProcessor.from_pretrained(MODEL_ID_K, trust_remote_code=True)
-model_k = VisionEncoderDecoderModel.from_pretrained(
-    MODEL_ID_K,
-    trust_remote_code=True,
-    torch_dtype=torch.float16
-).to(device).eval()
-
-# Load SmolDocling-256M-preview
-MODEL_ID_X = "ds4sd/SmolDocling-256M-preview"
-processor_x = AutoProcessor.from_pretrained(MODEL_ID_X, trust_remote_code=True)
-model_x = AutoModelForVision2Seq.from_pretrained(
-    MODEL_ID_X,
-    trust_remote_code=True,
-    torch_dtype=torch.float16
-).to(device).eval()
+# Global variables for Dolphin model
+model_k = None
+processor_k = None
+tokenizer_k = None
+
+# Load models
+def initialize_models():
+    global model_k, processor_k, tokenizer_k
+    # Load olmOCR-7B-0225-preview
+    MODEL_ID_M = "allenai/olmOCR-7B-0225-preview"
+    processor_m = AutoProcessor.from_pretrained(MODEL_ID_M, trust_remote_code=True)
+    model_m = Qwen2VLForConditionalGeneration.from_pretrained(
+        MODEL_ID_M, trust_remote_code=True, torch_dtype=torch.float16
+    ).to(device).eval()
+
+    # Load ByteDance's Dolphin
+    MODEL_ID_K = "ByteDance/Dolphin"
+    processor_k = AutoProcessor.from_pretrained(MODEL_ID_K, trust_remote_code=True)
+    if model_k is None:
+        model_k = VisionEncoderDecoderModel.from_pretrained(
+            MODEL_ID_K, trust_remote_code=True, torch_dtype=torch.float16
+        ).to(device).eval()
+        tokenizer_k = processor_k.tokenizer
+
+    # Load SmolDocling-256M-preview
+    MODEL_ID_X = "ds4sd/SmolDocling-256M-preview"
+    processor_x = AutoProcessor.from_pretrained(MODEL_ID_X, trust_remote_code=True)
+    model_x = AutoModelForVision2Seq.from_pretrained(
+        MODEL_ID_X, trust_remote_code=True, torch_dtype=torch.float16
+    ).to(device).eval()
+
+    return processor_m, model_m, processor_x, model_x
+
+processor_m, model_m, processor_x, model_x = initialize_models()
 
 # Preprocessing functions for SmolDocling-256M
 def add_random_padding(image, min_percent=0.1, max_percent=0.10):
@@ -71,7 +80,7 @@ def add_random_padding(image, min_percent=0.1, max_percent=0.10):
     pad_h_percent = random.uniform(min_percent, max_percent)
     pad_w = int(width * pad_w_percent)
     pad_h = int(height * pad_h_percent)
-    corner_pixel = image.getpixel((0, 0))  # Top-left corner
+    corner_pixel = image.getpixel((0, 0))
     padded_image = ImageOps.expand(image, border=(pad_w, pad_h, pad_w, pad_h), fill=corner_pixel)
     return padded_image
 
@@ -109,11 +118,12 @@ def downsample_video(video_path):
     return frames
 
 # Dolphin-specific functions
+@spaces.GPU
 def model_chat(prompt, image, is_batch=False):
     """Use Dolphin model for inference, supporting both single and batch processing."""
-    processor = processor_k
-    model = model_k
-    device = "cuda" if torch.cuda.is_available() else "cpu"
+    global model_k, processor_k, tokenizer_k
+    if model_k is None:
+        initialize_models()
 
     if not is_batch:
         images = [image]
@@ -122,33 +132,30 @@ def model_chat(prompt, image, is_batch=False):
         images = image
         prompts = prompt if isinstance(prompt, list) else [prompt] * len(images)
 
-    inputs = processor(images, return_tensors="pt", padding=True).to(device)
+    inputs = processor_k(images, return_tensors="pt", padding=True).to(device)
     pixel_values = inputs.pixel_values.half()
 
     prompts = [f"<s>{p} <Answer/>" for p in prompts]
-    prompt_inputs = processor.tokenizer(
-        prompts,
-        add_special_tokens=False,  # Explicitly set to False
-        return_tensors="pt",
-        padding=True
+    prompt_inputs = tokenizer_k(
+        prompts, add_special_tokens=False, return_tensors="pt", padding=True
     ).to(device)
 
-    outputs = model.generate(
+    outputs = model_k.generate(
         pixel_values=pixel_values,
         decoder_input_ids=prompt_inputs.input_ids,
         decoder_attention_mask=prompt_inputs.attention_mask,
         min_length=1,
         max_length=4096,
-        pad_token_id=processor.tokenizer.pad_token_id,
-        eos_token_id=processor.tokenizer.eos_token_id,
+        pad_token_id=tokenizer_k.pad_token_id,
+        eos_token_id=tokenizer_k.eos_token_id,
         use_cache=True,
-        bad_words_ids=[[processor.tokenizer.unk_token_id]],
+        bad_words_ids=[[tokenizer_k.unk_token_id]],
         return_dict_in_generate=True,
         do_sample=False,
         num_beams=1,
         repetition_penalty=1.1
     )
-    sequences = processor.tokenizer.batch_decode(outputs.sequences, skip_special_tokens=False)
+    sequences = tokenizer_k.batch_decode(outputs.sequences, skip_special_tokens=False)
     
     results = []
     for i, sequence in enumerate(sequences):
@@ -157,6 +164,7 @@ def model_chat(prompt, image, is_batch=False):
 
     return results[0] if not is_batch else results
 
+@spaces.GPU
 def process_element_batch(elements, prompt, max_batch_size=16):
     """Process a batch of elements with the same prompt."""
     results = []
@@ -244,24 +252,41 @@ def generate_markdown(recognition_results):
             markdown += f"{element['text']}\n\n"
     return markdown.strip()
 
+def convert_to_image(image):
+    """Convert uploaded file to PIL Image, handling PDFs by extracting the first page."""
+    if isinstance(image, str):  # File path from Gradio
+        if image.lower().endswith('.pdf'):
+            doc = pymupdf.open(image)
+            page = doc[0]
+            pix = page.get_pixmap()
+            img_data = pix.tobytes("png")
+            pil_image = Image.open(io.BytesIO(img_data)).convert("RGB")
+            doc.close()
+            return pil_image
+        else:
+            return Image.open(image).convert("RGB")
+    elif isinstance(image, Image.Image):  # Already a PIL Image
+        return image.convert("RGB")
+    return None
+
 def process_image_with_dolphin(image):
     """Process a single image with Dolphin model."""
-    layout_output = model_chat("Parse the reading order of this document.", image)
-    elements = process_elements(layout_output, image)
+    pil_image = convert_to_image(image)
+    if pil_image is None:
+        return "Error: Unable to process the uploaded file."
+    layout_output = model_chat("Parse the reading order of this document.", pil_image)
+    elements = process_elements(layout_output, pil_image)
     markdown_content = generate_markdown(elements)
     return markdown_content
 
 @spaces.GPU
 def generate_image(model_name: str, text: str, image: Image.Image,
-                   max_new_tokens: int = 1024,
-                   temperature: float = 0.6,
-                   top_p: float = 0.9,
-                   top_k: int = 50,
-                   repetition_penalty: float = 1.2):
+                   max_new_tokens: int = 1024, temperature: float = 0.6,
+                   top_p: float = 0.9, top_k: int = 50, repetition_penalty: float = 1.2):
     """Generate responses for image input using the selected model."""
     if model_name == "ByteDance-s-Dolphin":
         if image is None:
-            yield "Please upload an image."
+            yield "Please upload an image or PDF (first page will be processed)."
             return
         markdown_content = process_image_with_dolphin(image)
         yield markdown_content
@@ -280,7 +305,10 @@ def generate_image(model_name: str, text: str, image: Image.Image,
             yield "Please upload an image."
             return
 
-        images = [image]
+        images = [convert_to_image(image)]
+        if images[0] is None:
+            yield "Error: Unable to process the uploaded file."
+            return
 
         if model_name == "SmolDocling-256M-preview":
             if "OTSL" in text or "code" in text:
@@ -334,11 +362,8 @@ def generate_image(model_name: str, text: str, image: Image.Image,
 
 @spaces.GPU
 def generate_video(model_name: str, text: str, video_path: str,
-                   max_new_tokens: int = 1024,
-                   temperature: float = 0.6,
-                   top_p: float = 0.9,
-                   top_k: int = 50,
-                   repetition_penalty: float = 1.2):
+                   max_new_tokens: int = 1024, temperature: float = 0.6,
+                   top_p: float = 0.9, top_k: int = 50, repetition_penalty: float = 1.2):
     """Generate responses for video input using the selected model."""
     if model_name == "ByteDance-s-Dolphin":
         if video_path is None:
@@ -346,10 +371,10 @@ def generate_video(model_name: str, text: str, video_path: str,
             return
         frames = downsample_video(video_path)
         markdown_contents = []
-        for frame, _ in frames:
+        for idx, (frame, _) in enumerate(frames):
             markdown_content = process_image_with_dolphin(frame)
-            markdown_contents.append(markdown_content)
-        combined_markdown = "\n\n".join(markdown_contents)
+            markdown_contents.append(f"**Frame {idx + 1}:**\n{markdown_content}")
+        combined_markdown = "\n\n---\n\n".join(markdown_contents)
         yield combined_markdown
     else:
         if model_name == "olmOCR-7B-0225-preview":
@@ -419,7 +444,7 @@ def generate_video(model_name: str, text: str, video_path: str,
             else:
                 yield cleaned_output
 
-# Define examples for image and video inference
+# Define examples
 image_examples = [
     ["Convert this page to docling", "images/1.png"],
     ["OCR the image", "images/2.jpg"],
@@ -441,28 +466,23 @@ css = """
 }
 """
 
-# Create the Gradio Interface
+# Create Gradio Interface
 with gr.Blocks(css=css, theme="bethecloud/storj_theme") as demo:
     gr.Markdown("# **[Docling-VLMs](https://huggingface.co/collections/prithivMLmods/multimodal-implementations-67c9982ea04b39f0608badb0)**")
+    gr.Markdown("**Note:** For Dolphin model, the text query is ignored, and PDFs are processed by parsing the first page.")
     with gr.Row():
         with gr.Column():
             with gr.Tabs():
                 with gr.TabItem("Image Inference"):
                     image_query = gr.Textbox(label="Query Input", placeholder="Enter your query here...")
-                    image_upload = gr.Image(type="pil", label="Image")
+                    image_upload = gr.Image(type="pil", label="Image or PDF")
                     image_submit = gr.Button("Submit", elem_classes="submit-btn")
-                    gr.Examples(
-                        examples=image_examples,
-                        inputs=[image_query, image_upload]
-                    )
+                    gr.Examples(examples=image_examples, inputs=[image_query, image_upload])
                 with gr.TabItem("Video Inference"):
                     video_query = gr.Textbox(label="Query Input", placeholder="Enter your query here...")
                     video_upload = gr.Video(label="Video")
                     video_submit = gr.Button("Submit", elem_classes="submit-btn")
-                    gr.Examples(
-                        examples=video_examples,
-                        inputs=[video_query, video_upload]
-                    )
+                    gr.Examples(examples=video_examples, inputs=[video_query, video_upload])
             with gr.Accordion("Advanced options", open=False):
                 max_new_tokens = gr.Slider(label="Max new tokens", minimum=1, maximum=MAX_MAX_NEW_TOKENS, step=1, value=DEFAULT_MAX_NEW_TOKENS)
                 temperature = gr.Slider(label="Temperature", minimum=0.1, maximum=4.0, step=0.1, value=0.6)