visualize.py

import os
import torch
from PIL import Image
import matplotlib.pyplot as plt
from datasets import load_dataset
from transformers import AutoProcessor, Qwen2VLForConditionalGeneration, BitsAndBytesConfig
from peft import PeftModel

# =========================
# CONFIG
# =========================
MODEL_NAME = "Qwen/Qwen2-VL-2B-Instruct"
CHECKPOINT_PATH = "./qlora-vlm"   # your saved model
JSONL_PATH = "0508_clean.jsonl"

NUM_SAMPLES = 3   # show 3 samples
IMAGE_SIZE = 512

# =========================
# LOAD MODEL (IMPORTANT)
# =========================
bnb_config = BitsAndBytesConfig(
    load_in_4bit=True,
    bnb_4bit_compute_dtype=torch.float16,
    bnb_4bit_use_double_quant=True,
    bnb_4bit_quant_type="nf4"
)

processor = AutoProcessor.from_pretrained(MODEL_NAME)

base_model = Qwen2VLForConditionalGeneration.from_pretrained(
    MODEL_NAME,
    quantization_config=bnb_config,
    device_map="auto",
)

model = PeftModel.from_pretrained(base_model, CHECKPOINT_PATH)
model.eval()

# =========================
# LOAD DATASET
# =========================
dataset = load_dataset("json", data_files=JSONL_PATH)

# =========================
# GENERATION FUNCTION
# =========================
def generate_markdown(image_path):
    image = Image.open(image_path).convert("RGB")
    image = image.resize((IMAGE_SIZE, IMAGE_SIZE))

    prompt = "Convert this document image into structured Markdown."

    messages = [
        {
            "role": "user",
            "content": [
                {"type": "image"},
                {"type": "text", "text": prompt}
            ]
        }
    ]

    text = processor.apply_chat_template(
        messages,
        tokenize=False,
        add_generation_prompt=True
    )

    inputs = processor(
        text=text,
        images=image,
        return_tensors="pt"
    ).to(model.device)

    with torch.no_grad():
        output = model.generate(
            **inputs,
            max_new_tokens=512,
            do_sample=False
        )

    result = processor.decode(output[0], skip_special_tokens=True)
    return result

# =========================
# VISUALIZATION FUNCTION
# =========================
def visualize_sample(sample, title="Sample"):
    image_path = sample["image"]
    gt_markdown = sample["markdown"]

    pred_markdown = generate_markdown(image_path)

    image = Image.open(image_path).convert("RGB")

    plt.figure(figsize=(10, 6))
    plt.imshow(image)
    plt.axis("off")
    plt.title(title)
    plt.show()

    print("\n" + "="*80)
    print("GROUND TRUTH:\n")
    print(gt_markdown[:2000])

    print("\n" + "-"*80)
    print("GENERATED:\n")
    print(pred_markdown[:2000])
    print("="*80)

# =========================
# RUN VISUALIZATION
# =========================
print("\n===== TRAIN SAMPLES =====\n")
for i in range(NUM_SAMPLES):
    visualize_sample(dataset[i], title=f"Train Sample {i+1}")