app.py

from __future__ import annotations

import PIL
import gradio as gr
from pathlib import Path

import torch
import torchvision

from gator.datasets.shard.transforms import get_pair_transforms_gator
from gator.models.gator_2view.gator_wrapper import GatorWrapper
from gator.models.gator_2view.model_gator import Gator, GatorConfig
from gator.models.gator_losses.classification import GatorClassificationLoss
from gator.models.gator_visualizer.classification import ClassificationVis
from gator.models.jigsaw_1view.jigsaw_wrapper import OptimizationParameters
from gator.scripts.show.show_gator import shuffle_patches

_current_dir = Path(__file__).parent

class GatorGradioDemo:
    def __init__(self):
        self._config = GatorConfig(
            enc_emb_dim=384,
            enc_num_heads=6,
            dec_emb_dim=384,
            dec_num_heads=6,
        )

        self._checkpoint_path = _current_dir / "./checkpoints/gator-small-classification.ckpt"

        visualizer = ClassificationVis(grid_size=(14, 14), patch_size=16)
        loss_fn = GatorClassificationLoss(grid_size=(14, 14), patch_size=16)

        model = Gator(config=self._config)
        self._model_wrapper = GatorWrapper.load_from_checkpoint(
            checkpoint_path=self._checkpoint_path,
            model=model,
            visualizer=visualizer,
            loss_fn=loss_fn,
            optimization_config=OptimizationParameters(),
        )
        self._model_wrapper.eval()
        print(f"Loaded model from {self._checkpoint_path}!")

        self._transformation = get_pair_transforms_gator("resize224+crop224")
        self._to_pil = torchvision.transforms.ToPILImage()

    def forward(self, image1_pil: PIL.Image, image2_pil: PIL.Image) -> list[PIL.Image, PIL.Image]:
        img1, img2 = self._transformation(image1_pil, image2_pil)
        img1 = img1[None]
        img2 = img2[None]
        img1 = shuffle_patches(img1, self._model_wrapper._visualizer)

        with torch.inference_mode():
            out, chosen_ids, num_register_tokens = self._model_wrapper.forward(
                torch.stack([img1, img2], dim=1), shuffle_ratio=1.0,
            )

        images_pred = self._model_wrapper._visualizer.forward(
            pred=out.float(),
            gt_pos=chosen_ids,
            gt_image=img1,
            num_register_tokens=num_register_tokens,
        )

        return (
            self._to_pil(img1[0]),
            self._to_pil(images_pred[0]), 
        )

def main():
    print("Launching Gradio demo...")
    print("If running locally, open http://localhost:7860 in your browser.")
    
    title = "Gator Demo"
    description = "Upload two images and see how Gator unshuffles patches from one of them to match the other!"

    demo = GatorGradioDemo()
    gr.Interface(
        fn=demo.forward, 
        inputs=[
            gr.Image(type="pil", label="To-be shuffled"), 
            gr.Image(type="pil", label="Reference")
        ], 
        outputs=[
            gr.Image(type="pil", label="Shuffled"),
            gr.Image(type="pil", label="Unshuffled"), 
        ], 
        title=title,
        description=description,
        examples=[
            [
                PIL.Image.open(_current_dir / f"./gradio_examples/img{i}_1.jpg"),
                PIL.Image.open(_current_dir / f"./gradio_examples/img{i}_2.jpg"),
            ] for i in range(5)
        ]
    ).launch()

if __name__ == "__main__":
    main()