app.py

import gradio as gr
import tensorflow as tf
import tensorflow_hub as hub
from PIL import Image

import utils

_RESOLUTION = 224
_MODEL_URL = "https://tfhub.dev/sayakpaul/deit_tiny_patch16_224/1"


def get_model() -> tf.keras.Model:
    """Initiates a tf.keras.Model from TF-Hub."""
    inputs = tf.keras.Input((_RESOLUTION, _RESOLUTION, 3))
    hub_module = hub.KerasLayer(_MODEL_URL)

    logits, attention_scores_dict = hub_module(
        inputs
    )  # Second output in the tuple is a dictionary containing attention scores.

    return tf.keras.Model(inputs, [logits, attention_scores_dict])


_MODEL = get_model()


def show_rollout(image):
    """Function to be called when user hits submit on the UI."""
    _, preprocessed_image = utils.preprocess_image(
        image, "deit_tiny_patch16_224"
    )
    _, attention_scores_dict = _MODEL.predict(preprocessed_image)
    result = utils.attention_rollout_map(
        image, attention_scores_dict, "deit_tiny_patch16_224"
    )
    return Image.fromarray(result)


title = "Generate Attention Rollout Plots"
article = "Attention Rollout was proposed by [Abnar et al.](https://arxiv.org/abs/2005.00928) to quantify the information that flows through self-attention layers. In the original ViT paper ([Dosovitskiy et al.](https://arxiv.org/abs/2010.11929)), the authors use it to investigate the representations learned by ViTs. The model used in the backend is `deit_tiny_patch16_224`. For more details about it, refer [here](https://tfhub.dev/sayakpaul/collections/deit/1). DeiT was proposed by [Touvron et al.](https://arxiv.org/abs/2012.12877)"

iface = gr.Interface(
    show_rollout,
    inputs=gr.inputs.Image(type="pil", label="Input Image"),
    outputs="image",
    title=title,
    article=article,
    allow_flagging="never",
    examples=[["./car.jpeg", "./bulbul.jpeg"]],
)
iface.launch()