Feat: App file and visualize.py

.gitignore

@@ -0,0 +1 @@
+.idea/

app.py

@@ -0,0 +1,237 @@
+#!/usr/bin/env python3
+"""
+Gradio Application for model trained on CIFAR10 dataset
+Author: Shilpaj Bhalerao
+Date: Aug 06, 2023
+"""
+# Standard Library Imports
+import os
+from collections import OrderedDict
+
+# Third-Party Imports
+import gradio as gr
+import numpy as np
+import torch
+from torchvision import transforms
+from pytorch_grad_cam import GradCAM
+from pytorch_grad_cam.utils.image import show_cam_on_image
+
+# Local Imports
+from resnet import LITResNet
+from visualize import FeatureMapVisualizer
+
+# Directory Path
+example_directory = 'examples/'
+model_path = 'epoch=23-step=2112.ckpt'
+
+classes = ('plane', 'car', 'bird', 'cat', 'deer',
+           'dog', 'frog', 'horse', 'ship', 'truck')
+
+model = LITResNet.load_from_checkpoint(model_path, map_location=torch.device('cpu'), strict=False, class_names=classes)
+model.eval()
+
+# Create an object of the Class
+viz = FeatureMapVisualizer(model)
+
+
+def inference(input_img,
+              transparency=0.5,
+              number_of_top_classes=3,
+              target_layer_number=4):
+    """
+    Function to run inference on the input image
+    :param input_img: Image provided by the user
+    :parma transparency: Percentage of cam overlap over the input image
+    :param number_of_top_classes: Number of top predictions for the input image
+    :param target_layer_number: Layer for which GradCam to be shown
+    """
+    # Calculate mean over each channel of input image
+    mean_r, mean_g, mean_b = np.mean(input_img[:, :, 0]/255.), np.mean(input_img[:, :, 1]/255.), np.mean(input_img[:, :, 2]/255.)
+
+    # Calculate Standard deviation over each channel
+    std_r, std_g, std_b = np.std(input_img[:, :, 0]/255.), np.std(input_img[:, :, 1]/255.), np.std(input_img[:, :, 2]/255.)
+
+    # Convert img to tensor and normalize it
+    _transform = transforms.Compose([
+        transforms.ToTensor(),
+        transforms.Normalize((mean_r, mean_g, mean_b), (std_r, std_g, std_b))
+        ])
+
+    # Save a copy of input img
+    org_img = input_img
+
+    # Apply the transforms on the input image
+    input_img = _transform(input_img)
+
+    # Add batch dimension to perform inference
+    input_img = input_img.unsqueeze(0)
+
+    # Get Model Predictions
+    with torch.no_grad():
+        outputs = model(input_img)
+        o = torch.exp(outputs)[0]
+        confidences = {classes[i]: float(o[i]) for i in range(10)}
+
+    # Select the top classes based on user input
+    sorted_confidences = sorted(confidences.items(), key=lambda val: val[1], reverse=True)
+    show_confidences = OrderedDict(sorted_confidences[:number_of_top_classes])
+
+    # Name of layers defined in the model
+    _layers = ['prep_layer', 'custom_block1', 'resnet_block1',
+               'custom_block2', 'custom_block3', 'resnet_block3']
+    target_layers = [eval(f'model.{_layers[target_layer_number-1]}[0]')]
+
+    # Get the class activations from the selected layer
+    cam = GradCAM(model=model, target_layers=target_layers, use_cuda=False)
+    grayscale_cam = cam(input_tensor=input_img, targets=None)
+    grayscale_cam = grayscale_cam[0, :]
+
+    # Overlay input image with Class activations
+    visualization = show_cam_on_image(org_img/255, grayscale_cam, use_rgb=True, image_weight=transparency)
+    return show_confidences, visualization
+
+
+def display_misclassified_images(number: int = 1):
+    """
+    Display the misclassified images saved during training
+    :param number: Number of images to display
+    """
+    # List to store names of misclassified images
+    data = []
+
+    # Get the names of all the files from Misclassified directory
+    file_names = os.listdir('misclassified/')
+
+    # Save the correct name and misclassified class name as a tuple in the `data` list
+    for file in file_names:
+        file_name, extension = file.split('.')
+        correct_label, misclassified = file_name.split('_')
+        data.append((correct_label, misclassified))
+
+    # Create a path to the images for Gradio to access them
+    file_path = ['misclassified/' + file for file in file_names]
+
+    # Return the file path and names of correct and misclassified images
+    return file_path[:number], data[:number]
+
+
+def feature_maps(input_img, kernel_number=32):
+    """
+    Function to return feature maps for the selected image
+    :param input_img: User input image
+    :param kernel_number: Number of kernel in all 6 layers
+    """
+    # Calculate mean over each channel of input image
+    mean_r, mean_g, mean_b = np.mean(input_img[:, :, 0]/255.), np.mean(input_img[:, :, 1]/255.), np.mean(input_img[:, :, 2]/255.)
+
+    # Calculate Standard deviation over each channel
+    std_r, std_g, std_b = np.std(input_img[:, :, 0]/255.), np.std(input_img[:, :, 1]/255.), np.std(input_img[:, :, 2]/255.)
+
+    # Convert img to tensor and normalize it
+    _transform = transforms.Compose([
+        transforms.ToTensor(),
+        transforms.Normalize((mean_r, mean_g, mean_b), (std_r, std_g, std_b))
+        ])
+
+    # Apply transforms on the input image
+    input_img = _transform(input_img)
+
+    # Visualize feature maps for kernel number 32
+    plt = viz.visualize_feature_map_of_kernel(image=input_img, kernel_number=kernel_number)
+    return plt
+
+
+def get_kernels(layer_number):
+    """
+    Function to get the kernels from the layer
+    :param layer_number: Number of layer from which kernels to be visualized
+    """
+    # Visualize kernels from layer
+    plt = viz.visualize_kernels_from_layer(layer_number=layer_number)
+    return plt
+
+
+if __name__ == '__main__':
+    with gr.Blocks() as demo:
+        gr.Markdown(
+            """
+            # CIFAR10 trained on ResNet18 Model
+               - A model architecture by [David C](https://github.com/davidcpage) which is trained on CIFAR10 for 24 Epochs to achieve accuracy of 90+%
+               - One Cycle Policy is used during training to speed up the trainig process
+               - The model works for following classes: `plane`, `car`, `bird`, `cat`, `deer`, `dog`, `frog`, `horse`, `ship`, `truck`
+
+            ### A simple Gradio interface
+            - To infer what exactly the model is looking at using GradCAM results
+            - To display the misclassified images from the 10% of test data of CIFAR10 dataset
+            - To visualize the feature maps from each of the six convolutional block's first layer
+            - To visualize the kernels from each of the six convolutional block's first layer
+            """
+        )
+
+        # #############################################################################
+        # ################################ GradCam Tab ################################
+        # #############################################################################
+        with gr.Tab("GradCam"):
+            with gr.Row():
+                img_input = [gr.Image(shape=(32, 32), label="Input Image")]
+                gradcam_outputs = [gr.Label(),
+                                   gr.Image(shape=(32, 32), label="Output").style(width=128, height=128)]
+
+            with gr.Row():
+                gradcam_inputs = [gr.Slider(0, 1, value=0.5,
+                                            label="How much percentage overlap of what model is looking at in the image?"),
+                                  gr.Slider(1, 10, value=3, step=1, label="How many top predictions you want to see?"),
+                                  gr.Slider(1, 6, value=4, step=1,
+                                            label="From 6 layers of the model, which layer's class activation you want to see?")]
+
+            gradcam_button = gr.Button("Submit")
+            gradcam_button.click(inference, inputs=img_input + gradcam_inputs, outputs=gradcam_outputs)
+
+            gr.Markdown("## Examples")
+            gr.Examples([example_directory + 'dog.jpg', example_directory + 'cat.jpg', example_directory + 'frog.jpg', example_directory + 'bird.jpg', example_directory + 'shark-plane.jpg',
+                         example_directory + 'car.jpg', example_directory + 'truck.jpg', example_directory + 'horse.jpg', example_directory + 'plane.jpg', example_directory + 'ship.png'],
+                        inputs=img_input, fn=inference)
+
+        # ###########################################################################################
+        # ################################ Misclassified Images Tab #################################
+        # ###########################################################################################
+        with gr.Tab("Misclassified Images"):
+            with gr.Row():
+                mis_inputs = [gr.Slider(1, 10, value=1, step=1,
+                                        label="Select the Number of Misclassified Images you want to see")]
+                mis_outputs = [
+                    gr.Gallery(label="Misclassified Images", show_label=False, elem_id="gallery").style(columns=[2],
+                                                                                                        rows=[2],
+                                                                                                        object_fit="contain",
+                                                                                                        height="auto"),
+                    gr.Dataframe(headers=["Correct Label", "Misclassified Label"], type="array", datatype="str",
+                                 row_count=10, col_count=2)]
+            mis_button = gr.Button("Display Misclassified Images")
+            mis_button.click(display_misclassified_images, inputs=mis_inputs, outputs=mis_outputs)
+
+        # ################################################################################################
+        # ################################ Feature Maps Visualization Tab ################################
+        # ################################################################################################
+        with gr.Tab("Feature Map Visualization"):
+            with gr.Column():
+                feature_map_input = [gr.Image(shape=(32, 32), label="Feature Map Input Image"),
+                                     gr.Slider(1, 32, value=16, step=1,
+                                               label="Select a Kernel number for which features maps from all 6 layers to be shown")]
+                map = gr.Plot().style()
+                feature_map_button = gr.Button("Visualize FeatureMaps")
+            feature_map_button.click(feature_maps, inputs=feature_map_input, outputs=map)
+
+        # ##########################################################################################
+        # ################################ Kernel Visualization Tab ################################
+        # ##########################################################################################
+        with gr.Tab("Kernel Visualization"):
+            with gr.Column():
+                kernel_input = [
+                    gr.Slider(1, 4, value=2, step=1, label="Select a layer number from which the kernels to be shown")]
+                map = gr.Plot().style()
+                kernel_button = gr.Button("Visualize Kernels")
+
+            kernel_button.click(get_kernels, inputs=kernel_input, outputs=map)
+
+    gr.close_all()
+    demo.launch(debug=True, share=True)

visualize.py

@@ -0,0 +1,386 @@
+#!/usr/bin/env python3
+"""
+Function used for visualization of data and results
+Author: Shilpaj Bhalerao
+Date: Jun 21, 2023
+"""
+# Standard Library Imports
+import math
+from dataclasses import dataclass
+from typing import NoReturn
+
+# Third-Party Imports
+import numpy as np
+import matplotlib.pyplot as plt
+import pandas as pd
+import seaborn as sn
+import torch
+import torch.nn as nn
+from torchvision import transforms
+from sklearn.metrics import confusion_matrix
+
+
+# ---------------------------- DATA SAMPLES ----------------------------
+def display_mnist_data_samples(dataset: 'DataLoader object', number_of_samples: int) -> NoReturn:
+    """
+    Function to display samples for dataloader
+    :param dataset: Train or Test dataset transformed to Tensor
+    :param number_of_samples: Number of samples to be displayed
+    """
+    # Get batch from the data_set
+    batch_data = []
+    batch_label = []
+    for count, item in enumerate(dataset):
+        if not count <= number_of_samples:
+            break
+        batch_data.append(item[0])
+        batch_label.append(item[1])
+
+    # Plot the samples from the batch
+    fig = plt.figure()
+    x_count = 5
+    y_count = 1 if number_of_samples <= 5 else math.floor(number_of_samples / x_count)
+
+    # Plot the samples from the batch
+    for i in range(number_of_samples):
+        plt.subplot(y_count, x_count, i + 1)
+        plt.tight_layout()
+        plt.imshow(batch_data[i].squeeze(), cmap='gray')
+        plt.title(batch_label[i])
+        plt.xticks([])
+        plt.yticks([])
+
+
+def display_cifar_data_samples(data_set, number_of_samples: int, classes: list):
+    """
+    Function to display samples for data_set
+    :param data_set: Train or Test data_set transformed to Tensor
+    :param number_of_samples: Number of samples to be displayed
+    :param classes: Name of classes to be displayed
+    """
+    # Get batch from the data_set
+    batch_data = []
+    batch_label = []
+    for count, item in enumerate(data_set):
+        if not count <= number_of_samples:
+            break
+        batch_data.append(item[0])
+        batch_label.append(item[1])
+    batch_data = torch.stack(batch_data, dim=0).numpy()
+
+    # Plot the samples from the batch
+    fig = plt.figure()
+    x_count = 5
+    y_count = 1 if number_of_samples <= 5 else math.floor(number_of_samples / x_count)
+
+    for i in range(number_of_samples):
+        plt.subplot(y_count, x_count, i + 1)
+        plt.tight_layout()
+        plt.imshow(np.transpose(batch_data[i].squeeze(), (1, 2, 0)))
+        plt.title(classes[batch_label[i]])
+        plt.xticks([])
+        plt.yticks([])
+
+
+# ---------------------------- MISCLASSIFIED DATA ----------------------------
+def display_cifar_misclassified_data(data: list,
+                                     classes: list[str],
+                                     inv_normalize: transforms.Normalize,
+                                     number_of_samples: int = 10):
+    """
+    Function to plot images with labels
+    :param data: List[Tuple(image, label)]
+    :param classes: Name of classes in the dataset
+    :param inv_normalize: Mean and Standard deviation values of the dataset
+    :param number_of_samples: Number of images to print
+    """
+    fig = plt.figure(figsize=(10, 10))
+
+    x_count = 5
+    y_count = 1 if number_of_samples <= 5 else math.floor(number_of_samples / x_count)
+
+    for i in range(number_of_samples):
+        plt.subplot(y_count, x_count, i + 1)
+        img = data[i][0].squeeze().to('cpu')
+        img = inv_normalize(img)
+        plt.imshow(np.transpose(img, (1, 2, 0)))
+        plt.title(r"Correct: " + classes[data[i][1].item()] + '\n' + 'Output: ' + classes[data[i][2].item()])
+        plt.xticks([])
+        plt.yticks([])
+
+
+def display_mnist_misclassified_data(data: list,
+                                     number_of_samples: int = 10):
+    """
+    Function to plot images with labels
+    :param data: List[Tuple(image, label)]
+    :param number_of_samples: Number of images to print
+    """
+    fig = plt.figure(figsize=(8, 5))
+
+    x_count = 5
+    y_count = 1 if number_of_samples <= 5 else math.floor(number_of_samples / x_count)
+
+    for i in range(number_of_samples):
+        plt.subplot(y_count, x_count, i + 1)
+        img = data[i][0].squeeze(0).to('cpu')
+        plt.imshow(np.transpose(img, (1, 2, 0)), cmap='gray')
+        plt.title(r"Correct: " + str(data[i][1].item()) + '\n' + 'Output: ' + str(data[i][2].item()))
+        plt.xticks([])
+        plt.yticks([])
+
+
+# ---------------------------- AUGMENTATION SAMPLES ----------------------------
+def visualize_cifar_augmentation(data_set, data_transforms):
+    """
+    Function to visualize the augmented data
+    :param data_set: Dataset without transformations
+    :param data_transforms: Dictionary of transforms
+    """
+    sample, label = data_set[6]
+    total_augmentations = len(data_transforms)
+
+    fig = plt.figure(figsize=(10, 5))
+    for count, (key, trans) in enumerate(data_transforms.items()):
+        if count == total_augmentations - 1:
+            break
+        plt.subplot(math.ceil(total_augmentations / 5), 5, count + 1)
+        augmented = trans(image=sample)['image']
+        plt.imshow(augmented)
+        plt.title(key)
+        plt.xticks([])
+        plt.yticks([])
+
+
+def visualize_mnist_augmentation(data_set, data_transforms):
+    """
+    Function to visualize the augmented data
+    :param data_set: Dataset to visualize the augmentations
+    :param data_transforms: Dictionary of transforms
+    """
+    sample, label = data_set[6]
+    total_augmentations = len(data_transforms)
+
+    fig = plt.figure(figsize=(10, 5))
+    for count, (key, trans) in enumerate(data_transforms.items()):
+        if count == total_augmentations - 1:
+            break
+        plt.subplot(math.ceil(total_augmentations / 5), 5, count + 1)
+        img = trans(sample).to('cpu')
+        plt.imshow(np.transpose(img, (1, 2, 0)), cmap='gray')
+        plt.title(key)
+        plt.xticks([])
+        plt.yticks([])
+
+
+# ---------------------------- LOSS AND ACCURACIES ----------------------------
+def display_loss_and_accuracies(train_losses: list,
+                                train_acc: list,
+                                test_losses: list,
+                                test_acc: list,
+                                plot_size: tuple = (10, 10)) -> NoReturn:
+    """
+    Function to display training and test information(losses and accuracies)
+    :param train_losses: List containing training loss of each epoch
+    :param train_acc: List containing training accuracy of each epoch
+    :param test_losses: List containing test loss of each epoch
+    :param test_acc: List containing test accuracy of each epoch
+    :param plot_size: Size of the plot
+    """
+    # Create a plot of 2x2 of size
+    fig, axs = plt.subplots(2, 2, figsize=plot_size)
+
+    # Plot the training loss and accuracy for each epoch
+    axs[0, 0].plot(train_losses)
+    axs[0, 0].set_title("Training Loss")
+    axs[1, 0].plot(train_acc)
+    axs[1, 0].set_title("Training Accuracy")
+
+    # Plot the test loss and accuracy for each epoch
+    axs[0, 1].plot(test_losses)
+    axs[0, 1].set_title("Test Loss")
+    axs[1, 1].plot(test_acc)
+    axs[1, 1].set_title("Test Accuracy")
+
+
+# ---------------------------- Feature Maps and Kernels ----------------------------
+
+@dataclass
+class ConvLayerInfo:
+    """
+    Data Class to store Conv layer's information
+    """
+    layer_number: int
+    weights: torch.nn.parameter.Parameter
+    layer_info: torch.nn.modules.conv.Conv2d
+
+
+class FeatureMapVisualizer:
+    """
+    Class to visualize Feature Map of the Layers
+    """
+
+    def __init__(self, model):
+        """
+        Contructor
+        :param model: Model Architecture
+        """
+        self.conv_layers = []
+        self.outputs = []
+        self.layerwise_kernels = None
+
+        # Disect the model
+        counter = 0
+        model_children = model.children()
+        for children in model_children:
+            if type(children) == nn.Sequential:
+                for child in children:
+                    if type(child) == nn.Conv2d:
+                        counter += 1
+                        self.conv_layers.append(ConvLayerInfo(layer_number=counter,
+                                                              weights=child.weight,
+                                                              layer_info=child)
+                                                )
+
+    def get_model_weights(self):
+        """
+        Method to get the model weights
+        """
+        model_weights = [layer.weights for layer in self.conv_layers]
+        return model_weights
+
+    def get_conv_layers(self):
+        """
+        Get the convolution layers
+        """
+        conv_layers = [layer.layer_info for layer in self.conv_layers]
+        return conv_layers
+
+    def get_total_conv_layers(self) -> int:
+        """
+        Get total number of convolution layers
+        """
+        out = self.get_conv_layers()
+        return len(out)
+
+    def feature_maps_of_all_kernels(self, image: torch.Tensor) -> dict:
+        """
+        Get feature maps from all the kernels of all the layers
+        :param image: Image to be passed to the network
+        """
+        image = image.unsqueeze(0)
+        image = image.to('cpu')
+
+        outputs = {}
+
+        layers = self.get_conv_layers()
+        for index, layer in enumerate(layers):
+            image = layer(image)
+            outputs[str(layer)] = image
+        self.outputs = outputs
+        return outputs
+
+    def visualize_feature_map_of_kernel(self, image: torch.Tensor, kernel_number: int) -> None:
+        """
+        Function to visualize feature map of kernel number from each layer
+        :param image: Image passed to the network
+        :param kernel_number: Number of kernel in each layer (Should be less than or equal to the minimum number of kernel in the network)
+        """
+        # List to store processed feature maps
+        processed = []
+
+        # Get feature maps from all kernels of all the conv layers
+        outputs = self.feature_maps_of_all_kernels(image)
+
+        # Extract the n_th kernel's output from each layer and convert it to grayscale
+        for feature_map in outputs.values():
+            try:
+                feature_map = feature_map[0][kernel_number]
+            except IndexError:
+                print("Filter number should be less than the minimum number of channels in a network")
+                break
+            finally:
+                gray_scale = feature_map / feature_map.shape[0]
+                processed.append(gray_scale.data.numpy())
+
+        # Plot the Feature maps with layer and kernel number
+        x_range = len(outputs) // 5 + 4
+        fig = plt.figure(figsize=(10, 10))
+        for i in range(len(processed)):
+            a = fig.add_subplot(x_range, 5, i + 1)
+            imgplot = plt.imshow(processed[i])
+            a.axis("off")
+            title = f"{list(outputs.keys())[i].split('(')[0]}_l{i + 1}_k{kernel_number}"
+            a.set_title(title, fontsize=10)
+        return fig
+
+    def get_max_kernel_number(self):
+        """
+        Function to get maximum number of kernels in the network (for a layer)
+        """
+        layers = self.get_conv_layers()
+        channels = [layer.out_channels for layer in layers]
+        self.layerwise_kernels = channels
+        return max(channels)
+
+    def visualize_kernels_from_layer(self, layer_number: int):
+        """
+        Visualize Kernels from a layer
+        :param layer_number: Number of layer from which kernels are to be visualized
+        """
+        # Get the kernels number for each layer
+        self.get_max_kernel_number()
+
+        # Zero Indexing
+        layer_number = layer_number - 1
+        _kernels = self.layerwise_kernels[layer_number]
+
+        grid = math.ceil(math.sqrt(_kernels))
+
+        fig = plt.figure(figsize=(5, 4))
+        model_weights = self.get_model_weights()
+        _layer_weights = model_weights[layer_number].cpu()
+        for i, filter in enumerate(_layer_weights):
+            plt.subplot(grid, grid, i + 1)
+            plt.imshow(filter[0, :, :].detach(), cmap='gray')
+            plt.axis('off')
+        # plt.show()
+        return fig
+
+
+# ---------------------------- Confusion Matrix ----------------------------
+def visualize_confusion_matrix(classes: list[str], device: str, model: 'DL Model',
+                               test_loader: torch.utils.data.DataLoader):
+    """
+    Function to generate and visualize confusion matrix
+    :param classes: List of class names
+    :param device: cuda/cpu
+    :param model: Model Architecture
+    :param test_loader: DataLoader for test set
+    """
+    nb_classes = len(classes)
+    device = 'cuda'
+    cm = torch.zeros(nb_classes, nb_classes)
+
+    model.eval()
+    with torch.no_grad():
+        for inputs, labels in test_loader:
+            inputs = inputs.to(device)
+            labels = labels.to(device)
+            model = model.to(device)
+
+            preds = model(inputs)
+            preds = preds.argmax(dim=1)
+
+        for t, p in zip(labels.view(-1), preds.view(-1)):
+            cm[t, p] = cm[t, p] + 1
+
+    # Build confusion matrix
+    labels = labels.to('cpu')
+    preds = preds.to('cpu')
+    cf_matrix = confusion_matrix(labels, preds)
+    df_cm = pd.DataFrame(cf_matrix / np.sum(cf_matrix, axis=1)[:, None],
+                         index=[i for i in classes],
+                         columns=[i for i in classes])
+    plt.figure(figsize=(12, 7))
+    sn.heatmap(df_cm, annot=True)