Upload 10 files

app.py

@@ -1,25 +1,19 @@
-#%% TODO: Very important
-# Class-agnostic sensitivity measurement
-# instead of softmax, get the logits
-# make them unit-norm
-# This means that every model output is a point
-# in 1000-dimensional hyper-sphere. So it could be possible
-# to measure the sensitivity of the needle.
-#%% Libraries
-import cv2
-from segment_anything import SamAutomaticMaskGenerator, sam_model_registry
-import matplotlib.pyplot as plt
+import gradio as gd 
 import numpy as np
-from torchvision.io import read_image
-from torchvision.models import resnet50, ResNet50_Weights
+import os
 import torch
-from itertools import combinations
-import seaborn as sns
-import gradio as gr
-from PIL import Image
+import torchvision
+import torchvision.models as models
+from lime import lime_image
+import matplotlib.pyplot as plt
 import matplotlib
-matplotlib.use('agg')
+import torch.nn.functional as F
+from skimage.segmentation import mark_boundaries
+from PIL import Image
+from segment_anything import SamAutomaticMaskGenerator, sam_model_registry
 import wget
+import cv2
+matplotlib.use('agg')
 
 # Vanilla Legendre between [0,1]
 def Pn(m, x):    
@@ -29,205 +23,306 @@ def Pn(m, x):
         return x
     else:
         return (2*m-1)*x*Pn(m-1, x)/m - (m-1)*Pn(m-2, x)/m
+
 # Legendre between [a,b]    
 def L(a,b,m,x):
     return np.sqrt((2*m+1)/(b-a))*Pn(m, 2*(x-b)/(b-a)+1)
 
-URL = "https://dl.fbaipublicfiles.com/segment_anything/sam_vit_b_01ec64.pth"
-response = wget.download(URL, "sam_vit_b_01ec64.pth")
+def run_lime(input_image, 
+             model_name: str, 
+             top_labels: int,
+             num_samples: int,
+             num_features: int,
+             batch_size: int):
+    # input_image is a numpy array of shape (height, width, channels) 
+    # range is [0, 255]
+    print('model_name', model_name)
+    print('top_labels', top_labels)
+    print('num_samples', num_samples)
+    print('num_features', num_features)
+    print('batch_size', batch_size)
+    print('input image', type(input_image), input_image.shape)
+
+    model, weights = fetch_model(model_name)
+    preprocess = weights.transforms(antialias=True)
 
-#%% show segments on the image 
-def segment_heatmap_image(img, masks, mask_weights, alpha=0.4):
-    w, h, c = img.shape
+    input_image_processed = preprocess(torch.from_numpy(input_image.transpose(2,0,1))).unsqueeze(0)
+    logits = model(input_image_processed)
+    probs = F.softmax(logits, dim=1)
+    names = weights.meta['categories']
+
+    top_10_classes = []
+    print('probs', type(probs), probs.shape)
+    for x in probs.argsort(descending=True)[0][:10]:
+        print(x.item(), names[x], probs[0,x].item())
+        top_10_classes.append([x.item(), names[x], f'{probs[0,x].item():.4f}'])
+
+    def classifier_fn(images):
+        print('classifier_fn', type(images), images.shape)
+        
+        zz = preprocess(torch.from_numpy(images[0].transpose(2,0,1)))
+        c, w, h = zz.shape 
+        batch = torch.zeros(batch_size, c, w, h)
+        print('len(images)', len(images))
+        for i in range(batch_size):
+            batch[i] = preprocess(torch.from_numpy(images[i].transpose(2,0,1)))
+        print('batch', type(batch), batch.shape)
+
+        logits = model(batch)
+        probs = F.softmax(logits, dim=1)
+        print('probs', type(probs), probs.shape)
+        return probs.detach().cpu().numpy()
+
+    explainer = lime_image.LimeImageExplainer()
+    explanation = explainer.explain_instance(
+        input_image, 
+        classifier_fn,
+        top_labels=top_labels, 
+        hide_color=0, 
+        num_samples=num_samples,
+        num_features=num_features,
+        batch_size=batch_size) 
     
-    img_gray = 	cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)
+    temp, mask = explanation.get_image_and_mask(
+        explanation.top_labels[0], 
+        positive_only=False, num_features=num_features, hide_rest=False)
+    lime_output = mark_boundaries(temp/255.0, mask)
+    return lime_output, top_10_classes
+
+def segmented_image(img, masks, alpha=0.7):
+    segment_image = img.copy()
+    for mask in masks:
+        segment_image[mask['segmentation'] == 1] = 255*np.random.random(3)
+    cv2.addWeighted(segment_image, alpha, img, 1.0-alpha, 0, segment_image)
+    return segment_image
+
+def segment_heatmap_image(img, masks, mask_weights, num_features_hdmr):
+    w, h, c = img.shape
+    img_gray = 	cv2.cvtColor(img, cv2.COLOR_RGB2GRAY)
+    # increase brightness of gray image
+    img_gray = cv2.convertScaleAbs(img_gray, alpha=10, beta=0)
+
     img_grad3d = np.dstack([img_gray, img_gray, img_gray])
     print(img.shape, img_gray.shape)
     segment_image = np.zeros((w,h)).astype(np.uint8)
-    for mask, weight in zip(masks, mask_weights):
+
+    important_segment_indices = mask_weights.argsort()[-num_features_hdmr:]
+    for i in important_segment_indices:
+        mask = masks[i]
+        weight = mask_weights[i]
         segment_image[mask['segmentation'] == True] = int(255*weight)
-    #blur = cv2.GaussianBlur(segment_image,(13,13), 11)
     heatmap_img = cv2.applyColorMap(segment_image, cv2.COLORMAP_JET)
-    super_imposed_img = cv2.addWeighted(heatmap_img, alpha, img_grad3d, 1-alpha, 0)
-    return super_imposed_img, segment_image
-
-def run(gui_input_image_WHC):
-    print("hello----------",type(gui_input_image_WHC))
-    print(gui_input_image_WHC.shape)
-    input_image_torch_CWH = gui_input_image_WHC.transpose(2,0,1)
-    print(input_image_torch_CWH.shape)
-    input_image_torch = torch.from_numpy(input_image_torch_CWH)
-
-
-    # %% Step A: we have an input image
-    # torch version uses CWH format 
-    #input_image_torch = read_image(gui_input_image)
-    #input_image_torch = input_image_torch[:3,:,:]
+    super_imposed_img = cv2.addWeighted(heatmap_img, 1, img_grad3d, 0.6, 0)
+    return super_imposed_img, heatmap_img
 
-    # copy WHC format for SAM algorithm
-    input_image = gui_input_image_WHC #input_image_torch.permute(1,2,0).numpy()
+def sobol(x, y, m):
+    print(x.shape, y.shape)
+    N, n = x.shape 
+    f0 = np.mean(y)
+    alpha = np.zeros((m, n))
 
-    # %% Step B: we load pre-trained model and classify input image
-    weights = ResNet50_Weights.DEFAULT
+    for r in range(m):  
+        for i in range(n):  
+            alpha[r, i] = np.mean((y-f0) * L(0, 1, r+1, np.array(x[:, i])))
+    
+    global_D = np.mean(y ** 2) - np.mean(y) ** 2
+    D_first_order = np.zeros((n,m))
+    S_first_order = np.zeros((n,m))
+    for degree in range(m):
+        for k in range(n):
+            D_first_order[k,degree] = sum(alpha[r,k] ** 2 for r in range(degree+1))
+            S_first_order[k,degree] = D_first_order[k,degree]/global_D   
+
+    return S_first_order
+
+def run_hdmr(input_image, 
+             model_name: str, 
+             sam_model_name: str,
+             num_samples_hdmr: int,
+             num_legendre: int,
+             num_features_hdmr: int):
+    # input_image is a numpy array of shape (height, width, channels) 
+    # range is [0, 255]
+    print('model_name', model_name)
+    print('sam_model_name', sam_model_name)
+    print('num_samples_hdmr', num_samples_hdmr)
+    print('num_features_hdmr', num_features_hdmr)
+    print('input image', type(input_image), input_image.shape)
+
+    model, weights = fetch_model(model_name)
     preprocess = weights.transforms(antialias=True)
-    model = resnet50(weights=weights)
-    model.eval()
-    # feed the model and get the logits
-    batch = preprocess(input_image_torch).unsqueeze(0)
-    # Unit normalize the logits
-    logits = model(batch)
-    logits = logits.detach().numpy()
-    logits_length = np.linalg.norm(logits, ord=1, axis=1)
-    logits_normalized = logits[0] / logits_length
 
-    #%% Step C: Segmentation
-    sam = sam_model_registry["vit_b"](checkpoint='sam_vit_b_01ec64.pth')
-    mask_generator = SamAutomaticMaskGenerator(sam)
+    sam_model = fetch_sam_model(sam_model_name)
+    mask_generator = SamAutomaticMaskGenerator(sam_model)
     masks = mask_generator.generate(input_image)
 
-    # %% show segments on the image 
-    def segmented_image(img, masks, alpha=0.7):
-        segment_image = img.copy()
-        for mask in masks:
-            segment_image[mask['segmentation'] == 1] = 255*np.random.random(3)
-        cv2.addWeighted(segment_image, alpha, img, 1.0-alpha, 0, segment_image)
-        return segment_image
+    sam_segmented_image = segmented_image(input_image, masks, alpha=0.9)
 
-    gui_sam = segmented_image(input_image, masks)
-    #plt.imshow(gui_sam)
+    batch = preprocess(torch.from_numpy(input_image.transpose(2,0,1))).unsqueeze(0)
+    # Unit normalize the logits
+    logits = model(batch)
+    logits = logits[0].detach().numpy()
+    logits_length = np.linalg.norm(logits)
+    logits_normalized = logits / logits_length
+    print('logits_normalized',logits_normalized.shape)
 
-    #%% Step D: Perturbations
-    N = 200 # number of perturbations
-    n = len(masks) # number of masks
+    N = num_samples_hdmr
+    n = len(masks)
 
     x = np.random.rand(N, n)
-    y = np.zeros(N)
+    y = np.zeros((3,N)) # cosine, l1, l2
 
+    # TODO: implement batch_size 
     for sample in range(N):
-        x_input = input_image_torch.clone().type(torch.float64)
-
+        x_input = input_image.copy()
         for i, mask in enumerate(masks):
             x_seg = mask['segmentation']
-            x_input[:,x_seg == 1] =  x_input[:,x_seg == 1] * np.power(x[sample,i],2)
+            x_input[x_seg == 1] =  x_input[x_seg == 1] * np.power(x[sample,i],2)
 
-        x_input = x_input.type(torch.uint8)
-        batch_2 = preprocess(x_input).unsqueeze(0)
+        batch = preprocess(torch.from_numpy(x_input.transpose(2,0,1))).unsqueeze(0)
         # Unit normalize the logits
-        logits_2 = model(batch_2)
-        probs = logits_2.squeeze(0).softmax(0)
-        logits_2 = logits_2.detach().numpy()
-        logits_length_2 = np.linalg.norm(logits_2, ord=1, axis=1)
-        logits_normalized_2 = logits_2[0] / logits_length_2
-
-        # Calculate cosine distance between logits_normalized and logits_normalized_2
-        # This is the cosine distance between the two models
-        # The closer the distance is to 1, the more similar the models are
-        cosine_distance = np.dot(logits_normalized, logits_normalized_2)
-        eucledean_distance = np.linalg.norm(logits_normalized - logits_normalized_2)
-        l1_norm = np.sum(np.abs(logits_normalized - logits_normalized_2))
-
+        logits_sample = model(batch)
+        probs = logits_sample.squeeze(0).softmax(0)
+        logits_sample = logits_sample[0].detach().numpy()
+        logits_sample_length = np.linalg.norm(logits_sample)
+        logits_sample_normalized = logits_sample / logits_sample_length
+
+        cosine_distance = np.dot(logits_normalized, logits_sample_normalized)
+        l1_distance = np.sum(np.abs(logits_normalized - logits_sample_normalized))
+        l2_distance = np.linalg.norm(logits_normalized - logits_sample_normalized)
         class_id = probs.argmax().item()
         score = probs[class_id].item()
         category_name = weights.meta["categories"][class_id]
-        print(f"sample:{sample:2d} l1norm: {l1_norm:.5f} cosine distance: {cosine_distance:.5f} eucledean distance: {eucledean_distance:.5f} {category_name}: {100 * score:.1f}%")
-
-        # Store output in y
-        y[sample] = l1_norm
-
-    #fig, axes = plt.subplots(3,1)
-    #axes[0].imshow(x_input.permute(1,2,0).numpy()/255)
-    #axes[1].plot(logits_normalized_2)
-    #axes[2].plot(probs.detach().numpy())
-
-    m = 3
-    f0 = np.mean(y)
-    alpha = np.zeros((m, n))
-
-    num_pairs = len(list(combinations(range(n), 2)))
-    beta = np.zeros((m,num_pairs))
-
-    for r in range(m):  # for each input feature
-        ct = 0
-        for i in range(n):  # for each Legendre polynomial up to order m
-            alpha[r, i] = np.mean((y-f0) * L(0, 1, r+1, np.array(x[:, i])))
-            for j in range(i+1,n):
-                beta[r, ct] = np.mean((y-f0)*
-                                    L(0, 1, r+1, np.array(x[:, i]))*
-                                    L(0, 1, r+1, np.array(x[:, j])))
-                ct += 1
-
-    # Calculate Sobol indices
-    global_D = np.mean(y ** 2) - np.mean(y) ** 2
-    D_first_order = np.zeros((m,n))
-    S_first_order = np.zeros((m,n))
-    D_second_order = np.zeros((m,num_pairs))
-    S_second_order = np.zeros((m,num_pairs))
-    for degree in range(m):
-        for k in range(n):
-            D_first_order[degree,k] = sum(alpha[r,k] ** 2 for r in range(degree+1))
-            S_first_order[degree,k] = D_first_order[degree,k]/global_D   
-
-
-    for deg in range(m):
-        for pair in range(num_pairs):
-            D_second_order[deg,pair] = sum(beta[r,pair] ** 2 for r in range(deg+1))
-            S_second_order[deg,pair] = D_second_order[deg,pair]/global_D
-
-    selection_degree = m
-    calculated_sobol = np.diag(S_first_order[m-1,:])
-
-    ct = 0
-    for i in range(n):
-        for j in range(i+1,n):
-                calculated_sobol[i, j] = S_second_order[m-1, ct]
-                ct += 1
-
-    sum_calculated_sobol = np.sum(calculated_sobol)   
-    print("Sum of calculated sobol coefficients: ", sum_calculated_sobol)    
-
-    sns.heatmap(calculated_sobol, annot=False,cbar=False)
-    sns.set(font_scale=2)
-    plt.savefig('sensitivity.png')
-
-    gui_sensitivity = calculated_sobol
-
-    mask_weights = S_first_order[-1]/np.max(S_first_order[-1])
-
-    # Segment heatmap
-
-    hdmr_map_image, heatmap, = segment_heatmap_image(input_image, masks, mask_weights)
-
-    #plt.imshow(input_image)
-    #plt.show()
-    #plt.imshow(hdmr_map_image)
-    #plt.figure()
-    #plt.imshow(heatmap)
-    #
-    #plt.show()
-
-    gui_hdmr = hdmr_map_image
-
-    gui_sensitivity3d = np.dstack((gui_sensitivity,gui_sensitivity,gui_sensitivity))
-    
-    gui_sensitivity3d = (gui_sensitivity3d * 255).astype(np.uint8) 
-
-    print('gui_hdmr',type(gui_hdmr),gui_hdmr.shape)
-    print('gui_sensitivity',type(gui_sensitivity),gui_sensitivity.shape)
-
-    return Image.open('sensitivity.png'), gui_sam, gui_hdmr, heatmap
-
-with gr.Blocks() as demo:
-    gui_input_image = gr.Image(label="Yükleyin")
-    submit_button = gr.Button(label="Yükle", value="Yükle")
-    damage_score = gr.Textbox(label="Hasar Skoru")
-    with gr.Row():
-        gui_sam = gr.Image(label="SAM segments",
-        shape=(256,256))
-        gui_hdmr = gr.Image(label="HDMR map",
-        shape=(256,256))
-    heatmap = gr.Image(label="heatmap")
-    gui_sensitivity = gr.Image(label="Sensitivity Scores")
-    submit_button.click(fn=run, inputs=gui_input_image,
-                        outputs=[gui_sensitivity, gui_sam, gui_hdmr, heatmap])
+        print(f"sample:{sample:2d} cosine: {cosine_distance:.5f} l1: {l1_distance:.5f}  l2: {l2_distance:.5f} {category_name}: {100 * score:.1f}%")
+
+        y[:,sample] = [cosine_distance, l1_distance, l2_distance]
+
+    sobol_indices_cosine = sobol(x, y[0], num_legendre)
+    sobol_indices_l1 = sobol(x, y[1], num_legendre)
+    sobol_indices_l2 = sobol(x, y[2], num_legendre)
+
+    hdmr_indices_cosine_df = [[f'{b:.4f}' for b in a] for a in sobol_indices_cosine]
+    hdmr_indices_l1_df     = [[f'{b:.4f}' for b in a] for a in sobol_indices_l1]
+    hdmr_indices_l2_df     = [[f'{b:.4f}' for b in a] for a in sobol_indices_l2]
+
+    weight_cosine = sobol_indices_cosine[:,-1] / np.max(sobol_indices_cosine[:,-1])
+    weight_l1 = sobol_indices_l1[:,-1] / np.max(sobol_indices_l1[:,-1])
+    weight_l2 = sobol_indices_l2[:,-1] / np.max(sobol_indices_l2[:,-1])
+
+    print('weight_cosine',weight_cosine.shape)
+    _, hdmr_cosine = segment_heatmap_image(input_image, masks, weight_cosine,num_features_hdmr)
+    _, hdmr_l1 = segment_heatmap_image(input_image, masks, weight_l1,num_features_hdmr)
+    _, hdmr_l2, = segment_heatmap_image(input_image, masks, weight_l2,num_features_hdmr)
+
+    return hdmr_cosine, hdmr_l1, hdmr_l2, hdmr_indices_cosine_df, hdmr_indices_l1_df, hdmr_indices_l2_df,sam_segmented_image
+
+def fetch_sam_model(sam_model_name_checkpoint):
+    sam_model_name, sam_checkpoint = sam_model_name_checkpoint.split(' ')
+    URL = f"https://dl.fbaipublicfiles.com/segment_anything/{sam_checkpoint}"
+    if not os.path.isfile(sam_checkpoint):
+        response = wget.download(URL, sam_checkpoint)
+    sam = sam_model_registry[sam_model_name](checkpoint=sam_checkpoint)
+    return sam
+
+def fetch_model_names():
+    return models.list_models(module=torchvision.models)
+
+def fetch_model(model_name):
+    print('Retrieving model ', model_name)
+    weights_enum = models.get_model_weights(model_name)
+    for w in weights_enum:
+        if "IMAGENET1K" in w.name:
+            weights = w
+            model = models.get_model(model_name, weights=weights)
+            print('Model weights loaded', w.name)
+            return model, weights
+    return None, None
+
+with gd.Blocks() as demo:
+    with gd.Column():
+        gd.Markdown(value='''
+        # xAI with a Meta-Modelling Algorithm
+        And its comparison with LIME.
+        LIME implementation is based on:
+        * [LIME](https://github.com/marcotcr/lime)
+        * [LIME tutorial](https://github.com/marcotcr/lime/blob/master/tutorials/lime_image.ipynb)
+        ''')
+        with gd.Row():
+            with gd.Column():
+                input_image = gd.Image(label="Input Image. Please upload an image that you want LIME to explain")
+                model_name = gd.Dropdown(label="Model", 
+                                         info='''
+                                         Select the image classification model to use for LIME.
+                                         The list is automatically populated by using torchvision library.
+                                         ''',
+                                         value='convnext_tiny',
+                                         choices=fetch_model_names())
+                sam_model_name = gd.Dropdown(label="SAM model",
+                            info='Select the SAM model',
+                            value='vit_b sam_vit_b_01ec64.pth',
+                            choices=['vit_b sam_vit_b_01ec64.pth'])
+            with gd.Column():
+                top_labels = gd.Number(label='top_labels',info='''
+                        use the first <top_labels> labels to create explanations.
+                        For example, setting top_labels=5 will create explanations 
+                        for the top 5 most likely classes.''',
+                                             precision=0, value=5)
+                num_samples = gd.Number(label="num_samples", 
+                                        info="How many samples to be created to build the linear model inside LIME",
+                                             precision=0, value=100)
+                num_features = gd.Number(label="num_features", 
+                                         info='Among the most important superpixels (features), how many to be shown in the explanation image',
+                                             precision=0, value=2)
+                batch_size = gd.Number(label="batch_size", 
+                                       info='how many images in the samples to be processed at once',
+                                             precision=0, value=20)
+            with gd.Column():
+                num_samples_hdmr = gd.Number(label="num_samples_hdmr", 
+                                        info="How many samples in HDMR",
+                                             precision=0, value=10)
+                num_legendre = gd.Number(label="num_legendre",
+                                         info='Number of Legendre Bases for HDMR',
+                                         precision=0,value=3)
+                num_features_hdmr = gd.Number(label="num_features_hdmr", 
+                                         info='Among the most important segments, how many to be shown in the explanation image',
+                                             precision=0, value=2)
+                run_button = gd.Button(label="Run")
+        with gd.Row():
+            top_10_classes = gd.DataFrame(label="Top 10 classes",
+                                         info="Top-10 classes for the input image calculated by using the selected model",
+                                     headers=["class_id","label","probability"],
+                                     datatype=["number","str","number"])
+            lime_output = gd.Image(label="Lime Explanation",
+                                   info="The explanation image for the input image calculated by LIME for the selected model")
+            sam_segmented_image = gd.Image(label="SAM Segmentation",
+                                        info="The segmentation image for the input image calculated by SAM")
+        with gd.Row():
+            hdmr_cosine = gd.Image(label="HDMR Explanation via Cosine Distance")
+            hdmr_l1 = gd.Image(label="HDMR Explanation via L1 Distance")
+            hdmr_l2 = gd.Image(label="HDMR Explanation via L2 Distance")
+        with gd.Row():
+            hdmr_cosine_indices = gd.DataFrame(label="HDMR Cosine Indices")
+            hdmr_l1_indices = gd.DataFrame(label="HDMR L1 Indices")
+            hdmr_l2_indices = gd.DataFrame(label="HDMR L2 Indices")
+        gd.Examples(
+            label="Some examples images and parameters",
+            examples=[["jeep.png","convnext_tiny",5,20,2,20],
+                      ["IMG_0154.jpg","convnext_tiny",5,100,2,20],
+                      ["IMG_0155.jpg","convnext_tiny",5,100,2,20],
+                      ["IMG_0156.jpg","convnext_tiny",5,100,2,20],
+                      ["IMG_0157.jpg","convnext_tiny",5,100,2,20],
+                      ["IMG_0158.jpg","convnext_tiny",5,100,2,20],
+                      ["IMG_0159.jpg","convnext_tiny",5,100,2,20],
+                      ["IMG_0160.jpg","convnext_tiny",5,100,2,20]],
+            inputs=[input_image,model_name,top_labels,num_samples,num_features,batch_size])
+        
+    run_button.click(fn=run_lime,inputs=[input_image, model_name, top_labels,num_samples,num_features,batch_size],
+                     outputs=[lime_output,top_10_classes]) 
+    run_button.click(fn=run_hdmr,inputs=[input_image, model_name, sam_model_name, num_samples_hdmr, num_legendre, num_features_hdmr],
+                     outputs=[hdmr_cosine, hdmr_l1, hdmr_l2,
+                            hdmr_cosine_indices, hdmr_l1_indices, hdmr_l2_indices, 
+                            sam_segmented_image])
+  
 if __name__ == "__main__":
     demo.launch()
+    

requirements.txt

@@ -1,7 +1,4 @@
 segment-anything==1.0
 torchvision==0.15.1
 opencv-python==4.7.0.72
-fastai==2.7.12
-seaborn==0.12.2
-opencv-python==4.7.0.72
-wget
+fastai==2.7.12