Upload 3 files

CLIP_Explainability/image_utils.py

@@ -1,22 +1,38 @@
 import numpy as np
 import cv2
+from matplotlib import pyplot as plt
+
+
+def get_mpl_colormap(cmap_name):
+    cmap = plt.get_cmap(cmap_name)
+
+    # Initialize the matplotlib color map
+    sm = plt.cm.ScalarMappable(cmap=cmap)
+
+    # Obtain linear color range
+    color_range = sm.to_rgba(np.linspace(0, 1, 256), bytes=True)[:, 2::-1]
+
+    return color_range.reshape(256, 1, 3)
+
 
 def show_cam_on_image(img, mask, neg_saliency=False):
-   
     heatmap = cv2.applyColorMap(np.uint8(255 * mask), cv2.COLORMAP_JET)
-    
+
     heatmap = np.float32(heatmap) / 255
     cam = heatmap + np.float32(img)
     cam = cam / np.max(cam)
     return cam
 
+
 def show_overlapped_cam(img, neg_mask, pos_mask):
-    neg_heatmap = cv2.applyColorMap(np.uint8(255 * neg_mask), cv2.COLORMAP_RAINBOW)
-    pos_heatmap = cv2.applyColorMap(np.uint8(255 * pos_mask), cv2.COLORMAP_JET)
+    # neg_heatmap = cv2.applyColorMap(np.uint8(255 * neg_mask), cv2.COLORMAP_RAINBOW)
+    # pos_heatmap = cv2.applyColorMap(np.uint8(255 * pos_mask), cv2.COLORMAP_JET)
+    neg_heatmap = cv2.applyColorMap(np.uint8(255 * neg_mask), get_mpl_colormap("Blues"))
+    pos_heatmap = cv2.applyColorMap(np.uint8(255 * pos_mask), get_mpl_colormap("Reds"))
     neg_heatmap = np.float32(neg_heatmap) / 255
     pos_heatmap = np.float32(pos_heatmap) / 255
     # try different options: sum, average, ...
     heatmap = neg_heatmap + pos_heatmap
     cam = heatmap + np.float32(img)
     cam = cam / np.max(cam)
-    return cam
+    return cam

CLIP_Explainability/rn_cam.py

@@ -1,7 +1,8 @@
 import torch
 import numpy as np
-from PIL import Image
-import matplotlib.pyplot as plt
+
+# from PIL import Image
+# import matplotlib.pyplot as plt
 import cv2
 import re
 
@@ -22,7 +23,7 @@ def rn_relevance(
     cam = method(
         model=img_encoder,
         target_layers=target_layers,
-        use_cuda=torch.cuda.is_available(),
+        use_cuda=torch.cuda.is_available() and device != "cpu",
     )
 
     if neg_saliency:
@@ -127,12 +128,13 @@ def rn_perword_relevance(
     masked_text = re.sub(masked_word, "", text)
     masked_text = clip_tokenizer(masked_text).to(device)
 
-    image_features = clip_model.encode_image(image)
+    # image_features = clip_model.encode_image(image)
     main_text_features = clip_model.encode_text(main_text)
     masked_text_features = clip_model.encode_text(masked_text)
 
-    image_features_norm = image_features.norm(dim=-1, keepdim=True)
-    image_features_new = image_features / image_features_norm
+    # image_features_norm = image_features.norm(dim=-1, keepdim=True)
+    # image_features_new = image_features / image_features_norm
+
     main_text_features_norm = main_text_features.norm(dim=-1, keepdim=True)
     main_text_features_new = main_text_features / main_text_features_norm
 
@@ -146,10 +148,10 @@ def rn_perword_relevance(
     cam = method(
         model=clip_model.visual,
         target_layers=target_layers,
-        use_cuda=torch.cuda.is_available(),
+        use_cuda=torch.cuda.is_available() and device != "cpu",
     )
 
-    image_features = clip_model.visual(image)
+    # image_features = clip_model.visual(image)
 
     image_relevance = cam(input_tensor=image, target_encoding=target_encoding)[
         0
@@ -175,7 +177,7 @@ def rn_perword_relevance(
     image = image[0].permute(1, 2, 0).data.cpu().numpy()
     image = (image - image.min()) / (image.max() - image.min())
 
-    return image_relevance, image
+    return image_relevance
 
 
 def interpret_perword_rn(
@@ -189,7 +191,7 @@ def interpret_perword_rn(
     data_only=False,
     img_dim=224,
 ):
-    image_relevance, image = rn_perword_relevance(
+    image_relevance = rn_perword_relevance(
         image,
         text,
         clip_model,

CLIP_Explainability/vit_cam.py

@@ -1,7 +1,8 @@
 import torch
 import numpy as np
-from PIL import Image
-import matplotlib.pyplot as plt
+
+# from PIL import Image
+# import matplotlib.pyplot as plt
 import cv2
 import regex as re
 
@@ -71,7 +72,7 @@ def vit_block_vis(
     cam = cam[0].permute(1, 2, 0).data.cpu().numpy()
     cam = np.float32(cam)
 
-    plt.imshow(cam)
+    # plt.imshow(cam)
 
     return new_score
 
@@ -90,8 +91,10 @@ def vit_relevance(
 
     image_features_norm = image_features.norm(dim=-1, keepdim=True)
     image_features_new = image_features / image_features_norm
+
     target_features_norm = target_features.norm(dim=-1, keepdim=True)
     target_features_new = target_features / target_features_norm
+
     similarity = image_features_new[0].dot(target_features_new[0])
     if neg_saliency:
         objective = 1 - similarity
@@ -154,7 +157,7 @@ def vit_relevance(
     image = image[0].permute(1, 2, 0).data.cpu().numpy()
     image = (image - image.min()) / (image.max() - image.min())
 
-    return image_relevance, image
+    return image_relevance, image, image_features, similarity
 
 
 def interpret_vit(
@@ -166,7 +169,7 @@ def interpret_vit(
     neg_saliency=False,
     img_dim=224,
 ):
-    image_relevance, image = vit_relevance(
+    image_relevance, image, image_features, similarity = vit_relevance(
         image,
         target_features,
         img_encoder,
@@ -180,14 +183,14 @@ def interpret_vit(
     vis = np.uint8(255 * vis)
     vis = cv2.cvtColor(np.array(vis), cv2.COLOR_RGB2BGR)
 
-    return vis
+    return vis, image_features, similarity
     # plt.imshow(vis)
 
 
 def interpret_vit_overlapped(
     image, target_features, img_encoder, device, method="last grad", img_dim=224
 ):
-    pos_image_relevance, _ = vit_relevance(
+    pos_image_relevance, _, image_features, similarity = vit_relevance(
         image,
         target_features,
         img_encoder,
@@ -196,7 +199,7 @@ def interpret_vit_overlapped(
         neg_saliency=False,
         img_dim=img_dim,
     )
-    neg_image_relevance, image = vit_relevance(
+    neg_image_relevance, image, _, _ = vit_relevance(
         image,
         target_features,
         img_encoder,
@@ -210,19 +213,18 @@ def interpret_vit_overlapped(
     vis = np.uint8(255 * vis)
     vis = cv2.cvtColor(np.array(vis), cv2.COLOR_RGB2BGR)
 
-    return vis
+    return vis, image_features, similarity
     # plt.imshow(vis)
 
 
 def vit_perword_relevance(
-    image,
+    image_features,
     text,
     clip_model,
     clip_tokenizer,
     device,
     masked_word="",
     use_last_grad=True,
-    data_only=False,
     img_dim=224,
 ):
     clip_model.eval()
@@ -232,12 +234,13 @@ def vit_perword_relevance(
     masked_text = re.sub(masked_word, "", text)
     masked_text = clip_tokenizer(masked_text).to(device)
 
-    image_features = clip_model.encode_image(image)
     main_text_features = clip_model.encode_text(main_text)
     masked_text_features = clip_model.encode_text(masked_text)
 
+    # image_features = clip_model.encode_image(image)
     image_features_norm = image_features.norm(dim=-1, keepdim=True)
     image_features_new = image_features / image_features_norm
+
     main_text_features_norm = main_text_features.norm(dim=-1, keepdim=True)
     main_text_features_new = main_text_features / main_text_features_norm
 
@@ -290,38 +293,9 @@ def vit_perword_relevance(
         image_relevance.max() - image_relevance.min()
     )
 
-    if data_only:
-        return image_relevance
+    # image = image[0].permute(1, 2, 0).data.cpu().numpy()
+    # image = (image - image.min()) / (image.max() - image.min())
 
-    image = image[0].permute(1, 2, 0).data.cpu().numpy()
-    image = (image - image.min()) / (image.max() - image.min())
-
-    return image_relevance, image
-
-
-def interpret_perword_vit(
-    image,
-    text,
-    clip_model,
-    clip_tokenizer,
-    device,
-    masked_word="",
-    use_last_grad=True,
-    img_dim=224,
-):
-    image_relevance, image = vit_perword_relevance(
-        image,
-        text,
-        clip_model,
-        clip_tokenizer,
-        device,
-        masked_word,
-        use_last_grad,
-        img_dim=img_dim,
-    )
-    vis = show_cam_on_image(image, image_relevance)
-    vis = np.uint8(255 * vis)
-    vis = cv2.cvtColor(np.array(vis), cv2.COLOR_RGB2BGR)
+    # return image_relevance, image
 
-    return vis
-    # plt.imshow(vis)
+    return image_relevance