Upload 8 files

app.py

@@ -0,0 +1,60 @@
+from flask import Flask, jsonify, request, send_file, render_template
+from flask_cors import CORS
+from lrp_pipeline import lrp_main
+from utils import create_folders, delete_folders, create_zip_file
+from cam_pipeline import cam_main
+
+app = Flask(__name__)
+CORS(app)
+
+
+@app.route("/api/upload", methods=["GET"])
+def get_data():
+    data = {"message": "Hello from Flask backend!"}
+    return jsonify(data)
+
+
+@app.route("/api/upload", methods=["POST"])
+def submit_data():
+    # first clear all the existing files in uploads, heatmaps, segmentations, tables, cell_descriptors folders
+    folder_names = [
+        "uploads",
+        "heatmaps",
+        "segmentations",
+        "tables",
+        "cell_descriptors",
+    ]
+    delete_folders(folder_names)
+    create_folders(folder_names)
+
+    # then upload the submitted file(s)
+    file = list(dict(request.files).values())[0]
+    print(file)
+    file.save(
+        f"uploads/{file.filename}"
+    )  # Replace 'uploads' with your desired directory
+
+    # Process data here
+    return jsonify({"message": "Data received successfully!"})
+
+
+@app.route("/api/inputform", methods=["POST"])
+def submit_form():
+    data = dict(request.json)  # format of data: {'model': 'VGGNet', 'xaiMethod': 'LRP'}
+    print(data)
+    if "LRP" in data["xaiMethod"]:
+        # pixel_ratio = data['pixelRatio']
+        return lrp_main(float(data["magval"]))  # pixel_ratio
+    elif "GradCAM++" in data["xaiMethod"]:
+        # pixel_ratio = data['pixelRatio']
+        return cam_main(float(data["magval"]))  # pixel_ratio
+
+
+@app.route("/api/zip", methods=["GET"])
+def get_csv():
+    create_zip_file()
+    return send_file("outputs.zip", as_attachment=True)
+
+
+if __name__ == "__main__":
+    app.run(debug=True)

cam_pipeline.py

@@ -0,0 +1,431 @@
+import random
+import numpy as np
+from PIL import Image
+import os
+import tensorflow as tf
+import cv2
+from sklearn.mixture import GaussianMixture
+import base64
+import csv
+from tensorflow.keras.applications.efficientnet import preprocess_input
+from tensorflow.keras.preprocessing import image
+from tensorflow.keras.models import Model
+from tensorflow.keras.layers import Lambda
+from tensorflow.keras.models import load_model
+from utils import (
+    select_sample_images,
+    create_cell_descriptors_table,
+    calculate_cell_descriptors,
+)
+
+preprocessed_folder = 'uploads/'
+segmentation_folder = 'segmentations/'
+intermediate_folder = 'heatmaps/'
+tables_folder = "tables/"
+cell_descriptors_path = "cell_descriptors/cell_descriptors.csv"
+saved_model_path = 'xception_model_81.h5'  # Replace with the path to your saved model
+model = load_model(saved_model_path)
+
+
+def preprocess_image(img_path):
+    img = image.load_img(img_path, target_size=(224, 224))
+    x = image.img_to_array(img)
+    x = np.expand_dims(x, axis=0)
+    x = preprocess_input(x)
+    return x
+
+def generate_grad_cam_plus_plus(img_path, model, last_conv_layer_name, classifier_layer_names):
+    # image = edge_finding(img_path)
+    img = image.load_img(img_path, target_size=(224, 224))
+    x = image.img_to_array(img)
+    x = np.expand_dims(x, axis=0)
+    img = tf.keras.applications.xception.preprocess_input(x)
+    grad_model = Model(
+        inputs=[model.inputs],
+        outputs=[model.get_layer(last_conv_layer_name).output, model.output]
+    )
+
+    # print(grad_model)
+
+
+    with tf.GradientTape() as tape1, tf.GradientTape() as tape2:
+        last_conv_output, preds = grad_model(img)
+        # print(last_conv_output.shape)
+        class_idx = np.argmax(preds[0])
+        # max_value = tf.reduce_max(preds)
+
+        # Reshape to get the desired shape (1,)
+        # loss = tf.reshape(max_value, shape=(1,))
+        loss = preds[:,0]
+        # print('loss')
+        # print(loss)
+        grads = tape1.gradient(loss, last_conv_output)
+
+        first_derivative = tf.exp(loss) * grads
+        # print('grads')
+        # print(first_derivative)
+
+        second_derivative = tape2.gradient(grads, last_conv_output)
+        second_derivative = tf.exp(loss) * second_derivative
+        # print('grads2')
+        # print(second_derivative)
+
+    global_sum = tf.reduce_sum(first_derivative, axis=(0, 1, 2), keepdims=True)
+    alpha_num = second_derivative
+    alpha_denom = second_derivative * 2.0 + first_derivative * global_sum
+    alphas = alpha_num / (alpha_denom + 1e-7)
+
+    # print(alphas)
+
+
+    weights = tf.maximum(0, global_sum)
+    alpha_normalization_constant = tf.reduce_sum(alphas, axis=(0, 1), keepdims=True)
+    alphas /= (alpha_normalization_constant + 1e-7)
+
+    deep_linearization_weights = tf.reduce_sum(weights * alphas, axis=(0, 3))
+
+    # Reshape the deep_linearization_weights to match the shape of last_conv_output
+    deep_linearization_weights = tf.reshape(deep_linearization_weights, (1,7,7,-1))
+
+    # print(deep_linearization_weights.shape)
+
+    # Compute the CAM by taking a weighted sum of the convolutional layer output
+    cam = tf.reduce_sum(deep_linearization_weights * last_conv_output, axis=3)
+
+
+
+    # Normalize the CAM
+    cam = tf.maximum(cam, 0)
+    cam /= tf.reduce_max(cam)
+
+    heatmap = tf.reduce_mean(cam, axis=0)  # Take mean along the channel axis
+    # heatmap = tf.squeeze(cam)
+
+    heatmap=heatmap.numpy()
+
+
+    return heatmap
+
+
+def GMM_abnormal_method(heatmap):
+  heatmap = cv2.resize(heatmap, (224, 224))
+  flat_heatmap = heatmap.flatten().reshape(-1, 1)
+
+  # Define the number of clusters (segments)
+  n_clusters = 4 # Adjust based on your requirements
+
+  # Apply Gaussian Mixture Model clustering
+  gmm = GaussianMixture(n_components=n_clusters, random_state=0)
+  gmm.fit(flat_heatmap)
+  labels = gmm.predict(flat_heatmap).reshape(heatmap.shape[:2])
+
+  # Assign labels to the regions based on their intensity
+  sorted_labels = np.argsort(gmm.means_.flatten())
+  label_mapping = {sorted_labels[0]: 0, sorted_labels[1]: 1, sorted_labels[2]: 2,sorted_labels[3]: 3}
+  labels_mapped = np.vectorize(label_mapping.get)(labels)
+
+  colour_list=[[0,0,255],[128,0,0],[255,0,0],[255,0,0]]
+
+
+  colors = np.array(colour_list)  # BGR format
+  colored_labels = colors[labels_mapped]
+
+  return labels_mapped,colored_labels
+
+def GMM_normal_method(heatmap):
+  heatmap = cv2.resize(heatmap, (224, 224))
+  flat_heatmap = heatmap.flatten().reshape(-1, 1)
+
+  # Define the number of clusters (segments)
+  n_clusters = 4 # Adjust based on your requirements
+
+  # Apply Gaussian Mixture Model clustering
+  gmm = GaussianMixture(n_components=n_clusters, random_state=0)
+  gmm.fit(flat_heatmap)
+  labels = gmm.predict(flat_heatmap).reshape(heatmap.shape[:2])
+
+  # Assign labels to the regions based on their intensity
+  sorted_labels = np.argsort(gmm.means_.flatten())
+  label_mapping = {sorted_labels[0]: 0, sorted_labels[1]: 1, sorted_labels[2]: 2,sorted_labels[3]: 3}
+  labels_mapped = np.vectorize(label_mapping.get)(labels)
+
+  colour_list=[[0,0,255],[128,0,0],[128,0,0],[255,0,0]]
+
+
+  colors = np.array(colour_list)  # BGR format
+  colored_labels = colors[labels_mapped]
+
+  return labels_mapped,colored_labels
+
+def create_nucelus(img,colored_segmentation_mask):
+  mask=colored_segmentation_mask
+  # Define the colors
+  color_to_extract = [255, 0, 0]
+  background_color = [0, 0, 255]
+
+  # Create masks for the components and the background
+  component_mask = np.all(mask == color_to_extract, axis=-1)
+  background_mask = ~component_mask
+
+  # Create an image with the extracted components in red and the background in blue
+  result = np.zeros_like(mask)
+
+  # cv2_imshow(result)
+  result[component_mask] = color_to_extract
+  result[background_mask] = background_color
+
+  img= cv2.resize(img, (224,224))
+
+  fgModel = np.zeros((1, 65), dtype="float")
+  bgModel = np.zeros((1, 65), dtype="float")
+
+  mask = np.zeros(result.shape[:2], np.uint8)
+  mask[(result == [255, 0, 0]).all(axis=2)] = cv2.GC_PR_FGD  # Foreground
+  mask[(result == [0, 0, 255]).all(axis=2)] = cv2.GC_PR_BGD  # Background
+
+  # mask = np.mean(result, axis=2)
+  # mask=mask.astype("uint8")
+
+  rect = (0, 0, img.shape[1], img.shape[0])
+
+  (mask, bgModel, fgModel) = cv2.grabCut(img, mask, rect, bgModel,
+	fgModel, iterCount=10, mode=cv2.GC_INIT_WITH_MASK)
+
+  output_image_1 = np.zeros((mask.shape[0], mask.shape[1], 3), dtype=np.uint8)
+
+  # Replace black pixels with red and white pixels with blue
+  output_image_1[mask == 2] = [0, 0, 255]  # Black to red
+  output_image_1[mask == 3] = [255, 0, 0]  # White to blue
+
+  return output_image_1
+
+def create_colored_segmentation_mask(labels):
+  colour_list=[0,0,0,0]
+
+  # first_unique, second_unique, third_unique = find_unique_values(labels)
+  colour_list[0]=[0,0,255]
+  colour_list[1]=[255,0,0]
+  colour_list[2]=[255,0,0]
+  colour_list[3]=[255,0,0]
+  # colour_list[4]=[255,0,0]
+
+  colors = np.array(colour_list)  # BGR format
+  colored_labels = colors[labels]
+
+  return colored_labels
+
+def create_background(img,heatmap,labels):
+  colored_labels = create_colored_segmentation_mask(labels)
+  mask=colored_labels
+  # Define the colors
+  color_to_extract = [255, 0, 0]
+  background_color = [0, 0, 255]
+
+  # Create masks for the components and the background
+  component_mask = np.all(mask == color_to_extract, axis=-1)
+  background_mask = ~component_mask
+
+  # Create an image with the extracted components in red and the background in blue
+  result = np.zeros_like(mask)
+  result[component_mask] = color_to_extract
+  result[background_mask] = background_color
+
+  fgModel = np.zeros((1, 65), dtype="float")
+  bgModel = np.zeros((1, 65), dtype="float")
+
+  mask1 = np.zeros(result.shape[:2], np.uint8)
+  mask1[(result == [255, 0, 0]).all(axis=2)] = cv2.GC_PR_FGD  # Foreground
+  mask1[(result == [0, 0, 255]).all(axis=2)] = cv2.GC_PR_BGD  # Background
+
+  # mask = np.mean(result, axis=2)
+  # mask=mask.astype("uint8")
+
+  rect = (1, 1, img.shape[1], img.shape[0])
+
+
+  (mask1, bgModel, fgModel) = cv2.grabCut(img, mask1, rect, bgModel,
+	fgModel, iterCount=10, mode=cv2.GC_INIT_WITH_MASK)
+
+
+  output_image = np.zeros((mask1.shape[0], mask1.shape[1], 3), dtype=np.uint8)
+
+  # Replace black pixels with red and white pixels with blue
+  output_image[mask1 == 2] = [0, 0, 255]  # Black to red
+  output_image[mask1 == 3] = [128, 0, 0]  # White to blue
+
+  return output_image
+
+# def remove_nucleus(image, blue_mask):
+#   #expand the nucleus mask
+#   image1 = cv2.resize(image, (224,224))
+#   blue_mask1 = cv2.resize(blue_mask, (224,224))
+#   kernel = np.ones((5, 5), np.uint8)  # Adjust the kernel size as needed
+#   expandedmask = cv2.dilate(blue_mask1, kernel, iterations=1)
+#   simple_lama = SimpleLama()
+#   image_pil = Image.fromarray(cv2.cvtColor(image1, cv2.COLOR_BGR2RGB))
+#   mask_pil = Image.fromarray(expandedmask)
+#   result = simple_lama(image_pil, mask_pil)
+#   result_cv2 = np.array(result)
+#   result_cv2 = cv2.cvtColor(result_cv2, cv2.COLOR_RGB2BGR)
+#   # result_cv2 = cv2.resize(result_cv2, (x,y))
+#   return expandedmask, result_cv2
+
+# def get_nucleus_mask(nucleus): #image_path, x, y
+#   # nucleus = cv2.imread(nucleus)
+#   # Convert image to HSV color space
+#   hsv_image = cv2.cvtColor(nucleus, cv2.COLOR_BGR2HSV)
+#   # Define lower and upper bounds for blue color in HSV
+#   lower_blue = np.array([100, 50, 50])
+#   upper_blue = np.array([130, 255, 255])
+#   # Create a mask for blue color
+#   blue_mask = cv2.inRange(hsv_image, lower_blue, upper_blue)
+#   return blue_mask #, image
+
+def save_heatmap(heatmap,img_path,heatmap_path):
+  img = cv2.imread(img_path)
+  heatmap_1 = cv2.resize(heatmap, (img.shape[1], img.shape[0]))
+
+  heatmap_1 = np.uint8(255 * heatmap_1)
+
+  heatmap_1 = cv2.applyColorMap(heatmap_1, cv2.COLORMAP_JET)
+
+  superimposed_img = cv2.addWeighted(heatmap_1, 0.4,img, 0.6,  0)
+  superimposed_img = np.uint8(superimposed_img)
+  
+  superimposed_img = cv2.cvtColor(superimposed_img, cv2.COLOR_BGR2RGB)
+
+  cv2.imwrite(heatmap_path,superimposed_img)
+
+
+def cam_main(pixel_conversion):
+  count=0
+
+  return_dict_count = 1
+  return_dict = {}
+  selected_indices = select_sample_images()
+  print('selected_indices')
+  print(selected_indices)
+  resized_shape = (224,224)
+  cell_descriptors = [
+      ["Image Name", "Nucleus Area", "Cytoplasm Area", "Nucleus to Cytoplasm Ratio"]
+  ]
+
+  image_files = [f for f in os.listdir(preprocessed_folder) if not f.startswith('.DS_Store')]
+
+  for imagefile in image_files:
+    if (
+        "MACOSX".lower() in imagefile.lower()
+        or "." == imagefile[0]
+        or "_" == imagefile[0]
+    ):
+        print(imagefile)
+        continue
+    image_path = (
+        preprocessed_folder + imagefile
+    )
+    intermediate_path = (
+        intermediate_folder
+        + os.path.splitext(imagefile)[0].lower()
+        + "_heatmap.png"
+    )
+    save_path = (
+        segmentation_folder + os.path.splitext(imagefile)[0].lower() + "_mask.png"
+    )
+    table_path = (
+        tables_folder + os.path.splitext(imagefile)[0].lower() + "_table.png"
+    )
+    # img_path=input_folder+'/'+a
+
+    # print(a)
+
+    # count+=1
+
+    # input_image = preprocess_image(img_path)
+
+    heatmap = generate_grad_cam_plus_plus(image_path, model, 'block14_sepconv2_act', ['dense_1'])
+
+    save_heatmap(heatmap,image_path,intermediate_path)
+    
+
+    pred_class = model.predict(preprocess_image(image_path))
+    pred_class = pred_class.argmax(axis=1)[0]
+
+    # print(pred_class)
+
+    if pred_class == 0:
+      labels,colored_segmentation_mask = GMM_abnormal_method(heatmap)
+    else:
+      labels,colored_segmentation_mask = GMM_normal_method(heatmap)
+
+    image=cv2.imread(image_path)
+    original_shape = image.shape
+    image= cv2.resize(image, (224,224))
+
+    nucleus= create_nucelus(image,colored_segmentation_mask)
+
+    # blue_mask = get_nucleus_mask(nucleus)
+
+    # expandedmask, result_cv2 = remove_nucleus(image, blue_mask)
+
+    background=create_background(image,heatmap,labels)
+
+    combined_mask = background & nucleus
+  
+
+    for i in range(combined_mask.shape[0]):
+      for j in range(combined_mask.shape[1]):
+          original_color = tuple(combined_mask[i, j])
+          if original_color == (128,0,0):
+              combined_mask[i, j] = np.array((255,0,0))
+          elif original_color == (0,0,0):
+              combined_mask[i, j] = np.array((128,0,0))
+
+    # combined_mask = cv2.resize(combined_mask, (224,224))
+
+    cv2.imwrite(save_path,combined_mask)
+
+
+    nucleus_area, cytoplasm_area, ratio = calculate_cell_descriptors(
+            original_shape, resized_shape, pixel_conversion, combined_mask
+        )
+    cell_descriptors.append(
+        [
+            os.path.splitext(imagefile)[0].lower(),
+            nucleus_area,
+            cytoplasm_area,
+            ratio,
+        ]
+    )
+
+    create_cell_descriptors_table(table_path, nucleus_area, cytoplasm_area, ratio)
+
+    if count in selected_indices:
+        return_dict[f"image{return_dict_count}"] = str(
+            base64.b64encode(open(image_path, "rb").read()).decode("utf-8")
+        )
+        return_dict[f"inter{return_dict_count}"] = str(
+            base64.b64encode(open(intermediate_path, "rb").read()).decode("utf-8")
+        )
+        return_dict[f"mask{return_dict_count}"] = str(
+            base64.b64encode(open(save_path, "rb").read()).decode("utf-8")
+        )
+        return_dict[f"table{return_dict_count}"] = str(
+            base64.b64encode(open(table_path, "rb").read()).decode("utf-8")
+        )
+        return_dict_count += 1
+
+    count+=1
+
+    print(count)
+
+  with open(cell_descriptors_path, "w", newline="") as csv_file:
+        writer = csv.writer(csv_file)
+        writer.writerows(cell_descriptors)
+
+  print(list(return_dict.keys()))
+
+  return return_dict
+
+    
+
+# cam_main(0.2)

herlev_best_adam_vgg16_modified12_final.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2c2570970458b76afa0a6741077fedc8f9e60692c970594b4e69b846c1dc8543
+size 260263734

lrp_pipeline.py

@@ -0,0 +1,415 @@
+import cv2
+import torch
+import torch.nn as nn
+import numpy as np
+import torchvision
+import os
+import copy
+from sklearn.mixture import GaussianMixture as GMM
+from sklearn.cluster import KMeans
+from simple_lama_inpainting import SimpleLama
+from PIL import Image
+from matplotlib.colors import ListedColormap
+import matplotlib.pyplot as plt
+import matplotlib
+import csv
+
+matplotlib.use("Agg")
+
+import base64
+
+from utils import (
+    select_sample_images,
+    create_cell_descriptors_table,
+    calculate_cell_descriptors,
+)
+
+preprocessed_folder = "uploads/"
+intermediate_folder = "heatmaps/"
+segmentation_folder = "segmentations/"
+tables_folder = "tables/"
+cell_descriptors_path = "cell_descriptors/cell_descriptors.csv"
+imgclasses = {0: "abnormal", 1: "normal"}
+
+
+def toconv(layers):
+    newlayers = []
+    for i, layer in enumerate(layers):
+        if isinstance(layer, nn.Linear):
+            newlayer = None
+            if i == 0:
+                m, n = 512, layer.weight.shape[0]
+                newlayer = nn.Conv2d(m, n, 4)
+                newlayer.weight = nn.Parameter(layer.weight.reshape(n, m, 4, 4))
+            else:
+                m, n = layer.weight.shape[1], layer.weight.shape[0]
+                newlayer = nn.Conv2d(m, n, 1)
+                newlayer.weight = nn.Parameter(layer.weight.reshape(n, m, 1, 1))
+            newlayer.bias = nn.Parameter(layer.bias)
+            newlayers += [newlayer]
+        else:
+            newlayers += [layer]
+    return newlayers
+
+
+def newlayer(layer, g):
+    layer = copy.deepcopy(layer)
+    try:
+        layer.weight = nn.Parameter(g(layer.weight))
+    except AttributeError:
+        pass
+    try:
+        layer.bias = nn.Parameter(g(layer.bias))
+    except AttributeError:
+        pass
+    return layer
+
+
+def heatmap(R, sx, sy, intermediate_path):
+    b = 10 * ((np.abs(R) ** 3.0).mean() ** (1.0 / 3))
+    my_cmap = plt.cm.seismic(np.arange(plt.cm.seismic.N))
+    my_cmap[:, 0:3] *= 0.85
+    my_cmap = ListedColormap(my_cmap)
+    plt.figure(figsize=(sx, sy))
+    plt.subplots_adjust(left=0, right=1, bottom=0, top=1)
+    plt.axis("off")
+    plt.imshow(R, cmap=my_cmap, vmin=-b, vmax=b, interpolation="nearest")
+    # plt.show()
+    plt.savefig(intermediate_path, bbox_inches="tight", pad_inches=0)
+    plt.close()
+
+
+def get_LRP_heatmap(image, L, layers, imgclasses, intermediate_path):
+    img = np.array(image)[..., ::-1] / 255.0
+    mean = torch.FloatTensor([0.485, 0.456, 0.406]).reshape(1, -1, 1, 1)  # torch.cuda
+    std = torch.FloatTensor([0.229, 0.224, 0.225]).reshape(1, -1, 1, 1)  # torch.cuda
+    X = (torch.FloatTensor(img[np.newaxis].transpose([0, 3, 1, 2]) * 1) - mean) / std
+
+    A = [X] + [None] * L
+    for l in range(L):
+        A[l + 1] = layers[l].forward(A[l])
+
+    scores = np.array(A[-1].cpu().data.view(-1))
+    ind = np.argsort(-scores)
+    for i in ind[:2]:
+        print("%20s (%3d): %6.3f" % (imgclasses[i], i, scores[i]))
+
+    T = torch.FloatTensor(
+        (1.0 * (np.arange(2) == ind[0]).reshape([1, 2, 1, 1]))
+    )  # SET FOR THE HIGHEST SCORE CLASS
+    R = [None] * L + [(A[-1] * T).data]
+    for l in range(1, L)[::-1]:
+        A[l] = (A[l].data).requires_grad_(True)
+        if isinstance(layers[l], torch.nn.MaxPool2d):
+            layers[l] = torch.nn.AvgPool2d(2)
+        if isinstance(layers[l], torch.nn.Conv2d) or isinstance(
+            layers[l], torch.nn.AvgPool2d
+        ):
+            rho = lambda p: p + 0.25 * p.clamp(min=0)
+            incr = lambda z: z + 1e-9  # USE ONLY THE GAMMA RULE FOR ALL LAYERS
+
+            z = incr(newlayer(layers[l], rho).forward(A[l]))  # step 1
+            # adding epsilon
+            epsilon = 1e-9
+            z_nonzero = torch.where(z == 0, torch.tensor(epsilon, device=z.device), z)
+            s = (R[l + 1] / z_nonzero).data
+            # s = (R[l+1]/z).data                                    # step 2
+            (z * s).sum().backward()
+            c = A[l].grad  # step 3
+            R[l] = (A[l] * c).data  # step 4
+        else:
+            R[l] = R[l + 1]
+
+    A[0] = (A[0].data).requires_grad_(True)
+    lb = (A[0].data * 0 + (0 - mean) / std).requires_grad_(True)
+    hb = (A[0].data * 0 + (1 - mean) / std).requires_grad_(True)
+
+    z = layers[0].forward(A[0]) + 1e-9  # step 1 (a)
+    z -= newlayer(layers[0], lambda p: p.clamp(min=0)).forward(lb)  # step 1 (b)
+    z -= newlayer(layers[0], lambda p: p.clamp(max=0)).forward(hb)  # step 1 (c)
+
+    # adding epsilon
+    epsilon = 1e-9
+    z_nonzero = torch.where(z == 0, torch.tensor(epsilon, device=z.device), z)
+    s = (R[1] / z_nonzero).data  # step 2
+
+    (z * s).sum().backward()
+    c, cp, cm = A[0].grad, lb.grad, hb.grad  # step 3
+    R[0] = (A[0] * c + lb * cp + hb * cm).data  # step 4
+    heatmap(
+        np.array(R[0][0].cpu()).sum(axis=0), 2, 2, intermediate_path
+    )  # HEATMAPPING TO SEE LRP MAPS WITH NEW RULE
+    return R[0][0].cpu()
+
+
+def get_nucleus_mask_for_graphcut(R):
+    res = np.array(R).sum(axis=0)
+    # Reshape the data to a 1D array
+    data_1d = res.flatten().reshape(-1, 1)
+    n_clusters = 2
+    kmeans = KMeans(n_clusters=n_clusters, random_state=0)
+    # kmeans.fit(data_1d)
+    kmeans.fit(data_1d)
+    # Step 4: Assign data points to clusters
+    cluster_assignments = kmeans.labels_
+    # Step 5: Reshape cluster assignments into a 2D binary matrix
+    binary_matrix = cluster_assignments.reshape(128, 128)
+    # Now, binary_matrix contains 0s and 1s, separating the data into two classes using K-Means clustering
+    rel_grouping = np.zeros((128, 128, 3), dtype=np.uint8)
+    rel_grouping[binary_matrix == 1] = [255, 0, 0]  # Main object (Blue)
+    rel_grouping[binary_matrix == 2] = [128, 0, 0]  # Second label (Dark Blue)
+    rel_grouping[binary_matrix == 0] = [0, 0, 255]  # Background (Red)
+    return rel_grouping
+
+
+def segment_nucleus(image, rel_grouping):  # clustered = rel_grouping
+
+    # GET THE BOUNDING BOX FROM CLUSTERED
+    blue_pixels = np.sum(np.all(rel_grouping == [255, 0, 0], axis=-1))
+    red_pixels = np.sum(np.all(rel_grouping == [0, 0, 255], axis=-1))
+    if red_pixels > blue_pixels:
+        color = np.array([255, 0, 0])
+    else:
+        color = np.array([0, 0, 255])
+    mask = cv2.inRange(rel_grouping, color, color)
+    contours, _ = cv2.findContours(mask, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE)
+    contour_areas = []
+    for contour in contours:
+        x, y, w, h = cv2.boundingRect(contour)
+        contour_areas.append(cv2.contourArea(contour))
+    contour_areas.sort()
+    contour_areas = np.array(contour_areas)
+    quartile_50 = np.percentile(contour_areas, 50)
+    selected_contours = [
+        contour for contour in contours if cv2.contourArea(contour) >= quartile_50
+    ]
+    x, y, w, h = cv2.boundingRect(np.concatenate(selected_contours))
+
+    # APPLY GRABCUT
+    fgModel = np.zeros((1, 65), dtype="float")
+    bgModel = np.zeros((1, 65), dtype="float")
+    mask = np.zeros(image.shape[:2], np.uint8)
+    rect = (x, y, x + w, y + h)
+
+    # IF BOUNDING BOX IS THE WHOLE IMAGE, THEN BOUNDING BOX METHOD WONT'T WORK -> SO USE INIT WITH MASK METHOD ITSELF
+    if (x, y, x + w, y + h) == (0, 0, 128, 128):
+
+        if (
+            red_pixels > blue_pixels
+        ):  # red is the dominant color and thus the background
+            mask[(rel_grouping == [255, 0, 0]).all(axis=2)] = (
+                cv2.GC_PR_FGD
+            )  # Probable Foreground
+            mask[(rel_grouping == [0, 0, 255]).all(axis=2)] = (
+                cv2.GC_PR_BGD
+            )  # Probable Background
+        else:  # blue is the dominant color and thus the background
+            mask[(rel_grouping == [0, 0, 255]).all(axis=2)] = (
+                cv2.GC_PR_FGD
+            )  # Probable Foreground
+            mask[(rel_grouping == [255, 0, 0]).all(axis=2)] = (
+                cv2.GC_PR_BGD
+            )  # Probable Background
+
+        (mask, bgModel, fgModel) = cv2.grabCut(
+            image,
+            mask,
+            rect,
+            bgModel,
+            fgModel,
+            iterCount=10,
+            mode=cv2.GC_INIT_WITH_MASK,
+        )
+
+    # ELSE PASS THE BOUNDING BOX FOR GRABCUT
+    else:
+        (mask, bgModel, fgModel) = cv2.grabCut(
+            image,
+            mask,
+            rect,
+            bgModel,
+            fgModel,
+            iterCount=10,
+            mode=cv2.GC_INIT_WITH_RECT,
+        )
+
+    # FORM THE COLORED SEGMENTATION MASK
+    clean_binary_mask = np.where(
+        (mask == cv2.GC_FGD) | (mask == cv2.GC_PR_FGD), 1, 0
+    ).astype("uint8")
+    nucleus_segment = np.zeros((128, 128, 3), dtype=np.uint8)
+    nucleus_segment[clean_binary_mask == 1] = [255, 0, 0]  # Main object (Blue)
+    nucleus_segment[clean_binary_mask == 0] = [0, 0, 255]  # Background (Red)
+    return nucleus_segment, clean_binary_mask
+
+
+def remove_nucleus(image1, blue_mask1):  # image, blue_mask, x, y
+    # expand the nucleus mask
+    # image1 = cv2.resize(image, (128,128))
+    # blue_mask1 = cv2.resize(blue_mask, (128,128))
+    kernel = np.ones((5, 5), np.uint8)  # Adjust the kernel size as needed
+    expandedmask = cv2.dilate(blue_mask1, kernel, iterations=1)
+    simple_lama = SimpleLama()
+    image_pil = Image.fromarray(cv2.cvtColor(image1, cv2.COLOR_BGR2RGB))
+    mask_pil = Image.fromarray(expandedmask)
+    result = simple_lama(image_pil, mask_pil)
+    result_cv2 = np.array(result)
+    result_cv2 = cv2.cvtColor(result_cv2, cv2.COLOR_RGB2BGR)
+    # result_cv2 = cv2.resize(result_cv2, (x,y))
+    return expandedmask, result_cv2
+
+
+def get_final_mask(nucleus_removed_img, blue_mask, expanded_mask):
+    # apply graphcut - init with rectangle (not mask approximation mask)
+    fgModel = np.zeros((1, 65), dtype="float")
+    bgModel = np.zeros((1, 65), dtype="float")
+
+    rect = (1, 1, nucleus_removed_img.shape[1], nucleus_removed_img.shape[0])
+
+    (mask, bgModel, fgModel) = cv2.grabCut(
+        nucleus_removed_img,
+        expanded_mask,
+        rect,
+        bgModel,
+        fgModel,
+        iterCount=20,
+        mode=cv2.GC_INIT_WITH_RECT,
+    )
+
+    clean_binary_mask = np.where(
+        (mask == cv2.GC_FGD) | (mask == cv2.GC_PR_FGD), 1, 0
+    ).astype("uint8")
+    colored_segmentation_mask = np.zeros((128, 128, 3), dtype=np.uint8)
+    colored_segmentation_mask[clean_binary_mask == 1] = [
+        128,
+        0,
+        0,
+    ]  # Main object (Blue)
+    colored_segmentation_mask[clean_binary_mask == 0] = [0, 0, 255]  # Background (Red)
+    colored_segmentation_mask[blue_mask > 0] = [255, 0, 0]
+    return colored_segmentation_mask
+
+
+def lrp_main(pixel_conversion):
+    i = 0
+    return_dict_count = 1
+    return_dict = {}
+    selected_indices = select_sample_images()
+    resized_shape = (128, 128)
+    cell_descriptors = [
+        ["Image Name", "Nucleus Area", "Cytoplasm Area", "Nucleus to Cytoplasm Ratio"]
+    ]
+
+    for imagefile in os.listdir(preprocessed_folder):
+        if (
+            "MACOSX".lower() in imagefile.lower()
+            or "." == imagefile[0]
+            or "_" == imagefile[0]
+        ):
+            print(imagefile)
+            continue
+        image_path = (
+            preprocessed_folder + os.path.splitext(imagefile)[0].lower() + ".png"
+        )
+        intermediate_path = (
+            intermediate_folder
+            + os.path.splitext(imagefile)[0].lower()
+            + "_heatmap.png"
+        )
+        save_path = (
+            segmentation_folder + os.path.splitext(imagefile)[0].lower() + "_mask.png"
+        )
+        table_path = (
+            tables_folder + os.path.splitext(imagefile)[0].lower() + "_table.png"
+        )
+
+        # print(i, imagefile)
+        image = cv2.imread(image_path)
+        original_shape = image.shape
+
+        image = cv2.resize(image, (128, 128))
+
+        # MODEL SECTION STARTS FOR NEW MODEL
+        vgg16 = torchvision.models.vgg16(pretrained=True)
+        new_avgpool = nn.AdaptiveAvgPool2d(output_size=(4, 4))
+        vgg16.avgpool = new_avgpool
+        classifier_list = [
+            nn.Linear(8192, vgg16.classifier[0].out_features)
+        ]  # vgg16.classifier[0].out_features = 4096
+        classifier_list += list(vgg16.classifier.children())[
+            1:-1
+        ]  # Remove the first and last layers
+        classifier_list += [
+            nn.Linear(vgg16.classifier[6].in_features, 2)
+        ]  # vgg16.classifier[6].in_features = 4096
+        vgg16.classifier = nn.Sequential(
+            *classifier_list
+        )  # Replace the model classifier
+
+        PATH = "herlev_best_adam_vgg16_modified12_final.pth"
+        checkpoint = torch.load(PATH, map_location=torch.device("cpu"))
+        vgg16.load_state_dict(checkpoint)
+        # vgg16.to(torch.device('cuda'))
+        vgg16.eval()
+
+        layers = list(vgg16._modules["features"]) + toconv(
+            list(vgg16._modules["classifier"])
+        )
+        L = len(layers)
+        # MODEL SECTION ENDS
+
+        R = get_LRP_heatmap(image, L, layers, imgclasses, intermediate_path)
+
+        rel_grouping = get_nucleus_mask_for_graphcut(R)
+
+        nucleus_segment, clean_binary_mask = segment_nucleus(image, rel_grouping)
+
+        expanded_mask, nucleus_removed_image = remove_nucleus(image, clean_binary_mask)
+
+        colored_segmentation_mask = get_final_mask(
+            nucleus_removed_image, clean_binary_mask, expanded_mask
+        )
+
+        cv2.imwrite(save_path, colored_segmentation_mask)
+
+        nucleus_area, cytoplasm_area, ratio = calculate_cell_descriptors(
+            original_shape, resized_shape, pixel_conversion, colored_segmentation_mask
+        )
+        cell_descriptors.append(
+            [
+                os.path.splitext(imagefile)[0].lower(),
+                nucleus_area,
+                cytoplasm_area,
+                ratio,
+            ]
+        )
+
+        create_cell_descriptors_table(table_path, nucleus_area, cytoplasm_area, ratio)
+
+        if i in selected_indices:
+            return_dict[f"image{return_dict_count}"] = str(
+                base64.b64encode(open(image_path, "rb").read()).decode("utf-8")
+            )
+            return_dict[f"inter{return_dict_count}"] = str(
+                base64.b64encode(open(intermediate_path, "rb").read()).decode("utf-8")
+            )
+            return_dict[f"mask{return_dict_count}"] = str(
+                base64.b64encode(open(save_path, "rb").read()).decode("utf-8")
+            )
+            return_dict[f"table{return_dict_count}"] = str(
+                base64.b64encode(open(table_path, "rb").read()).decode("utf-8")
+            )
+            return_dict_count += 1
+
+        i += 1
+
+        # Visualization
+        # for im in [image, gt2, rel_grouping, nucleus_segment, clean_binary_mask*255, nucleus_removed_image, colored_segmentation_mask]:
+        #   cv2_imshow(im)
+
+    # write cell_descriptors list to csv file
+    with open(cell_descriptors_path, "w", newline="") as csv_file:
+        writer = csv.writer(csv_file)
+        writer.writerows(cell_descriptors)
+
+    return return_dict

preprocessing_pipeline.py

@@ -0,0 +1,80 @@
+import cv2
+import numpy as np
+import os
+
+image_height=224
+image_width=224
+
+
+def read_image(image_path,image_height,image_width):
+  image=cv2.imread(image_path)
+  image=cv2.resize(image, (image_height,image_width))
+  image=cv2.cvtColor(image, cv2.COLOR_RGB2BGR)
+
+  return image
+
+def min_max_normalization (image):
+    float_image = image.astype(np.float32)
+
+    # Calculate the minimum and maximum pixel values
+    min_value = np.min(float_image)
+    max_value = np.max(float_image)
+
+    # Perform Min-Max normalization
+    normalized_image = (float_image - min_value) / (max_value - min_value)
+
+    return normalized_image
+
+
+def apply_histogram_normalization(image):
+
+  b_channel, g_channel, r_channel = cv2.split(image)
+
+  normalized_b = cv2.normalize(b_channel, None, alpha=0, beta=255, norm_type=cv2.NORM_MINMAX)
+  normalized_g = cv2.normalize(g_channel, None, alpha=0, beta=255, norm_type=cv2.NORM_MINMAX)
+  normalized_r = cv2.normalize(r_channel, None, alpha=0, beta=255, norm_type=cv2.NORM_MINMAX)
+
+
+  normalized_image = cv2.merge((normalized_b, normalized_g, normalized_r))
+
+  return normalized_image
+
+
+def remove_noise(image):
+
+  median = cv2.medianBlur(image,5)
+
+  return median
+
+def adaptive_gamma_correction(image):
+  def apply_adaptive_gamma_correction(channel, gamma):
+    corrected_channel = np.power((channel / 255.0), 1.0 / gamma)
+    return cv2.normalize(corrected_channel, None, alpha=0, beta=255, norm_type=cv2.NORM_MINMAX)
+
+
+  b_channel, g_channel, r_channel = cv2.split(image)
+
+
+  gamma = 1.5
+  gamma_corrected_b = apply_adaptive_gamma_correction(b_channel, gamma)
+  gamma_corrected_g = apply_adaptive_gamma_correction(g_channel, gamma)
+  gamma_corrected_r = apply_adaptive_gamma_correction(r_channel, gamma)
+
+
+  gamma_corrected_image = cv2.merge((gamma_corrected_b, gamma_corrected_g, gamma_corrected_r))
+
+
+  gamma_corrected_image=min_max_normalization(gamma_corrected_image)
+
+  return gamma_corrected_image
+
+
+def preprocess_image(img_path):
+
+    image = read_image(img_path,image_height,image_width)
+
+    normalized_image= apply_histogram_normalization(image)
+    median= remove_noise(normalized_image)
+    gamma_corrected_image=adaptive_gamma_correction(median)
+
+    return gamma_corrected_image*255

requirements.txt

@@ -0,0 +1,13 @@
+Flask==3.0.3
+Flask_Cors==4.0.0
+matplotlib==3.8.1
+# matplotlib==3.8.4
+numpy==1.26.4
+opencv_python==4.9.0.80
+pandas==2.2.2
+Pillow==9.5.0
+scikit_learn==1.3.2
+simple_lama_inpainting==0.1.2
+tensorflow==2.15.0
+torch==2.2.2
+torchvision==0.17.2

utils.py

@@ -0,0 +1,194 @@
+import os
+import zipfile
+import random
+import pandas as pd
+import matplotlib.pyplot as plt
+import shutil
+import numpy as np
+import cv2
+
+preprocessed_folder = "uploads/"
+intermediate_folder = "heatmaps/"
+segmentation_folder = "segmentations/"
+tables_folder = "tables/"
+cell_descriptors_path = "cell_descriptors/cell_descriptors.csv"
+zip_file_path = "outputs.zip"
+
+
+def select_sample_images():
+    # first check if a zip file has been uploaded and extract images from it
+    for file_name in os.listdir(preprocessed_folder):
+        if file_name.endswith(".zip"):
+            zip_file_path = os.path.join(preprocessed_folder, file_name)
+            with zipfile.ZipFile(zip_file_path, "r") as zip_ref:
+                # Extract all contents of the zip file to the preprocessed_folder
+                zip_ref.extractall(path=preprocessed_folder)
+            # Remove the original zip file
+            os.remove(zip_file_path)
+            # print("Contents of the zip file extracted to the folder.")
+            break
+
+    # Get a list of all subfolders in the main folder
+    subfolders = [
+        f
+        for f in os.listdir(preprocessed_folder)
+        if os.path.isdir(os.path.join(preprocessed_folder, f))
+    ]
+    # Iterate through each subfolder and move its files to the main folder
+    for subfolder in subfolders:
+        if "MACOSX" not in subfolder:
+            subfolder_path = os.path.join(preprocessed_folder, subfolder)
+            for file_name in os.listdir(subfolder_path):
+                source_path = os.path.join(subfolder_path, file_name)
+                destination_path = os.path.join(preprocessed_folder, file_name)
+                shutil.move(source_path, destination_path)
+            # print(f"Moved file '{file_name}' from '{subfolder}' to '{main_folder}'")
+    # Delete empty subfolders
+    for subfolder in subfolders:
+        if "MACOSX" not in subfolder:
+            subfolder_path = os.path.join(preprocessed_folder, subfolder)
+            try:
+                os.rmdir(subfolder_path)
+                # print(f"Deleted empty folder '{subfolder}'")
+            except OSError as e:
+                print(f"Error deleting folder '{subfolder}': {e}")
+
+    # next check the count of images in the folder
+    image_extensions = [
+        ".jpg",
+        ".jpeg",
+        ".png",
+        ".gif",
+        ".bmp",
+    ]  # Add more extensions if needed
+    image_count = 0
+    for file_name in os.listdir(preprocessed_folder):
+        if any(file_name.lower().endswith(ext) for ext in image_extensions):
+            image_count += 1
+
+    # if count > 5, return 5 random indices
+    # else, return all 5 indices
+    if image_count > 5:
+        indices = random.sample(range(image_count), 5)
+        indices.sort()
+        return indices
+    else:
+        return list(range(image_count))
+
+
+def create_cell_descriptors_table(table_path, nucleus_area, cytoplasm_area, ratio):
+    # Sample data for the table
+    data = {
+        "Metric": ["Nucleus Area", "Cytoplasm Area", "N:C Ratio"],
+        "Value": [
+            str(round(nucleus_area, 5)),
+            str(round(cytoplasm_area, 5)),
+            str(round(ratio, 5)),
+        ],
+    }
+
+    # Define cell colors
+    cell_colors = [
+        ["lightgrey", "lightblue"],
+        ["lightgrey", "lightgreen"],
+        ["lightgrey", "lightyellow"],
+    ]
+
+    # Create a DataFrame
+    df = pd.DataFrame(data)
+
+    # Plot table
+    fig = plt.figure(figsize=(2, 2))
+    table = plt.table(
+        cellText=df.values,
+        colLabels=df.columns,
+        loc="center",
+        cellLoc="center",
+        cellColours=cell_colors,
+    )
+
+    # Set cell heights
+    table.auto_set_font_size(False)
+    table.set_fontsize(6)  # Adjust font size if needed
+    table.scale(1, 2)  # Increase cell heights
+
+    # Hide axes
+    plt.axis("off")
+
+    fig.tight_layout()
+    # Save as image
+    fig.savefig(table_path)  # pad_inches=(0.1, 0.1, 0.1, 0.1)  bbox_inches="tight"
+    plt.close()
+    # plt.show()
+
+
+def delete_folders(folder_names):
+    for folder_name in folder_names:
+        try:
+            shutil.rmtree(folder_name)
+            # print(f"Folder deleted: {folder_name}")
+        except FileNotFoundError:
+            print(f"Folder does not exist: {folder_name}")
+        except Exception as e:
+            print(f"Error deleting folder {folder_name}: {e}")
+
+
+def create_folders(folder_names):
+    for folder_name in folder_names:
+        try:
+            os.makedirs(folder_name)
+            # print(f"Folder created: {folder_name}")
+        except FileExistsError:
+            print(f"Folder already exists: {folder_name}")
+        except Exception as e:
+            print(f"Error creating folder {folder_name}: {e}")
+
+
+def calculate_cell_descriptors(
+    original_shape, resized_shape, pixel_conversion, segmentation_mask
+):
+    area_of_pixel = (
+        original_shape[0]
+        * original_shape[1]
+        * (pixel_conversion**2)
+        / (resized_shape[0] * resized_shape[1])
+    )
+
+    binary_nucleus = np.zeros(resized_shape, dtype=np.uint8)
+    binary_cytoplasm = np.zeros(resized_shape, dtype=np.uint8)
+    binary_nucleus[(segmentation_mask == [255, 0, 0]).all(axis=2)] = 1
+    binary_cytoplasm[(segmentation_mask == [128, 0, 0]).all(axis=2)] = 1
+
+    # Find contours in the binary masks
+    nucleus_contours, _ = cv2.findContours(
+        binary_nucleus, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE
+    )  # nucleus
+    cytoplasm_contours, _ = cv2.findContours(
+        binary_cytoplasm, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE
+    )  # cytoplasm
+
+    # Calculate area for nucleus and cytoplasm
+    nucleus_area = sum(cv2.contourArea(contour) for contour in nucleus_contours)
+    cytoplasm_area = sum(cv2.contourArea(contour) for contour in cytoplasm_contours)
+    if cytoplasm_area == 0:
+        ratio = np.NaN
+    else:
+        ratio = nucleus_area / cytoplasm_area
+
+    return nucleus_area * area_of_pixel, cytoplasm_area * area_of_pixel, ratio
+
+
+def create_zip_file():
+    folders = [intermediate_folder, segmentation_folder]
+    csv_file = cell_descriptors_path
+    with zipfile.ZipFile(zip_file_path, "w") as zipf:
+        # Add folders to the zip file
+        for folder in folders:
+            for root, dirs, files in os.walk(folder):
+                for file in files:
+                    file_path = os.path.join(root, file)
+                    arcname = os.path.relpath(file_path, os.path.join(folder, ".."))
+                    zipf.write(file_path, arcname=arcname)
+
+        # Add the CSV file to the zip file
+        zipf.write(csv_file, arcname=os.path.basename(csv_file))

xception_model_81.h5

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:dc8dc9524a83d4e90bd0b854639e7ec87e8ba6c09b4f6d11010baf6fefde88ac
+size 90259112