feat:workbench page

app.py

@@ -21,7 +21,8 @@ from inference_resnet import get_triplet_model
 from inference_beit import get_triplet_model_beit
 import pathlib
 import tensorflow as tf
-from closest_sample import get_images
+from closest_sample import get_images,get_diagram
+
 
 if not os.path.exists('images'):
     REPO_ID='Serrelab/image_examples_gradio'
@@ -35,6 +36,57 @@ if not os.path.exists('dataset'):
      print("warning! A read token in env variables is needed for authentication.")
   snapshot_download(repo_id=REPO_ID, token=token,repo_type='dataset',local_dir='dataset')
 
+HEADER = '''
+<h2><b>Official Gradio Demo</b></h2><h2><a href='https://huggingface.co/spaces/Serrelab/fossil_app' target='_blank'><b>Identifying Florissant Leaf Fossils to Family using Deep Neural Networks </b></a></h2>
+Code: <a href='https://github.com/orgs/serre-lab/projects/2' target='_blank'>GitHub</a>. Paper: <a href='' target='_blank'>ArXiv</a>.
+
+
+'''
+
+"""
+**Fossil** a brief intro to the project.
+# ❗️❗️❗️**Important Notes:**
+# - some notes to users some notes to users some notes to users some notes to users some notes to users some notes to users .
+# - some notes to users some notes to users some notes to users some notes to users some notes to users some notes to users.
+
+"""
+
+USER_GUIDE = """
+                <div style='background-color: #f0f0f0; padding: 20px; border-radius: 10px;'>
+                    <h2>❗️ User Guide</h2>
+                    <p>Welcome to the interactive fossil exploration tool. Here's how to get started:</p>
+                    <ul>
+                        <li><strong>Upload an Image:</strong> Drag and drop or choose from given samples to upload images of fossils.</li>
+                        <li><strong>Process Image:</strong> After uploading, click the 'Process Image' button to analyze the image.</li>
+                        <li><strong>Explore Results:</strong> Switch to the 'Workbench' tab to check out detailed analysis and results.</li>
+                    </ul>
+                    <h3>Tips</h3>
+                    <ul>
+                        <li>Zoom into images on the workbench for finer details.</li>
+                        <li>Use the examples below as references for what types of images to upload.</li>
+                    </ul>
+                    <p>Enjoy exploring! 🌟</p>
+                </div>
+                """
+                
+TIPS = """
+                ## Tips
+                - Zoom into images on the workbench for finer details.
+                - Use the examples below as references for what types of images to upload.
+                
+                Enjoy exploring! 
+                """
+CITATION = '''
+📧 **Contact** <br>
+If you have any questions, feel free to contact us at <b>ivan_felipe_rodriguez@brown.edu</b>.
+'''
+"""
+📝 **Citation**
+cite using this bibtex:...
+```
+```
+📋 **License**
+"""
 def get_model(model_name):
    
         
@@ -61,6 +113,13 @@ def get_model(model_name):
                                   embedding_depth = 2,
                                   n_classes = n_classes)
         model.load_weights('model_classification/fossil-142.h5')
+    elif model_name == 'Fossils new':
+        n_classes = 142
+        model = get_triplet_model_beit(input_shape = (384, 384, 3),
+                                  embedding_units = 256,
+                                  embedding_depth = 2,
+                                  n_classes = n_classes)
+        model.load_weights('model_classification/fossil-new.h5')
     else:
         raise ValueError(f"Model name '{model_name}' is not recognized") 
     return model,n_classes
@@ -82,7 +141,12 @@ def classify_image(input_image, model_name):
         model, n_classes= get_model(model_name)
         result = inference_resnet_finer(input_image,model,size=600,n_classes=n_classes)
         return result 
-    if 'Fossils 142' ==model_name:
+    elif 'Fossils 142' ==model_name:
+        from inference_beit import inference_resnet_finer_beit
+        model,n_classes = get_model(model_name)
+        result = inference_resnet_finer_beit(input_image,model,size=384,n_classes=n_classes)
+        return result
+    elif 'Fossils new' ==model_name:
         from inference_beit import inference_resnet_finer_beit
         model,n_classes = get_model(model_name)
         result = inference_resnet_finer_beit(input_image,model,size=384,n_classes=n_classes)
@@ -100,7 +164,12 @@ def get_embeddings(input_image,model_name):
         model, n_classes= get_model(model_name)
         result = inference_resnet_embedding(input_image,model,size=600,n_classes=n_classes)
         return result 
-    if 'Fossils 142' ==model_name:
+    elif 'Fossils 142' ==model_name:
+        from inference_beit import inference_resnet_embedding_beit
+        model,n_classes = get_model(model_name)
+        result = inference_resnet_embedding_beit(input_image,model,size=384,n_classes=n_classes)
+        return result
+    elif 'Fossils new' ==model_name:
         from inference_beit import inference_resnet_embedding_beit
         model,n_classes = get_model(model_name)
         result = inference_resnet_embedding_beit(input_image,model,size=384,n_classes=n_classes)
@@ -114,30 +183,103 @@ def find_closest(input_image,model_name):
     #outputs = classes+paths
     return classes,paths
 
-def explain_image(input_image,model_name):
+def generate_diagram_closest(input_image,model_name,top_k):
+    embedding = get_embeddings(input_image,model_name)
+    diagram_path = get_diagram(embedding,top_k)
+    return diagram_path
+
+def explain_image(input_image,model_name,explain_method,nb_samples):
     model,n_classes= get_model(model_name)
-    if model_name=='Fossils 142':
+    if model_name=='Fossils 142' or 'Fossils new':
         size = 384
     else:
         size = 600
     #saliency, integrated, smoothgrad,
-    exp_list = explain(model,input_image,size = size, n_classes=n_classes)
+    classes,exp_list = explain(model,input_image,explain_method,nb_samples,size = size, n_classes=n_classes)
     #original =  saliency + integrated + smoothgrad 
     print('done')
-    sobol1,sobol2,sobol3,sobol4,sobol5 = exp_list[0],exp_list[1],exp_list[2],exp_list[3],exp_list[4]
-    rise1,rise2,rise3,rise4,rise5 = exp_list[5],exp_list[6],exp_list[7],exp_list[8],exp_list[9]
-    hsic1,hsic2,hsic3,hsic4,hsic5 = exp_list[10],exp_list[11],exp_list[12],exp_list[13],exp_list[14]
-    saliency1,saliency2,saliency3,saliency4,saliency5 = exp_list[15],exp_list[16],exp_list[17],exp_list[18],exp_list[19]
-    return sobol1,sobol2,sobol3,sobol4,sobol5,rise1,rise2,rise3,rise4,rise5,hsic1,hsic2,hsic3,hsic4,hsic5,saliency1,saliency2,saliency3,saliency4,saliency5
     
+    return classes,exp_list
+
+def setup_examples():
+    paths = sorted(pathlib.Path('images/').rglob('*.jpg'))
+    samples = [path.as_posix() for path in paths if 'fossils' in str(path)][:19]
+    examples_fossils = gr.Examples(samples, inputs=input_image,examples_per_page=5,label='Fossils Examples from the dataset')
+    samples=[[path.as_posix()] for path in paths  if 'leaves' in  str(path) ][:19]
+    examples_leaves = gr.Examples(samples, inputs=input_image,examples_per_page=5,label='Leaves Examples from the dataset')
+    return examples_fossils,examples_leaves
+
+def preprocess_image(image, output_size=(300, 300)):
+    #shape (height, width, channels)
+    h, w = image.shape[:2]
+
+    #padding
+    if h > w:
+        padding = (h - w) // 2
+        image_padded = cv2.copyMakeBorder(image, 0, 0, padding, padding, cv2.BORDER_CONSTANT, value=[0, 0, 0])
+    else:
+        padding = (w - h) // 2
+        image_padded = cv2.copyMakeBorder(image, padding, padding, 0, 0, cv2.BORDER_CONSTANT, value=[0, 0, 0])
+    
+    # resize
+    image_resized = cv2.resize(image_padded, output_size, interpolation=cv2.INTER_AREA)
+
+    return image_resized
+
+def update_display(image):
+    processed_image = preprocess_image(image) 
+    instruction = "Image ready. Please switch to the 'Specimen Workbench' tab to check out further analysis and outputs."
+    model_name = gr.Dropdown(
+                    ["Mummified 170", "Rock 170","Fossils 142","Fossils new"],
+                    multiselect=False,
+                    value="Fossils new", # default option
+                    label="Model",
+                    interactive=True,
+                    info="Choose the model you'd like to use"
+                )
+    explain_method = gr.Dropdown(
+        ["Sobol", "HSIC","Rise","Saliency"],
+        multiselect=False,
+        value="Rise", # default option
+        label="Explain method",
+        interactive=True,
+        info="Choose one method to explain the model"
+    )
+    sampling_size = gr.Slider(1, 5000, value=2000, label="Sampling Size in Rise",interactive=True,visible=True,
+                                          info="Choose between 1 and 5000")
+                
+    top_k = gr.Slider(10,200,value=50,label="Number of Closest Samples for Distribution Chart",interactive=True,info="Choose between 10 and 200")
+    class_predicted = gr.Label(label='Class Predicted',num_top_classes=10)
+    exp_gallery = gr.Gallery(label="Explanation Heatmaps for top 5 predicted classes", show_label=False,elem_id="gallery",columns=[5], rows=[1],height='auto', allow_preview=True, preview=None)
+    closest_gallery = gr.Gallery(label="Closest Images", show_label=False,elem_id="gallery",columns=[5], rows=[1],height='auto', allow_preview=True, preview=None)
+    diagram= gr.Image(label = 'Bar Chart')
+    return processed_image,processed_image,instruction,model_name,explain_method,sampling_size,top_k,class_predicted,exp_gallery,closest_gallery,diagram
+def update_slider_visibility(explain_method):
+    bool = explain_method=="Rise"
+    return {sampling_size: gr.Slider(1, 5000, value=2000, label="Sampling Size in Rise", visible=bool, interactive=True)}
+
 #minimalist theme
 with gr.Blocks(theme='sudeepshouche/minimalist') as demo:
     
     with gr.Tab(" Florrissant Fossils"):
-    
+        gr.Markdown(HEADER)
         with gr.Row():
             with gr.Column():
-                input_image = gr.Image(label="Input")
+                gr.Markdown(USER_GUIDE)
+            with gr.Column(scale=2):
+                with gr.Column(scale=2):
+                    instruction_text = gr.Textbox(label="Instructions", value="Upload/Choose an image and click 'Process Image'.")
+                    input_image = gr.Image(label="Input",width="100%",container=True)
+                    process_button = gr.Button("Process Image")
+                with gr.Column(scale=1):
+                    examples_fossils,examples_leaves = setup_examples()
+                    
+        gr.Markdown(CITATION)
+                
+    with gr.Tab("Specimen Workbench"):
+        with gr.Row():
+            with gr.Column():
+                workbench_image = gr.Image(label="Workbench Image")
                 classify_image_button = gr.Button("Classify Image")
             
             # with gr.Column():
@@ -148,21 +290,101 @@ with gr.Blocks(theme='sudeepshouche/minimalist') as demo:
                 
             with gr.Column():
                 model_name = gr.Dropdown(
-                    ["Mummified 170", "Rock 170","Fossils 142"],
+                    ["Mummified 170", "Rock 170","Fossils 142","Fossils new"],
                     multiselect=False,
-                    value="Fossils 142", # default option
+                    value="Fossils new", # default option
                     label="Model",
                     interactive=True,
+                    info="Choose the model you'd like to use"
+                )
+                explain_method = gr.Dropdown(
+                    ["Sobol", "HSIC","Rise","Saliency"],
+                    multiselect=False,
+                    value="Rise", # default option
+                    label="Explain method",
+                    interactive=True,
+                    info="Choose one method to explain the model"
                 )
+                # explain_method = gr.CheckboxGroup(["Sobol", "HSIC","Rise","Saliency"],
+                #                                   label="explain method",
+                #                                   value="Rise",
+                #                                   multiselect=False,
+                #                                   interactive=True,)
+                sampling_size = gr.Slider(1, 5000, value=2000, label="Sampling Size in Rise",interactive=True,visible=True,
+                                          info="Choose between 1 and 5000")
+                
+                top_k = gr.Slider(10,200,value=50,label="Number of Closest Samples for Distribution Chart",interactive=True,info="Choose between 10 and 200")
+                explain_method.change(
+                    fn=update_slider_visibility, 
+                    inputs=explain_method, 
+                    outputs=sampling_size
+                )
+        with gr.Row():
+            with gr.Column(scale=1):
                 class_predicted = gr.Label(label='Class Predicted',num_top_classes=10)
+            with gr.Column(scale=4):
+                with gr.Accordion("Explanations "):
+                    gr.Markdown("Computing Explanations from the model")
+                    with gr.Column():
+                        with gr.Row():
+
+                        #original_input = gr.Image(label="Original Frame")
+                        #saliency  = gr.Image(label="saliency")
+                        #gradcam = gr.Image(label='integraged gradients')
+                        #guided_gradcam = gr.Image(label='gradcam')
+                        #guided_backprop = gr.Image(label='guided backprop')
+                            # exp1 = gr.Image(label = 'Class_name1')
+                            # exp2= gr.Image(label = 'Class_name2')
+                            # exp3= gr.Image(label = 'Class_name3')
+                            # exp4= gr.Image(label = 'Class_name4')
+                            # exp5= gr.Image(label = 'Class_name5')
+                        
+                            exp_gallery = gr.Gallery(label="Explanation Heatmaps for top 5 predicted classes", show_label=False,elem_id="gallery",columns=[5], rows=[1],height='auto', allow_preview=True, preview=None)
+                    
+                    generate_explanations = gr.Button("Generate Explanations")
+                        
+                # with gr.Accordion('Closest Images'):
+                #     gr.Markdown("Finding the closest images in the dataset")
+                #     with gr.Row():
+                #         with gr.Column():
+                #             label_closest_image_0 = gr.Markdown('')
+                #             closest_image_0 = gr.Image(label='Closest Image',image_mode='contain',width=200, height=200)
+                #         with gr.Column():
+                #             label_closest_image_1 = gr.Markdown('')
+                #             closest_image_1 = gr.Image(label='Second Closest Image',image_mode='contain',width=200, height=200)
+                #         with gr.Column():
+                #             label_closest_image_2 = gr.Markdown('')
+                #             closest_image_2 = gr.Image(label='Third Closest Image',image_mode='contain',width=200, height=200)
+                #         with gr.Column():
+                #             label_closest_image_3 = gr.Markdown('')
+                #             closest_image_3 = gr.Image(label='Forth Closest Image',image_mode='contain', width=200, height=200)
+                #         with gr.Column():
+                #             label_closest_image_4 = gr.Markdown('')
+                #             closest_image_4 = gr.Image(label='Fifth Closest Image',image_mode='contain',width=200, height=200)
+                #     find_closest_btn = gr.Button("Find Closest Images")
+                with gr.Accordion('Closest Fossil Images'):
+                    gr.Markdown("Finding the closest images in the dataset")
+                    
+                    with gr.Row():
+                        closest_gallery = gr.Gallery(label="Closest Images", show_label=False,elem_id="gallery",columns=[5], rows=[1],height='auto', allow_preview=True, preview=None)
+                        #.style(grid=[1, 5], height=200, width=200)
+
+                    find_closest_btn = gr.Button("Find Closest Images")
+                    
+                #segment_button.click(segment_image, inputs=input_image, outputs=segmented_image)
+                classify_image_button.click(classify_image, inputs=[input_image,model_name], outputs=class_predicted)
+                # generate_exp.click(exp_image, inputs=[input_image,model_name,explain_method,sampling_size], outputs=[exp1,exp2,exp3,exp4,exp5]) #
+                with gr.Accordion('Closest Leaves Images'):
+                    gr.Markdown("5 closest leaves")
+                with gr.Accordion("Class Distribution of Closest Samples "):
+                    gr.Markdown("Visualize class distribution of top-k closest samples in our dataset")
+                    with gr.Column():
+                        with gr.Row():
+                            diagram= gr.Image(label = 'Bar Chart')
+                    
+                    generate_diagram = gr.Button("Generate Diagram")
+
                
-        with gr.Row():
-           
-            paths = sorted(pathlib.Path('images/').rglob('*.jpg'))
-            samples=[[path.as_posix()] for path in paths  if 'fossils' in  str(path) ][:19]
-            examples_fossils = gr.Examples(samples, inputs=input_image,examples_per_page=10,label='Fossils Examples from the dataset')
-            samples=[[path.as_posix()] for path in paths  if 'leaves' in  str(path) ][:19]
-            examples_leaves = gr.Examples(samples, inputs=input_image,examples_per_page=5,label='Leaves Examples from the dataset')
             
         # with gr.Accordion("Using Diffuser"):
         #     with gr.Column(): 
@@ -173,80 +395,20 @@ with gr.Blocks(theme='sudeepshouche/minimalist') as demo:
         #         class_predicted2 = gr.Label(label='Class Predicted from diffuser')
         #         classify_button = gr.Button("Classify Image")
 
-        
-        with gr.Accordion("Explanations "):
-            gr.Markdown("Computing Explanations from the model")
-            with gr.Column():
-                with gr.Row():
-
-                #original_input = gr.Image(label="Original Frame")
-                #saliency  = gr.Image(label="saliency")
-                #gradcam = gr.Image(label='integraged gradients')
-                #guided_gradcam = gr.Image(label='gradcam')
-                #guided_backprop = gr.Image(label='guided backprop')
-                    sobol1 = gr.Image(label = 'Sobol1')
-                    sobol2= gr.Image(label = 'Sobol2')
-                    sobol3= gr.Image(label = 'Sobol3')
-                    sobol4= gr.Image(label = 'Sobol4')
-                    sobol5= gr.Image(label = 'Sobol5')
-                    
-                with gr.Row():
-                    rise1 = gr.Image(label = 'Rise1')
-                    rise2 = gr.Image(label = 'Rise2')
-                    rise3 = gr.Image(label = 'Rise3')
-                    rise4 = gr.Image(label = 'Rise4')
-                    rise5 = gr.Image(label = 'Rise5')
-                
-                with gr.Row():
-                    hsic1 = gr.Image(label = 'HSIC1')
-                    hsic2 = gr.Image(label = 'HSIC2')
-                    hsic3 = gr.Image(label = 'HSIC3')
-                    hsic4 = gr.Image(label = 'HSIC4')
-                    hsic5 = gr.Image(label = 'HSIC5')
-
-                with gr.Row():
-                    saliency1 = gr.Image(label = 'Saliency1')
-                    saliency2 = gr.Image(label = 'Saliency2')
-                    saliency3 = gr.Image(label = 'Saliency3')
-                    saliency4 = gr.Image(label = 'Saliency4')
-                    saliency5 = gr.Image(label = 'Saliency5')
-
-
-            generate_explanations = gr.Button("Generate Explanations")
-                
-        # with gr.Accordion('Closest Images'):
-        #     gr.Markdown("Finding the closest images in the dataset")
-        #     with gr.Row():
-        #         with gr.Column():
-        #             label_closest_image_0 = gr.Markdown('')
-        #             closest_image_0 = gr.Image(label='Closest Image',image_mode='contain',width=200, height=200)
-        #         with gr.Column():
-        #             label_closest_image_1 = gr.Markdown('')
-        #             closest_image_1 = gr.Image(label='Second Closest Image',image_mode='contain',width=200, height=200)
-        #         with gr.Column():
-        #             label_closest_image_2 = gr.Markdown('')
-        #             closest_image_2 = gr.Image(label='Third Closest Image',image_mode='contain',width=200, height=200)
-        #         with gr.Column():
-        #             label_closest_image_3 = gr.Markdown('')
-        #             closest_image_3 = gr.Image(label='Forth Closest Image',image_mode='contain', width=200, height=200)
-        #         with gr.Column():
-        #             label_closest_image_4 = gr.Markdown('')
-        #             closest_image_4 = gr.Image(label='Fifth Closest Image',image_mode='contain',width=200, height=200)
-        #     find_closest_btn = gr.Button("Find Closest Images")
-        with gr.Accordion('Closest Images'):
-            gr.Markdown("Finding the closest images in the dataset")
-            
-            with gr.Row():
-                gallery = gr.Gallery(label="Closest Images", show_label=False,elem_id="gallery",columns=[5], rows=[1],height='auto', allow_preview=True, preview=None)
-                #.style(grid=[1, 5], height=200, width=200)
 
-            find_closest_btn = gr.Button("Find Closest Images")
-            
-        #segment_button.click(segment_image, inputs=input_image, outputs=segmented_image)
-        classify_image_button.click(classify_image, inputs=[input_image,model_name], outputs=class_predicted)
-        generate_explanations.click(explain_image, inputs=[input_image,model_name], outputs=[sobol1,sobol2,sobol3,sobol4,sobol5,rise1,rise2,rise3,rise4,rise5,hsic1,hsic2,hsic3,hsic4,hsic5,saliency1,saliency2,saliency3,saliency4,saliency5]) #
+        def update_exp_outputs(input_image,model_name,explain_method,nb_samples):
+            labels, images = explain_image(input_image,model_name,explain_method,nb_samples)
+            #labels_html = "".join([f'<div style="display: inline-block; text-align: center; width: 18%;">{label}</div>' for label in labels])
+            #labels_markdown = f"<div style='width: 100%; text-align: center;'>{labels_html}</div>"
+            image_caption=[]
+            for i in range(5):
+                image_caption.append((images[i],"Predicted Class "+str(i)+": "+labels[i]))
+            return image_caption
+
+        generate_explanations.click(fn=update_exp_outputs, inputs=[input_image,model_name,explain_method,sampling_size], outputs=[exp_gallery])
+
         #find_closest_btn.click(find_closest, inputs=[input_image,model_name], outputs=[label_closest_image_0,label_closest_image_1,label_closest_image_2,label_closest_image_3,label_closest_image_4,closest_image_0,closest_image_1,closest_image_2,closest_image_3,closest_image_4])
-        def update_outputs(input_image,model_name):
+        def update_closest_outputs(input_image,model_name):
             labels, images = find_closest(input_image,model_name)
             #labels_html = "".join([f'<div style="display: inline-block; text-align: center; width: 18%;">{label}</div>' for label in labels])
             #labels_markdown = f"<div style='width: 100%; text-align: center;'>{labels_html}</div>"
@@ -255,8 +417,19 @@ with gr.Blocks(theme='sudeepshouche/minimalist') as demo:
                 image_caption.append((images[i],labels[i]))
             return image_caption
 
-        find_closest_btn.click(fn=update_outputs, inputs=[input_image,model_name], outputs=[gallery])
+        find_closest_btn.click(fn=update_closest_outputs, inputs=[input_image,model_name], outputs=[closest_gallery])
         #classify_segmented_button.click(classify_image, inputs=[segmented_image,model_name], outputs=class_predicted)
+
+        generate_diagram.click(generate_diagram_closest, inputs=[input_image,model_name,top_k], outputs=diagram)
+
+    process_button.click(
+                        fn=update_display,
+                        inputs=input_image,
+                        outputs=[input_image,workbench_image,instruction_text,model_name,explain_method,sampling_size,top_k,class_predicted,exp_gallery,closest_gallery,diagram]
+                    )
+
+    
+        
         
 demo.queue()     # manage multiple incoming requests
    

closest_sample.py

@@ -5,6 +5,8 @@ import pandas as pd
 import os
 from huggingface_hub import snapshot_download
 import requests
+import matplotlib.pyplot as plt
+from collections import Counter
 
 
 pca_fossils = pk.load(open('pca_fossils_170_finer.pkl','rb'))
@@ -23,7 +25,7 @@ embedding_fossils = np.load('dataset/embedding_fossils_170_finer.npy')
 
 fossils_pd= pd.read_csv('fossils_paths.csv')
 
-def pca_distance(pca,sample,embedding):
+def pca_distance(pca,sample,embedding,top_k):
     """
     Args:
         pca:fitted PCA model
@@ -35,7 +37,7 @@ def pca_distance(pca,sample,embedding):
     s = pca.transform(sample.reshape(1,-1))
     all = pca.transform(embedding[:,-1])
     distances = np.linalg.norm(all - s, axis=1)
-    return np.argsort(distances)[:5]
+    return np.argsort(distances)[:top_k]
 
 def return_paths(argsorted,files):
     paths= []
@@ -56,7 +58,7 @@ def get_images(embedding):
     
     #pca_embedding_fossils = pca_fossils.transform(embedding_fossils[:,-1])
     
-    pca_d =pca_distance(pca_fossils,embedding,embedding_fossils)
+    pca_d =pca_distance(pca_fossils,embedding,embedding_fossils,top_k=5)
     
     fossils_paths = fossils_pd['file_name'].values
     
@@ -87,3 +89,54 @@ def get_images(embedding):
     #                     '/media/data_cifs/projects/prj_fossils/data/processed_data/leavesdb-v1_1/images/Fossil/Florissant_Fossil/original/full/jpg/') for path in paths]
 
     return classes, local_paths
+
+def get_diagram(embedding,top_k):
+    
+    #pca_embedding_fossils = pca_fossils.transform(embedding_fossils[:,-1])
+    
+    pca_d =pca_distance(pca_fossils,embedding,embedding_fossils,top_k=top_k)
+    
+    fossils_paths = fossils_pd['file_name'].values
+    
+    paths = return_paths(pca_d,fossils_paths)
+    #print(paths)
+
+    folder_florissant = 'https://storage.googleapis.com/serrelab/prj_fossils/2024/Florissant_Fossil_v2.0/'
+    folder_general = 'https://storage.googleapis.com/serrelab/prj_fossils/2024/General_Fossil_v2.0/'
+    
+    classes = []
+    for i, path in enumerate(paths):
+        local_file_path = f'image_{i}.jpg'
+        if 'Florissant_Fossil/512/full/jpg/' in path:
+            public_path = path.replace('/gpfs/data/tserre/irodri15/Fossils/new_data/leavesdb-v1_1/images/Fossil/Florissant_Fossil/512/full/jpg/', folder_florissant)
+        elif 'General_Fossil/512/full/jpg/' in path:
+            public_path = path.replace('/gpfs/data/tserre/irodri15/Fossils/new_data/leavesdb-v1_1/images/Fossil/General_Fossil/512/full/jpg/', folder_general)
+        else:
+            print("no match found")
+        print(public_path)
+        #download_public_image(public_path, local_file_path)
+        parts = [part for part in public_path.split('/') if part]
+        part = parts[-2]
+        classes.append(part)
+        #local_paths.append(local_file_path)
+    #paths= [path.replace('/gpfs/data/tserre/irodri15/Fossils/new_data/leavesdb-v1_1/images/Fossil/Florissant_Fossil/512/full/jpg/',
+    #                     '/media/data_cifs/projects/prj_fossils/data/processed_data/leavesdb-v1_1/images/Fossil/Florissant_Fossil/original/full/jpg/') for path in paths]
+    class_counts = Counter(classes)
+
+    sorted_class_counts = sorted(class_counts.items(), key=lambda item: item[1], reverse=True)
+    sorted_classes, sorted_frequencies = zip(*sorted_class_counts)
+    colors = plt.cm.viridis(np.linspace(0, 1, len(sorted_classes)))
+    fig, ax = plt.subplots()
+    ax.bar(sorted_classes, sorted_frequencies,color=colors)
+    ax.set_xlabel('Class Label')
+    ax.set_ylabel('Frequency')
+    ax.set_title('Distribution of '+str(top_k) +' Closest Sample Classes')
+    ax.set_xticklabels(class_counts.keys(), rotation=45, ha='right')
+
+    # Save the diagram to a file
+    diagram_path = 'class_distribution_chart.png'
+    plt.tight_layout()  # Adjust layout to make room for rotated x-axis labels
+    plt.savefig(diagram_path)
+    plt.close()  # Close the figure to free up memory
+
+    return diagram_path

explanations.py

@@ -7,6 +7,7 @@ from xplique.attributions.global_sensitivity_analysis import LatinHypercube
 import numpy as np
 import matplotlib.pyplot as plt
 from inference_resnet import inference_resnet_finer, preprocess, _clever_crop
+from labels import lookup_140
 BATCH_SIZE = 1
 
 def show(img, p=False, **kwargs):
@@ -35,7 +36,7 @@ def show(img, p=False, **kwargs):
     
 
 
-def explain(model, input_image,size=600, n_classes=171) :
+def explain(model, input_image,explain_method,nb_samples,size=600, n_classes=171) :
     """
     Generate explanations for a given model and dataset.
     :param model: The model to explain.
@@ -45,31 +46,55 @@ def explain(model, input_image,size=600, n_classes=171) :
     :param batch_size: The batch size to use.
     :return: The explanations.
     """
-    
+    print('using explain_method:',explain_method)
     # we only need the classification part of the model
     class_model = tf.keras.Model(model.input, model.output[1]) 
     
-    explainers = [
-        #Sobol, RISE, HSIC, Saliency
-             #IntegratedGradients(class_model, steps=50, batch_size=BATCH_SIZE),
-             #SmoothGrad(class_model, nb_samples=50, batch_size=BATCH_SIZE),
-             #GradCAM(class_model),
-             SobolAttributionMethod(class_model, grid_size=8, nb_design=32),
-             Rise(class_model,nb_samples = 5000, batch_size = BATCH_SIZE,grid_size=15,
-                 preservation_probability=0.5),
-             HsicAttributionMethod(class_model, 
+    explainers = []
+    if explain_method=="Sobol":
+        explainers.append(SobolAttributionMethod(class_model, grid_size=8, nb_design=32))
+    if explain_method=="HSIC":
+        explainers.append(HsicAttributionMethod(class_model, 
                                   grid_size=7, nb_design=1500,
-                                  sampler = LatinHypercube(binary=True)),
-             Saliency(class_model),
-             #
-    ]
-  
+                                  sampler = LatinHypercube(binary=True)))
+    if explain_method=="Rise":
+        explainers.append(Rise(class_model,nb_samples = nb_samples, batch_size = BATCH_SIZE,grid_size=15,
+                 preservation_probability=0.5))
+    if explain_method=="Saliency":
+        explainers.append(Saliency(class_model))
+
+    # explainers = [
+    #     #Sobol, RISE, HSIC, Saliency
+    #          #IntegratedGradients(class_model, steps=50, batch_size=BATCH_SIZE),
+    #          #SmoothGrad(class_model, nb_samples=50, batch_size=BATCH_SIZE),
+    #          #GradCAM(class_model),
+    #          SobolAttributionMethod(class_model, grid_size=8, nb_design=32),
+    #          HsicAttributionMethod(class_model, 
+    #                               grid_size=7, nb_design=1500,
+    #                               sampler = LatinHypercube(binary=True)),
+    #          Saliency(class_model),
+    #          Rise(class_model,nb_samples = 5000, batch_size = BATCH_SIZE,grid_size=15,
+    #              preservation_probability=0.5),
+    #          #
+    # ]
+
     cropped,repetitions = _clever_crop(input_image,(size,size))
-    size_repetitions = int(size//(repetitions.numpy()+1))
+    # size_repetitions = int(size//(repetitions.numpy()+1))
+    # print(size)
+    # print(type(input_image))
+    # print(input_image.shape)
+    # size_repetitions = int(size//(repetitions+1))
+    # print(type(repetitions))
+    # print(repetitions)
+    # print(size_repetitions)
+    # print(type(size_repetitions))
     X = preprocess(cropped,size=size)
     predictions = class_model.predict(np.array([X]))
     #Y = np.argmax(predictions)
     top_5_indices = np.argsort(predictions[0])[-5:][::-1]
+    classes = []
+    for index in top_5_indices:
+        classes.append(lookup_140[index])
     #print(top_5_indices)
     X = np.expand_dims(X, 0)
     explanations = []
@@ -81,8 +106,10 @@ def explain(model, input_image,size=600, n_classes=171) :
             phi = np.abs(explainer(X, Y))[0]
             if len(phi.shape) == 3:
                 phi = np.mean(phi, -1)
-            show(X[0][:,size_repetitions:2*size_repetitions,:])
-            show(phi[:,size_repetitions:2*size_repetitions], p=1, alpha=0.4)
+            show(X[0])
+            show(phi, p=1, alpha=0.4)
+            # show(X[0][:,size_repetitions:2*size_repetitions,:])
+            # show(phi[:,size_repetitions:2*size_repetitions], p=1, alpha=0.4)
             plt.savefig(f'phi_{e}{i}.png')
             explanations.append(f'phi_{e}{i}.png')
     # avg=[]
@@ -101,4 +128,4 @@ def explain(model, input_image,size=600, n_classes=171) :
     if len(explanations)==1:
         explanations = explanations[0]
     # return explanations,avg
-    return explanations
+    return classes,explanations

inference_resnet.py

@@ -7,7 +7,7 @@ else:
 
 from keras.applications import resnet
 import tensorflow.keras.layers as L
-import os 
+import os
 
 from tensorflow.keras.layers import Dense, GlobalAveragePooling2D
 import matplotlib.pyplot as plt