initial version of the damage classification app

app.py

@@ -0,0 +1,177 @@
+import gradio as gr
+import numpy as np
+import pandas as pd
+
+# our own helper tools
+import clustering
+import utils
+
+import logging
+logging.getLogger().setLevel(logging.INFO)
+
+from tensorflow import keras
+
+#image_threshold = 20
+damage_sites = {}
+
+model1_windowsize = [250,250]
+#model1_threshold = 0.7
+
+model1 = keras.models.load_model('rwthmaterials_dp800_network1_inclusion.h5')
+model1.compile()
+
+damage_classes = {3: "Martensite",2: "Interface",0:"Notch",1:"Shadowing"}
+
+model2_windowsize = [100,100]
+#model2_threshold = 0.5
+
+model2 = keras.models.load_model('rwthmaterials_dp800_newtwork2_damage.h5')
+model2.compile()
+
+
+
+##
+## Function to do the actual damage classification
+##
+def damage_classification(SEM_image,image_threshold, model1_threshold, model2_threshold):
+
+
+    ##
+    ## clustering
+    ##
+    logging.debug('---------------: clustering :=====================')
+    all_centroids = clustering.get_centroids(SEM_image, image_threshold=image_threshold,
+                                            fill_holes=True, filter_close_centroids=True)
+    
+    for i in range(len(all_centroids)) :
+        key = (all_centroids[i][0],all_centroids[i][1])
+        damage_sites[key] = 'Not Classified'
+    
+    ##
+    ## Inclusions vs the rest
+    ##
+    logging.debug('---------------: prepare model 1 :=====================')
+    images_model1 = utils.prepare_classifier_input(SEM_image, all_centroids, window_size=model1_windowsize)
+    
+    logging.debug('---------------: run model 1 :=====================')
+    y1_pred = model1.predict(np.asarray(images_model1, float))
+
+    logging.debug('---------------: model1 threshold :=====================')
+    inclusions = y1_pred[:,0].reshape(len(y1_pred),1)
+    inclusions = np.where(inclusions > model1_threshold)
+
+    logging.debug('---------------: model 1 update dict :=====================')
+    for i in range(len(inclusions[0])):
+        centroid_id = inclusions[0][i]
+        coordinates = all_centroids[centroid_id]
+        key = (coordinates[0], coordinates[1])
+        damage_sites[key] = 'Inclusion'
+    logging.debug('Damage sites after model 1')
+    logging.debug(damage_sites)
+
+    ##
+    ## Martensite cracking, etc
+    ##
+    logging.debug('---------------: prepare model 2 :=====================')
+    centroids_model2 = []
+    for key, value in damage_sites.items():
+        if value == 'Not Classified':
+            coordinates = list([key[0],key[1]])
+            centroids_model2.append(coordinates)
+    logging.debug('Centroids model 2')
+    logging.debug(centroids_model2)
+
+    logging.debug('---------------: prepare model 2 :=====================')
+    images_model2 = utils.prepare_classifier_input(SEM_image, centroids_model2, window_size=model2_windowsize)
+    logging.debug('Images model 2')
+    logging.debug(images_model2)
+    
+    logging.debug('---------------: run model 2 :=====================')
+    y2_pred = model2.predict(np.asarray(images_model2, float))
+
+    damage_index = np.asarray(y2_pred > model2_threshold).nonzero()
+
+
+    for i in range(len(damage_index[0])):
+        index = damage_index[0][i]
+        identified_class = damage_index[1][i]
+        label = damage_classes[identified_class]
+        coordinates = centroids_model2[index]
+        #print('Damage {} \t identified as {}, \t coordinates {}'.format(i, label, coordinates))   
+        key = (coordinates[0], coordinates[1])
+        damage_sites[key] = label
+
+    ##
+    ## show the damage sites on the image
+    ##
+    logging.debug("-----------------: final damage sites :=================")
+    logging.debug(damage_sites)
+
+    image_path = 'classified_damage_sites.png'
+    image = utils.show_boxes(SEM_image, damage_sites, 
+                             save_image=True, 
+                             image_path=image_path)
+
+    ##
+    ## export data
+    ##
+    csv_path = 'classified_damage_sites.csv'
+    cols = ['x', 'y', 'damage_type']
+    
+    data = []
+    for key, value in damage_sites.items():
+        data.append([key[0], key[1], value])
+
+    df = pd.DataFrame(columns=cols, data=data)
+    
+    df.to_csv(csv_path)
+
+
+    return  image, image_path, csv_path
+
+## ---------------------------------------------------------------------------------------------------------------
+## main app interface
+## -----------------------------------------------------------------------------------------------------------------
+with gr.Blocks() as app:
+    gr.Markdown('# Damage Classification in Dual Phase Steels')
+    gr.Markdown('This app classifies damage types in dual phase steels. Two models are used. The first model is used to identify inclusions in the steel. The second model is used to identify the remaining damage types: Martensite cracking, Interface Decohesion, Notch effect and Shadows.')
+
+    gr.Markdown('If you use this app, kindly cite the following papers:')
+    gr.Markdown('Kusche, C., Reclik, T., Freund, M., Al-Samman, T., Kerzel, U., & Korte-Kerzel, S. (2019). Large-area, high-resolution characterisation and classification of damage mechanisms in dual-phase steel using deep learning. PloS one, 14(5), e0216493. [Link](https://doi.org/10.1371/journal.pone.0216493)')
+    gr.Markdown('Medghalchi, S., Kusche, C. F., Karimi, E., Kerzel, U., & Korte-Kerzel, S. (2020). Damage analysis in dual-phase steel using deep learning: transfer from uniaxial to biaxial straining conditions by image data augmentation. Jom, 72, 4420-4430. [Link](https://link.springer.com/article/10.1007/s11837-020-04404-0)')
+
+    image_input = gr.Image()
+    with gr.Row():
+        cluster_threshold_input = gr.Number(label='Cluster Threshold', value = 20, 
+                                            info='Grayscale value at which a pixel is attributed to a potential damage site')
+        model1_threshold_input = gr.Number(label='Model 1 Threshold', value = 0.7, info='Threshold for the model identifying inclusions')   
+        model2_threshold_input = gr.Number(label='Model 2 Threshold', value = 0.5, info='Thrshold for the model identifying the remaining damage types')
+
+
+    button = gr.Button("Classify")
+
+    
+    output_image = gr.Image()
+    with gr.Row():
+        download_image = gr.DownloadButton(label='Download Image')
+        download_csv = gr.DownloadButton(label='Download Damage List')
+
+    
+    button.click(damage_classification, 
+                 inputs=[image_input, cluster_threshold_input, model1_threshold_input, model2_threshold_input],
+                 outputs=[output_image, download_image, download_csv])
+
+
+
+# simple interface, no title, etc
+# app = gr.Interface(damage_classification, 
+#                    inputs=[gr.Image(),
+#                             gr.Number(label='Cluster Threshold', value = 20, info='Grayscale value at which a pixel is attributed to a potential damage site'),    
+#                             gr.Number(label='Model 1 Threshold', value = 0.7, info='Threshold for the model identifying inclusions'),
+#                             gr.Number(label='Model 2 Threshold', value = 0.5, info='Thrshold for the model identifying the remaining damage types')
+#                    ], 
+#                    outputs=[gr.Image(), 
+#                             gr.DownloadButton(label='Download Image'),
+#                             gr.DownloadButton(label='Download Damage List')])
+if __name__ == "__main__":
+    app.launch()

clustering.py

@@ -0,0 +1,103 @@
+"""
+    Before we can identify damage sites, we need to look for suitable regions in the image.
+    Typically, damage sites appear as dark regions in the image. Instead of simple thresholding, we use 
+    a clustering approach to identify regions that belong together and form damage site candidates.
+"""
+
+import numpy as np
+
+
+import scipy.ndimage as ndi
+from scipy.spatial import KDTree
+
+from sklearn.cluster import DBSCAN
+
+import logging
+
+
+def get_centroids(image : np.ndarray, image_threshold = 20,
+                  eps=1, min_samples=5, metric='euclidean',
+                  min_size = 20, fill_holes = False,
+                  filter_close_centroids = False, filter_radius = 50) -> list:
+    """    Determine centroids of clusters corresponding to potential damage sites.
+    In a first step, a threshold is applied to the input image to identify areas of potential damage sites.
+    Using DBSCAN, these agglomerations of pixels are fitted into clusters. Then, the mean x/y values are determined
+    from pixels belonging to one cluster. If the number of pixels in a given cluster excees the threshold given by min_size, this cluster is added
+    to the list of (x,y) coordinates that is returned as the final list potential damage sites.
+
+    Sometimes, clusters may be found in very close proximity to each other, we can reject those to avoid 
+    classifying the same event multiple times (which may distort our statistics).
+
+    DBScan documentation: https://scikit-learn.org/stable/modules/generated/sklearn.cluster.DBSCAN.html
+
+    Args:
+        image (np.ndarray): Input SEM image 
+        image_threshold (int, optional): Threshold to be applied to the image to identify candidates for damage sites. Defaults to 20.
+        eps (int, optional): parameter eps of DBSCAN: The maximum distance between two samples for one to be considered as in the neighborhood of the other. Defaults to 1.
+        min_samples (int, optional): parameter min_samples of DBSCAN: The number of samples (or total weight) in a neighborhood for a point to be considered as a core point. Defaults to 5.
+        metric (str, optional): parameter metric of DBSCAN. Defaults to 'euclidean'.
+        min_size (int, optional): Minimum number of pixels in a cluster for the damage site candidate to be considered in the final list. Defaults to 20.
+        fill_holes (bool, optional): Fill small holes in damage sites clusters using binary_fill_holes. Defaults to False.
+        filter_close_centroids (book optional): Filter cluster centroids within a given radius. Defaults to False
+        filter_radius (float, optional): Radius within which centroids are considered to be the same. Defaults to 50
+
+    Returns:
+        list: list of (x,y) coordinates of the centroids of the clusters of accepted damage site candidates.
+    """
+
+
+    centroids = []
+
+    # apply the threshold to identify regions of "dark" pixels
+    # the result is a binary mask (true/false) whether a given pixel is above or below the threshold
+    cluster_candidates = image < image_threshold
+
+    # sometimes the clusters have small holes in them, for example, individual pixels
+    # inside a region below the threshold. This may confuse the clustering algorith later on
+    # and we can use the following to fill these holes
+    # https://docs.scipy.org/doc/scipy/reference/generated/scipy.ndimage.binary_fill_holes.html
+    # N.B. the algorith only works on binay data
+    if fill_holes:
+        cluster_candidates = ndi.binary_fill_holes(cluster_candidates)
+
+    # apply the treshold to the image to identify regions of "dark" pixels
+    #cluster_candidates = np.asarray(image < image_threshold).nonzero()
+
+    # transform image format into a numpy array to pass on to DBSCAN clustering
+    cluster_candidates = np.asarray(cluster_candidates).nonzero()
+    cluster_candidates = np.transpose(cluster_candidates)
+
+
+    # run the DBSCAN clustering algorithm, candidate sites that are not attributed to a cluster are labelled as "-1", i.e. "noise"
+    # (e.g. they are too small, etc)
+    # For the remaining pixels, a label is assigned to each pixel, indicating to which cluster (or noise) they belong to.
+    dbscan = DBSCAN(eps=eps, min_samples=min_samples, metric='euclidean')
+
+    dbscan.fit(cluster_candidates)
+
+    labels = dbscan.labels_
+    # Number of clusters in labels, ignoring noise if present.
+    n_clusters = len(set(labels)) - (1 if -1 in labels else 0)
+    n_noise = list(labels).count(-1)
+    logging.debug('# clusters {}, #noise {}'.format(n_clusters, n_noise))
+
+
+    # now loop over all labels found by DBSCAN, i.e. all identified clusters and the noise
+    # we use "set" here, as the labels are attributed to individual pixels, i.e. they appear as often as we have pixels
+    # in the cluster candidates
+    for i in set(labels):
+        if i>-1:
+            # all points belonging to a given cluster
+            cluster_points = cluster_candidates[labels==i, :]
+            if len(cluster_points) > min_size:
+                x_mean=np.mean(cluster_points, axis=0)[0]
+                y_mean=np.mean(cluster_points, axis=0)[1]
+                centroids.append([x_mean,y_mean])
+
+    if filter_close_centroids:
+        proximity_tree = KDTree(centroids)
+        pairs = proximity_tree.query_pairs(filter_radius)
+        for p in pairs:
+            item = centroids.pop(p[0])
+
+    return centroids

requirements.txt

@@ -0,0 +1,102 @@
+absl-py==2.1.0 ; python_version >= "3.10" and python_version < "3.12"
+aiofiles==23.2.1 ; python_version >= "3.10" and python_version < "3.12"
+altair==5.2.0 ; python_version >= "3.10" and python_version < "3.12"
+annotated-types==0.6.0 ; python_version >= "3.10" and python_version < "3.12"
+anyio==4.3.0 ; python_version >= "3.10" and python_version < "3.12"
+astunparse==1.6.3 ; python_version >= "3.10" and python_version < "3.12"
+attrs==23.2.0 ; python_version >= "3.10" and python_version < "3.12"
+cachetools==5.3.3 ; python_version >= "3.10" and python_version < "3.12"
+certifi==2024.2.2 ; python_version >= "3.10" and python_version < "3.12"
+charset-normalizer==3.3.2 ; python_version >= "3.10" and python_version < "3.12"
+click==8.1.7 ; python_version >= "3.10" and python_version < "3.12"
+colorama==0.4.6 ; python_version >= "3.10" and python_version < "3.12"
+contourpy==1.2.0 ; python_version >= "3.10" and python_version < "3.12"
+cycler==0.12.1 ; python_version >= "3.10" and python_version < "3.12"
+exceptiongroup==1.2.0 ; python_version >= "3.10" and python_version < "3.11"
+fastapi==0.110.0 ; python_version >= "3.10" and python_version < "3.12"
+ffmpy==0.3.2 ; python_version >= "3.10" and python_version < "3.12"
+filelock==3.13.1 ; python_version >= "3.10" and python_version < "3.12"
+flatbuffers==24.3.7 ; python_version >= "3.10" and python_version < "3.12"
+fonttools==4.50.0 ; python_version >= "3.10" and python_version < "3.12"
+fsspec==2024.3.1 ; python_version >= "3.10" and python_version < "3.12"
+gast==0.5.4 ; python_version >= "3.10" and python_version < "3.12"
+google-auth-oauthlib==1.2.0 ; python_version >= "3.10" and python_version < "3.12"
+google-auth==2.29.0 ; python_version >= "3.10" and python_version < "3.12"
+google-pasta==0.2.0 ; python_version >= "3.10" and python_version < "3.12"
+gradio-client==0.13.0 ; python_version >= "3.10" and python_version < "3.12"
+gradio==4.22.0 ; python_version >= "3.10" and python_version < "3.12"
+grpcio==1.62.1 ; python_version >= "3.10" and python_version < "3.12"
+h11==0.14.0 ; python_version >= "3.10" and python_version < "3.12"
+h5py==3.10.0 ; python_version >= "3.10" and python_version < "3.12"
+httpcore==1.0.4 ; python_version >= "3.10" and python_version < "3.12"
+httpx==0.27.0 ; python_version >= "3.10" and python_version < "3.12"
+huggingface-hub==0.21.4 ; python_version >= "3.10" and python_version < "3.12"
+idna==3.6 ; python_version >= "3.10" and python_version < "3.12"
+imageio==2.34.0 ; python_version >= "3.10" and python_version < "3.12"
+importlib-resources==6.4.0 ; python_version >= "3.10" and python_version < "3.12"
+jinja2==3.1.3 ; python_version >= "3.10" and python_version < "3.12"
+joblib==1.3.2 ; python_version >= "3.10" and python_version < "3.12"
+jsonschema-specifications==2023.12.1 ; python_version >= "3.10" and python_version < "3.12"
+jsonschema==4.21.1 ; python_version >= "3.10" and python_version < "3.12"
+keras==2.15.0 ; python_version >= "3.10" and python_version < "3.12"
+kiwisolver==1.4.5 ; python_version >= "3.10" and python_version < "3.12"
+libclang==18.1.1 ; python_version >= "3.10" and python_version < "3.12"
+markdown-it-py==3.0.0 ; python_version >= "3.10" and python_version < "3.12"
+markdown==3.6 ; python_version >= "3.10" and python_version < "3.12"
+markupsafe==2.1.5 ; python_version >= "3.10" and python_version < "3.12"
+matplotlib==3.8.3 ; python_version >= "3.10" and python_version < "3.12"
+mdurl==0.1.2 ; python_version >= "3.10" and python_version < "3.12"
+ml-dtypes==0.2.0 ; python_version >= "3.10" and python_version < "3.12"
+numpy==1.26.4 ; python_version < "3.12" and python_version >= "3.10"
+oauthlib==3.2.2 ; python_version >= "3.10" and python_version < "3.12"
+opt-einsum==3.3.0 ; python_version >= "3.10" and python_version < "3.12"
+orjson==3.9.15 ; python_version >= "3.10" and python_version < "3.12"
+packaging==24.0 ; python_version >= "3.10" and python_version < "3.12"
+pandas==2.2.1 ; python_version >= "3.10" and python_version < "3.12"
+pillow==10.2.0 ; python_version >= "3.10" and python_version < "3.12"
+protobuf==4.25.3 ; python_version >= "3.10" and python_version < "3.12"
+pyasn1-modules==0.3.0 ; python_version >= "3.10" and python_version < "3.12"
+pyasn1==0.5.1 ; python_version >= "3.10" and python_version < "3.12"
+pydantic-core==2.16.3 ; python_version >= "3.10" and python_version < "3.12"
+pydantic==2.6.4 ; python_version >= "3.10" and python_version < "3.12"
+pydub==0.25.1 ; python_version >= "3.10" and python_version < "3.12"
+pygments==2.17.2 ; python_version >= "3.10" and python_version < "3.12"
+pyparsing==3.1.2 ; python_version >= "3.10" and python_version < "3.12"
+python-dateutil==2.9.0.post0 ; python_version >= "3.10" and python_version < "3.12"
+python-multipart==0.0.9 ; python_version >= "3.10" and python_version < "3.12"
+pytz==2024.1 ; python_version >= "3.10" and python_version < "3.12"
+pyyaml==6.0.1 ; python_version >= "3.10" and python_version < "3.12"
+referencing==0.34.0 ; python_version >= "3.10" and python_version < "3.12"
+requests-oauthlib==1.4.0 ; python_version >= "3.10" and python_version < "3.12"
+requests==2.31.0 ; python_version >= "3.10" and python_version < "3.12"
+rich==13.7.1 ; python_version >= "3.10" and python_version < "3.12"
+rpds-py==0.18.0 ; python_version >= "3.10" and python_version < "3.12"
+rsa==4.9 ; python_version >= "3.10" and python_version < "3.12"
+ruff==0.3.4 ; python_version >= "3.10" and python_version < "3.12"
+scikit-learn==1.4.1.post1 ; python_version >= "3.10" and python_version < "3.12"
+scipy==1.12.0 ; python_version >= "3.10" and python_version < "3.12"
+semantic-version==2.10.0 ; python_version >= "3.10" and python_version < "3.12"
+setuptools==69.2.0 ; python_version >= "3.10" and python_version < "3.12"
+shellingham==1.5.4 ; python_version >= "3.10" and python_version < "3.12"
+six==1.16.0 ; python_version >= "3.10" and python_version < "3.12"
+sniffio==1.3.1 ; python_version >= "3.10" and python_version < "3.12"
+starlette==0.36.3 ; python_version >= "3.10" and python_version < "3.12"
+tensorboard-data-server==0.7.2 ; python_version >= "3.10" and python_version < "3.12"
+tensorboard==2.15.2 ; python_version >= "3.10" and python_version < "3.12"
+tensorflow-estimator==2.15.0 ; python_version >= "3.10" and python_version < "3.12"
+tensorflow-io-gcs-filesystem==0.36.0 ; python_version >= "3.10" and python_version < "3.12"
+tensorflow==2.15.0 ; python_version >= "3.10" and python_version < "3.12"
+termcolor==2.4.0 ; python_version >= "3.10" and python_version < "3.12"
+threadpoolctl==3.4.0 ; python_version >= "3.10" and python_version < "3.12"
+tomlkit==0.12.0 ; python_version >= "3.10" and python_version < "3.12"
+toolz==0.12.1 ; python_version >= "3.10" and python_version < "3.12"
+tqdm==4.66.2 ; python_version >= "3.10" and python_version < "3.12"
+typer[all]==0.9.0 ; python_version >= "3.10" and python_version < "3.12"
+typing-extensions==4.10.0 ; python_version >= "3.10" and python_version < "3.12"
+tzdata==2024.1 ; python_version >= "3.10" and python_version < "3.12"
+urllib3==2.2.1 ; python_version >= "3.10" and python_version < "3.12"
+uvicorn==0.29.0 ; python_version >= "3.10" and python_version < "3.12"
+websockets==11.0.3 ; python_version >= "3.10" and python_version < "3.12"
+werkzeug==3.0.1 ; python_version >= "3.10" and python_version < "3.12"
+wheel==0.43.0 ; python_version >= "3.10" and python_version < "3.12"
+wrapt==1.14.1 ; python_version >= "3.10" and python_version < "3.12"

rwthmaterials_dp800_network1_inclusion.h5

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7005f07ed2108e18c38b3f0b547b7255215d61fe12cf69daed2337b6cc6c24f5
+size 87724360

rwthmaterials_dp800_network2_damage.h5

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a4c363089c7c2b04d4f36488720373a6b4b5eb136bac8a4e35c94358ff093bed
+size 4048504

utils.py

@@ -0,0 +1,189 @@
+"""
+Collection of various utils 
+"""
+
+import numpy as np
+
+import imageio.v3 as iio
+from PIL import Image
+# we may have very large images (e.g. panoramic SEM images), allow to read them w/o warnings
+Image.MAX_IMAGE_PIXELS = 933120000
+
+import matplotlib.pyplot as plt
+import matplotlib.patches as patches
+from matplotlib.lines import Line2D
+
+
+import math
+
+
+###
+### load SEM images
+### 
+def load_image(filename : str) -> np.ndarray :
+    """Load an SEM image 
+
+    Args:
+        filename (str): full path and name of the image file to be loaded
+
+    Returns:
+        np.ndarray: file as numpy ndarray
+    """
+    image =  iio.imread(filename,mode='F')
+
+    return image
+
+
+
+###
+### show SEM image with boxes in various colours around each damage site
+###
+def show_boxes(image : np.ndarray, damage_sites : dict, box_size = [250,250],
+               save_image = False, image_path : str = None) :
+    """_summary_
+
+    Args:
+        image (np.ndarray): SEM image to be shown
+        damage_sites (dict): python dictionary using the coordinates as key (x,y), and the label as value
+        box_size (list, optional): size of the rectangle drawn around each centroid. Defaults to [250,250].
+        save_image (bool, optional): save the image with the boxes or not. Defaults to False.
+        image_path (str, optional) : Full path and name of the output file to be saved
+    """
+
+    _, ax = plt.subplots(1)
+    fig = plt.imshow(image,cmap='gray')
+    # do not show axis ticks (indicating pixels)
+    plt.xticks([])
+    plt.yticks([]) 
+
+    for key, label in damage_sites.items():
+        
+        position = list([key[0],key[1]])
+
+        # define colours of the rectangles overlaid on the image per damage type
+        match label:
+            case 'Inclusion':
+                edgecolor = 'b'
+            case 'Interface' :
+                edgecolor = 'g'
+            case 'Martensite' :
+                edgecolor = 'r'
+            case 'Notch':
+                edgecolor = 'y'
+            case 'Shadowing' :
+                edgecolor = 'm'
+            case _:
+                edgecolor = 'k'
+        
+            
+        rectangle = patches.Rectangle((position[1]-box_size[1]/2., position[0]-box_size[0]/2),
+                                       box_size[0],box_size[1],
+                                       linewidth=1,edgecolor=edgecolor,facecolor='none')
+        ax.add_patch(rectangle)
+
+
+    legend_elements = [Line2D([0], [0], color='b', lw=4, label='Inclusion'),
+                       Line2D([0], [0], color='g', lw=4, label='Interface'),
+                       Line2D([0], [0], color='r', lw=4, label='Martensite'),
+                       Line2D([0], [0], color='y', lw=4, label='Notch'),
+                       Line2D([0], [0], color='m', lw=4, label='Shadow'),
+                       Line2D([0], [0], color='k', lw=4, label='Not Classified')
+        ]
+
+    ax.legend(handles=legend_elements,bbox_to_anchor=(1.04, 1), loc="upper left")
+
+    if save_image:
+        plt.savefig(image_path,dpi=1200,bbox_inches='tight' )
+
+    canvas = plt.gca().figure.canvas
+    canvas.draw()
+    data = np.frombuffer(canvas.tostring_rgb(), dtype=np.uint8)
+    image = data.reshape(canvas.get_width_height()[::-1] + (3,))
+
+    plt.show()
+
+
+
+    return image
+
+
+###
+### cut out small images from panorama, append colour information
+###
+def prepare_classifier_input(panorama : np.ndarray, centroids : list, window_size = [250,250]) -> list :
+    """Create a list of smaller images from the SEM panoramic image. 
+       The neural networks expect images of a given size that are centered around a single damage site candiates.
+       For each centroid (from the clustering step before), we cut out a smaller image from the panorama of the size
+       expected by the classfier network.
+       Since the networks expect colour images, we repeat the gray-scale image 3 times for a given candiate site.
+
+    Args:
+        panorama (np.ndarray): SEM input image
+        centroids (list): list of centroids for the damage site candidates
+        window_size (list, optional): Size of the image expected by the neural network later. Defaults to [250,250].
+
+    Returns:
+        list: List of "colour" images cut out from the SEM panorama, one per damage site candidate
+    """
+
+    panorama_shape = panorama.shape
+
+    # list of the small images cut out from the panorama,
+    # each of these is then fed into the classfier model
+    images = []
+
+    for i in range(len(centroids)):
+        x1 = int(math.floor(centroids[i][0] - window_size[0]/2))
+        y1 = int(math.floor(centroids[i][1] - window_size[1]/2))
+        x2 = int(math.floor(centroids[i][0] + window_size[0]/2))
+        y2 = int(math.floor(centroids[i][1] + window_size[1]/2))
+    
+
+        ##
+        ## Catch the cases in which the extract would go
+        ## over the boundaries of the original image
+        ##
+        if x1<0:
+            x1 = 0
+            x2 = window_size[0]
+        if x2>= panorama_shape[0]:
+            x1 = panorama_shape[0] - window_size[0]
+            x2 = panorama_shape[0]
+        if y1<0:
+            y1 = 0
+            y2 = window_size[1]
+        if y2>= panorama_shape[1]:
+            y1 = panorama_shape[1] - window_size[1]
+            y2 = panorama_shape[1]
+
+        # we now need to create the image path from the panoramic image that corresponds to the 
+        # centroid, with the size determined by the window_size. 
+        # First, we create an empty container with np.zeros()
+        tmp_img = np.zeros((window_size[1],  window_size[0],1), dtype=float)
+
+        # Then we copy over the patch of the panomaric image.
+        # The later classfier expects colour images, i.e. 3 colour channels for RGB
+        # Since we use gray-scale images, we only have one colour information, so we add the image to the first colour channel
+        tmp_img[:,:,0] = panorama[x1:x2,y1:y2,0]
+
+        # rescale the colour values
+        tmp_img = tmp_img*2./255. - 1.
+
+        # The classifier expects colour images, i.e. 3 colour channels.
+        # We "fake" this by repeating the same gray-scale information 3 times, once per colour channel
+        tmp_img_colour = np.repeat(tmp_img,3, axis=2) #3
+
+        images.append(tmp_img_colour)
+    
+
+    return images
+
+
+
+
+
+
+
+        
+    
+