first commit

app.py

@@ -0,0 +1,284 @@
+import gradio as gr
+import torch
+import onnxruntime as ort
+import numpy as np
+from PIL import Image
+from torchvision import transforms
+import torch.nn.functional as F
+import matplotlib.pyplot as plt
+
+
+_metadata_columns = [
+    "age", "usePesticide_I", "usePesticide_False", "usePesticide_True", "gender_M", "gender_F", "gender_O",
+    "familySkinCancerHistory_False", "familySkinCancerHistory_True", "familySkinCancerHistory_I", "familyCancerHistory_True", 
+    "familyCancerHistory_False", "familyCancerHistory_I", "fitzpatrickSkinType_2.0", "fitzpatrickSkinType_1.0", 
+    "fitzpatrickSkinType_4.0", "fitzpatrickSkinType_3.0", "fitzpatrickSkinType_5.0", "macroBodyRegion_PEITORAL",
+    "macroBodyRegion_NARIZ", "macroBodyRegion_LABIOS", "macroBodyRegion_DORSO", "macroBodyRegion_ANTEBRACO", "macroBodyRegion_BRACO",
+    "macroBodyRegion_PERNA", "macroBodyRegion_FACE", "macroBodyRegion_MAO", "macroBodyRegion_COURO CABELUDO", "macroBodyRegion_PESCOCO",
+    "macroBodyRegion_PE", "macroBodyRegion_ORELHA", "macroBodyRegion_COXA", "macroBodyRegion_ABDOME",
+    "hasItched_True", "hasItched_False", "hasItched_I", "hasGrown_I", "hasGrown_False", "hasGrown_True", "hasHurt_True", "hasHurt_False",
+    "hasHurt_I", "hasChanged_I", "hasChanged_False", "hasChanged_True", "hasBled_False", "hasBled_True", "hasBled_I", "hasElevation_I",
+    "hasElevation_False", "hasElevation_True"
+    ]
+
+metadata_mapping = {
+    "age": "age",
+    "usePesticide_I": "usePesticide_I",
+    "usePesticide_False": "pesticide_False",
+    "usePesticide_True": "pesticide_True",
+    "gender_M": "gender_MALE",
+    "gender_F": "gender_FEMALE",
+    "gender_O": "gender_OTHER",
+    "familySkinCancerHistory_False": "skin_cancer_history_False",
+    "familySkinCancerHistory_True": "skin_cancer_history_True",
+    "familySkinCancerHistory_I": "familySkinCancerHistory_I",
+    "familyCancerHistory_True": "cancer_history_True",
+    "familyCancerHistory_False": "cancer_history_False",
+    "familyCancerHistory_I": "familyCancerHistory_I",
+    "fitzpatrickSkinType_2.0": "fitspatrick_2.0",
+    "fitzpatrickSkinType_1.0": "fitspatrick_1.0",
+    "fitzpatrickSkinType_4.0": "fitspatrick_4.0",
+    "fitzpatrickSkinType_3.0": "fitspatrick_3.0",
+    "fitzpatrickSkinType_5.0": "fitspatrick_5.0",
+    "fitzpatrickSkinType_6.0": "fitspatrick_6.0",
+    "macroBodyRegion_PEITORAL": "region_CHEST",
+    "macroBodyRegion_NARIZ": "region_NOSE",
+    "macroBodyRegion_LABIOS": "region_LIP",
+    "macroBodyRegion_DORSO": "region_BACK",
+    "macroBodyRegion_ANTEBRACO": "region_FOREARM",
+    "macroBodyRegion_BRACO": "region_ARM",
+    "macroBodyRegion_PERNA": "region_THIGH",
+    "macroBodyRegion_FACE": "region_FACE",
+    "macroBodyRegion_MAO": "region_HAND",
+    "macroBodyRegion_COURO CABELUDO": "region_SCALP",
+    "macroBodyRegion_PESCOCO": "region_NECK",
+    "macroBodyRegion_PE": "region_FOOT",
+    "macroBodyRegion_ORELHA": "region_EAR",
+    "macroBodyRegion_COXA": "region_THIGH",
+    "macroBodyRegion_ABDOME": "region_ABDOMEN",
+    "hasItched_True": "itch_True",
+    "hasItched_False": "itch_False",
+    "hasItched_I": "itch_UNK",
+    "hasGrown_I": "grew_UNK",
+    "hasGrown_False": "grew_False",
+    "hasGrown_True": "grew_True",
+    "hasHurt_True": "hurt_True",
+    "hasHurt_False": "hurt_False",
+    "hasHurt_I": "hurt_UNK",
+    "hasChanged_I": "changed_UNK",
+    "hasChanged_False": "changed_False",
+    "hasChanged_True": "changed_True",
+    "hasBled_False": "bleed_False",
+    "hasBled_True": "bleed_True",
+    "hasBled_I": "bleed_UNK",
+    "hasElevation_I": "elevation_UNK",
+    "hasElevation_False": "elevation_False",
+    "hasElevation_True": "elevation_True"
+}
+
+_metadata_columns = [metadata_mapping[col] for col in _metadata_columns if metadata_mapping[col] is not None]
+try:
+    ort_session = ort.InferenceSession("./pad25_mobilenetv3_folder_1.onnx")
+    print("ONNX model loaded successfully.")
+except Exception as e:
+    print(f"Error loading ONNX model: {e}")
+    ort_session = None
+
+LABELS = ['ACK', 'BCC', 'MEL', 'NEV', 'SCC', 'SEK']
+
+def create_plot(probs_history, steps_labels):
+    fig, ax = plt.subplots(figsize=(10, 6))
+    
+    class_data = {label: [] for label in LABELS}
+    for step_probs in probs_history:
+        for label, prob in step_probs.items():
+            class_data[label].append(prob * 100)
+
+    # Identify top 3 classes based on final probability
+    final_probs = {label: values[-1] for label, values in class_data.items()}
+    top_classes = sorted(final_probs, key=final_probs.get, reverse=True)[:3]
+
+    annotations = {} 
+
+    # Plot every class
+    for name, values in class_data.items():
+        x_vals = range(len(values))
+        
+        # Style logic
+        if name in top_classes: # Highlight top classes
+             line, = ax.plot(x_vals, values, label=name, linewidth=2, marker='o')
+             color = line.get_color()
+             
+             # Collect Text Annotations
+             for x, y in zip(x_vals, values):
+                if x not in annotations:
+                    annotations[x] = []
+                annotations[x].append((y, f"{y:.1f}", color))
+        else:
+            # Other low prob classes (faded)
+            ax.plot(x_vals, values, label=name, alpha=1, linewidth=1)
+
+    # Process annotations to avoid overlap
+    for x in sorted(annotations.keys()):
+        points = sorted(annotations[x], key=lambda p: p[0])
+        
+        min_dist = 5
+        last_text_y = -100
+        
+        for i, (y, text, color) in enumerate(points):
+            text_y = y + 3
+            
+            if text_y < last_text_y + min_dist:
+                text_y = last_text_y + min_dist
+            
+            ax.text(x, text_y, text, ha='center', fontweight='bold', fontsize=10, color='black')
+            last_text_y = text_y
+
+    ax.set_xticks(range(len(steps_labels)))
+    ax.set_xticklabels(steps_labels, rotation=30, ha='right')
+    ax.set_ylabel("Probability (%)")
+    ax.set_xlabel("Incremental Features Added")
+    ax.set_ylim(0, 115)
+    ax.grid(True, linestyle='--', alpha=0.3)
+    ax.legend(loc='upper right', bbox_to_anchor=(1.10, 1), borderaxespad=0., framealpha=0.8)
+    
+    plt.tight_layout()
+    return fig
+
+def predict(image, age, region, cancer_history, skin_cancer_history, bleed, hurt, itch, grown, changed, elevation):
+    if ort_session is None:
+        return "Model not loaded", None
+
+    steps = [
+        ("Baseline (Image only)", {}),
+        (f"Age ({age})", {"age": age}),
+        (f"Region ({region})", {"region": region}),
+    ]
+    
+    symptoms_map = {
+        "Cancer History": ("cancer_history", cancer_history),
+        "Skin Cancer History": ("skin_cancer_history", skin_cancer_history),
+        "Bleed": ("bleed", bleed),
+        "Hurt": ("hurt", hurt),
+        "Itch": ("itch", itch),
+        "Grew": ("grew", grown),
+        "Changed": ("changed", changed),
+        "Elevation": ("elevation", elevation)
+    }
+
+    for label, (key, val) in symptoms_map.items():
+        steps.append((f"{label} ({val})", {key: val}))
+
+    probs_history = []
+    steps_labels = []
+    
+    if image is not None:
+        transform = transforms.Compose([
+            transforms.Resize((224, 224)),
+            transforms.ToTensor(),
+            transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225])
+        ])
+        image_pil = Image.open(image).convert('RGB')
+        image_tensor = transform(image_pil).unsqueeze(0)
+    else:
+        image_tensor = torch.zeros(1, 3, 224, 224)
+
+    def set_feature(vector, feature_name, value):
+        col_name = f"{feature_name}_{value}"
+        if col_name in _metadata_columns:
+            idx = _metadata_columns.index(col_name)
+            vector[idx] = 1.0
+
+    accumulated_features = {}
+    
+    for step_name, new_features in steps:
+        steps_labels.append(step_name)
+        accumulated_features.update(new_features)
+        
+        metadata_vector = np.zeros(len(_metadata_columns), dtype=np.float32)
+        
+        if "age" in accumulated_features and accumulated_features["age"] is not None:
+             if "age" in _metadata_columns:
+                val = accumulated_features["age"]
+                metadata_vector[_metadata_columns.index("age")] = float(val) if val is not None else np.nan
+        else:
+             if "age" in _metadata_columns:
+                metadata_vector[_metadata_columns.index("age")] = np.nan
+
+        if "region" in accumulated_features and accumulated_features["region"]:
+            set_feature(metadata_vector, "region", accumulated_features["region"])
+            
+        symptom_keys = ["cancer_history", "skin_cancer_history", "bleed", "hurt", "itch", "grew", "changed", "elevation"]
+        for key in symptom_keys:
+            if key in accumulated_features:
+                val = accumulated_features[key]
+                if val != "None":
+                    set_feature(metadata_vector, key, val)
+        
+        metadata_tensor = torch.tensor(metadata_vector, dtype=torch.float32).unsqueeze(0)
+        
+        ort_inputs = {
+            ort_session.get_inputs()[0].name: image_tensor.numpy(),
+            ort_session.get_inputs()[1].name: metadata_tensor.numpy()
+        }
+        ort_outs = ort_session.run(None, ort_inputs)
+        log_probs = ort_outs[0][0]
+        probs = F.softmax(torch.tensor(log_probs), dim=0).numpy()
+        
+        probs_dict = {LABELS[i]: float(probs[i]) for i in range(len(LABELS))}
+        probs_history.append(probs_dict)
+
+    final_result = probs_history[-1]
+    
+    plot = create_plot(probs_history, steps_labels)
+    
+    return final_result, plot
+
+def clear_func():
+    return None, None, None, "None", "None", "None", "None", "None", "None", "None", "None", None, None
+
+with gr.Blocks() as demo:
+    with gr.Row():
+        gr.Markdown("# PRISM: A Clinically Interpretable Stepwise Framework for Multimodal Skin Cancer Diagnosis (DOI: TODO)")
+              
+    with gr.Row():
+        with gr.Column():
+            image = gr.Image(type="filepath", height=534, label="Input Image",)
+        with gr.Column():
+            age = gr.Number(label="Age", value=None)
+            dropdown = gr.Dropdown(multiselect=False, allow_custom_value=False, label="Region", choices=['ARM', 'NECK', 'FACE', 'HAND', 'FOREARM', 'CHEST', 'NOSE', 'THIGH', 'SCALP', 'EAR', 'BACK', 'FOOT', 'ABDOMEN', 'LIP', 'TORSO', None])
+
+            with gr.Row():
+                with gr.Column():
+                    cancer_history = gr.Radio(label="Cancer history", choices=["True", "False", "None"], value="None")
+                    skin_cancer_history = gr.Radio(label="Skin cancer history", choices=["True", "False", "None"], value="None")
+                    bleed = gr.Radio(label="Bled", choices=["True", "False", "None"], value="None")
+                    hurt = gr.Radio(label="Pain", choices=["True", "False", "None"], value="None")
+                with gr.Column():
+                    itch = gr.Radio(label="Itch", choices=["True", "False", "None"], value="None")
+                    grown = gr.Radio(label="Grew", choices=["True", "False", "None"], value="None")
+                    changed = gr.Radio(label="Changed", choices=["True", "False", "None"], value="None")
+                    elevation = gr.Radio(label="Elevation", choices=["True", "False", "None"], value="None")
+    
+    examples = [
+        [None, 45, "ARM", "True", "False", "True", "False", "True", "True", "False", "True"],
+        [None, 30, "TORSO", "False", "False", "False", "False", "False", "False", "False", "False"],
+        [None, 60, "ARM", "False", "True", "False", "True", "False", "True", "True", "False"],
+    ]
+    gr.Examples(examples=examples, inputs=[image, age, dropdown, cancer_history, skin_cancer_history, bleed, hurt, itch, grown, changed, elevation])
+    
+    with gr.Row():
+        with gr.Column():
+            output_plot = gr.Plot(label="Incremental Prediction Change")
+        with gr.Column():
+            output = gr.Label(label="Output", num_top_classes=6)
+                
+    with gr.Row():
+        with gr.Column():
+            submit = gr.Button("Submit")
+            submit.click(predict, inputs=[image, age, dropdown, cancer_history, skin_cancer_history, bleed, hurt, itch, grown, changed, elevation], outputs=[output, output_plot])
+
+            clear = gr.Button("Clear")
+            clear.click(clear_func, inputs=[], outputs=[image, age, dropdown, cancer_history, skin_cancer_history, bleed, hurt, itch, grown, changed, elevation, output, output_plot])
+
+demo.launch(share=True)