copy from GH

README.md

@@ -1,19 +1,29 @@
----
-title: Team Project Gui
-emoji: 🚀
-colorFrom: red
-colorTo: red
-sdk: docker
-app_port: 8501
-tags:
-- streamlit
-pinned: false
-short_description: Streamlit template space
----
+# By Kolia Aimilia, Kontoudakis Nikos, Skiada Kyriaki, Lampropoulou Nancy
+The project aims to classify lesions.
 
-# Welcome to Streamlit!
+# Project Structure
+The project follows a modular structure to separate configuration, data processing, computer vision logic, and model training.
 
-Edit `/src/streamlit_app.py` to customize this app to your heart's desire. :heart:
-
-If you have any questions, checkout our [documentation](https://docs.streamlit.io) and [community
-forums](https://discuss.streamlit.io).
+```text
+ham10000_project/
+│
+├── data/
+│   ├── images/              # Original dermoscopy images (e.g., ISIC_0024306.jpg)
+│   └── GroundTruth.csv      # Metadata and labels
+│
+├── models/                  # Generated automatically during training
+│   ├── skin_cancer_model.pkl # Trained Random Forest/SVM model
+│   ├── scaler.pkl           # StandardScaler for feature normalization
+│   ├── classes.pkl          # List of class names (MEL, NV, etc.)
+│   └── comparison_results.png # Confusion matrix plot
+│
+├── src/                     # Core Logic Package
+│   ├── __init__.py          # Makes this folder a Python package
+│   ├── config.py            # Configuration, Constants, and Hyperparameters
+│   ├── data.py              # Data loading and stratified splitting logic
+│   ├── features.py          # Computer Vision pipeline (CLAHE, Otsu, Sobel, etc.)
+│   └── model.py             # Model training, evaluation, and saving
+│
+├── train_main.py            # Script 1: Main entry point to train the model
+├── app.py                   # Script 2: Streamlit Web Interface for inference
+└── requirements.txt         # Project dependencies

app.py

@@ -0,0 +1,196 @@
+import streamlit as st
+import cv2
+import numpy as np
+import pandas as pd
+import joblib
+import os
+import matplotlib.pyplot as plt
+import lime
+import lime.lime_tabular
+from src import features, config
+
+st.set_page_config(page_title="DermAI Classification", layout="centered")
+
+# Custom styling to widen the center container
+st.markdown(
+    """
+    <style>
+    .block-container {
+        max-width: 1000px;
+        padding-top: 2rem;
+        padding-bottom: 2rem;
+    }
+    </style>
+    """,
+    unsafe_allow_html=True
+)
+
+st.title("🔬 Skin Lesion Classification")
+st.markdown("""
+This system uses **Classical Machine Vision** techniques (CLAHE, Otsu Thresholding, Morphology) 
+to classify skin lesions.
+""")
+
+try:
+    model_path = os.path.join(config.MODEL_DIR, 'skin_cancer_model.pkl')
+    scaler_path = os.path.join(config.MODEL_DIR, 'scaler.pkl')
+    classes_path = os.path.join(config.MODEL_DIR, 'classes.pkl')
+
+    model = joblib.load(model_path)
+    scaler = joblib.load(scaler_path)
+    classes = joblib.load(classes_path)
+    st.success("System Ready: Model Loaded Successfully")
+except FileNotFoundError:
+    st.error("Model files not found. Please run 'train_main.py' first.")
+    st.stop()
+
+uploaded_file = st.file_uploader("Choose a dermoscopy image...", type=["jpg", "jpeg", "png"])
+
+if uploaded_file is not None:
+    file_bytes = np.asarray(bytearray(uploaded_file.read()), dtype=np.uint8)
+    image = cv2.imdecode(file_bytes, 1)
+
+    col1, col2 = st.columns(2)
+    with col1:
+        st.image(image, channels="BGR", caption="Uploaded Image", use_container_width=True)
+
+    with st.spinner('Extracting Handcrafted Features...'):
+        feat_vector = features.extract_all_features_pipeline(image)
+
+        # Reshape for model input
+        feat_vector_reshaped = feat_vector.reshape(1, -1)
+        feat_scaled = scaler.transform(feat_vector_reshaped)
+
+        # Predict
+        probs = model.predict_proba(feat_scaled)
+        pred_idx = np.argmax(probs)
+        pred_label = classes[pred_idx]
+
+    with col2:
+        st.subheader(f"Prediction: **{pred_label}**")
+        st.metric("Confidence", f"{probs[0][pred_idx] * 100:.2f}%")
+
+    # --- Charts ---
+    st.subheader("Class Probabilities")
+    chart_data = pd.DataFrame({"Class": classes, "Probability": probs[0] * 100})
+    st.bar_chart(chart_data.set_index("Class"))
+
+    with st.expander("Abbreviation information"):
+        df_legend = pd.DataFrame(config.LEGEND_DATA)
+        st.dataframe(
+            df_legend,
+            column_config={
+                "More Info": st.column_config.LinkColumn(
+                    "More",
+                    help="Click to visit Wikipedia page",
+                    display_text="🔍"
+                )
+            },
+            hide_index=True,
+            use_container_width=True
+        )
+
+    # --- LIME EXPLANATION (Local XAI) ---
+    st.divider()
+    st.subheader("Explainable AI (LIME)")
+    st.write(f"#### Why was this specific image classified as **{pred_label}**?")
+    st.write(
+        "The charts below show which features supported (Green) or contradicted (Red) the decision for **EACH** possible class.")
+
+    try:
+        # 1. Load the training sample (needed to initialize LIME)
+        train_sample_path = os.path.join(config.MODEL_DIR, 'X_train_sample.npy')
+        if os.path.exists(train_sample_path):
+            X_train_sample = np.load(train_sample_path)
+            feature_names = features.get_feature_names()
+
+            # Check for feature mismatch
+            if X_train_sample.shape[1] != len(feature_names):
+                st.warning(
+                    f"Feature count mismatch (Model: {X_train_sample.shape[1]}, Code: {len(feature_names)}). Falling back to generic names.")
+                feature_names = [f"Feature_{i}" for i in range(X_train_sample.shape[1])]
+
+            # 2. Initialize Explainer
+            explainer = lime.lime_tabular.LimeTabularExplainer(
+                training_data=X_train_sample,
+                feature_names=feature_names,
+                class_names=classes,
+                mode='classification',
+                verbose=False
+            )
+
+            # 3. Explain this specific instance for ALL classes
+            # We pass labels=range(len(classes)) to calculate explanations for every class index
+            exp = explainer.explain_instance(
+                data_row=feat_scaled[0],
+                predict_fn=model.predict_proba,
+                num_features=10,
+                labels=range(len(classes))
+            )
+
+            # 4. Plot using Tabs
+            # Create a tab for each class so the user can switch between them
+            tabs = st.tabs(list(classes))
+
+            for i, class_name in enumerate(classes):
+                with tabs[i]:
+                    st.write(f"**Evidence For/Against: {class_name}**")
+                    # LIME uses the index (i) to retrieve the specific explanation
+                    fig = exp.as_pyplot_figure(label=i)
+                    st.pyplot(fig)
+
+        else:
+            st.warning("LIME initialization data (X_train_sample.npy) not found. Re-run training.")
+
+    except Exception as e:
+        st.error(f"Could not generate explanation: {type(e).__name__}: {e}")
+
+    # --- Pipeline Visualization ---
+    st.divider()
+    with st.expander("See Internal Logic (Computer Vision Pipeline Steps)", expanded=True):
+        st.info("Visualizing the exact steps performed by `src.features.py`")
+
+        img_resized, img_gray, img_eq, img_blur = features.preprocess_image(image)
+        mask_raw, mask_clean, mask_connected = features.segment_lesion(img_blur)
+        mask_final, _, _, _ = features.isolate_largest_component(mask_connected)
+        _, texture_vis = features.compute_texture_canny(img_gray, mask=mask_final)
+        img_lesion_only = cv2.bitwise_and(img_resized, img_resized, mask=mask_final)
+
+        # Row 1: Preprocessing
+        st.markdown("### Phase 1: Preprocessing")
+        c1, c2, c3, c4 = st.columns(4)
+        c1.image(img_resized, channels="BGR", caption="1. Resize")
+        c2.image(img_gray, caption="2. Grayscale")
+        c3.image(img_eq, caption="3. CLAHE (Smart Contrast)")
+        c4.image(img_blur, caption="4. Blur (Reduce Noise)")
+        st.divider()
+
+        # Row 2: Segmentation
+        st.markdown("### Phase 2: Segmentation")
+        c5, c6 = st.columns(2)
+        c5.image(mask_raw, caption="5. Otsu Threshold")
+        c6.image(mask_clean, caption="6. Morph Opening")
+        st.divider()
+
+        # Row 3: Connection & Selection
+        c7, c8 = st.columns(2)
+        c7.image(mask_connected, caption="7. Morph Dilation")
+        c8.image(mask_final, caption="8. Final Mask")
+        st.divider()
+
+        # Row 4: Analysis
+        st.markdown("### Phase 3: Analysis")
+        c9, c10 = st.columns(2)
+        c9.image(img_lesion_only, channels="BGR", caption="9. Masked Source")
+        c10.image(texture_vis, caption="10. Canny Edges (Masked)")
+
+        # Histogram
+        st.write("**11. Lesion Color Histogram**")
+        fig, ax = plt.subplots(figsize=(10, 3))
+        colors = ('b', 'g', 'r')
+        for i, color in enumerate(colors):
+            hist = cv2.calcHist([img_resized], [i], mask_final, [256], [0, 256])
+            ax.plot(hist, color=color)
+            ax.set_xlim([0, 256])
+        ax.set_title("Color Frequency")
+        st.pyplot(fig)

models/X_train_sample.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:dacc23f02577d428bb68c9f15def2373ee2dfac9ed3e55f4f328add123f54384
+size 11027608

models/classes.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:be4e4643483c57dcb12b3052e4f1bfc75ef9d81569c95216c93ad4047d0baafb
+size 59

models/scaler.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:27ae467c5aa5c69269cb2c2524518b1985eda83b9bed02ebf0fd1ad16e61d8a2
+size 1487

models/skin_cancer_model.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7867c457b12b6d092ccadcb34d0c6c2921620b2416f6f1e53b925a0ad24d700c
+size 118655649

requirements.txt

@@ -1,3 +1,8 @@
-altair
+opencv-python-headless
+numpy
 pandas
-streamlit
+scikit-learn
+matplotlib
+seaborn
+streamlit
+joblib

src/augmentation.py

@@ -0,0 +1,64 @@
+import numpy as np
+import cv2
+from tensorflow.keras.preprocessing.image import ImageDataGenerator
+from . import config
+
+# Define the generator with your specific settings
+# Note: We removed 'preprocessing_function' because our feature pipeline handles color/contrast.
+# This generator focuses on GEOMETRIC variations.
+datagen = ImageDataGenerator(
+    rotation_range=30,
+    width_shift_range=0.1,
+    height_shift_range=0.1,
+    shear_range=0.1,
+    zoom_range=0.2,
+    horizontal_flip=True,
+    fill_mode='nearest'
+)
+
+
+def get_augmentation_factor(class_name, class_counts, max_count):
+    """
+    Calculates how many augmented versions we need per image
+    to reach the majority class count.
+    """
+    current_count = class_counts.get(class_name, 0)
+    if current_count == 0: return 0
+
+    # Example: If Max=1000 and Current=100, factor is 10.
+    # We need 9 new images for every 1 original image.
+    factor = int(max_count / current_count)
+    return factor
+
+
+def generate_augmented_images(img, count=1):
+    """
+    Takes an OpenCV image, converts to Keras format, generates 'count' variations,
+    and returns them as a list of OpenCV images.
+    """
+    if count <= 0: return []
+
+    # 1. Keras expects RGB, OpenCV is BGR. Convert for safety (though geometric ops don't care)
+    img_rgb = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)
+
+    # 2. Keras expects 4D array (Batch Size, Height, Width, Channels)
+    img_expanded = np.expand_dims(img_rgb, 0)
+
+    augmented_images = []
+
+    # 3. Generate
+    # flow() generates batches indefinitely, so we loop 'count' times
+    i = 0
+    for batch in datagen.flow(img_expanded, batch_size=1):
+        # Retrieve the single image from batch
+        aug_img = batch[0].astype('uint8')
+
+        # Convert back to BGR for our feature pipeline
+        aug_img_bgr = cv2.cvtColor(aug_img, cv2.COLOR_RGB2BGR)
+
+        augmented_images.append(aug_img_bgr)
+        i += 1
+        if i >= count:
+            break
+
+    return augmented_images

src/config.py

@@ -0,0 +1,55 @@
+# src/config.py
+import os
+
+# Paths
+base_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+IMAGE_FOLDER = os.path.join(base_dir, 'data', 'images')
+CSV_PATH = os.path.join(base_dir, 'data', 'GroundTruth.csv')
+MODEL_DIR = os.path.join(base_dir, 'models')
+
+# Constants
+IMG_SIZE = 224
+CLASSES = ['MEL', 'NV', 'BCC', 'AKIEC', 'BKL', 'DF', 'VASC']
+
+# Hyperparameters
+BLUR_KERNEL = (9, 9)
+MORPH_OPEN_KERNEL = (5, 5)   # Increase to remove more noise but risk losing information
+MORPH_DILATE_KERNEL = (5, 5)
+
+# CLAHE Settings (Smart Equalization)
+CLAHE_CLIP = 2.0             # Threshold for contrast limiting
+CLAHE_GRID = (8, 8)          # Grid size for local equalization
+
+# Histogram Config
+HIST_BINS = 8
+
+LEGEND_DATA = {
+    "Abbreviation": ["MEL", "NV", "BCC", "AKIEC", "BKL", "DF", "VASC"],
+    "Full Diagnosis": [
+        "Melanoma",
+        "Melanocytic nevus",
+        "Basal cell carcinoma",
+        "Actinic keratoses",
+        "Benign keratosis-like lesions",
+        "Dermatofibroma",
+        "Vascular lesions"
+    ],
+    "Description": [
+        "Malignant skin tumor (Cancerous).",
+        "Benign melanocytic proliferations (Moles).",
+        "Common variant of skin cancer (Cancerous).",
+        "Pre-cancerous skin lesions.",
+        "Non-cancerous skin growths (e.g., solar lentigines).",
+        "Benign skin lesion (nodules).",
+        "Benign blood vessel lesions."
+    ],
+    "More Info": [
+        "https://en.wikipedia.org/wiki/Melanoma",
+        "https://en.wikipedia.org/wiki/Melanocytic_nevus",
+        "https://en.wikipedia.org/wiki/Basal-cell_carcinoma",
+        "https://en.wikipedia.org/wiki/Actinic_keratosis",
+        "https://en.wikipedia.org/wiki/Seborrheic_keratosis",
+        "https://en.wikipedia.org/wiki/Dermatofibroma",
+        "https://en.wikipedia.org/wiki/Cherry_angioma"
+    ]
+}

src/data.py

@@ -0,0 +1,71 @@
+import pandas as pd
+import os
+from sklearn.model_selection import train_test_split
+from sklearn.utils import resample
+from . import config
+
+
+def balance_dataset(df):
+    """
+    Upsamples minority classes to match the count of the majority class.
+    This ensures the model sees an equal number of examples for each lesion type.
+    """
+    print("Balancing dataset (Upsampling)...")
+
+    # 1. Find the maximum class count
+    max_count = df['target'].value_counts().max()
+
+    balanced_dfs = []
+
+    # 2. Resample each class
+    for class_name in df['target'].unique():
+        class_subset = df[df['target'] == class_name]
+
+        # Upsample (replace=True means we duplicate rows)
+        df_resampled = resample(
+            class_subset,
+            replace=True,
+            n_samples=max_count,
+            random_state=42
+        )
+        balanced_dfs.append(df_resampled)
+
+    # 3. Combine back together
+    df_balanced = pd.concat(balanced_dfs)
+
+    # Shuffle the dataset so classes aren't grouped together
+    df_balanced = df_balanced.sample(frac=1, random_state=42).reset_index(drop=True)
+
+    print(f"Original size: {len(df)} -> Balanced size: {len(df_balanced)}")
+    return df_balanced
+
+
+def load_metadata(limit=None, balance=True):
+    """
+    Loads CSV, parses classes, prepares file paths, and optionally balances data.
+    """
+    print("Loading Metadata...")
+    if not os.path.exists(config.CSV_PATH):
+        raise FileNotFoundError(f"CSV not found at {config.CSV_PATH}")
+
+    df = pd.read_csv(config.CSV_PATH)
+
+    # Validate Classes
+    classes = config.CLASSES
+    available = [c for c in classes if c in df.columns]
+
+    df['target'] = df[available].idxmax(axis=1)
+    df['label_idx'] = df['target'].apply(lambda x: available.index(x))
+    df['path'] = df['image'].apply(lambda x: os.path.join(config.IMAGE_FOLDER, x + '.jpg'))
+
+    # Apply limit first (if testing)
+    if limit:
+        print(f"Subsampling to {limit}...")
+        actual_limit = min(limit, len(df))
+        df, _ = train_test_split(df, train_size=actual_limit, stratify=df['label_idx'], random_state=42)
+
+    # Apply balancing (Upsampling)
+    # if balance:
+    #     df = balance_dataset(df)
+
+    return df, available

src/explainability.py

@@ -0,0 +1,80 @@
+import matplotlib.pyplot as plt
+import pandas as pd
+import numpy as np
+import os
+import lime
+import lime.lime_tabular
+from . import config
+
+
+def plot_rf_feature_importance(model, feature_names):
+    """
+    Plots and saves the Feature Importance for a Random Forest model.
+    """
+    if not hasattr(model, 'feature_importances_'):
+        return None
+
+    importances = model.feature_importances_
+    indices = np.argsort(importances)[::-1]
+    sorted_names = [feature_names[i] for i in indices]
+
+    plt.figure(figsize=(14, 8))
+    plt.title("Random Forest: Feature Importance (Global XAI)")
+    plt.bar(range(len(importances)), importances[indices], align="center", color='teal')
+    plt.xticks(range(len(importances)), sorted_names, rotation=90)
+    plt.xlim([-1, len(importances)])
+    plt.ylabel("Relative Importance")
+    plt.tight_layout()
+
+    save_path = os.path.join(config.MODEL_DIR, 'rf_feature_importance.png')
+    plt.savefig(save_path)
+    plt.close()
+    print(f"Global XAI Plot saved to {save_path}")
+
+
+def generate_lime_explanations(model, X_train, X_test, y_test, feature_names, class_names, model_name, num_samples=3):
+    """
+    Generates LIME (Local Interpretable Model-agnostic Explanations) for specific test instances.
+    This works for ANY model (RF, SVM, etc.).
+    """
+    print(f"  Initializing LIME Explainer for {model_name}...")
+
+    # 1. Initialize Explainer
+    # LIME needs the training data to learn the distribution of features (mean, std, etc.)
+    explainer = lime.lime_tabular.LimeTabularExplainer(
+        training_data=np.array(X_train),
+        feature_names=feature_names,
+        class_names=class_names,
+        mode='classification',
+        verbose=False
+    )
+
+    # 2. Pick sample indices to explain
+    # We pick evenly spaced samples from the test set to get a variety
+    indices = np.linspace(0, len(X_test) - 1, num_samples, dtype=int)
+
+    output_dir = os.path.join(config.MODEL_DIR, 'lime_explanations')
+    os.makedirs(output_dir, exist_ok=True)
+
+    for i in indices:
+        # 3. Generate Explanation
+        # LIME perturbs this specific instance and sees how the model's prediction changes
+        exp = explainer.explain_instance(
+            data_row=X_test[i],
+            predict_fn=model.predict_proba,
+            num_features=10,
+            top_labels=1
+        )
+
+        # 4. Save Plot
+        # We title it with the True Label for context
+        true_label = class_names[y_test[i]]
+        fig = exp.as_pyplot_figure()
+        plt.title(f"LIME ({model_name}): Test Instance {i} | True Label: {true_label}")
+        plt.tight_layout()
+
+        save_path = os.path.join(output_dir, f'{model_name}_inst_{i}_lime.png')
+        plt.savefig(save_path)
+        plt.close()
+
+    print(f"  LIME explanations saved to {output_dir}")

src/features.py

@@ -0,0 +1,196 @@
+import cv2
+import numpy as np
+from scipy.stats import skew
+from . import config
+
+
+def get_feature_names():
+    """
+    Returns the list of feature names in the exact order they are extracted
+    by the pipeline. Used for Explainable AI plots.
+    """
+    names = []
+
+    # 1. Color Stats (Mean, Std, Skew for B, G, R)
+    # OpenCV loads images as BGR
+    for c in ['Blue', 'Green', 'Red']:
+        names.extend([f'{c}_Mean', f'{c}_Std', f'{c}_Skew'])
+
+    # 2. Histogram (Bins for B, G, R)
+    for c in ['Blue', 'Green', 'Red']:
+        for i in range(config.HIST_BINS):
+            names.append(f'{c}_Hist_Bin_{i}')
+
+    # 3. Shape
+    names.extend(['Area', 'Perimeter', 'Compactness'])
+
+    # 4. Texture
+    names.append('Texture_EdgeDensity')
+
+    return names
+
+def center_crop_and_resize(img, target_size=224):
+    """
+    Take the largest possible center square from the image (no distortion)
+    Resize that square to (target_size x target_size)
+    """
+
+    h, w = img.shape[:2]
+
+    # Determine the size of the largest possible center square
+    min_side = min(h, w)
+
+    # Starting points for center crop
+    start_x = (w - min_side) // 2
+    start_y = (h - min_side) // 2
+
+    # Perform center crop
+    img_cropped = img[start_y:start_y + min_side,
+                      start_x:start_x + min_side]
+
+    # Resize the center crop to target_size x target_size
+    img_resized = cv2.resize(
+        img_cropped,
+        (target_size, target_size),
+        interpolation=cv2.INTER_AREA if min_side > target_size else cv2.INTER_CUBIC
+    )
+
+    return img_resized
+
+
+def preprocess_image(img):
+    """Standardizes, Grayscale, CLAHE, and Blur."""
+    if img is None: return None, None, None, None
+
+    img_resized = center_crop_and_resize(img, target_size=config.IMG_SIZE)
+    img_gray = cv2.cvtColor(img_resized, cv2.COLOR_BGR2GRAY)
+
+    clahe = cv2.createCLAHE(clipLimit=config.CLAHE_CLIP, tileGridSize=config.CLAHE_GRID)
+    img_eq = clahe.apply(img_gray)
+
+    img_blur = cv2.GaussianBlur(img_eq, config.BLUR_KERNEL, 0)
+
+    return img_resized, img_gray, img_eq, img_blur
+
+
+def extract_color_stats(img, mask=None):
+    """Calculates Mean, Std, Skew for R, G, B."""
+    stats = []
+    for i in range(3):
+        channel = img[:, :, i]
+        if mask is not None:
+            pixels = channel[mask > 0]
+        else:
+            pixels = channel.flatten()
+
+        if len(pixels) == 0:
+            stats.extend([0, 0, 0])
+        else:
+            stats.append(np.mean(pixels))
+            stats.append(np.std(pixels))
+            stats.append(skew(pixels))
+    return stats
+
+
+def extract_histogram_features(img, mask=None):
+    """Calculates Color Histogram for lesion area."""
+    hist_features = []
+    for i in range(3):
+        hist = cv2.calcHist([img], [i], mask, [config.HIST_BINS], [0, 256])
+        cv2.normalize(hist, hist)
+        hist_features.extend(hist.flatten())
+    return hist_features
+
+
+def segment_lesion(img_blur):
+    """Pipeline: Otsu Thresholding -> Open (Clean) -> Dilate (Connect)."""
+    _, mask_raw = cv2.threshold(img_blur, 0, 255, cv2.THRESH_BINARY_INV + cv2.THRESH_OTSU)
+
+    kernel_open = cv2.getStructuringElement(cv2.MORPH_RECT, config.MORPH_OPEN_KERNEL)
+    mask_clean = cv2.morphologyEx(mask_raw, cv2.MORPH_OPEN, kernel_open, iterations=2)
+
+    kernel_dilate = cv2.getStructuringElement(cv2.MORPH_RECT, config.MORPH_DILATE_KERNEL)
+    mask_connected = cv2.dilate(mask_clean, kernel_dilate, iterations=2)
+
+    return mask_raw, mask_clean, mask_connected
+
+
+def isolate_largest_component(mask):
+    """Filters all blobs except the largest one."""
+    contours, _ = cv2.findContours(mask, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE)
+    final_mask = np.zeros_like(mask)
+    area, perimeter, compactness = 0, 0, 0
+
+    if contours:
+        sorted_contours = sorted(contours, key=cv2.contourArea, reverse=True)
+        cnt = sorted_contours[0]
+        area = cv2.contourArea(cnt)
+
+        img_area = mask.shape[0] * mask.shape[1]
+
+        if 50 < area < (img_area * 0.95):
+            cv2.drawContours(final_mask, [cnt], -1, 255, -1)
+            perimeter = cv2.arcLength(cnt, True)
+            if perimeter > 0:
+                compactness = (4 * np.pi * area) / (perimeter ** 2)
+        elif len(sorted_contours) > 1:
+            cnt2 = sorted_contours[1]
+            area2 = cv2.contourArea(cnt2)
+            if area2 > 50:
+                cv2.drawContours(final_mask, [cnt2], -1, 255, -1)
+                area = area2
+                perimeter = cv2.arcLength(cnt2, True)
+                if perimeter > 0:
+                    compactness = (4 * np.pi * area) / (perimeter ** 2)
+
+    return final_mask, area, perimeter, compactness
+
+
+def compute_texture_canny(img_gray, mask=None):
+    """Calculates texture score using Canny Edge Detection."""
+    edges = cv2.Canny(img_gray, 100, 200)
+
+    if mask is not None:
+        lesion_edges = edges[mask > 0]
+        if len(lesion_edges) > 0:
+            texture_score = np.mean(lesion_edges)
+        else:
+            texture_score = 0
+    else:
+        texture_score = np.mean(edges)
+
+    edges_vis = edges.copy()
+    if mask is not None:
+        edges_vis = cv2.bitwise_and(edges_vis, edges_vis, mask=mask)
+
+    return texture_score, edges_vis
+
+
+def extract_all_features_pipeline(image_path_or_array):
+    """Master Orchestrator."""
+    if isinstance(image_path_or_array, str):
+        img = cv2.imread(image_path_or_array)
+    else:
+        img = image_path_or_array
+
+    if img is None: return None
+
+    # Preprocess
+    img_resized, img_gray, img_eq, img_blur = preprocess_image(img)
+
+    # Segmentation
+    _, _, mask_connected = segment_lesion(img_blur)
+    mask_final, area, perimeter, compactness = isolate_largest_component(mask_connected)
+
+    # Texture
+    texture_score, _ = compute_texture_canny(img_gray, mask=mask_final)
+
+    features = []
+    # Color Analysis
+    features.extend(extract_color_stats(img_resized, mask=mask_final))
+    features.extend(extract_histogram_features(img_resized, mask=mask_final))
+    # Shape
+    features.extend([area, perimeter, compactness])
+    features.append(texture_score)
+
+    return np.array(features)

src/model.py

@@ -0,0 +1,103 @@
+import os
+import joblib
+import numpy as np
+import matplotlib.pyplot as plt
+import seaborn as sns
+from sklearn.ensemble import RandomForestClassifier
+from sklearn.svm import SVC
+from sklearn.preprocessing import StandardScaler
+from sklearn.metrics import accuracy_score
+from . import config, plots, explainability, features
+
+
+def train_and_evaluate_split(X_train, y_train, X_test, y_test, classes):
+    """
+    Accepts PRE-SPLIT and PRE-AUGMENTED data.
+    Trains models, generates plots, and saves artifacts.
+    """
+
+    # 1. Define Models
+    techniques = {
+        "RF": RandomForestClassifier(n_estimators=100, class_weight='balanced', random_state=42),
+        "SVM": SVC(probability=True, class_weight='balanced', random_state=42)
+    }
+
+    best_score = 0
+    best_model = None
+    best_scaler = None
+    best_name = ""
+
+    os.makedirs(config.MODEL_DIR, exist_ok=True)
+
+    # 2. Prepare Feature Names for XAI
+    # We fetch these once so we can use them for LIME (all models) and RF Importance
+    feature_names = features.get_feature_names()
+    # Safety fallback if feature count mismatches name list
+    if len(feature_names) != X_train.shape[1]:
+        print(f"Warning: Feature names count ({len(feature_names)}) != Data columns ({X_train.shape[1]})")
+        feature_names = [f"Feature_{i}" for i in range(X_train.shape[1])]
+
+    # 3. Scaling
+    # Important: Fit on Train, Transform Test
+    scaler = StandardScaler()
+    X_train_s = scaler.fit_transform(X_train)
+    X_test_s = scaler.transform(X_test)
+
+    # --- NEW: Save Training Sample for LIME in App ---
+    # We save a subset (e.g., 500 samples) to keep the app lightweight and fast.
+    # LIME needs this to understand the "background" distribution of features.
+    print("Saving training sample for App LIME initialization...")
+    # if X_train_s.shape[0] > 500:
+    #     indices = np.random.choice(X_train_s.shape[0], 500, replace=False)
+    #     X_sample = X_train_s[indices]
+    # else:
+    # keep all results for XAI
+    X_sample = X_train_s
+    np.save(os.path.join(config.MODEL_DIR, 'X_train_sample.npy'), X_sample)
+    # -------------------------------------------------
+
+    # 4. Training Loop
+    for name, model in techniques.items():
+        print(f"\n--- Training {name} ---")
+        model.fit(X_train_s, y_train)
+        preds = model.predict(X_test_s)
+        acc = accuracy_score(y_test, preds)
+
+        print(f"--> {name} Accuracy on Test Set: {acc:.4f}")
+
+        # --- PLOTTING METRICS ---
+        print(f"Generating ROC and Confusion Matrix for {name}...")
+        plots.plot_confusion_matrix(y_test, preds, classes, name)
+        plots.plot_multiclass_roc(model, X_test_s, y_test, classes, name)
+        plots.save_classification_report(y_test, preds, classes, name)
+
+        # --- EXPLAINABLE AI (Global: Feature Importance) ---
+        if name == "RF":
+            print("Generating Global Feature Importance Plot (RF)...")
+            explainability.plot_rf_feature_importance(model, feature_names)
+
+        # --- EXPLAINABLE AI (Local: LIME) ---
+        # This works for BOTH RF and SVM
+        print(f"Generating Local LIME Explanations for {name}...")
+        explainability.generate_lime_explanations(
+            model=model,
+            X_train=X_train_s,  # LIME needs training distribution
+            X_test=X_test_s,  # Instances to explain
+            y_test=y_test,  # For labeling plots
+            feature_names=feature_names,
+            class_names=classes,
+            model_name=name
+        )
+
+        # Track Best
+        if acc > best_score:
+            best_score = acc
+            best_model = model
+            best_name = name
+            best_scaler = scaler
+
+            # Save Artifacts
+    print(f"\nSaving Best Model: {best_name}")
+    joblib.dump(best_model, os.path.join(config.MODEL_DIR, 'skin_cancer_model.pkl'))
+    joblib.dump(best_scaler, os.path.join(config.MODEL_DIR, 'scaler.pkl'))
+    joblib.dump(classes, os.path.join(config.MODEL_DIR, 'classes.pkl'))

src/plots.py

@@ -0,0 +1,77 @@
+import matplotlib.pyplot as plt
+import seaborn as sns
+import numpy as np
+import os
+from sklearn.metrics import confusion_matrix, roc_curve, auc, classification_report
+from sklearn.preprocessing import label_binarize
+from . import config
+
+
+def plot_confusion_matrix(y_true, y_pred, classes, model_name):
+    """Generates and saves a confusion matrix heatmap."""
+    cm = confusion_matrix(y_true, y_pred)
+
+    plt.figure(figsize=(10, 8))
+    sns.heatmap(cm, annot=True, fmt='d', cmap='Blues',
+                xticklabels=classes, yticklabels=classes)
+    plt.title(f"{model_name} Confusion Matrix")
+    plt.ylabel('True Label')
+    plt.xlabel('Predicted Label')
+    plt.tight_layout()
+
+    filename = f"{model_name.lower()}_confusion_matrix.png"
+    plt.savefig(os.path.join(config.MODEL_DIR, filename))
+    plt.close()
+
+
+def plot_multiclass_roc(model, X_test, y_test, classes, model_name):
+    """Generates and saves a Multi-class ROC Curve."""
+    # 1. Binarize labels (One-vs-Rest)
+    y_test_bin = label_binarize(y_test, classes=range(len(classes)))
+    n_classes = y_test_bin.shape[1]
+
+    # 2. Get probabilities
+    if hasattr(model, "predict_proba"):
+        y_score = model.predict_proba(X_test)
+    else:
+        print(f"{model_name} does not support probability prediction. Skipping ROC.")
+        return
+
+    # 3. Compute ROC curve and ROC area for each class
+    fpr = dict()
+    tpr = dict()
+    roc_auc = dict()
+
+    for i in range(n_classes):
+        fpr[i], tpr[i], _ = roc_curve(y_test_bin[:, i], y_score[:, i])
+        roc_auc[i] = auc(fpr[i], tpr[i])
+
+    # 4. Plot
+    plt.figure(figsize=(10, 8))
+    colors = plt.cm.rainbow(np.linspace(0, 1, n_classes))
+
+    for i, color in zip(range(n_classes), colors):
+        plt.plot(fpr[i], tpr[i], color=color, lw=2,
+                 label=f'{classes[i]} (AUC = {roc_auc[i]:.2f})')
+
+    plt.plot([0, 1], [0, 1], 'k--', lw=2)  # Diagonal line
+    plt.xlim([0.0, 1.0])
+    plt.ylim([0.0, 1.05])
+    plt.xlabel('False Positive Rate')
+    plt.ylabel('True Positive Rate')
+    plt.title(f'{model_name} Multi-class ROC Curve')
+    plt.legend(loc="lower right")
+    plt.tight_layout()
+
+    filename = f"{model_name.lower()}_roc_curve.png"
+    plt.savefig(os.path.join(config.MODEL_DIR, filename))
+    plt.close()
+
+
+def save_classification_report(y_true, y_pred, classes, model_name):
+    """Saves the text classification report."""
+    report = classification_report(y_true, y_pred, target_names=classes)
+    filename = f"{model_name.lower()}_report.txt"
+    with open(os.path.join(config.MODEL_DIR, filename), "w") as f:
+        f.write(report)
+    print(f"Report saved to {filename}")

src/streamlit_app.py

@@ -1,40 +0,0 @@
-import altair as alt
-import numpy as np
-import pandas as pd
-import streamlit as st
-
-"""
-# Welcome to Streamlit!
-
-Edit `/streamlit_app.py` to customize this app to your heart's desire :heart:.
-If you have any questions, checkout our [documentation](https://docs.streamlit.io) and [community
-forums](https://discuss.streamlit.io).
-
-In the meantime, below is an example of what you can do with just a few lines of code:
-"""
-
-num_points = st.slider("Number of points in spiral", 1, 10000, 1100)
-num_turns = st.slider("Number of turns in spiral", 1, 300, 31)
-
-indices = np.linspace(0, 1, num_points)
-theta = 2 * np.pi * num_turns * indices
-radius = indices
-
-x = radius * np.cos(theta)
-y = radius * np.sin(theta)
-
-df = pd.DataFrame({
-    "x": x,
-    "y": y,
-    "idx": indices,
-    "rand": np.random.randn(num_points),
-})
-
-st.altair_chart(alt.Chart(df, height=700, width=700)
-    .mark_point(filled=True)
-    .encode(
-        x=alt.X("x", axis=None),
-        y=alt.Y("y", axis=None),
-        color=alt.Color("idx", legend=None, scale=alt.Scale()),
-        size=alt.Size("rand", legend=None, scale=alt.Scale(range=[1, 150])),
-    ))

train_main.py

@@ -0,0 +1,102 @@
+import numpy as np
+import cv2
+import pandas as pd
+from sklearn.model_selection import train_test_split
+from src import data, features, model, augmentation
+
+
+def process_dataset_with_augmentation(df, is_training=False):
+    """
+    Loops through the dataframe.
+    If is_training=True, it augments minority classes to balance the data.
+    """
+    X = []
+    y = []
+
+    # 1. Calculate Statistics for Balancing (Only needed for training)
+    if is_training:
+        class_counts = df['target'].value_counts().to_dict()
+        max_count = max(class_counts.values())
+        print(f"  [Augmentation] Balancing classes to match majority count: {max_count}")
+
+    total = len(df)
+
+    for idx, row in df.iterrows():
+        if idx % 100 == 0: print(f"  Processing image {idx}/{total}...")
+
+        # Load Original Image
+        img = cv2.imread(row['path'])
+        if img is None: continue
+
+        # A. Extract Features for Original Image
+        feats = features.extract_all_features_pipeline(img)
+        if feats is not None:
+            X.append(feats)
+            y.append(row['label_idx'])
+
+        # B. Augmentation Logic (Training Only)
+        if is_training:
+            # Check how many extra copies we need
+            class_name = row['target']
+
+            # Calculate factor. e.g., if factor is 5, we generate 4 NEW images
+            # so total = 1 original + 4 augmented = 5
+            factor = augmentation.get_augmentation_factor(class_name, class_counts, max_count)
+            num_new_images = factor - 1
+
+            if num_new_images > 0:
+                # Generate variations
+                aug_imgs = augmentation.generate_augmented_images(img, count=num_new_images)
+
+                # Extract features for every augmented variation
+                for aug_img in aug_imgs:
+                    aug_feats = features.extract_all_features_pipeline(aug_img)
+                    if aug_feats is not None:
+                        X.append(aug_feats)
+                        y.append(row['label_idx'])
+
+    return np.array(X), np.array(y)
+
+
+def main():
+    # 1. Load Data (Metadata only)
+    df, classes = data.load_metadata(limit=None)  # Adjust limit as needed
+
+    print("-" * 50)
+    print("STEP 1: Splitting Data (Train/Test) on File Paths")
+    print("-" * 50)
+
+    # Split DataFrame FIRST to avoid data leakage
+    df_train, df_test = train_test_split(
+        df, test_size=0.2, stratify=df['label_idx'], random_state=42
+    )
+
+    print(f"Training Samples (Files): {len(df_train)}")
+    print(f"Test Samples (Files): {len(df_test)}")
+
+    # 2. Process Test Data (No Augmentation, just feature extraction)
+    print("\n" + "-" * 50)
+    print("STEP 2: Extracting Test Features (Standard)")
+    print("-" * 50)
+    X_test, y_test = process_dataset_with_augmentation(df_test, is_training=False)
+
+    # 3. Process Training Data (WITH Augmentation)
+    print("\n" + "-" * 50)
+    print("STEP 3: Extracting Training Features (With Keras Augmentation)")
+    print("-" * 50)
+    X_train, y_train = process_dataset_with_augmentation(df_train, is_training=True)
+
+    print(f"\nFinal Feature Matrix Shapes:")
+    print(f"X_train: {X_train.shape}, y_train: {y_train.shape}")
+    print(f"X_test:  {X_test.shape}, y_test:  {y_test.shape}")
+
+    # 4. Train & Evaluate
+    # We pass the pre-split arrays directly to a modified train function
+    if len(X_train) > 0 and len(X_test) > 0:
+        model.train_and_evaluate_split(X_train, y_train, X_test, y_test, classes)
+    else:
+        print("Error: Feature extraction failed.")
+
+
+if __name__ == "__main__":
+    main()