app.py

import pandas as pd
from scipy.stats import ttest_rel

# Load results
df = pd.read_csv("THE_MODEL_RESULTS.csv")

# Define individual models
tree_models = ["RandomForest", "DecisionTree"]
non_tree_models = ["KNN", "SVM", "LogisticRegression", "PyTorchNN"]

print("="*80)
print("REPLICATION: Uddin & Lu (2024) - Pairwise Model Comparisons")
print("="*80)

# Store all results
all_results = []

# For each metric
for metric in ["accuracy", "precision", "recall", "f1_score"]:
    print(f"\n({metric.upper()})")
    print("-"*80)
    print(f"{'#':<3} {'Tree Model':<20} {'Non-Tree Model':<20} {'Mean 1':<10} {'Mean 2':<10} {'t':<8} {'p-value':<10} {'Sig.'}")
    print("-"*80)

    comparison_num = 1

    # Compare each tree model with each non-tree model
    for tree_model in tree_models:
        for non_tree_model in non_tree_models:

            # Get data for both models across all datasets
            tree_data = df[df['model'] == tree_model].set_index('dataset')[metric]
            non_tree_data = df[df['model'] == non_tree_model].set_index('dataset')[metric]

            # Align datasets (inner join - only datasets present for both models)
            combined = pd.DataFrame({
                'tree': tree_data,
                'non_tree': non_tree_data
            }).dropna()

            if len(combined) < 2:
                print(f"{comparison_num:<3} {tree_model:<20} {non_tree_model:<20} Insufficient data")
                comparison_num += 1
                continue

            # Paired t-test
            t_stat, p_val = ttest_rel(combined['tree'], combined['non_tree'])

            # Calculate means and stds
            mean1 = combined['tree'].mean()
            mean2 = combined['non_tree'].mean()
            std1 = combined['tree'].std()
            std2 = combined['non_tree'].std()
            n = len(combined)

            sig = "< 0.001" if p_val < 0.001 else f"{p_val:.3f}"

            print(f"{comparison_num:<3} {tree_model:<20} {non_tree_model:<20} {mean1:<10.5f} {mean2:<10.5f} {t_stat:<8.2f} {sig:<10} {'True' if p_val < 0.05 else 'False'}")

            all_results.append({
                'metric': metric,
                'tree_model': tree_model,
                'non_tree_model': non_tree_model,
                'tree_mean': mean1,
                'non_tree_mean': mean2,
                'tree_std': std1,
                'non_tree_std': std2,
                'n_datasets': n,
                't_statistic': t_stat,
                'p_value': p_val
            })

            comparison_num += 1

# Summary
print("\n" + "="*80)
print("SUMMARY")
print("="*80)
results_df = pd.DataFrame(all_results)
significant_count = (results_df['p_value'] < 0.05).sum()
total_count = len(results_df)
print(f"\nSignificant comparisons (p < 0.05): {significant_count}/{total_count}")
print(f"Tree models won in: {(results_df['tree_mean'] > results_df['non_tree_mean']).sum()} comparisons")

# Save detailed results
results_df.to_csv('FINAL_COMPARISON_RESULTS.csv', index=False)

import gradio as gr
import pandas as pd
from scipy.stats import ttest_rel
#import matplotlib.pyplot as plt
#import seaborn as sns
import numpy as np
import plotly.express as px


# DATASET_CATEGORIES is assumed to be defined globally or in an earlier cell
DATASET_CATEGORIES = {
    "Medical & Healthcare": {
        "D1": "Heart Disease (Comprehensive)",
        "D2": "Heart attack possibility",
        "D3": "Heart Disease Dataset",
        "D4": "Liver Disorders",
        "D5": "Diabetes Prediction",
        "D9": "Chronic Kidney Disease",
        "D10": "Breast Cancer Prediction",
        "D11": "Stroke Prediction",
        "D12": "Lung Cancer Prediction",
        "D13": "Hepatitis",
        "D15": "Thyroid Disease",
        "D16": "Heart Failure Prediction",
        "D17": "Parkinson's",
        "D18": "Indian Liver Patient",
        "D19": "COVID-19 Effect on Liver Cancer",
        "D20": "Liver Dataset",
        "D21": "Specht Heart",
        "D22": "Early-stage Diabetes",
        "D23": "Diabetic Retinopathy",
        "D24": "Breast Cancer Coimbra",
        "D25": "Chronic Kidney Disease",
        "D26": "Kidney Stone",
        "D28": "Echocardiogram",
        "D29": "Bladder Cancer Recurrence",
        "D31": "Prostate Cancer",
        "D46": "Real Breast Cancer Data",
        "D47": "Breast Cancer (Royston)",
        "D48": "Lung Cancer Dataset",
        "D52": "Cervical Cancer Risk",
        "D53": "Breast Cancer Wisconsin",
        "D61": "Breast Cancer Prediction",
        "D62": "Thyroid Disease",
        "D68": "Lung Cancer",
        "D69": "Cancer Patients Data",
        "D70": "Labor Relations",
        "D71": "Glioma Grading",
        "D74": "Post-Operative Patient",
        "D80": "Heart Rate Stress Monitoring",
        "D82": "Diabetes 2019",
        "D87": "Personal Heart Disease Indicators",
        "D92": "Heart Disease (Logistic)",
        "D95": "Diabetes Prediction",
        "D97": "Cardiovascular Disease",
        "D98": "Diabetes 130 US Hospitals",
        "D99": "Heart Disease Dataset",
        "D181": "HCV Data",
        "D184": "Cardiotocography",
        "D189": "Mammographic Mass",
        "D199": "Easiest Diabetes",
        "D200": "Monkey-Pox Patients",
        "D54": "Breast Cancer Wisconsin",
        "D63": "Sick-euthyroid",
        "D64": "Ann-test",
        "D65": "Ann-train",
        "D66": "Hypothyroid",
        "D67": "New-thyroid",
        "D72": "Glioma Grading"
    },

    "Gaming & Sports": {
        "D27": "Chess King-Rook",
        "D36": "Tic-Tac-Toe",
        "D40": "IPL 2022 Matches",
        "D41": "League of Legends",
        "D55": "League of Legends Diamond",
        "D56": "Chess Game Dataset",
        "D57": "Game of Thrones",
        "D73": "Connect-4",
        "D75": "FIFA 2018",
        "D76": "Dota 2 Matches",
        "D77": "IPL Match Analysis",
        "D78": "CS:GO Professional",
        "D79": "IPL 2008-2022",
        "D114": "Video Games",
        "D115": "Video Games Sales",
        "D117": "Sacred Games",
        "D118": "PC Games Sales",
        "D119": "Popular Video Games",
        "D120": "Olympic Games 2021",
        "D121": "Video Games ESRB",
        "D122": "Top Play Store Games",
        "D123": "Steam Games",
        "D124": "PS4 Games",
        "D116": "Video Games Sales"
    },

    "Education & Students": {
        "D43": "Student Marks",
        "D44": "Student 2nd Year Result",
        "D45": "Student Mat Pass/Fail",
        "D103": "Academic Performance",
        "D104": "Student Academic Analysis",
        "D105": "Student Dropout Prediction",
        "D106": "Electronic Gadgets Impact",
        "D107": "Campus Recruitment",
        "D108": "End-Semester Performance",
        "D109": "Fitbits and Grades",
        "D110": "Student Time Management",
        "D111": "Student Feedback",
        "D112": "Depression & Performance",
        "D113": "University Rankings",
        "D126": "University Ranking CWUR",
        "D127": "University Ranking CWUR 2013-2014",
        "D128": "University Ranking CWUR 2014-2015",
        "D129": "University Ranking CWUR 2015-2016",
        "D130": "University Ranking CWUR 2016-2017",
        "D131": "University Ranking CWUR 2017-2018",
        "D132": "University Ranking CWUR 2018-2019",
        "D133": "University Ranking CWUR 2019-2020",
        "D134": "University Ranking CWUR 2020-2021",
        "D135": "University Ranking CWUR 2021-2022",
        "D136": "University Ranking CWUR 2022-2023",
        "D137": "University Ranking GM 2016",
        "D138": "University Ranking GM 2017",
        "D139": "University Ranking GM 2018",
        "D140": "University Ranking GM 2019",
        "D141": "University Ranking GM 2020",
        "D142": "University Ranking GM 2021",
        "D143": "University Ranking GM 2022",
        "D144": "University Ranking Webometric 2012",
        "D145": "University Ranking Webometric 2013",
        "D146": "University Ranking Webometric 2014",
        "D147": "University Ranking Webometric 2015",
        "D148": "University Ranking Webometric 2016",
        "D149": "University Ranking Webometric 2017",
        "D150": "University Ranking Webometric 2018",
        "D151": "University Ranking Webometric 2019",
        "D152": "University Ranking Webometric 2020",
        "D153": "University Ranking Webometric 2021",
        "D154": "University Ranking Webometric 2022",
        "D155": "University Ranking Webometric 2023",
        "D156": "University Ranking URAP 2018-2019",
        "D157": "University Ranking URAP 2019-2020",
        "D158": "University Ranking URAP 2020-2021",
        "D159": "University Ranking URAP 2021-2022",
        "D160": "University Ranking URAP 2022-2023",
        "D161": "University Ranking THE 2011",
        "D162": "University Ranking THE 2012",
        "D163": "University Ranking THE 2013",
        "D164": "University Ranking THE 2014",
        "D165": "University Ranking THE 2015",
        "D166": "University Ranking THE 2016",
        "D167": "University Ranking THE 2017",
        "D168": "University Ranking THE 2018",
        "D169": "University Ranking THE 2019",
        "D170": "University Ranking THE 2020",
        "D171": "University Ranking THE 2021",
        "D172": "University Ranking THE 2022",
        "D173": "University Ranking THE 2023",
        "D174": "University Ranking QS 2022",
        "D190": "Student Academics Performance"
    },

    "Banking & Finance": {
        "D6": "Bank Marketing 1",
        "D7": "Bank Marketing 2",
        "D30": "Adult Income",
        "D32": "Telco Customer Churn",
        "D35": "Credit Approval",
        "D50": "Term Deposit Prediction",
        "D96": "Credit Card Fraud",
        "D188": "South German Credit",
        "D193": "Credit Risk Classification",
        "D195": "Credit Score Classification",
        "D196": "Banking Classification"
    },

    "Science & Engineering": {
        "D8": "Mushroom",
        "D14": "Ionosphere",
        "D33": "EEG Eye State",
        "D37": "Steel Plates Faults",
        "D39": "Fertility",
        "D51": "Darwin",
        "D58": "EEG Emotions",
        "D81": "Predictive Maintenance",
        "D84": "Oranges vs Grapefruit",
        "D90": "Crystal System Li-ion",
        "D183": "Drug Consumption",
        "D49": "Air Pressure System Failures",
        "D93": "Air Pressure System Failures",
        "D185": "Toxicity",
        "D186": "Toxicity"
    },

    "Social & Lifestyle": {
        "D38": "Online Shoppers",
        "D59": "Red Wine Quality",
        "D60": "White Wine Quality",
        "D88": "Airline Passenger Satisfaction",
        "D94": "Go Emotions Google",
        "D100": "Spotify East Asian",
        "D125": "Suicide Rates",
        "D182": "Obesity Levels",
        "D187": "Blood Transfusion",
        "D191": "Obesity Classification",
        "D192": "Gender Classification",
        "D194": "Happiness Classification",
        "D42": "Airline customer Holiday Booking dataset"
    },

    "ML Benchmarks & Synthetic": {
        "D34": "Spambase",
        "D85": "Synthetic Binary",
        "D89": "Naive Bayes Data",
        "D175": "Monk's Problems 1",
        "D176": "Monk's Problems 2",
        "D177": "Monk's Problems 3",
        "D178": "Monk's Problems 4",
        "D179": "Monk's Problems 5",
        "D180": "Monk's Problems 6"
    },

    "Other": {
        "D83": "Paris Housing",
        "D91": "Fake Bills",
        "D197": "Star Classification"
    }
}

try:
    df = pd.read_csv("THE_MODEL_RESULTS.csv")
except FileNotFoundError:
    raise FileNotFoundError("THE_MODEL_RESULTS.csv not found. Please run the previous steps to generate it.")

#models and accuracy
available_models = df['model'].unique().tolist()
available_metrics = ["accuracy", "precision", "recall", "f1_score"]

# Helper functions
def update_datasets_choices(category, select_all):
    if category not in DATASET_CATEGORIES:
        return gr.update(choices=[], value=[])
    items = DATASET_CATEGORIES[category]
    options = [f"{key}: {value}" for key, value in items.items()]
    if select_all:
        return gr.update(choices=options, value=options)
    else:
        return gr.update(choices=options, value=options[:1])

def update_metrics_choices(select_all):
    if select_all:
        return gr.update(value=available_metrics)
    else:
        return gr.update(value=["accuracy"])

def update_models_choices(select_all):
    if select_all:
        return gr.update(value=available_models)
    else:
        return gr.update(value=available_models[:2])  # default: first two models


def run_evaluation(selected_datasets, models, primary_metrics):
    if not selected_datasets:
        return ["Select at least one dataset"] + [None]*4 + [pd.DataFrame()]
    if not models or not primary_metrics:
        return ["Please select models and metrics"] + [None]*4 + [pd.DataFrame()]
    if not isinstance(primary_metrics, list):
        primary_metrics = [primary_metrics]

    dataset_ids = [d.split(":")[0].strip() for d in selected_datasets]

    # Map dataset ids to names
    dataset_name_map = {key: name for cat, datasets in DATASET_CATEGORIES.items() for key, name in datasets.items()}

    #only include rows that exist in both the selected datasets and selected models.
    filtered = df[df['dataset'].isin(dataset_ids) & df['model'].isin(models)].copy()
    if filtered.empty:
        return ["No data available for selected datasets/models."] + [None]*4 + [pd.DataFrame()]

    filtered['dataset_name'] = filtered['dataset'].map(lambda x: dataset_name_map.get(x, x))

    figs = {}
    num_datasets = len(dataset_ids)

    for metric in primary_metrics:
        if num_datasets <= 3:
            fig = px.bar(
                filtered,
                x='dataset_name',
                y=metric,
                color='model',
                barmode='group',
                text=filtered[metric].round(3),
                labels={'dataset_name': 'Dataset', metric: metric},
                title=f"Model Performance: {metric}"
            )
            fig.update_yaxes(autorange=True)
            fig.update_xaxes(autorange=True)
        else:
            pivot_table = filtered.pivot_table(index='dataset_name', columns='model', values=metric).round(3)
            fig = px.imshow(
                pivot_table,
                text_auto=True,
                labels={"color": metric},
                title=f"Model Performance Heatmap: {metric}"
            )
            fig.update_xaxes(autorange=True)
            fig.update_yaxes(autorange=True)
            fig.update_traces(zmin=None, zmax=None)

        figs[metric] = fig

    test_results = []
    for metric in primary_metrics:
        for i, m1 in enumerate(models):
            for j, m2 in enumerate(models):
                if j <= i:
                    continue
                scores1 = filtered[filtered['model'] == m1][metric].values
                scores2 = filtered[filtered['model'] == m2][metric].values
                if len(scores1) == 0 or len(scores2) == 0:
                    continue
                t_stat, p_val = ttest_rel(scores1, scores2)
                test_results.append({
                    "metric": metric,
                    "model A": m1,
                    "model B": m2,
                    "mean(A)": round(scores1.mean(), 4),
                    "mean(B)": round(scores2.mean(), 4),
                    "mean diff": round(scores1.mean() - scores2.mean(), 4),
                    "t-statistic": round(t_stat, 4),
                    "p-value": round(p_val, 4)
                })

    results_df = pd.DataFrame(test_results)

    return [f"Generated {len(primary_metrics)} chart(s).",
            figs.get("accuracy"), figs.get("precision"), figs.get("recall"), figs.get("f1_score"),
            results_df]

# Build Gradio app
"""theme = Soft(
    primary_hue="blue",
    secondary_hue="pink",
    font="poppins"
)"""

with gr.Blocks(title="Model Evaluation Platform") as demo:
    gr.Markdown("## Model Evaluation Platform")

    # Dataset selection
    with gr.Group():
        gr.Markdown("### 1. Select Datasets")
        category_dropdown = gr.Dropdown(
            choices=list(DATASET_CATEGORIES.keys()),
            value=list(DATASET_CATEGORIES.keys())[0],
            label="Category"
        )
        select_all_box = gr.Checkbox(label="Select ALL datasets in this category", value=False)
        datasets_in_category = gr.CheckboxGroup(choices=[], label="Datasets", interactive=True)

    # Model & metrics selection
    with gr.Group():
        gr.Markdown("### 2. Select Models & Metrics")
        select_all_models = gr.Checkbox(label="Select ALL models", value=False)
        models_input = gr.CheckboxGroup(choices=available_models, value=["RandomForest", "KNN"], label="Models")
        select_all_metrics = gr.Checkbox(label="Select ALL metrics", value=False)
        metric_input = gr.CheckboxGroup(choices=available_metrics, value=["accuracy"], label="Metrics")
        run_button = gr.Button("Run Evaluation", variant="primary")

    # Outputs
        gr.Markdown(
        "### Test Results Explanation\n"
        "- **Mean(A)**: average metric score for Model A over the selected datasets\n"
        "- **Mean(B)**: average metric score for Model B over the selected datasets\n"
        "- **Mean Diff**: difference between Model A and Model B\n"
        "- **t-statistic / p-value**: results of pairwise t-test"
    )
    with gr.Group():
        gr.Markdown("### 3. Outputs")
        output_text = gr.Textbox(label="Status")
        with gr.Tabs() as output_tabs:
            with gr.Tab("Accuracy"):
                plot_accuracy = gr.Plot()
            with gr.Tab("Precision"):
                plot_precision = gr.Plot()
            with gr.Tab("Recall"):
                plot_recall = gr.Plot()
            with gr.Tab("F1 Score"):
                plot_f1 = gr.Plot()
        output_table = gr.Dataframe(label="Test Results (t-test, p-values)", wrap=True)

    category_dropdown.change(
        fn=update_datasets_choices,
        inputs=[category_dropdown, select_all_box],
        outputs=datasets_in_category)

    select_all_box.change(
        fn=update_datasets_choices,
        inputs=[category_dropdown, select_all_box],
        outputs=datasets_in_category)

    select_all_models.change(
      fn=update_models_choices,
      inputs=[select_all_models],
      outputs=[models_input]
    )

    select_all_metrics.change(
        fn=update_metrics_choices,
        inputs=[select_all_metrics],
        outputs=[metric_input])

    run_button.click(fn=run_evaluation,
        inputs=[datasets_in_category, models_input, metric_input],
        outputs=[output_text, plot_accuracy, plot_precision, plot_recall, plot_f1, output_table])

demo.launch(debug=True)