import pandas as pd from scipy.stats import ttest_rel # Load results df = pd.read_csv("THE_MODEL_RESULTS.csv") # Define individual models tree_models = ["RandomForest", "DecisionTree"] non_tree_models = ["KNN", "SVM", "LogisticRegression", "PyTorchNN"] print("="*80) print("REPLICATION: Uddin & Lu (2024) - Pairwise Model Comparisons") print("="*80) # Store all results all_results = [] # For each metric for metric in ["accuracy", "precision", "recall", "f1_score"]: print(f"\n({metric.upper()})") print("-"*80) print(f"{'#':<3} {'Tree Model':<20} {'Non-Tree Model':<20} {'Mean 1':<10} {'Mean 2':<10} {'t':<8} {'p-value':<10} {'Sig.'}") print("-"*80) comparison_num = 1 # Compare each tree model with each non-tree model for tree_model in tree_models: for non_tree_model in non_tree_models: # Get data for both models across all datasets tree_data = df[df['model'] == tree_model].set_index('dataset')[metric] non_tree_data = df[df['model'] == non_tree_model].set_index('dataset')[metric] # Align datasets (inner join - only datasets present for both models) combined = pd.DataFrame({ 'tree': tree_data, 'non_tree': non_tree_data }).dropna() if len(combined) < 2: print(f"{comparison_num:<3} {tree_model:<20} {non_tree_model:<20} Insufficient data") comparison_num += 1 continue # Paired t-test t_stat, p_val = ttest_rel(combined['tree'], combined['non_tree']) # Calculate means and stds mean1 = combined['tree'].mean() mean2 = combined['non_tree'].mean() std1 = combined['tree'].std() std2 = combined['non_tree'].std() n = len(combined) sig = "< 0.001" if p_val < 0.001 else f"{p_val:.3f}" print(f"{comparison_num:<3} {tree_model:<20} {non_tree_model:<20} {mean1:<10.5f} {mean2:<10.5f} {t_stat:<8.2f} {sig:<10} {'True' if p_val < 0.05 else 'False'}") all_results.append({ 'metric': metric, 'tree_model': tree_model, 'non_tree_model': non_tree_model, 'tree_mean': mean1, 'non_tree_mean': mean2, 'tree_std': std1, 'non_tree_std': std2, 'n_datasets': n, 't_statistic': t_stat, 'p_value': p_val }) comparison_num += 1 # Summary print("\n" + "="*80) print("SUMMARY") print("="*80) results_df = pd.DataFrame(all_results) significant_count = (results_df['p_value'] < 0.05).sum() total_count = len(results_df) print(f"\nSignificant comparisons (p < 0.05): {significant_count}/{total_count}") print(f"Tree models won in: {(results_df['tree_mean'] > results_df['non_tree_mean']).sum()} comparisons") # Save detailed results results_df.to_csv('FINAL_COMPARISON_RESULTS.csv', index=False) import gradio as gr import pandas as pd from scipy.stats import ttest_rel #import matplotlib.pyplot as plt #import seaborn as sns import numpy as np import plotly.express as px # DATASET_CATEGORIES is assumed to be defined globally or in an earlier cell DATASET_CATEGORIES = { "Medical & Healthcare": { "D1": "Heart Disease (Comprehensive)", "D2": "Heart attack possibility", "D3": "Heart Disease Dataset", "D4": "Liver Disorders", "D5": "Diabetes Prediction", "D9": "Chronic Kidney Disease", "D10": "Breast Cancer Prediction", "D11": "Stroke Prediction", "D12": "Lung Cancer Prediction", "D13": "Hepatitis", "D15": "Thyroid Disease", "D16": "Heart Failure Prediction", "D17": "Parkinson's", "D18": "Indian Liver Patient", "D19": "COVID-19 Effect on Liver Cancer", "D20": "Liver Dataset", "D21": "Specht Heart", "D22": "Early-stage Diabetes", "D23": "Diabetic Retinopathy", "D24": "Breast Cancer Coimbra", "D25": "Chronic Kidney Disease", "D26": "Kidney Stone", "D28": "Echocardiogram", "D29": "Bladder Cancer Recurrence", "D31": "Prostate Cancer", "D46": "Real Breast Cancer Data", "D47": "Breast Cancer (Royston)", "D48": "Lung Cancer Dataset", "D52": "Cervical Cancer Risk", "D53": "Breast Cancer Wisconsin", "D61": "Breast Cancer Prediction", "D62": "Thyroid Disease", "D68": "Lung Cancer", "D69": "Cancer Patients Data", "D70": "Labor Relations", "D71": "Glioma Grading", "D74": "Post-Operative Patient", "D80": "Heart Rate Stress Monitoring", "D82": "Diabetes 2019", "D87": "Personal Heart Disease Indicators", "D92": "Heart Disease (Logistic)", "D95": "Diabetes Prediction", "D97": "Cardiovascular Disease", "D98": "Diabetes 130 US Hospitals", "D99": "Heart Disease Dataset", "D181": "HCV Data", "D184": "Cardiotocography", "D189": "Mammographic Mass", "D199": "Easiest Diabetes", "D200": "Monkey-Pox Patients", "D54": "Breast Cancer Wisconsin", "D63": "Sick-euthyroid", "D64": "Ann-test", "D65": "Ann-train", "D66": "Hypothyroid", "D67": "New-thyroid", "D72": "Glioma Grading" }, "Gaming & Sports": { "D27": "Chess King-Rook", "D36": "Tic-Tac-Toe", "D40": "IPL 2022 Matches", "D41": "League of Legends", "D55": "League of Legends Diamond", "D56": "Chess Game Dataset", "D57": "Game of Thrones", "D73": "Connect-4", "D75": "FIFA 2018", "D76": "Dota 2 Matches", "D77": "IPL Match Analysis", "D78": "CS:GO Professional", "D79": "IPL 2008-2022", "D114": "Video Games", "D115": "Video Games Sales", "D117": "Sacred Games", "D118": "PC Games Sales", "D119": "Popular Video Games", "D120": "Olympic Games 2021", "D121": "Video Games ESRB", "D122": "Top Play Store Games", "D123": "Steam Games", "D124": "PS4 Games", "D116": "Video Games Sales" }, "Education & Students": { "D43": "Student Marks", "D44": "Student 2nd Year Result", "D45": "Student Mat Pass/Fail", "D103": "Academic Performance", "D104": "Student Academic Analysis", "D105": "Student Dropout Prediction", "D106": "Electronic Gadgets Impact", "D107": "Campus Recruitment", "D108": "End-Semester Performance", "D109": "Fitbits and Grades", "D110": "Student Time Management", "D111": "Student Feedback", "D112": "Depression & Performance", "D113": "University Rankings", "D126": "University Ranking CWUR", "D127": "University Ranking CWUR 2013-2014", "D128": "University Ranking CWUR 2014-2015", "D129": "University Ranking CWUR 2015-2016", "D130": "University Ranking CWUR 2016-2017", "D131": "University Ranking CWUR 2017-2018", "D132": "University Ranking CWUR 2018-2019", "D133": "University Ranking CWUR 2019-2020", "D134": "University Ranking CWUR 2020-2021", "D135": "University Ranking CWUR 2021-2022", "D136": "University Ranking CWUR 2022-2023", "D137": "University Ranking GM 2016", "D138": "University Ranking GM 2017", "D139": "University Ranking GM 2018", "D140": "University Ranking GM 2019", "D141": "University Ranking GM 2020", "D142": "University Ranking GM 2021", "D143": "University Ranking GM 2022", "D144": "University Ranking Webometric 2012", "D145": "University Ranking Webometric 2013", "D146": "University Ranking Webometric 2014", "D147": "University Ranking Webometric 2015", "D148": "University Ranking Webometric 2016", "D149": "University Ranking Webometric 2017", "D150": "University Ranking Webometric 2018", "D151": "University Ranking Webometric 2019", "D152": "University Ranking Webometric 2020", "D153": "University Ranking Webometric 2021", "D154": "University Ranking Webometric 2022", "D155": "University Ranking Webometric 2023", "D156": "University Ranking URAP 2018-2019", "D157": "University Ranking URAP 2019-2020", "D158": "University Ranking URAP 2020-2021", "D159": "University Ranking URAP 2021-2022", "D160": "University Ranking URAP 2022-2023", "D161": "University Ranking THE 2011", "D162": "University Ranking THE 2012", "D163": "University Ranking THE 2013", "D164": "University Ranking THE 2014", "D165": "University Ranking THE 2015", "D166": "University Ranking THE 2016", "D167": "University Ranking THE 2017", "D168": "University Ranking THE 2018", "D169": "University Ranking THE 2019", "D170": "University Ranking THE 2020", "D171": "University Ranking THE 2021", "D172": "University Ranking THE 2022", "D173": "University Ranking THE 2023", "D174": "University Ranking QS 2022", "D190": "Student Academics Performance" }, "Banking & Finance": { "D6": "Bank Marketing 1", "D7": "Bank Marketing 2", "D30": "Adult Income", "D32": "Telco Customer Churn", "D35": "Credit Approval", "D50": "Term Deposit Prediction", "D96": "Credit Card Fraud", "D188": "South German Credit", "D193": "Credit Risk Classification", "D195": "Credit Score Classification", "D196": "Banking Classification" }, "Science & Engineering": { "D8": "Mushroom", "D14": "Ionosphere", "D33": "EEG Eye State", "D37": "Steel Plates Faults", "D39": "Fertility", "D51": "Darwin", "D58": "EEG Emotions", "D81": "Predictive Maintenance", "D84": "Oranges vs Grapefruit", "D90": "Crystal System Li-ion", "D183": "Drug Consumption", "D49": "Air Pressure System Failures", "D93": "Air Pressure System Failures", "D185": "Toxicity", "D186": "Toxicity" }, "Social & Lifestyle": { "D38": "Online Shoppers", "D59": "Red Wine Quality", "D60": "White Wine Quality", "D88": "Airline Passenger Satisfaction", "D94": "Go Emotions Google", "D100": "Spotify East Asian", "D125": "Suicide Rates", "D182": "Obesity Levels", "D187": "Blood Transfusion", "D191": "Obesity Classification", "D192": "Gender Classification", "D194": "Happiness Classification", "D42": "Airline customer Holiday Booking dataset" }, "ML Benchmarks & Synthetic": { "D34": "Spambase", "D85": "Synthetic Binary", "D89": "Naive Bayes Data", "D175": "Monk's Problems 1", "D176": "Monk's Problems 2", "D177": "Monk's Problems 3", "D178": "Monk's Problems 4", "D179": "Monk's Problems 5", "D180": "Monk's Problems 6" }, "Other": { "D83": "Paris Housing", "D91": "Fake Bills", "D197": "Star Classification" } } try: df = pd.read_csv("THE_MODEL_RESULTS.csv") except FileNotFoundError: raise FileNotFoundError("THE_MODEL_RESULTS.csv not found. Please run the previous steps to generate it.") #models and accuracy available_models = df['model'].unique().tolist() available_metrics = ["accuracy", "precision", "recall", "f1_score"] # Helper functions def update_datasets_choices(category, select_all): if category not in DATASET_CATEGORIES: return gr.update(choices=[], value=[]) items = DATASET_CATEGORIES[category] options = [f"{key}: {value}" for key, value in items.items()] if select_all: return gr.update(choices=options, value=options) else: return gr.update(choices=options, value=options[:1]) def update_metrics_choices(select_all): if select_all: return gr.update(value=available_metrics) else: return gr.update(value=["accuracy"]) def update_models_choices(select_all): if select_all: return gr.update(value=available_models) else: return gr.update(value=available_models[:2]) # default: first two models def run_evaluation(selected_datasets, models, primary_metrics): if not selected_datasets: return ["Select at least one dataset"] + [None]*4 + [pd.DataFrame()] if not models or not primary_metrics: return ["Please select models and metrics"] + [None]*4 + [pd.DataFrame()] if not isinstance(primary_metrics, list): primary_metrics = [primary_metrics] dataset_ids = [d.split(":")[0].strip() for d in selected_datasets] # Map dataset ids to names dataset_name_map = {key: name for cat, datasets in DATASET_CATEGORIES.items() for key, name in datasets.items()} #only include rows that exist in both the selected datasets and selected models. filtered = df[df['dataset'].isin(dataset_ids) & df['model'].isin(models)].copy() if filtered.empty: return ["No data available for selected datasets/models."] + [None]*4 + [pd.DataFrame()] filtered['dataset_name'] = filtered['dataset'].map(lambda x: dataset_name_map.get(x, x)) figs = {} num_datasets = len(dataset_ids) for metric in primary_metrics: if num_datasets <= 3: fig = px.bar( filtered, x='dataset_name', y=metric, color='model', barmode='group', text=filtered[metric].round(3), labels={'dataset_name': 'Dataset', metric: metric}, title=f"Model Performance: {metric}" ) fig.update_yaxes(autorange=True) fig.update_xaxes(autorange=True) else: pivot_table = filtered.pivot_table(index='dataset_name', columns='model', values=metric).round(3) fig = px.imshow( pivot_table, text_auto=True, labels={"color": metric}, title=f"Model Performance Heatmap: {metric}" ) fig.update_xaxes(autorange=True) fig.update_yaxes(autorange=True) fig.update_traces(zmin=None, zmax=None) figs[metric] = fig test_results = [] for metric in primary_metrics: for i, m1 in enumerate(models): for j, m2 in enumerate(models): if j <= i: continue scores1 = filtered[filtered['model'] == m1][metric].values scores2 = filtered[filtered['model'] == m2][metric].values if len(scores1) == 0 or len(scores2) == 0: continue t_stat, p_val = ttest_rel(scores1, scores2) test_results.append({ "metric": metric, "model A": m1, "model B": m2, "mean(A)": round(scores1.mean(), 4), "mean(B)": round(scores2.mean(), 4), "mean diff": round(scores1.mean() - scores2.mean(), 4), "t-statistic": round(t_stat, 4), "p-value": round(p_val, 4) }) results_df = pd.DataFrame(test_results) return [f"Generated {len(primary_metrics)} chart(s).", figs.get("accuracy"), figs.get("precision"), figs.get("recall"), figs.get("f1_score"), results_df] # Build Gradio app """theme = Soft( primary_hue="blue", secondary_hue="pink", font="poppins" )""" with gr.Blocks(title="Model Evaluation Platform") as demo: gr.Markdown("## Model Evaluation Platform") # Dataset selection with gr.Group(): gr.Markdown("### 1. Select Datasets") category_dropdown = gr.Dropdown( choices=list(DATASET_CATEGORIES.keys()), value=list(DATASET_CATEGORIES.keys())[0], label="Category" ) select_all_box = gr.Checkbox(label="Select ALL datasets in this category", value=False) datasets_in_category = gr.CheckboxGroup(choices=[], label="Datasets", interactive=True) # Model & metrics selection with gr.Group(): gr.Markdown("### 2. Select Models & Metrics") select_all_models = gr.Checkbox(label="Select ALL models", value=False) models_input = gr.CheckboxGroup(choices=available_models, value=["RandomForest", "KNN"], label="Models") select_all_metrics = gr.Checkbox(label="Select ALL metrics", value=False) metric_input = gr.CheckboxGroup(choices=available_metrics, value=["accuracy"], label="Metrics") run_button = gr.Button("Run Evaluation", variant="primary") # Outputs gr.Markdown( "### Test Results Explanation\n" "- **Mean(A)**: average metric score for Model A over the selected datasets\n" "- **Mean(B)**: average metric score for Model B over the selected datasets\n" "- **Mean Diff**: difference between Model A and Model B\n" "- **t-statistic / p-value**: results of pairwise t-test" ) with gr.Group(): gr.Markdown("### 3. Outputs") output_text = gr.Textbox(label="Status") with gr.Tabs() as output_tabs: with gr.Tab("Accuracy"): plot_accuracy = gr.Plot() with gr.Tab("Precision"): plot_precision = gr.Plot() with gr.Tab("Recall"): plot_recall = gr.Plot() with gr.Tab("F1 Score"): plot_f1 = gr.Plot() output_table = gr.Dataframe(label="Test Results (t-test, p-values)", wrap=True) category_dropdown.change( fn=update_datasets_choices, inputs=[category_dropdown, select_all_box], outputs=datasets_in_category) select_all_box.change( fn=update_datasets_choices, inputs=[category_dropdown, select_all_box], outputs=datasets_in_category) select_all_models.change( fn=update_models_choices, inputs=[select_all_models], outputs=[models_input] ) select_all_metrics.change( fn=update_metrics_choices, inputs=[select_all_metrics], outputs=[metric_input]) run_button.click(fn=run_evaluation, inputs=[datasets_in_category, models_input, metric_input], outputs=[output_text, plot_accuracy, plot_precision, plot_recall, plot_f1, output_table]) demo.launch(debug=True)