import pandas as pd import numpy as np from scipy import stats def bootstrap_ci(data, n_bootstraps=10000, ci=0.95): np.random.seed(42) # Agar hasilnya konsisten jika diulang bootstrapped_means = [] for _ in range(n_bootstraps): sample = np.random.choice(data, size=len(data), replace=True) bootstrapped_means.append(np.mean(sample)) lower_bound = np.percentile(bootstrapped_means, (1 - ci) / 2 * 100) upper_bound = np.percentile(bootstrapped_means, (1 + ci) / 2 * 100) return lower_bound, upper_bound def evaluate_metric(base_scores, prop_scores, metric_name, is_percentage=False): print("="*60) print(f"šŸ“Š HASIL UJI SIGNIFIKANSI STATISTIK: {metric_name.upper()}") print("="*60) differences = prop_scores - base_scores mean_diff = np.mean(differences) if is_percentage: print(f"Rata-rata {metric_name} Baseline : {np.mean(base_scores)*100:.2f}%") print(f"Rata-rata {metric_name} Proposed : {np.mean(prop_scores)*100:.2f}%") print(f"Rata-rata Kenaikan (Delta) : +{mean_diff*100:.2f}%") else: print(f"Rata-rata {metric_name} Baseline : {np.mean(base_scores):.4f}") print(f"Rata-rata {metric_name} Proposed : {np.mean(prop_scores):.4f}") print(f"Rata-rata Kenaikan (Delta) : {mean_diff:+.4f}") print("-" * 60) # Uji Statistik t_stat, p_value_t = stats.ttest_rel(prop_scores, base_scores) w_stat, p_value_w = stats.wilcoxon(prop_scores, base_scores, zero_method='zsplit') ci_lower, ci_upper = bootstrap_ci(differences) print("--- P-VALUES ---") print(f"Paired T-Test p-value : {p_value_t:.5e}") print(f"Wilcoxon p-value : {p_value_w:.5e}") print("\n--- CONFIDENCE INTERVAL (95%) ---") if is_percentage: print(f"Bootstrap 95% CI : [{ci_lower*100:.2f}%, {ci_upper*100:.2f}%]") else: print(f"Bootstrap 95% CI : [{ci_lower:.4f}, {ci_upper:.4f}]") print("\nšŸ“Œ KESIMPULAN:") if p_value_t < 0.05 and p_value_w < 0.05: print(f"āœ… Signifikan! Peningkatan {metric_name} dapat dibuktikan secara statistik.") else: print(f"āš ļø Tidak Signifikan. Perbedaan {metric_name} mungkin terjadi karena variasi acak.") print("\n") def run_significance_test(csv_path, sample_size=10): print(f"Membaca data dari: {csv_path}") df = pd.read_csv(csv_path) # 1. Menghitung MRR per Epoch dari Cumulative MRR (Reverse Engineering) # Total MRR sejauh ini = Cumulative MRR * run_id * sample_size df['Total_MRR_Base_so_far'] = df['Cum_MRR_Base'] * df['Run_ID'] * sample_size df['Total_MRR_Prop_so_far'] = df['Cum_MRR_Prop'] * df['Run_ID'] * sample_size # MRR Epoch saat ini = (Total saat ini - Total sebelumnya) / sample_size df['Epoch_MRR_Base'] = (df['Total_MRR_Base_so_far'] - df['Total_MRR_Base_so_far'].shift(1, fill_value=0)) / sample_size df['Epoch_MRR_Prop'] = (df['Total_MRR_Prop_so_far'] - df['Total_MRR_Prop_so_far'].shift(1, fill_value=0)) / sample_size hit_base = df['Epoch_HitRate_Base'].values hit_prop = df['Epoch_HitRate_Prop'].values mrr_base = df['Epoch_MRR_Base'].values mrr_prop = df['Epoch_MRR_Prop'].values # Jalankan Evaluasi evaluate_metric(hit_base, hit_prop, "Hit Rate", is_percentage=True) evaluate_metric(mrr_base, mrr_prop, "MRR", is_percentage=False) if __name__ == "__main__": run_significance_test("tests/reports/report_ml_50_runs_gemma3.csv", sample_size=10) run_significance_test("tests/reports/report_ml_50_runs_ministral3.csv", sample_size=10) run_significance_test("tests/reports/report_ml_50_runs_qwen.csv", sample_size=10)