Download tests/plot_significance.py from recommender-system/llmproccrs: direct link, hf CLI and curl.
- Browser
- Download file 3.67 kB
-
https://huggingface.co/recommender-system/llmproccrs/resolve/main/tests/plot_significance.py
- Command line
-
hf download hf://recommender-system/llmproccrs/tests/plot_significance.py
-
curl -L -o plot_significance.py https://huggingface.co/recommender-system/llmproccrs/resolve/main/tests/plot_significance.py
3.67 kB
| import pandas as pd | |
| import numpy as np | |
| from scipy import stats | |
| def bootstrap_ci(data, n_bootstraps=10000, ci=0.95): | |
| np.random.seed(42) # Agar hasilnya konsisten jika diulang | |
| bootstrapped_means = [] | |
| for _ in range(n_bootstraps): | |
| sample = np.random.choice(data, size=len(data), replace=True) | |
| bootstrapped_means.append(np.mean(sample)) | |
| lower_bound = np.percentile(bootstrapped_means, (1 - ci) / 2 * 100) | |
| upper_bound = np.percentile(bootstrapped_means, (1 + ci) / 2 * 100) | |
| return lower_bound, upper_bound | |
| def evaluate_metric(base_scores, prop_scores, metric_name, is_percentage=False): | |
| print("="*60) | |
| print(f"📊 HASIL UJI SIGNIFIKANSI STATISTIK: {metric_name.upper()}") | |
| print("="*60) | |
| differences = prop_scores - base_scores | |
| mean_diff = np.mean(differences) | |
| if is_percentage: | |
| print(f"Rata-rata {metric_name} Baseline : {np.mean(base_scores)*100:.2f}%") | |
| print(f"Rata-rata {metric_name} Proposed : {np.mean(prop_scores)*100:.2f}%") | |
| print(f"Rata-rata Kenaikan (Delta) : +{mean_diff*100:.2f}%") | |
| else: | |
| print(f"Rata-rata {metric_name} Baseline : {np.mean(base_scores):.4f}") | |
| print(f"Rata-rata {metric_name} Proposed : {np.mean(prop_scores):.4f}") | |
| print(f"Rata-rata Kenaikan (Delta) : {mean_diff:+.4f}") | |
| print("-" * 60) | |
| # Uji Statistik | |
| t_stat, p_value_t = stats.ttest_rel(prop_scores, base_scores) | |
| w_stat, p_value_w = stats.wilcoxon(prop_scores, base_scores, zero_method='zsplit') | |
| ci_lower, ci_upper = bootstrap_ci(differences) | |
| print("--- P-VALUES ---") | |
| print(f"Paired T-Test p-value : {p_value_t:.5e}") | |
| print(f"Wilcoxon p-value : {p_value_w:.5e}") | |
| print("\n--- CONFIDENCE INTERVAL (95%) ---") | |
| if is_percentage: | |
| print(f"Bootstrap 95% CI : [{ci_lower*100:.2f}%, {ci_upper*100:.2f}%]") | |
| else: | |
| print(f"Bootstrap 95% CI : [{ci_lower:.4f}, {ci_upper:.4f}]") | |
| print("\n📌 KESIMPULAN:") | |
| if p_value_t < 0.05 and p_value_w < 0.05: | |
| print(f"✅ Signifikan! Peningkatan {metric_name} dapat dibuktikan secara statistik.") | |
| else: | |
| print(f"⚠️ Tidak Signifikan. Perbedaan {metric_name} mungkin terjadi karena variasi acak.") | |
| print("\n") | |
| def run_significance_test(csv_path, sample_size=10): | |
| print(f"Membaca data dari: {csv_path}") | |
| df = pd.read_csv(csv_path) | |
| # 1. Menghitung MRR per Epoch dari Cumulative MRR (Reverse Engineering) | |
| # Total MRR sejauh ini = Cumulative MRR * run_id * sample_size | |
| df['Total_MRR_Base_so_far'] = df['Cum_MRR_Base'] * df['Run_ID'] * sample_size | |
| df['Total_MRR_Prop_so_far'] = df['Cum_MRR_Prop'] * df['Run_ID'] * sample_size | |
| # MRR Epoch saat ini = (Total saat ini - Total sebelumnya) / sample_size | |
| df['Epoch_MRR_Base'] = (df['Total_MRR_Base_so_far'] - df['Total_MRR_Base_so_far'].shift(1, fill_value=0)) / sample_size | |
| df['Epoch_MRR_Prop'] = (df['Total_MRR_Prop_so_far'] - df['Total_MRR_Prop_so_far'].shift(1, fill_value=0)) / sample_size | |
| hit_base = df['Epoch_HitRate_Base'].values | |
| hit_prop = df['Epoch_HitRate_Prop'].values | |
| mrr_base = df['Epoch_MRR_Base'].values | |
| mrr_prop = df['Epoch_MRR_Prop'].values | |
| # Jalankan Evaluasi | |
| evaluate_metric(hit_base, hit_prop, "Hit Rate", is_percentage=True) | |
| evaluate_metric(mrr_base, mrr_prop, "MRR", is_percentage=False) | |
| if __name__ == "__main__": | |
| run_significance_test("tests/reports/report_ml_50_runs_gemma3.csv", sample_size=10) | |
| run_significance_test("tests/reports/report_ml_50_runs_ministral3.csv", sample_size=10) | |
| run_significance_test("tests/reports/report_ml_50_runs_qwen.csv", sample_size=10) |