llmproccrs / tests /plot_significance.py
aitek230telu's picture
Upload 58 files
2055a21 verified
Raw History Blame Contribute Delete
3.67 kB
import pandas as pd
import numpy as np
from scipy import stats
def bootstrap_ci(data, n_bootstraps=10000, ci=0.95):
np.random.seed(42) # Agar hasilnya konsisten jika diulang
bootstrapped_means = []
for _ in range(n_bootstraps):
sample = np.random.choice(data, size=len(data), replace=True)
bootstrapped_means.append(np.mean(sample))
lower_bound = np.percentile(bootstrapped_means, (1 - ci) / 2 * 100)
upper_bound = np.percentile(bootstrapped_means, (1 + ci) / 2 * 100)
return lower_bound, upper_bound
def evaluate_metric(base_scores, prop_scores, metric_name, is_percentage=False):
print("="*60)
print(f"📊 HASIL UJI SIGNIFIKANSI STATISTIK: {metric_name.upper()}")
print("="*60)
differences = prop_scores - base_scores
mean_diff = np.mean(differences)
if is_percentage:
print(f"Rata-rata {metric_name} Baseline : {np.mean(base_scores)*100:.2f}%")
print(f"Rata-rata {metric_name} Proposed : {np.mean(prop_scores)*100:.2f}%")
print(f"Rata-rata Kenaikan (Delta) : +{mean_diff*100:.2f}%")
else:
print(f"Rata-rata {metric_name} Baseline : {np.mean(base_scores):.4f}")
print(f"Rata-rata {metric_name} Proposed : {np.mean(prop_scores):.4f}")
print(f"Rata-rata Kenaikan (Delta) : {mean_diff:+.4f}")
print("-" * 60)
# Uji Statistik
t_stat, p_value_t = stats.ttest_rel(prop_scores, base_scores)
w_stat, p_value_w = stats.wilcoxon(prop_scores, base_scores, zero_method='zsplit')
ci_lower, ci_upper = bootstrap_ci(differences)
print("--- P-VALUES ---")
print(f"Paired T-Test p-value : {p_value_t:.5e}")
print(f"Wilcoxon p-value : {p_value_w:.5e}")
print("\n--- CONFIDENCE INTERVAL (95%) ---")
if is_percentage:
print(f"Bootstrap 95% CI : [{ci_lower*100:.2f}%, {ci_upper*100:.2f}%]")
else:
print(f"Bootstrap 95% CI : [{ci_lower:.4f}, {ci_upper:.4f}]")
print("\n📌 KESIMPULAN:")
if p_value_t < 0.05 and p_value_w < 0.05:
print(f"✅ Signifikan! Peningkatan {metric_name} dapat dibuktikan secara statistik.")
else:
print(f"⚠️ Tidak Signifikan. Perbedaan {metric_name} mungkin terjadi karena variasi acak.")
print("\n")
def run_significance_test(csv_path, sample_size=10):
print(f"Membaca data dari: {csv_path}")
df = pd.read_csv(csv_path)
# 1. Menghitung MRR per Epoch dari Cumulative MRR (Reverse Engineering)
# Total MRR sejauh ini = Cumulative MRR * run_id * sample_size
df['Total_MRR_Base_so_far'] = df['Cum_MRR_Base'] * df['Run_ID'] * sample_size
df['Total_MRR_Prop_so_far'] = df['Cum_MRR_Prop'] * df['Run_ID'] * sample_size
# MRR Epoch saat ini = (Total saat ini - Total sebelumnya) / sample_size
df['Epoch_MRR_Base'] = (df['Total_MRR_Base_so_far'] - df['Total_MRR_Base_so_far'].shift(1, fill_value=0)) / sample_size
df['Epoch_MRR_Prop'] = (df['Total_MRR_Prop_so_far'] - df['Total_MRR_Prop_so_far'].shift(1, fill_value=0)) / sample_size
hit_base = df['Epoch_HitRate_Base'].values
hit_prop = df['Epoch_HitRate_Prop'].values
mrr_base = df['Epoch_MRR_Base'].values
mrr_prop = df['Epoch_MRR_Prop'].values
# Jalankan Evaluasi
evaluate_metric(hit_base, hit_prop, "Hit Rate", is_percentage=True)
evaluate_metric(mrr_base, mrr_prop, "MRR", is_percentage=False)
if __name__ == "__main__":
run_significance_test("tests/reports/report_ml_50_runs_gemma3.csv", sample_size=10)
run_significance_test("tests/reports/report_ml_50_runs_ministral3.csv", sample_size=10)
run_significance_test("tests/reports/report_ml_50_runs_qwen.csv", sample_size=10)