File size: 3,674 Bytes
2055a21
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
import pandas as pd
import numpy as np
from scipy import stats

def bootstrap_ci(data, n_bootstraps=10000, ci=0.95):
    np.random.seed(42) # Agar hasilnya konsisten jika diulang
    bootstrapped_means = []
    for _ in range(n_bootstraps):
        sample = np.random.choice(data, size=len(data), replace=True)
        bootstrapped_means.append(np.mean(sample))
    
    lower_bound = np.percentile(bootstrapped_means, (1 - ci) / 2 * 100)
    upper_bound = np.percentile(bootstrapped_means, (1 + ci) / 2 * 100)
    return lower_bound, upper_bound

def evaluate_metric(base_scores, prop_scores, metric_name, is_percentage=False):
    print("="*60)
    print(f"📊 HASIL UJI SIGNIFIKANSI STATISTIK: {metric_name.upper()}")
    print("="*60)
    
    differences = prop_scores - base_scores
    mean_diff = np.mean(differences)
    
    if is_percentage:
        print(f"Rata-rata {metric_name} Baseline : {np.mean(base_scores)*100:.2f}%")
        print(f"Rata-rata {metric_name} Proposed : {np.mean(prop_scores)*100:.2f}%")
        print(f"Rata-rata Kenaikan (Delta) : +{mean_diff*100:.2f}%")
    else:
        print(f"Rata-rata {metric_name} Baseline : {np.mean(base_scores):.4f}")
        print(f"Rata-rata {metric_name} Proposed : {np.mean(prop_scores):.4f}")
        print(f"Rata-rata Kenaikan (Delta) : {mean_diff:+.4f}")
    print("-" * 60)

    # Uji Statistik
    t_stat, p_value_t = stats.ttest_rel(prop_scores, base_scores)
    w_stat, p_value_w = stats.wilcoxon(prop_scores, base_scores, zero_method='zsplit')
    ci_lower, ci_upper = bootstrap_ci(differences)

    print("--- P-VALUES ---")
    print(f"Paired T-Test p-value  : {p_value_t:.5e}")
    print(f"Wilcoxon p-value       : {p_value_w:.5e}")
    
    print("\n--- CONFIDENCE INTERVAL (95%) ---")
    if is_percentage:
        print(f"Bootstrap 95% CI       : [{ci_lower*100:.2f}%, {ci_upper*100:.2f}%]")
    else:
        print(f"Bootstrap 95% CI       : [{ci_lower:.4f}, {ci_upper:.4f}]")
    
    print("\n📌 KESIMPULAN:")
    if p_value_t < 0.05 and p_value_w < 0.05:
        print(f"✅ Signifikan! Peningkatan {metric_name} dapat dibuktikan secara statistik.")
    else:
        print(f"⚠️ Tidak Signifikan. Perbedaan {metric_name} mungkin terjadi karena variasi acak.")
    print("\n")

def run_significance_test(csv_path, sample_size=10):
    print(f"Membaca data dari: {csv_path}")
    df = pd.read_csv(csv_path)
    
    # 1. Menghitung MRR per Epoch dari Cumulative MRR (Reverse Engineering)
    # Total MRR sejauh ini = Cumulative MRR * run_id * sample_size
    df['Total_MRR_Base_so_far'] = df['Cum_MRR_Base'] * df['Run_ID'] * sample_size
    df['Total_MRR_Prop_so_far'] = df['Cum_MRR_Prop'] * df['Run_ID'] * sample_size

    # MRR Epoch saat ini = (Total saat ini - Total sebelumnya) / sample_size
    df['Epoch_MRR_Base'] = (df['Total_MRR_Base_so_far'] - df['Total_MRR_Base_so_far'].shift(1, fill_value=0)) / sample_size
    df['Epoch_MRR_Prop'] = (df['Total_MRR_Prop_so_far'] - df['Total_MRR_Prop_so_far'].shift(1, fill_value=0)) / sample_size

    hit_base = df['Epoch_HitRate_Base'].values
    hit_prop = df['Epoch_HitRate_Prop'].values
    mrr_base = df['Epoch_MRR_Base'].values
    mrr_prop = df['Epoch_MRR_Prop'].values

    # Jalankan Evaluasi
    evaluate_metric(hit_base, hit_prop, "Hit Rate", is_percentage=True)
    evaluate_metric(mrr_base, mrr_prop, "MRR", is_percentage=False)

if __name__ == "__main__":
    run_significance_test("tests/reports/report_ml_50_runs_gemma3.csv", sample_size=10)
    run_significance_test("tests/reports/report_ml_50_runs_ministral3.csv", sample_size=10)
    run_significance_test("tests/reports/report_ml_50_runs_qwen.csv", sample_size=10)