Spaces:
Sleeping
Sleeping
File size: 2,119 Bytes
53bb779 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 | """
Calculates McNemar's test for significance between two models,
using the stated binarization threshold.
"""
import sys
import os
sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
from modules.reasoning_engine import ReasoningEngine
THRESHOLD = ReasoningEngine.MCNEMAR_BINARIZATION_THRESHOLD
def mcnemars_test(scores_model_A: list, scores_model_B: list):
"""
Computes McNemar's test p-value for paired nominal data.
scores are lists of float faithfulness scores.
"""
if len(scores_model_A) != len(scores_model_B):
raise ValueError("Must have same number of scores")
# Binarize
bin_A = [1 if s >= THRESHOLD else 0 for s in scores_model_A]
bin_B = [1 if s >= THRESHOLD else 0 for s in scores_model_B]
# Contingency table
# B correct | B wrong
# A correct | a | b
# A wrong | c | d
a, b, c, d = 0, 0, 0, 0
for a_val, b_val in zip(bin_A, bin_B):
if a_val == 1 and b_val == 1: a += 1
elif a_val == 1 and b_val == 0: b += 1
elif a_val == 0 and b_val == 1: c += 1
else: d += 1
# Chi-square statistic: (b - c)^2 / (b + c)
if b + c == 0:
print("Models are identical given the threshold.")
return 1.0 # No difference
chi_square = ((abs(b - c) - 1)**2) / (b + c) # with continuity correction
print(f"McNemar's Test Results:")
print(f"Binarization Threshold: {THRESHOLD}")
print(f"Contingency Table: a={a}, b={b}, c={c}, d={d}")
print(f"Chi-square: {chi_square:.3f}")
try:
from scipy.stats import chi2
p_value = 1 - chi2.cdf(chi_square, 1)
print(f"p-value: {p_value:.4f}")
return p_value
except ImportError:
print("Note: Install scipy ('pip install scipy') to automatically calculate the p-value.")
return chi_square
if __name__ == "__main__":
# Mock data
scores_mexar = [0.8, 0.9, 0.4, 0.7, 0.65, 0.8]
scores_baseline = [0.5, 0.7, 0.6, 0.4, 0.55, 0.8]
mcnemars_test(scores_mexar, scores_baseline)
|