import math from typing import List, Dict, Any def run_mcnemar_test(plain_correct: List[bool], rif_correct: List[bool]) -> Dict[str, Any]: """ Computes McNemar's test on paired binary outcomes. Returns: - contingency_table: {both_correct, plain_only, rif_only, both_incorrect} - statistic: McNemar chi-square value - p_value: calculated using normal approximation & error function - significance: True if p_value < 0.05 - confidence_interval: [lower, upper] for RIF_acc - Plain_acc """ n = len(plain_correct) if n == 0 or len(rif_correct) != n: return { "error": "Mismatched or empty prediction sizes", "sample_size": n, "contingency_table": {"both_correct": 0, "plain_only": 0, "rif_only": 0, "both_incorrect": 0}, "statistic": 0.0, "p_value": 1.0, "significance": False, "accuracy_difference": 0.0, "ci_lower": 0.0, "ci_upper": 0.0 } # Contingency elements both_correct = 0 # a plain_only = 0 # b (Plain correct, RIF wrong) rif_only = 0 # c (RIF correct, Plain wrong) both_incorrect = 0 # d for p, r in zip(plain_correct, rif_correct): if p and r: both_correct += 1 elif p and not r: plain_only += 1 elif not p and r: rif_only += 1 else: both_incorrect += 1 b = plain_only c = rif_only # Accuracy rates plain_acc = (both_correct + plain_only) / n rif_acc = (both_correct + rif_only) / n acc_diff = rif_acc - plain_acc # Chi-square with Yates' continuity correction if (b + c) > 0: statistic = ((abs(b - c) - 1) ** 2) / (b + c) if abs(b - c) > 0 else 0.0 # Calculate p-value from Z-score (sqrt of chi-square stat) z = math.sqrt(statistic) # Phi(z) = 0.5 * (1 + erf(z / sqrt(2))) # p_value for two-tailed = 2 * (1 - Phi(z)) = 1 - erf(z / sqrt(2)) p_value = 1.0 - math.erf(z / math.sqrt(2)) else: statistic = 0.0 p_value = 1.0 # Paired proportion confidence interval (Wald method) # SE = sqrt( (b + c) - (b - c)^2 / N ) / N se = math.sqrt(max(0.0, (b + c) - ((b - c) ** 2) / n)) / n margin_of_error = 1.96 * se ci_lower = acc_diff - margin_of_error ci_upper = acc_diff + margin_of_error return { "sample_size": n, "contingency_table": { "both_correct": both_correct, "plain_only": plain_only, "rif_only": rif_only, "both_incorrect": both_incorrect }, "statistic": round(statistic, 4), "p_value": round(p_value, 6), "significance": p_value < 0.05, "accuracy_difference": round(acc_diff, 4), "ci_lower": round(ci_lower, 4), "ci_upper": round(ci_upper, 4) }