benchmark/statistics.py

import math
from typing import List, Dict, Any

def run_mcnemar_test(plain_correct: List[bool], rif_correct: List[bool]) -> Dict[str, Any]:
    """
    Computes McNemar's test on paired binary outcomes.
    Returns:
        - contingency_table: {both_correct, plain_only, rif_only, both_incorrect}
        - statistic: McNemar chi-square value
        - p_value: calculated using normal approximation & error function
        - significance: True if p_value < 0.05
        - confidence_interval: [lower, upper] for RIF_acc - Plain_acc
    """
    n = len(plain_correct)
    if n == 0 or len(rif_correct) != n:
        return {
            "error": "Mismatched or empty prediction sizes",
            "sample_size": n,
            "contingency_table": {"both_correct": 0, "plain_only": 0, "rif_only": 0, "both_incorrect": 0},
            "statistic": 0.0,
            "p_value": 1.0,
            "significance": False,
            "accuracy_difference": 0.0,
            "ci_lower": 0.0,
            "ci_upper": 0.0
        }
        
    # Contingency elements
    both_correct = 0  # a
    plain_only = 0    # b (Plain correct, RIF wrong)
    rif_only = 0      # c (RIF correct, Plain wrong)
    both_incorrect = 0 # d
    
    for p, r in zip(plain_correct, rif_correct):
        if p and r:
            both_correct += 1
        elif p and not r:
            plain_only += 1
        elif not p and r:
            rif_only += 1
        else:
            both_incorrect += 1
            
    b = plain_only
    c = rif_only
    
    # Accuracy rates
    plain_acc = (both_correct + plain_only) / n
    rif_acc = (both_correct + rif_only) / n
    acc_diff = rif_acc - plain_acc
    
    # Chi-square with Yates' continuity correction
    if (b + c) > 0:
        statistic = ((abs(b - c) - 1) ** 2) / (b + c) if abs(b - c) > 0 else 0.0
        # Calculate p-value from Z-score (sqrt of chi-square stat)
        z = math.sqrt(statistic)
        # Phi(z) = 0.5 * (1 + erf(z / sqrt(2)))
        # p_value for two-tailed = 2 * (1 - Phi(z)) = 1 - erf(z / sqrt(2))
        p_value = 1.0 - math.erf(z / math.sqrt(2))
    else:
        statistic = 0.0
        p_value = 1.0
        
    # Paired proportion confidence interval (Wald method)
    # SE = sqrt( (b + c) - (b - c)^2 / N ) / N
    se = math.sqrt(max(0.0, (b + c) - ((b - c) ** 2) / n)) / n
    margin_of_error = 1.96 * se
    ci_lower = acc_diff - margin_of_error
    ci_upper = acc_diff + margin_of_error
    
    return {
        "sample_size": n,
        "contingency_table": {
            "both_correct": both_correct,
            "plain_only": plain_only,
            "rif_only": rif_only,
            "both_incorrect": both_incorrect
        },
        "statistic": round(statistic, 4),
        "p_value": round(p_value, 6),
        "significance": p_value < 0.05,
        "accuracy_difference": round(acc_diff, 4),
        "ci_lower": round(ci_lower, 4),
        "ci_upper": round(ci_upper, 4)
    }