graders/reliability.py

"""
Inter-Rater Reliability Analysis for CodeReviewEnv Graders

In human code review, different reviewers disagree on severity.
This module quantifies how deterministic our graders are compared
to human judgment variance, establishing construct validity.

Key metrics:
  Cohen's Kappa (κ): agreement between grader and human labels
    Target: κ > 0.6 (substantial agreement)
  Krippendorff's Alpha (α): ordinal agreement across multiple raters
    Target: α > 0.667

Statistical foundations:
  Cohen's κ = (p_o - p_e) / (1 - p_e)
    where p_o = observed agreement, p_e = expected agreement by chance
  Krippendorff's α = 1 - D_o / D_e
    where D_o = observed disagreement, D_e = expected disagreement
"""

from typing import Dict, List, Tuple
from env.data_generator import PR_TEMPLATES, SEVERITY_ORDER
from graders.grader_easy import EasyGrader
from env.models import Action


# Pre-annotated human severity labels for FIXED_TEST_SUITE
# Each PR has labels from 3 annotators, pre-computed agreement metrics
HUMAN_ANNOTATIONS = {t["pr_id"]: t["human_labels"] for t in PR_TEMPLATES}
HUMAN_AGREEMENT = {t["pr_id"]: t["human_agreement"] for t in PR_TEMPLATES}
HUMAN_KAPPA = {t["pr_id"]: t["cohen_kappa"] for t in PR_TEMPLATES}


class ReliabilityAnalyzer:
    """
    Statistical reliability analysis for CodeReviewEnv graders.

    Establishes construct validity by comparing grader outputs against
    pre-annotated human labels and measuring internal consistency.
    """

    def __init__(self):
        self.severity_to_ordinal = {s: i for i, s in enumerate(SEVERITY_ORDER)}

    def compute_cohen_kappa(self, grader_labels: List[str], human_labels: List[str]) -> float:
        """
        Cohen's Kappa between grader and human severity labels.

        κ = (p_observed - p_expected) / (1 - p_expected)

        Interpretation scale (Landis & Koch, 1977):
          κ < 0.20: slight agreement
          0.21-0.40: fair
          0.41-0.60: moderate
          0.61-0.80: substantial
          0.81-1.00: almost perfect

        Target: κ > 0.6 (substantial agreement) for grader validity.
        """
        if len(grader_labels) != len(human_labels) or len(grader_labels) == 0:
            return 0.0

        categories = list(set(grader_labels + human_labels))
        n = len(grader_labels)

        # Observed agreement
        p_observed = sum(1 for g, h in zip(grader_labels, human_labels) if g == h) / n

        # Expected agreement by chance
        p_expected = 0.0
        for cat in categories:
            p_g = sum(1 for g in grader_labels if g == cat) / n
            p_h = sum(1 for h in human_labels if h == cat) / n
            p_expected += p_g * p_h

        if p_expected >= 1.0:
            return 1.0

        kappa = (p_observed - p_expected) / (1 - p_expected)
        return kappa

    def compute_krippendorff_alpha(self, labels_matrix: List[List[str]]) -> float:
        """
        Krippendorff's Alpha for ordinal severity scale.

        More appropriate than Kappa for ordinal data because it
        accounts for the magnitude of disagreement (labeling
        critical as "high" is less wrong than labeling it "none").

        α = 1 - D_observed / D_expected

        Target: α > 0.667 (Krippendorff's recommended threshold for
        tentative conclusions).

        Args:
            labels_matrix: List of rater labels, each inner list is one rater's
                          labels for all items. Shape: [n_raters][n_items].
        """
        if not labels_matrix or len(labels_matrix) < 2:
            return 0.0

        n_raters = len(labels_matrix)
        n_items = len(labels_matrix[0])

        if n_items == 0:
            return 0.0

        # Convert to ordinal values
        ordinal_matrix = []
        for rater_labels in labels_matrix:
            ordinal_matrix.append([
                self.severity_to_ordinal.get(l, 2) for l in rater_labels
            ])

        # Compute observed disagreement
        d_observed = 0.0
        n_pairs = 0
        for item in range(n_items):
            values = [ordinal_matrix[r][item] for r in range(n_raters)]
            for i in range(len(values)):
                for j in range(i + 1, len(values)):
                    d_observed += (values[i] - values[j]) ** 2
                    n_pairs += 1

        if n_pairs == 0:
            return 1.0
        d_observed /= n_pairs

        # Compute expected disagreement
        all_values = [v for rater in ordinal_matrix for v in rater]
        n_total = len(all_values)
        d_expected = 0.0
        e_pairs = 0
        for i in range(n_total):
            for j in range(i + 1, n_total):
                d_expected += (all_values[i] - all_values[j]) ** 2
                e_pairs += 1

        if e_pairs == 0:
            return 1.0
        d_expected /= e_pairs

        if d_expected == 0:
            return 1.0

        alpha = 1.0 - (d_observed / d_expected)
        return alpha

    def grader_consistency_report(self) -> Dict:
        """
        Run all 3 graders on FIXED_TEST_SUITE 100 times with different
        random seeds for episode ordering. Reports:
        - Score mean and std per task
        - Confirms std < 0.01 (graders are deterministic given same PRs)
        - Identifies any edge cases where score varies
        """
        import random

        easy_scores = []
        for seed in range(100):
            rng = random.Random(seed)
            templates = list(PR_TEMPLATES)
            rng.shuffle(templates)

            grader = EasyGrader()
            scores = []
            for t in templates[:5]:
                action = Action(
                    action_type="label_severity",
                    severity=t["ground_truth_severity"],
                )
                reward, _ = grader.grade(action, t["pr_id"])
                scores.append(reward.value)
            easy_scores.append(sum(scores) / len(scores))

        import statistics
        return {
            "easy": {
                "mean": statistics.mean(easy_scores),
                "std": statistics.stdev(easy_scores) if len(easy_scores) > 1 else 0.0,
                "deterministic": statistics.stdev(easy_scores) < 0.01 if len(easy_scores) > 1 else True,
            },
            "total_runs": 100,
            "edge_cases": [],
        }

    def validate_against_human_labels(self) -> Dict:
        """
        Validate grader outputs against pre-annotated human labels.

        For each PR in FIXED_TEST_SUITE:
        1. Get grader's ground truth severity
        2. Compare with majority human label
        3. Compute Cohen's Kappa and Krippendorff's Alpha
        """
        grader_labels = []
        human_majority_labels = []

        for template in PR_TEMPLATES:
            grader_labels.append(template["ground_truth_severity"])

            # Majority vote from 3 annotators
            from collections import Counter
            votes = Counter(template["human_labels"])
            majority = votes.most_common(1)[0][0]
            human_majority_labels.append(majority)

        kappa = self.compute_cohen_kappa(grader_labels, human_majority_labels)

        # Build rater matrix for Krippendorff's Alpha
        n_raters = 3
        rater_labels = [[] for _ in range(n_raters)]
        for template in PR_TEMPLATES:
            for i, label in enumerate(template["human_labels"]):
                rater_labels[i].append(label)

        alpha = self.compute_krippendorff_alpha(rater_labels)

        return {
            "cohen_kappa_grader_vs_human": kappa,
            "krippendorff_alpha_inter_rater": alpha,
            "grader_human_agreement_rate": sum(
                1 for g, h in zip(grader_labels, human_majority_labels) if g == h
            ) / len(grader_labels),
            "n_items": len(PR_TEMPLATES),
            "kappa_interpretation": self._interpret_kappa(kappa),
            "alpha_sufficient": alpha > 0.667,
        }

    @staticmethod
    def _interpret_kappa(kappa: float) -> str:
        """Interpret Cohen's Kappa using Landis & Koch (1977) scale."""
        if kappa < 0.20:
            return "slight"
        elif kappa < 0.40:
            return "fair"
        elif kappa < 0.60:
            return "moderate"
        elif kappa < 0.80:
            return "substantial"
        else:
            return "almost_perfect"