env/hidden_tests.py

from __future__ import annotations

from typing import Any

from env.graders.deterministic import DeterministicGrader


class HiddenTestRunner:
    """Evaluates whether a fix generalizes across deterministic CI variants."""

    def __init__(self, grader: DeterministicGrader | None = None, pass_threshold: float = 0.65):
        self.grader = grader or DeterministicGrader()
        self.pass_threshold = pass_threshold

    def generate_variants(self, config_text: str) -> list[str]:
        base = config_text or ""
        variants: list[str] = []

        for replacements in self._variant_replacement_sets():
            variant = self._apply_replacements(base, replacements)
            if variant not in variants:
                variants.append(variant)

        return variants

    def evaluate_fix(
        self,
        fixed_config: str,
        task: dict[str, Any] | None = None,
        expected_config: str | None = None,
        metadata: dict[str, Any] | None = None,
    ) -> float:
        fixed_config = fixed_config or ""
        task = task or {}
        metadata = metadata or {}
        expected = expected_config or str(task.get("expected_config", ""))

        if not fixed_config.strip() or not expected.strip():
            return 0.0

        total = 0
        passed = 0

        for replacements in self._variant_replacement_sets():
            fixed_variant = self._apply_replacements(fixed_config, replacements)
            expected_variant = self._apply_replacements(expected, replacements)
            score = self.grader.grade(fixed_variant, expected_variant, metadata)
            total += 1
            if score >= self.pass_threshold:
                passed += 1

        if total == 0:
            return 0.0

        return round(passed / total, 4)

    def _variant_replacement_sets(self) -> list[tuple[tuple[str, str], ...]]:
        return [
            tuple(),
            (("ubuntu-latest", "windows-latest"),),
            (("windows-latest", "ubuntu-latest"),),
            (("node-version: 16", "node-version: 18"),),
            (("node-version: \"16\"", "node-version: \"18\""),),
            (("python-version: \"3.10\"", "python-version: \"3.12\""),),
            (("NODE_ENV=production", "NODE_ENV=development"),),
        ]

    def _apply_replacements(self, text: str, replacements: tuple[tuple[str, str], ...]) -> str:
        output = text
        for old, new in replacements:
            output = output.replace(old, new)
        return output