eval/test_eval_matching.py

"""
Unit tests for eval/run_eval.py — matching primitives only.

These tests import run_eval DIRECTLY (not via app) and never load model weights.
No BUREAUCAT_NO_MODEL needed here because run_eval.py does NOT import app at
module scope — only inside run_eval() (lazy import contract).

Test coverage:
  Task 1 primitives (normalize, value_found, extract_values_from_section):
    - normalize: collapses NBSP, thin space, narrow no-break space, lowercases
    - value_found: long ISO date found in transcription with surrounding text
    - value_found short-value boundary: "123" NOT matched inside "1234567"
    - value_found short-value boundary: "123" IS matched when standing alone
    - value_found whitespace tolerance: gold with thin-space found in transcription
    - extract_values_from_section: em-dash separator, strips bullets
    - extract_values_from_section: hyphen-with-spaces separator
    - extract_values_from_section: "None found." yields no values
    - importability: 'app' not in sys.modules after importing run_eval
      (checked in a standalone -c probe, not here — see verify block in plan)

  Task 2 functions (evaluate, beginner_invariant):
    - evaluate: clean result passes (invented=0, missing=0, all_sections_present, severity)
    - evaluate: invented value (not in transcription) → pass=False
    - evaluate: missing gold value (not in deadlines) → pass=False
    - evaluate: empty section (tldr="") → pass=False even with zero invented/missing
    - evaluate: severity MAE is computed but does not affect pass
    - beginner_invariant: True when all fields present
    - beginner_invariant: False when a section is empty
    - beginner_invariant: False when severity is None
    - beginner_invariant: False when transcription is empty
"""

import os
import sys

# Ensure project root and eval/ are on the path regardless of where pytest is run from.
_REPO_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
_EVAL_DIR = os.path.join(_REPO_ROOT, "eval")
if _REPO_ROOT not in sys.path:
    sys.path.insert(0, _REPO_ROOT)
if _EVAL_DIR not in sys.path:
    sys.path.insert(0, _EVAL_DIR)

import run_eval as e
from types import SimpleNamespace as S


# ---------------------------------------------------------------------------
# Task 1 primitive tests: normalize
# ---------------------------------------------------------------------------

def test_normalize_case_insensitive():
    """normalize lowercases input."""
    assert e.normalize("1 234,56 KR") == e.normalize("1 234,56 kr")


def test_normalize_collapses_nbsp():
    """U+00A0 non-breaking space collapsed to ASCII space."""
    #   between digits, as OCR might produce
    text_with_nbsp = "1 234 kr"
    text_plain = "1 234 kr"
    assert e.normalize(text_with_nbsp) == e.normalize(text_plain)


def test_normalize_collapses_thin_space():
    """U+2009 thin space collapsed to ASCII space."""
    text_thin = "1 234 kr"
    text_plain = "1 234 kr"
    assert e.normalize(text_thin) == e.normalize(text_plain)


def test_normalize_collapses_narrow_nobreak():
    """U+202F narrow no-break space collapsed to ASCII space."""
    text_narrow = "1 234 kr"
    text_plain = "1 234 kr"
    assert e.normalize(text_narrow) == e.normalize(text_plain)


def test_normalize_nfc():
    """NFC normalization handles combining characters from OCR."""
    # e + combining acute = é (NFC)
    composed = "é"     # é
    decomposed = "é"  # e + combining acute
    assert e.normalize(decomposed) == e.normalize(composed)


def test_normalize_collapses_multiple_spaces():
    """Multiple spaces (including mixed Unicode) collapse to single space."""
    assert e.normalize("a   b") == "a b"
    assert e.normalize("a   b") == "a b"


# ---------------------------------------------------------------------------
# Task 1 primitive tests: value_found
# ---------------------------------------------------------------------------

def test_value_found_long_date_in_transcription():
    """An ISO date (>= 6 digits) is found when present with surrounding text."""
    haystack = "Du ska betala senast 2026-06-15. Glöm inte."
    assert e.value_found("2026-06-15", haystack) is True


def test_value_found_long_date_absent():
    """A date not in the transcription is not found."""
    haystack = "Du ska betala senast 2026-06-15."
    assert e.value_found("2026-07-99", haystack) is False


def test_value_found_short_boundary_inside_longer():
    """A 3-digit ref "123" is NOT found inside "1234567" (boundary rule, Pitfall 5)."""
    haystack = "ref 1234567 i systemet"
    assert e.value_found("123", haystack) is False


def test_value_found_short_boundary_standalone():
    """A 3-digit ref "123" IS found when it stands alone."""
    haystack = "OCR-nummer 123 gäller"
    assert e.value_found("123", haystack) is True


def test_value_found_short_boundary_at_start():
    """A 3-digit ref at the start of the string is found."""
    haystack = "123 är ditt referensnummer"
    assert e.value_found("123", haystack) is True


def test_value_found_whitespace_tolerance():
    """Gold "1 234 kr" (thin space) is found in transcription with plain space."""
    haystack = "belopp 1 234 kr forfaller 2026-06-15"
    # Gold may have thin space; haystack has plain space — both normalize
    assert e.value_found("1 234 kr", haystack) is True


def test_value_found_case_insensitive():
    """Match is case-insensitive (normalize lowercases)."""
    assert e.value_found("JUNI", "betala i juni 2026") is True


def test_value_found_not_found_returns_false():
    """A value genuinely absent from haystack returns False."""
    assert e.value_found("2026-12-31", "betala senast 2026-06-15") is False


def test_value_found_six_digit_reference_uses_plain_substring():
    """A 6+ digit reference uses plain normalized substring (no boundary required)."""
    # "123456789" has 9 digits (>= 6) — substring match, no boundary needed
    assert e.value_found("123456789", "OCR: 123456789") is True


def test_value_found_long_value_not_matched_inside_longer_digit_run():
    """WR-01: a value must NOT match inside a longer run of digits, regardless of
    digit count — the non-digit boundary is applied universally."""
    # 6-digit value inside a 7-digit run — must NOT match
    assert e.value_found("123456", "ref 1234567") is False
    # 8-digit reference must NOT match as a superstring (the WR-01 case)
    assert e.value_found("12-345678", "ref 912-3456789") is False
    # ISO date must NOT match inside a longer leading-digit run
    assert e.value_found("2026-06-15", "x12026-06-15") is False
    # ...but the same values DO match when properly delimited
    assert e.value_found("123456", "ref 123456 end") is True
    assert e.value_found("12-345678", "dossier 12-345678.") is True
    assert e.value_found("2026-06-15", "senast 2026-06-15") is True


# ---------------------------------------------------------------------------
# Task 1 primitive tests: extract_values_from_section
# ---------------------------------------------------------------------------

def test_extract_em_dash_separator():
    """Em-dash separator lines: verbatim value before ' — '."""
    section = "- 15 juni 2026 — last day to file\n- 1 234 kr — amount owed"
    vals = e.extract_values_from_section(section)
    assert vals == ["15 juni 2026", "1 234 kr"]


def test_extract_hyphen_with_spaces_separator():
    """Hyphen-with-spaces separator lines: verbatim value before ' - '."""
    section = "- 2026-06-15 - sista betalningsdag\n- 1 234 kr - fakturabelopp"
    vals = e.extract_values_from_section(section)
    assert vals == ["2026-06-15", "1 234 kr"]


def test_extract_none_found_yields_empty():
    """'None found.' line (and variants) yields no values."""
    assert e.extract_values_from_section("None found.") == []
    assert e.extract_values_from_section("none found.") == []
    assert e.extract_values_from_section("None found") == []


def test_extract_blank_lines_skipped():
    """Blank lines between entries are skipped."""
    section = "\n- 2026-06-15 — sista dag\n\n- 1 234 kr — belopp\n"
    vals = e.extract_values_from_section(section)
    assert vals == ["2026-06-15", "1 234 kr"]


def test_extract_date_with_internal_hyphens_preserved():
    """ISO date 2026-06-15 is not split by internal hyphens — only ' - ' with spaces."""
    section = "- 2026-06-15 — sista dag"
    vals = e.extract_values_from_section(section)
    assert vals == ["2026-06-15"]


def test_extract_mixed_separators():
    """Mixed em-dash and hyphen-space separators in same section."""
    section = "- 15 juni 2026 — sista dag\n- 1 234 kr - belopp"
    vals = e.extract_values_from_section(section)
    assert vals == ["15 juni 2026", "1 234 kr"]


def test_extract_empty_section_yields_empty():
    """Empty section string yields empty list."""
    assert e.extract_values_from_section("") == []


# ---------------------------------------------------------------------------
# Task 2 tests: evaluate (SimpleNamespace stand-ins — no app import)
# ---------------------------------------------------------------------------

def _gold(
    deadlines=None, amounts=None, references=None, expected_severity=4
):
    """Helper: build a gold dict with the pattern-6 sidecar schema."""
    return {
        "expected_severity": expected_severity,
        "deadlines": deadlines or [],
        "amounts": amounts or [],
        "references": references or [],
    }


def _good_result():
    """A fully-populated StructuredResult stand-in that should PASS."""
    return S(
        transcription="betala 1 234 kr senast 2026-06-15",
        tldr="You must pay.",
        why="You owe money.",
        actions="Pay the invoice.",
        deadlines="- 2026-06-15 — sista dag\n- 1 234 kr — belopp",
        severity=4,
    )


def test_evaluate_clean_result_passes():
    """A clean, complete result passes all gate checks."""
    gold = _gold(
        deadlines=[{"verbatim_swedish": "2026-06-15"}],
        amounts=[{"verbatim_swedish": "1 234 kr"}],
    )
    v = e.evaluate(_good_result(), gold)
    assert v["pass"] is True
    assert v["invented_count"] == 0
    assert v["missing_count"] == 0
    assert v["all_sections_present"] is True


def test_evaluate_invented_value_fails():
    """A value in deadlines section absent from transcription → pass=False."""
    result = S(
        transcription="betala senast 2026-06-15",  # "2026-07-99" NOT in transcription
        tldr="You must pay.",
        why="Because.",
        actions="Pay it.",
        deadlines="- 2026-07-99 — invented\n- 2026-06-15 — real",
        severity=4,
    )
    gold = _gold(deadlines=[{"verbatim_swedish": "2026-06-15"}])
    v = e.evaluate(result, gold)
    assert v["pass"] is False
    assert v["invented_count"] >= 1


def test_evaluate_missing_gold_value_fails():
    """A gold verbatim_swedish value absent from deadlines section → pass=False."""
    result = S(
        transcription="betala 1 234 kr senast 2026-06-15",
        tldr="You must pay.",
        why="Because.",
        actions="Pay it.",
        # "1 234 kr" is NOT in this deadlines section
        deadlines="- 2026-06-15 — sista dag",
        severity=4,
    )
    gold = _gold(
        deadlines=[{"verbatim_swedish": "2026-06-15"}],
        amounts=[{"verbatim_swedish": "1 234 kr"}],
    )
    v = e.evaluate(result, gold)
    assert v["pass"] is False
    assert v["missing_count"] >= 1


def test_evaluate_empty_section_fails_sc1():
    """
    SC1 gate: an empty TL;DR section → pass=False even with zero invented/missing
    and a valid severity.
    """
    result = S(
        transcription="betala 1 234 kr senast 2026-06-15",
        tldr="",            # EMPTY — SC1 failure
        why="Because.",
        actions="Pay it.",
        deadlines="- 2026-06-15 — sista dag\n- 1 234 kr — belopp",
        severity=4,
    )
    gold = _gold(
        deadlines=[{"verbatim_swedish": "2026-06-15"}],
        amounts=[{"verbatim_swedish": "1 234 kr"}],
    )
    v = e.evaluate(result, gold)
    assert v["pass"] is False
    assert v["all_sections_present"] is False


def test_evaluate_empty_deadlines_section_fails_sc1():
    """An empty deadlines section → pass=False (SC1)."""
    result = S(
        transcription="betala 1 234 kr senast 2026-06-15",
        tldr="Summary.",
        why="Because.",
        actions="Pay.",
        deadlines="",       # EMPTY deadlines — SC1
        severity=4,
    )
    gold = _gold()          # No gold values — so 0 invented, 0 missing
    v = e.evaluate(result, gold)
    assert v["pass"] is False
    assert v["all_sections_present"] is False


def test_evaluate_severity_none_fails():
    """severity=None → pass=False (schema_complete=False)."""
    result = S(
        transcription="betala 1 234 kr",
        tldr="Summary.",
        why="Because.",
        actions="Pay.",
        deadlines="- 1 234 kr — belopp",
        severity=None,      # truncated output
    )
    gold = _gold(amounts=[{"verbatim_swedish": "1 234 kr"}])
    v = e.evaluate(result, gold)
    assert v["pass"] is False
    assert v["schema_complete"] is False


def test_evaluate_severity_mae_advisory_does_not_fail():
    """Severity MAE is reported but does NOT cause pass=False (D-15)."""
    result = S(
        transcription="betala 1 234 kr senast 2026-06-15",
        tldr="Summary.",
        why="Because.",
        actions="Pay.",
        deadlines="- 2026-06-15 — sista dag\n- 1 234 kr — belopp",
        severity=1,         # severity=1 vs expected=4 → MAE=3
    )
    gold = _gold(
        deadlines=[{"verbatim_swedish": "2026-06-15"}],
        amounts=[{"verbatim_swedish": "1 234 kr"}],
        expected_severity=4,
    )
    v = e.evaluate(result, gold)
    # Large MAE but everything else is correct → PASS
    assert v["pass"] is True
    assert v["severity_mae"] == 3


def test_evaluate_all_sections_present_in_verdict():
    """all_sections_present key is always present in the verdict dict."""
    v = e.evaluate(_good_result(), _gold())
    assert "all_sections_present" in v


def test_evaluate_references_checked_in_recall():
    """References in gold sidecar are checked in the recall (deadlines section)."""
    result = S(
        transcription="betala 1 234 kr OCR 123456789",
        tldr="Summary.",
        why="Because.",
        actions="Pay.",
        # "123456789" is absent from deadlines
        deadlines="- 1 234 kr — belopp",
        severity=3,
    )
    gold = _gold(
        amounts=[{"verbatim_swedish": "1 234 kr"}],
        references=[{"verbatim_swedish": "123456789"}],
    )
    v = e.evaluate(result, gold)
    assert v["pass"] is False
    assert v["missing_count"] >= 1


# ---------------------------------------------------------------------------
# Task 2 tests: beginner_invariant
# ---------------------------------------------------------------------------

def test_beginner_invariant_fully_populated_passes():
    """A fully-populated beginner-mode result passes all D-08 invariants."""
    result = S(
        transcription="betala 1 234 kr senast 2026-06-15",
        tldr="Summary.",
        why="Because.",
        actions="Pay.",
        deadlines="- 2026-06-15 — sista dag\n- 1 234 kr — belopp",
        severity=4,
    )
    ok, reasons = e.beginner_invariant(result)
    assert ok is True
    assert reasons == []


def test_beginner_invariant_empty_section_fails():
    """beginner_invariant returns False when a section is empty."""
    result = S(
        transcription="betala 1 234 kr senast 2026-06-15",
        tldr="",            # empty
        why="Because.",
        actions="Pay.",
        deadlines="- 2026-06-15 — sista dag",
        severity=4,
    )
    ok, reasons = e.beginner_invariant(result)
    assert ok is False
    assert len(reasons) >= 1


def test_beginner_invariant_severity_none_fails():
    """beginner_invariant returns False when severity is None (D-08: SEVERITY line dropped)."""
    result = S(
        transcription="betala 1 234 kr",
        tldr="Summary.",
        why="Because.",
        actions="Pay.",
        deadlines="- 1 234 kr — belopp",
        severity=None,
    )
    ok, reasons = e.beginner_invariant(result)
    assert ok is False
    assert any("severity" in r.lower() for r in reasons)


def test_beginner_invariant_empty_transcription_fails():
    """beginner_invariant returns False when transcription is empty (D-08: block dropped)."""
    result = S(
        transcription="",   # empty transcription block
        tldr="Summary.",
        why="Because.",
        actions="Pay.",
        deadlines="- 2026-06-15 — sista dag",
        severity=3,
    )
    ok, reasons = e.beginner_invariant(result)
    assert ok is False
    assert any("transcription" in r.lower() for r in reasons)


def test_beginner_invariant_multiple_failures_all_reported():
    """beginner_invariant reports ALL failures, not just the first."""
    result = S(
        transcription="",   # empty
        tldr="",            # empty
        why="Because.",
        actions="Pay.",
        deadlines="d",
        severity=None,
    )
    ok, reasons = e.beginner_invariant(result)
    assert ok is False
    assert len(reasons) >= 2  # at least transcription + severity


if __name__ == "__main__":
    import pytest
    pytest.main([__file__, "-v"])