tests/test_grader.py

"""Tests for the InvoiceGuard grading system."""

import sys
import os

sys.path.insert(0, os.path.join(os.path.dirname(__file__), ".."))

from models import (
    CaseData,
    CaseHistory,
    CompanyPolicy,
    DecisionType,
    Difficulty,
    ExceptionType,
    GRNLineItem,
    GoodsReceiptNote,
    GroundTruth,
    Invoice,
    InvoiceGuardState,
    InvoiceLineItem,
    POLineItem,
    PurchaseOrder,
    TaskID,
    VendorProfile,
)
from graders.scoring import grade_episode, _score_explanation


def _make_case(
    correct_decision=DecisionType.approve_for_payment,
    correct_exception=ExceptionType.clean_match,
    acceptable_evidence=None,
    key_findings=None,
    max_steps=10,
):
    return CaseData(
        case_id="TEST-001",
        task_id=TaskID.task_1_clean_match,
        difficulty=Difficulty.easy,
        max_steps=max_steps,
        invoice=Invoice(
            invoice_number="INV-001",
            supplier_name="Test",
            supplier_id="SUP-001",
            invoice_date="2024-01-01",
            po_reference="PO-001",
            line_items=[
                InvoiceLineItem(
                    item_code="A",
                    description="Widget",
                    quantity_billed=10,
                    unit_price_billed=5.0,
                    line_total_billed=50.0,
                )
            ],
            subtotal=50.0,
            tax=5.0,
            total_amount=55.0,
        ),
        purchase_order=PurchaseOrder(
            po_number="PO-001",
            supplier_id="SUP-001",
            order_date="2024-01-01",
            line_items=[
                POLineItem(
                    item_code="A",
                    description="Widget",
                    ordered_quantity=10,
                    unit_price_ordered=5.0,
                    line_total_ordered=50.0,
                )
            ],
        ),
        goods_receipt_note=GoodsReceiptNote(
            grn_number="GRN-001",
            po_reference="PO-001",
            receipt_date="2024-01-05",
            line_items=[
                GRNLineItem(
                    item_code="A",
                    quantity_received=10,
                    accepted_quantity=10,
                )
            ],
        ),
        vendor_profile=VendorProfile(
            supplier_id="SUP-001",
            supplier_name="Test",
            risk_tier="low",
        ),
        company_policy=CompanyPolicy(),
        case_history=CaseHistory(),
        ground_truth=GroundTruth(
            correct_decision=correct_decision,
            correct_exception_type=correct_exception,
            acceptable_evidence=acceptable_evidence or [
                "inspect_purchase_order",
                "compare_quantity",
                "compare_price",
            ],
            key_findings=key_findings or [
                "quantities match within tolerance",
                "prices match within 5% tolerance",
            ],
        ),
    )


def _make_state(
    decision=None,
    exception_type=None,
    actions=None,
    evidence=None,
    explanation="",
    step_count=5,
):
    s = InvoiceGuardState()
    s.step_count = step_count
    s.final_decision = decision
    s.final_exception_type = exception_type
    s.actions_taken = actions or []
    s.final_evidence = evidence or []
    s.final_explanation = explanation
    s.documents_revealed = [
        a.replace("inspect_", "").replace("_", " ")
        for a in (actions or [])
        if a.startswith("inspect_")
    ]
    s.repeated_action_counts = {}
    for a in (actions or []):
        s.repeated_action_counts[a] = s.repeated_action_counts.get(a, 0) + 1
    return s


class TestGradeEpisode:
    def test_perfect_score(self):
        case = _make_case()
        state = _make_state(
            decision="approve_for_payment",
            exception_type="clean_match",
            actions=["inspect_purchase_order", "compare_quantity", "compare_price"],
            evidence=["inspect_purchase_order", "compare_quantity", "compare_price"],
            explanation="All quantities and prices match within 5% tolerance per company policy.",
            step_count=4,
        )
        result = grade_episode(case, state)
        assert result.score >= 0.85
        assert result.decision_score == 1.0
        assert result.exception_type_score == 1.0

    def test_wrong_decision_zero_decision_score(self):
        case = _make_case()
        state = _make_state(
            decision="reject_invoice",
            exception_type="clean_match",
            actions=["inspect_purchase_order"],
            evidence=["inspect_purchase_order"],
            explanation="Rejecting.",
        )
        result = grade_episode(case, state)
        assert result.decision_score == 0.0

    def test_partial_decision_credit(self):
        case = _make_case(correct_decision=DecisionType.place_on_hold)
        state = _make_state(
            decision="escalate_for_supervisor_review",
            exception_type="partial_receipt",
            actions=["inspect_purchase_order"],
            evidence=["inspect_purchase_order"],
        )
        result = grade_episode(case, state)
        assert 0.0 < result.decision_score < 1.0

    def test_no_decision_scores_zero(self):
        case = _make_case()
        state = _make_state(decision=None, step_count=10)
        result = grade_episode(case, state)
        assert result.decision_score == 0.0
        assert result.score < 0.5

    def test_score_in_bounds(self):
        case = _make_case()
        state = _make_state(
            decision="approve_for_payment",
            exception_type="clean_match",
            actions=["inspect_purchase_order"],
            evidence=["inspect_purchase_order"],
        )
        result = grade_episode(case, state)
        assert 0.0 <= result.score <= 1.0

    def test_all_rubric_dimensions_present(self):
        case = _make_case()
        state = _make_state(
            decision="approve_for_payment",
            exception_type="clean_match",
            actions=["inspect_purchase_order"],
            evidence=["inspect_purchase_order"],
            explanation="Matches.",
        )
        result = grade_episode(case, state)
        assert result.decision_score >= 0
        assert result.exception_type_score >= 0
        assert result.evidence_score >= 0
        assert result.investigation_score >= 0
        assert result.explanation_score >= 0
        assert result.efficiency_score >= 0


class TestExplanationScoring:
    def test_empty_explanation(self):
        case = _make_case()
        score = _score_explanation("", case, ["quantities match"])
        assert score == 0.0

    def test_good_explanation(self):
        case = _make_case(
            key_findings=["quantities match within tolerance", "price variance 3%"],
        )
        score = _score_explanation(
            "All quantities match within the 5% tolerance threshold per company policy. "
            "Price variance of 3% is acceptable.",
            case,
            case.ground_truth.key_findings,
        )
        assert score >= 0.6

    def test_explanation_with_numbers(self):
        case = _make_case()
        score = _score_explanation(
            "The invoice total of $55.00 matches the PO within tolerance.",
            case,
            [],
        )
        assert score > 0.0

    def test_vague_explanation_scores_low(self):
        case = _make_case()
        score = _score_explanation("ok", case, ["quantities match"])
        assert score <= 0.4