tests/test_evaluation_cli.py

from __future__ import annotations

import io
import json
import os
import subprocess
import sys

from deberta_ime.evaluation_cli import run


def test_evaluation_cli_import_does_not_require_model_runtime() -> None:
    script = """
import builtins
real_import = builtins.__import__
def guarded_import(name, *args, **kwargs):
    if name == "torch" or name.startswith("transformers"):
        raise AssertionError(f"unexpected model runtime import: {name}")
    return real_import(name, *args, **kwargs)
builtins.__import__ = guarded_import
from deberta_ime.evaluation_cli import run
assert callable(run)
"""
    environment = {**os.environ, "PYTHONPATH": "src"}

    completed = subprocess.run(
        [sys.executable, "-c", script],
        cwd=os.getcwd(),
        env=environment,
        capture_output=True,
        text=True,
        check=False,
    )

    assert completed.returncode == 0, completed.stderr


def test_evaluation_cli_scores_finite_predictions_and_writes_receipts(tmp_path) -> None:
    items_path = tmp_path / "items.json"
    predictions_path = tmp_path / "predictions.json"
    items_path.write_text(
        json.dumps(
            [
                {
                    "id": "clean",
                    "input": "猫です",
                    "references": ["猫です"],
                    "label": "clean",
                },
                {
                    "id": "typo",
                    "input": "犬でし",
                    "references": ["犬です"],
                    "label": "typo",
                },
            ],
            ensure_ascii=False,
        ),
        encoding="utf-8",
    )
    predictions_path.write_text(
        json.dumps(
            [
                {
                    "id": "clean",
                    "candidates": [],
                    "provenance": "provider",
                    "reason": "baseline_best",
                    "margin": 0.0,
                },
                {
                    "id": "typo",
                    "candidates": ["犬です"],
                    "provenance": "deberta",
                    "reason": "accepted",
                    "margin": 1.25,
                },
            ],
            ensure_ascii=False,
        ),
        encoding="utf-8",
    )
    stdout = io.StringIO()

    exit_code = run(
        [
            "--items",
            str(items_path),
            "--predictions",
            str(predictions_path),
            "--dataset-name",
            "fixture-clean-typo",
            "--dataset-revision",
            "fixture-v1",
            "--dataset-license",
            "test-only",
            "--output-dir",
            str(tmp_path / "outputs"),
            "--stem",
            "fixture",
        ],
        stdout=stdout,
    )

    summary = json.loads(stdout.getvalue())
    report = json.loads((tmp_path / "outputs" / "fixture.json").read_text("utf-8"))
    assert exit_code == 0
    assert summary["ok"] is True
    assert report["schema_version"] == 2
    assert report["status"] == "LOCAL_FINITE_CANDIDATE_EVALUATION"
    assert report["evaluation"]["metrics"]["effective_acc_at_1"] == 1.0
    assert report["evaluation"]["metrics"]["overcorrection_rate"] == 0.0
    assert report["evaluation"]["metrics"]["declared_provenance_counts"] == {
        "deberta": 1,
        "provider": 1,
    }
    assert report["evaluation"]["metrics"]["mean_reported_margin"] == 0.625
    assert report["dataset"] == {
        "name": "fixture-clean-typo",
        "revision": "fixture-v1",
        "license": "test-only",
    }
    assert len(report["artifacts"]["items"]["sha256"]) == 64
    markdown = (tmp_path / "outputs" / "fixture.md").read_text("utf-8")
    assert "fixture-clean-typo" in markdown
    assert "Accepted candidate misses: 0" in markdown
    assert "Selection errors: 0" in markdown


def test_evaluation_cli_adapts_ajimee_without_inventing_clean_labels(tmp_path) -> None:
    items_path = tmp_path / "ajimee.json"
    predictions_path = tmp_path / "predictions.json"
    items_path.write_text(
        json.dumps(
            [
                {
                    "index": "7",
                    "input": "セイネンシ",
                    "expected_output": ["青年誌"],
                }
            ],
            ensure_ascii=False,
        ),
        encoding="utf-8",
    )
    predictions_path.write_text(
        json.dumps([{"index": "7", "candidates": ["青年誌"]}], ensure_ascii=False),
        encoding="utf-8",
    )

    run(
        [
            "--items",
            str(items_path),
            "--predictions",
            str(predictions_path),
            "--format",
            "ajimee",
            "--output-dir",
            str(tmp_path / "outputs"),
        ],
        stdout=io.StringIO(),
    )

    report = json.loads(
        (tmp_path / "outputs" / "finite_candidate_evaluation.json").read_text("utf-8")
    )
    assert report["evaluation"]["metrics"]["effective_acc_at_1"] == 1.0
    assert report["evaluation"]["metrics"]["overcorrection_rate"] is None
    assert report["evaluation"]["metrics"]["clean_rows"] == 0
    assert report["dataset"] == {
        "name": "AJIMEE-compatible input",
        "revision": "unverified-by-sha256",
        "license": "unspecified",
    }