tests/test_benchmark_v2_cli.py

from __future__ import annotations

import io
import json

import deberta_ime.benchmark_v2_cli as benchmark_v2_cli
from deberta_ime import RerankRequest
from deberta_ime.mozc import build_mozc_index
from deberta_ime.ud_gsd import CorpusArtifact

DEV_CONLLU = """\
# sent_id = dev
1\t橋\t橋\tNOUN\t名詞\t_\t0\troot\t_\tUnidicInfo=ハシ,橋,橋,橋,ハシ

"""


def test_v2_dev_cli_writes_machine_readable_and_markdown_reports(tmp_path) -> None:
    dictionary_dir = tmp_path / "dictionary_oss"
    dictionary_dir.mkdir()
    (dictionary_dir / "dictionary00.txt").write_text(
        "はし\t1\t1\t3500\t箸\nはし\t1\t1\t3800\t橋\n",
        encoding="utf-8",
    )
    index_path = tmp_path / "mozc.sqlite3"
    build_mozc_index(dictionary_dir, index_path, source_revision="fixture")
    artifact = CorpusArtifact(
        split="dev",
        path=tmp_path / "dev.conllu",
        url="https://example.invalid/dev",
        revision="dev-fixture",
        sha256="c" * 64,
        size_bytes=len(DEV_CONLLU.encode("utf-8")),
        text=DEV_CONLLU,
    )

    class ContextScorer:
        def score_candidates(self, request: RerankRequest) -> list[float]:
            return [-2.0, 0.0]

    stdout = io.StringIO()
    exit_code = benchmark_v2_cli.run(
        [
            "dev",
            "--index",
            str(index_path),
            "--context-mode",
            "left_only",
            "--output-dir",
            str(tmp_path / "outputs"),
            "--stem",
            "dev_fixture",
            "--bootstrap-samples",
            "50",
        ],
        stdout=stdout,
        scorer=ContextScorer(),
        artifact_loader=lambda stage, data_dir: artifact,
    )

    summary = json.loads(stdout.getvalue())
    report = json.loads((tmp_path / "outputs" / "dev_fixture.json").read_text("utf-8"))
    assert exit_code == 0
    assert summary["ok"] is True
    assert report["stage"] == "DEV_SELECTION"
    assert (tmp_path / "outputs" / "dev_fixture.md").is_file()


def test_external_cli_refuses_profiles_not_marked_frozen(tmp_path, monkeypatch) -> None:
    monkeypatch.setattr(benchmark_v2_cli.profile_config, "MOZC_PROFILE_STATE", "provisional")
    stdout = io.StringIO()
    stderr = io.StringIO()

    exit_code = benchmark_v2_cli.run(
        [
            "external",
            "--index",
            str(tmp_path / "unused.sqlite3"),
            "--profile",
            "incremental",
            "--stem",
            "must_not_run",
        ],
        stdout=stdout,
        stderr=stderr,
        scorer=object(),
        artifact_loader=lambda stage, data_dir: (_ for _ in ()).throw(
            AssertionError("PUD must not be loaded before freeze")
        ),
    )

    assert exit_code == 2
    assert stdout.getvalue() == ""
    assert stderr.getvalue() == "error: Mozc profiles are not frozen from GSD dev\n"