File size: 2,867 Bytes
f11438f | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 | from __future__ import annotations
import io
import json
import deberta_ime.benchmark_v2_cli as benchmark_v2_cli
from deberta_ime import RerankRequest
from deberta_ime.mozc import build_mozc_index
from deberta_ime.ud_gsd import CorpusArtifact
DEV_CONLLU = """\
# sent_id = dev
1\t橋\t橋\tNOUN\t名詞\t_\t0\troot\t_\tUnidicInfo=ハシ,橋,橋,橋,ハシ
"""
def test_v2_dev_cli_writes_machine_readable_and_markdown_reports(tmp_path) -> None:
dictionary_dir = tmp_path / "dictionary_oss"
dictionary_dir.mkdir()
(dictionary_dir / "dictionary00.txt").write_text(
"はし\t1\t1\t3500\t箸\nはし\t1\t1\t3800\t橋\n",
encoding="utf-8",
)
index_path = tmp_path / "mozc.sqlite3"
build_mozc_index(dictionary_dir, index_path, source_revision="fixture")
artifact = CorpusArtifact(
split="dev",
path=tmp_path / "dev.conllu",
url="https://example.invalid/dev",
revision="dev-fixture",
sha256="c" * 64,
size_bytes=len(DEV_CONLLU.encode("utf-8")),
text=DEV_CONLLU,
)
class ContextScorer:
def score_candidates(self, request: RerankRequest) -> list[float]:
return [-2.0, 0.0]
stdout = io.StringIO()
exit_code = benchmark_v2_cli.run(
[
"dev",
"--index",
str(index_path),
"--context-mode",
"left_only",
"--output-dir",
str(tmp_path / "outputs"),
"--stem",
"dev_fixture",
"--bootstrap-samples",
"50",
],
stdout=stdout,
scorer=ContextScorer(),
artifact_loader=lambda stage, data_dir: artifact,
)
summary = json.loads(stdout.getvalue())
report = json.loads((tmp_path / "outputs" / "dev_fixture.json").read_text("utf-8"))
assert exit_code == 0
assert summary["ok"] is True
assert report["stage"] == "DEV_SELECTION"
assert (tmp_path / "outputs" / "dev_fixture.md").is_file()
def test_external_cli_refuses_profiles_not_marked_frozen(tmp_path, monkeypatch) -> None:
monkeypatch.setattr(benchmark_v2_cli.profile_config, "MOZC_PROFILE_STATE", "provisional")
stdout = io.StringIO()
stderr = io.StringIO()
exit_code = benchmark_v2_cli.run(
[
"external",
"--index",
str(tmp_path / "unused.sqlite3"),
"--profile",
"incremental",
"--stem",
"must_not_run",
],
stdout=stdout,
stderr=stderr,
scorer=object(),
artifact_loader=lambda stage, data_dir: (_ for _ in ()).throw(
AssertionError("PUD must not be loaded before freeze")
),
)
assert exit_code == 2
assert stdout.getvalue() == ""
assert stderr.getvalue() == "error: Mozc profiles are not frozen from GSD dev\n"
|