test_nano_proofread.py

"""Tests that gate the science: identity examples exist, most inputs carry an error,
corrected phrases are clean (no doubled words), determinism, the context-free script
provably fails on a real fraction (the must-beat-a-script filter), and a regression
test that the PUBLISHED weights still fix the demo phrases.

    pytest test_nano_proofread.py -q
"""

from __future__ import annotations

import os

from data_proofread import naive_fix, proofread_pairs


def test_identity_examples_exist():
    pairs = proofread_pairs(7, 4000)
    ident = sum(1 for i, t in pairs if i[:-4] == t)
    assert 0.08 <= ident / len(pairs) <= 0.25


def test_most_inputs_have_an_error():
    pairs = proofread_pairs(3, 4000)
    changed = sum(1 for i, t in pairs if i[:-4] != t)
    assert changed / len(pairs) > 0.7


def test_targets_are_clean():
    for _, tgt in proofread_pairs(5, 2000):
        toks = tgt.split()
        assert all(toks[i] != toks[i + 1] for i in range(len(toks) - 1)), tgt


def test_deterministic():
    assert proofread_pairs(0, 300) == proofread_pairs(0, 300)


def test_context_free_script_cannot_be_exact():
    """The best context-free script must get a real fraction of examples wrong,
    because the homophone confusions are context-dependent."""
    pairs = proofread_pairs(11, 4000)
    wrong = sum(1 for inp, tgt in pairs if naive_fix(inp[:-4]) != tgt)
    assert wrong > 0.15 * len(pairs)


def test_published_weights_reproduce():
    if not (os.path.exists("model.safetensors") and os.path.exists("config.json")):
        import pytest
        pytest.skip("weights not present in this checkout")
    from modeling_nano_proofread import load, proofread
    m = load()
    for phrase, gold in [
        ("their going to win", "they're going to win"),
        ("its raining again", "it's raining again"),
        ("the the cat sat", "the cat sat"),
        ("we went they're", "we went there"),
        ("this is bigger then that", "this is bigger than that"),
        ("they're house is big", "their house is big"),
        ("she is happy today", "she is happy today"),  # already correct -> unchanged
    ]:
        assert proofread(m, phrase) == gold, (phrase, proofread(m, phrase))