tests/test_vocal.py

# tests/test_vocal.py
from fractus_vorax.cards import Card
from fractus_vorax.model.vocal import CharTokenizer, core_available, serialize_cards, verbalize

CARDS = [
    Card("FAIT", "what is the capital of japan => tokyo", 0.9, "cap.csv:4"),
    Card("ANALOGIE", "paris", 1.0, "cap"),
]


def test_serialize_cards_deterministic_and_sorted():
    s1 = serialize_cards("q?", CARDS)
    s2 = serialize_cards("q?", CARDS)
    assert s1 == s2
    assert s1.startswith("[FAIT]") or "[ANALOGIE]" in s1
    assert s1.endswith("Q: q?")
    lines = [l for l in s1.splitlines() if l]
    assert lines == sorted(lines, key=lambda l: (not l.startswith("["), l))  # cartes avant Q


def test_char_tokenizer_roundtrip():
    tok = CharTokenizer(vocab_size=1114112)
    assert tok.decode(tok.encode("héllo wörld")) == "héllo wörld"


def test_verbalize_returns_text_when_core_available():
    if not core_available():
        import pytest
        pytest.skip("torch indisponible dans ce python")
    from fractus_vorax.model.fractus_core import FractusCore, FractusCoreConfig
    import torch
    torch.manual_seed(0)
    core = FractusCore(FractusCoreConfig(vocab_size=1114112, d_model=64, n_layers=1, n_experts=2, rank=8, max_seq_len=512))
    out = verbalize(core, "what is the capital of japan?", CARDS[:1], CharTokenizer(vocab_size=1114112), max_new_tokens=8)
    assert isinstance(out, str) and len(out) >= 0  # le noyau non-né peut sortir du bruit — le contrat est le pipeline


def test_safe_decode_text_replaces_surrogates():
    from fractus_vorax.model.vocal import safe_decode_text
    bad = "ok\ud800\udfffend"
    out = safe_decode_text(bad)
    assert "\ud800" not in out and out.startswith("ok")


def test_strip_eos_drops_single_trailing_eos():
    # Directive de la revue T2 : generate inclut l'EOS dans la séquence
    # retournée (append-then-break) — _strip_eos retire UN unique EOS
    # final avant décodage, sans jamais toucher au reste.
    from fractus_vorax.model.vocal import _strip_eos
    assert _strip_eos([1, 2, 50256], 50256) == [1, 2]
    assert _strip_eos([1, 2], 50256) == [1, 2]
    assert _strip_eos([1, 2], None) == [1, 2]