tests/test_bpe_tokenizer.py

# tests/test_bpe_tokenizer.py
import pytest

bpt = pytest.importorskip("fractus_vorax.model.bpe_tokenizer")


def test_bpe_available_probe():
    from fractus_vorax.model.bpe_tokenizer import bpe_available
    assert isinstance(bpe_available(), bool)


def test_gpt2_known_encodings():
    if not bpt.bpe_available():
        pytest.skip("tokenizers indisponible")
    tok = bpt.Gpt2BpeTokenizer()
    assert tok.vocab_size == 50257
    assert tok.eos_token_id == 50256
    # encodages GPT-2 canoniques
    assert tok.encode("hello world")[0] == 31373
    assert tok.decode([31373, 995]) == "hello world"


def test_roundtrip():
    if not bpt.bpe_available():
        pytest.skip("tokenizers indisponible")
    tok = bpt.Gpt2BpeTokenizer()
    text = "What is the capital of france? ÀÉïö — naïve"
    assert tok.decode(tok.encode(text)) == text


def test_decode_out_of_range_ids_do_not_crash():
    if not bpt.bpe_available():
        pytest.skip("tokenizers indisponible")
    tok = bpt.Gpt2BpeTokenizer()
    out = tok.decode([31373, 99999999, -5, 50256])
    assert isinstance(out, str)  # filtre, pas d'exception