tests/test_core.py

import sys
from pathlib import Path

ROOT = Path(__file__).resolve().parent.parent
sys.path.insert(0, str(ROOT))

from indigo.bpe import BPETokenizer
from indigo.common import (
    _kandidat_akar,
    load_wordlist,
    save_meta,
    load_meta,
    word_known_ratio,
)
from indigo.tokenizer import CharTokenizer


def test_bpe_roundtrip():
    teks = "halo dunia, halo lagi! belajar bahasa indonesia bersama."
    tok = BPETokenizer.train(teks * 5, 280)
    ids = tok.encode(teks)
    assert tok.decode(ids) == teks
    assert 256 <= tok.vocab_size <= 280


def test_char_roundtrip():
    tok = CharTokenizer.from_text("abcba")
    assert tok.decode(tok.encode("abc")) == "abc"


def test_meta_save_load(tmp_path):
    base = str(tmp_path / "ck.safetensors")
    (tmp_path / "ck.safetensors").write_bytes(b"")
    cfg = {"vocab_size": 10, "block_size": 8}
    save_meta(base, cfg, None, 5, 1.23, backend="pytorch", tokenizer={"type": "char"})
    meta = load_meta(base)
    assert meta["config"] == cfg and meta["step"] == 5 and meta["val_loss"] == 1.23


def test_afiks_asimilasi():
    pref = {"meng", "meny", "men", "mem", "pem", "di", "ter"}
    suf = {"kan", "annya", "nya", "an", "i", "lah"}
    assert "sapu" in _kandidat_akar("menyapu", pref, suf)
    assert "pukul" in _kandidat_akar("pemukul", pref, suf)
    assert "ambil" in _kandidat_akar("mengambil", pref, suf)
    assert "terima" in _kandidat_akar("diterima", pref, suf)


def test_word_known_ratio_naik_dengan_afiks(tmp_path):
    kamus = tmp_path / "kamus.txt"
    kamus.write_text("paham\nterima\nringan\n", encoding="utf-8")
    ws = load_wordlist(str(kamus))
    pref = {"di"}
    suf = {"i", "nya"}
    teks = "paham dipahami ringannya"
    r0 = word_known_ratio(teks, ws)
    r1 = word_known_ratio(teks, ws, pref, suf)
    assert r0 < r1 <= 1.0


def test_load_wordlist_normalisasi(tmp_path):
    f = tmp_path / "w.txt"
    f.write_text("Apa\nBEBEK\n\nbebek\n", encoding="utf-8")
    ws = load_wordlist(str(f))
    assert ws == {"apa", "bebek"}