File size: 2,020 Bytes
9a82835 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 | import sys
from pathlib import Path
ROOT = Path(__file__).resolve().parent.parent
sys.path.insert(0, str(ROOT))
from indigo.bpe import BPETokenizer
from indigo.common import (
_kandidat_akar,
load_wordlist,
save_meta,
load_meta,
word_known_ratio,
)
from indigo.tokenizer import CharTokenizer
def test_bpe_roundtrip():
teks = "halo dunia, halo lagi! belajar bahasa indonesia bersama."
tok = BPETokenizer.train(teks * 5, 280)
ids = tok.encode(teks)
assert tok.decode(ids) == teks
assert 256 <= tok.vocab_size <= 280
def test_char_roundtrip():
tok = CharTokenizer.from_text("abcba")
assert tok.decode(tok.encode("abc")) == "abc"
def test_meta_save_load(tmp_path):
base = str(tmp_path / "ck.safetensors")
(tmp_path / "ck.safetensors").write_bytes(b"")
cfg = {"vocab_size": 10, "block_size": 8}
save_meta(base, cfg, None, 5, 1.23, backend="pytorch", tokenizer={"type": "char"})
meta = load_meta(base)
assert meta["config"] == cfg and meta["step"] == 5 and meta["val_loss"] == 1.23
def test_afiks_asimilasi():
pref = {"meng", "meny", "men", "mem", "pem", "di", "ter"}
suf = {"kan", "annya", "nya", "an", "i", "lah"}
assert "sapu" in _kandidat_akar("menyapu", pref, suf)
assert "pukul" in _kandidat_akar("pemukul", pref, suf)
assert "ambil" in _kandidat_akar("mengambil", pref, suf)
assert "terima" in _kandidat_akar("diterima", pref, suf)
def test_word_known_ratio_naik_dengan_afiks(tmp_path):
kamus = tmp_path / "kamus.txt"
kamus.write_text("paham\nterima\nringan\n", encoding="utf-8")
ws = load_wordlist(str(kamus))
pref = {"di"}
suf = {"i", "nya"}
teks = "paham dipahami ringannya"
r0 = word_known_ratio(teks, ws)
r1 = word_known_ratio(teks, ws, pref, suf)
assert r0 < r1 <= 1.0
def test_load_wordlist_normalisasi(tmp_path):
f = tmp_path / "w.txt"
f.write_text("Apa\nBEBEK\n\nbebek\n", encoding="utf-8")
ws = load_wordlist(str(f))
assert ws == {"apa", "bebek"}
|