Delete tes.py
Browse files
tes.py
DELETED
|
@@ -1,46 +0,0 @@
|
|
| 1 |
-
import sentencepiece as spm
|
| 2 |
-
import os
|
| 3 |
-
|
| 4 |
-
def interactive_test(model_path="aibys.model"):
|
| 5 |
-
if not os.path.exists(model_path):
|
| 6 |
-
print(f"❌ File model tidak ditemukan di: {model_path}")
|
| 7 |
-
return
|
| 8 |
-
|
| 9 |
-
# Load model
|
| 10 |
-
sp = spm.SentencePieceProcessor()
|
| 11 |
-
sp.load(model_path)
|
| 12 |
-
|
| 13 |
-
print("="*60)
|
| 14 |
-
print(" 🤖 AIBYS TOKENIZER INTERACTIVE TESTER")
|
| 15 |
-
print(" Type 'exit' atau 'keluar' untuk stop")
|
| 16 |
-
print("="*60)
|
| 17 |
-
print(f"Vocab Size: {sp.vocab_size()}")
|
| 18 |
-
|
| 19 |
-
while True:
|
| 20 |
-
print("\n" + "-"*50)
|
| 21 |
-
text = input("📝 Masukkan kalimat: ")
|
| 22 |
-
|
| 23 |
-
if text.lower() in ['exit', 'keluar', 'q']:
|
| 24 |
-
print("👋 Sampai jumpa, Syahril!")
|
| 25 |
-
break
|
| 26 |
-
|
| 27 |
-
if not text.strip():
|
| 28 |
-
continue
|
| 29 |
-
|
| 30 |
-
# Proses Tokenisasi
|
| 31 |
-
tokens = sp.encode_as_pieces(text)
|
| 32 |
-
ids = sp.encode_as_ids(text)
|
| 33 |
-
|
| 34 |
-
print(f"\n📊 Hasil Analisis:")
|
| 35 |
-
print(f" Input : {text}")
|
| 36 |
-
print(f" Tokens : {tokens}")
|
| 37 |
-
print(f" Token IDs : {ids}")
|
| 38 |
-
print(f" Count : {len(ids)} Token")
|
| 39 |
-
|
| 40 |
-
# Cek Efisiensi (Estimasi: makin sedikit token vs jumlah kata, makin bagus)
|
| 41 |
-
kata_count = len(text.split())
|
| 42 |
-
efisiensi = "SANGAT BAGUS" if len(ids) <= kata_count + 2 else "STANDAR"
|
| 43 |
-
print(f" Efisiensi : {efisiensi}")
|
| 44 |
-
|
| 45 |
-
if __name__ == "__main__":
|
| 46 |
-
interactive_test()
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|