syhrlhyn commited on
Commit
9bd3418
·
verified ·
1 Parent(s): e9a0d1c

Delete tes.py

Browse files
Files changed (1) hide show
  1. tes.py +0 -46
tes.py DELETED
@@ -1,46 +0,0 @@
1
- import sentencepiece as spm
2
- import os
3
-
4
- def interactive_test(model_path="aibys.model"):
5
- if not os.path.exists(model_path):
6
- print(f"❌ File model tidak ditemukan di: {model_path}")
7
- return
8
-
9
- # Load model
10
- sp = spm.SentencePieceProcessor()
11
- sp.load(model_path)
12
-
13
- print("="*60)
14
- print(" 🤖 AIBYS TOKENIZER INTERACTIVE TESTER")
15
- print(" Type 'exit' atau 'keluar' untuk stop")
16
- print("="*60)
17
- print(f"Vocab Size: {sp.vocab_size()}")
18
-
19
- while True:
20
- print("\n" + "-"*50)
21
- text = input("📝 Masukkan kalimat: ")
22
-
23
- if text.lower() in ['exit', 'keluar', 'q']:
24
- print("👋 Sampai jumpa, Syahril!")
25
- break
26
-
27
- if not text.strip():
28
- continue
29
-
30
- # Proses Tokenisasi
31
- tokens = sp.encode_as_pieces(text)
32
- ids = sp.encode_as_ids(text)
33
-
34
- print(f"\n📊 Hasil Analisis:")
35
- print(f" Input : {text}")
36
- print(f" Tokens : {tokens}")
37
- print(f" Token IDs : {ids}")
38
- print(f" Count : {len(ids)} Token")
39
-
40
- # Cek Efisiensi (Estimasi: makin sedikit token vs jumlah kata, makin bagus)
41
- kata_count = len(text.split())
42
- efisiensi = "SANGAT BAGUS" if len(ids) <= kata_count + 2 else "STANDAR"
43
- print(f" Efisiensi : {efisiensi}")
44
-
45
- if __name__ == "__main__":
46
- interactive_test()