Upload folder using huggingface_hub

aibys.model

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:af9a5e0fc0216a887c5aa8d8299fa59cadb7e72c51f3efc93c97a67587fd8058
+size 770035

push_tokenizer.py

@@ -0,0 +1,21 @@
+from huggingface_hub import HfApi
+
+api = HfApi()
+
+# Ganti sesuai username kamu
+REPO_ID = "syhrlhyn/aibys-tokenizer" 
+# Folder tempat file aibys.model dan aibys.vocab berada
+FOLDER_TOKENIZER = "../tokenizer" 
+
+print(f"🚀 Memulai upload Tokenizer Aibys ke {REPO_ID}...")
+
+try:
+    api.upload_folder(
+        folder_path=FOLDER_TOKENIZER,
+        repo_id=REPO_ID,
+        repo_type="model", # Tipe Model!
+    )
+    print("✅ MANTAP! aibys.model dan aibys.vocab sudah online.")
+    print(f"Cek di: https://huggingface.co/{REPO_ID}")
+except Exception as e:
+    print(f"❌ Waduh error: {e}")

tes.py

@@ -0,0 +1,46 @@
+import sentencepiece as spm
+import os
+
+def interactive_test(model_path="aibys.model"):
+    if not os.path.exists(model_path):
+        print(f"❌ File model tidak ditemukan di: {model_path}")
+        return
+
+    # Load model
+    sp = spm.SentencePieceProcessor()
+    sp.load(model_path)
+
+    print("="*60)
+    print("        🤖 AIBYS TOKENIZER INTERACTIVE TESTER")
+    print("          Type 'exit' atau 'keluar' untuk stop")
+    print("="*60)
+    print(f"Vocab Size: {sp.vocab_size()}")
+    
+    while True:
+        print("\n" + "-"*50)
+        text = input("📝 Masukkan kalimat: ")
+        
+        if text.lower() in ['exit', 'keluar', 'q']:
+            print("👋 Sampai jumpa, Syahril!")
+            break
+
+        if not text.strip():
+            continue
+
+        # Proses Tokenisasi
+        tokens = sp.encode_as_pieces(text)
+        ids = sp.encode_as_ids(text)
+
+        print(f"\n📊 Hasil Analisis:")
+        print(f"   Input      : {text}")
+        print(f"   Tokens     : {tokens}")
+        print(f"   Token IDs  : {ids}")
+        print(f"   Count      : {len(ids)} Token")
+        
+        # Cek Efisiensi (Estimasi: makin sedikit token vs jumlah kata, makin bagus)
+        kata_count = len(text.split())
+        efisiensi = "SANGAT BAGUS" if len(ids) <= kata_count + 2 else "STANDAR"
+        print(f"   Efisiensi  : {efisiensi}")
+
+if __name__ == "__main__":
+    interactive_test()