Upload 3 files

icefire_spm.model

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:618c81434ac7381e299479573396f1020c352d6723b961e07596c0a442110f75
+size 743008

usage.py

@@ -0,0 +1,27 @@
+
+import os
+import json
+import sentencepiece as spm
+from tqdm import tqdm
+MODEL_PREFIX     = "icefire_spm"
+
+sp = spm.SentencePieceProcessor(model_file=f"{MODEL_PREFIX}.model")
+sample = "Daenerys Targaryen rides Drogon to Winterfell."
+ids   = sp.encode(sample)
+pieces = sp.encode(sample, out_type=str)
+print("\nTest encode:")
+print(f"   Text : {sample}")
+print(f"   IDs  : {ids}")
+print(f"   Pieces: {pieces}")
+print(f"   Decode: {sp.decode(ids)}")
+
+
+
+# check an entity
+entity =  "Winterfell"
+e_pieces = sp.encode(entity, out_type=str)
+print(f"\nEntity '{entity}' → {len(e_pieces)} piece(s): {e_pieces}")
+
+entity =  "Long Claw"
+e_pieces = sp.encode(entity, out_type=str)
+print(f"\nEntity '{entity}' → {len(e_pieces)} piece(s): {e_pieces}")