Upload 4 files

got_qa_transformer.h5

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fad12318a9e0ef690db12505c2ab0332b0b34d4938501dd13719b7cb300fa3b9
+size 517216632

part5.h5

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:87388001d309ce5442316aa5058fcce1a1d84a6f64dd5e05dd2e2b281fa0e4a9
+size 517216632

usage.py

@@ -0,0 +1,27 @@
+
+import os
+import json
+import sentencepiece as spm
+from tqdm import tqdm
+MODEL_PREFIX     = "icefire_spm"
+
+sp = spm.SentencePieceProcessor(model_file=f"{MODEL_PREFIX}.model")
+sample = "Daenerys Targaryen rides Drogon to Winterfell."
+ids   = sp.encode(sample)
+pieces = sp.encode(sample, out_type=str)
+print("\nTest encode:")
+print(f"   Text : {sample}")
+print(f"   IDs  : {ids}")
+print(f"   Pieces: {pieces}")
+print(f"   Decode: {sp.decode(ids)}")
+
+
+
+# check an entity
+entity =  "Winterfell"
+e_pieces = sp.encode(entity, out_type=str)
+print(f"\nEntity '{entity}' → {len(e_pieces)} piece(s): {e_pieces}")
+
+entity =  "Long Claw"
+e_pieces = sp.encode(entity, out_type=str)
+print(f"\nEntity '{entity}' → {len(e_pieces)} piece(s): {e_pieces}")