Self-contained PyTorch example, escaped regex ranges

README.md

@@ -60,8 +60,8 @@ sess = ort.InferenceSession(hf_hub_download(REPO, "ruq-30m.onnx"))
 
 # The tokenizer was trained on normalised text: diacritics and tatweel
 # removed. Skipping this step gives a different, worse tokenisation.
-_DIACRITICS = re.compile(r"[ً-ْٰٓ-ٕ]")
-_ZERO_WIDTH = re.compile(r"[​-‏‪-‮]")
+_DIACRITICS = re.compile(r"[\u064b-\u0652\u0670\u0653-\u0655]")
+_ZERO_WIDTH = re.compile(r"[\u200b-\u200f\u202a-\u202e\ufeff]")
 
 def normalize(text):
     text = unicodedata.normalize("NFC", text)
@@ -106,9 +106,22 @@ The same weights are published as safetensors alongside the architecture module,
 continued training or fine-tuning:
 
 ```python
-import json, torch
+import json, re, unicodedata, torch
 from huggingface_hub import hf_hub_download
 from safetensors.torch import load_file
+from tokenizers import Tokenizer
+
+REPO, EOS = "Ruqiya/ruqlm", 2
+
+tok = Tokenizer.from_file(hf_hub_download(REPO, "tokenizer.json"))
+
+_DIACRITICS = re.compile(r"[\u064b-\u0652\u0670\u0653-\u0655]")
+_ZERO_WIDTH = re.compile(r"[\u200b-\u200f\u202a-\u202e\ufeff]")
+
+def normalize(text):
+    text = unicodedata.normalize("NFC", text)
+    text = _ZERO_WIDTH.sub("", text).replace("\u0640", "")     # tatweel
+    return re.sub(r"\s+", " ", _DIACRITICS.sub("", text)).strip()
 
 # the architecture module has to be on the path before it can be imported
 hf_hub_download(REPO, "modeling_ruqlm.py", local_dir=".")
@@ -122,8 +135,9 @@ state["lm_head.weight"] = state["tok_emb.weight"]   # embeddings are tied
 model.load_state_dict(state)
 model.eval()
 
-out = model.generate(torch.tensor([tok.encode(normalize("كان يا ما كان")).ids[:-1]]),
-                     max_new_tokens=120, temperature=0.85, top_k=50, eos_id=EOS)
+ids = tok.encode(normalize("كان يا ما كان")).ids[:-1]   # drop the trailing </s>
+out = model.generate(torch.tensor([ids]), max_new_tokens=120,
+                     temperature=0.85, top_k=50, eos_id=EOS)
 print(tok.decode(out[0].tolist()))
 ```