Upload NyayaLM tokenizer folder

README.md

@@ -0,0 +1,35 @@
+---
+language:
+- ne
+- en
+license: apache-2.0
+tags:
+- tokenizer
+- sentencepiece
+- bpe
+- nepali
+- legal
+- nyayalm
+---
+
+# NyayaLM Tokenizer
+
+SentencePiece-BPE tokenizer (64,000 vocab) for **NyayaLM-200M**,
+a bilingual (Nepali/English) legal-domain model with a Qwen3-style architecture and
+ChatML-style special tokens.
+
+- Model type: BPE, byte-fallback enabled (never emits `<unk>`)
+- Character coverage: 0.9995
+- Curated special tokens: 17
+- Reserved tokens for future use: 32
+
+## Usage
+
+```python
+from transformers import AutoTokenizer
+tok = AutoTokenizer.from_pretrained("REPO_ID")
+tok.apply_chat_template(
+    [{'role': "user", "content": "नमस्ते"}],
+    tokenize=False, add_generation_prompt=True,
+)
+```

chat_template.jinja

@@ -0,0 +1,8 @@
+{% for message in messages %}{% if message['role'] == 'system' %}<|im_start|>system
+{{ message['content'] }}<|im_end|>
+{% elif message['role'] == 'user' %}<|im_start|>user
+{{ message['content'] }}<|im_end|>
+{% elif message['role'] == 'assistant' %}<|im_start|>assistant
+{{ message['content'] }}<|im_end|>
+{% endif %}{% endfor %}{% if add_generation_prompt %}<|im_start|>assistant
+{% endif %}

nyayalm_sp.model

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:de34c567d7cd811837bd6f7d5c332cdc2862c8df3437f9cbabce3672ce77507e
+size 1761018

tokenizer_build_metadata.json

@@ -0,0 +1,44 @@
+{
+  "model_type": "bpe",
+  "vocab_size": 64000,
+  "sp_trained_vocab_size": 63951,
+  "curated_special_tokens": [
+    "<|endoftext|>",
+    "<|im_start|>",
+    "<|im_end|>",
+    "<|user|>",
+    "<|assistant|>",
+    "<|system|>",
+    "<|thinking|>",
+    "<|/thinking|>",
+    "<|legal_doc|>",
+    "<|/legal_doc|>",
+    "<|nepali|>",
+    "<|english|>",
+    "<|court_order|>",
+    "<|judgement|>",
+    "<|statute|>",
+    "<|contract|>",
+    "<|tool_call|>"
+  ],
+  "num_reserved_tokens": 32,
+  "character_coverage": 0.9995,
+  "byte_fallback": true,
+  "training_lines": 90251,
+  "architecture_target": {
+    "params": "213M",
+    "layers": 20,
+    "hidden_size": 768,
+    "ffn_size": 3072,
+    "attn_heads_q": 12,
+    "attn_heads_kv": 4,
+    "head_dim": 64,
+    "context_train": 4096,
+    "context_infer": 8192,
+    "rope_base": 1000000,
+    "qk_norm": true,
+    "weight_tying": true,
+    "activation": "SwiGLU",
+    "norm": "RMSNorm(eps=1e-6)"
+  }
+}

tokenizer_config.json

@@ -0,0 +1,63 @@
+{
+  "backend": "tokenizers",
+  "bos_token": "<|endoftext|>",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|endoftext|>",
+  "extra_special_tokens": [
+    "<|endoftext|>",
+    "<|im_start|>",
+    "<|im_end|>",
+    "<|user|>",
+    "<|assistant|>",
+    "<|system|>",
+    "<|thinking|>",
+    "<|/thinking|>",
+    "<|legal_doc|>",
+    "<|/legal_doc|>",
+    "<|nepali|>",
+    "<|english|>",
+    "<|court_order|>",
+    "<|judgement|>",
+    "<|statute|>",
+    "<|contract|>",
+    "<|tool_call|>",
+    "<|reserved_special_token_0|>",
+    "<|reserved_special_token_1|>",
+    "<|reserved_special_token_2|>",
+    "<|reserved_special_token_3|>",
+    "<|reserved_special_token_4|>",
+    "<|reserved_special_token_5|>",
+    "<|reserved_special_token_6|>",
+    "<|reserved_special_token_7|>",
+    "<|reserved_special_token_8|>",
+    "<|reserved_special_token_9|>",
+    "<|reserved_special_token_10|>",
+    "<|reserved_special_token_11|>",
+    "<|reserved_special_token_12|>",
+    "<|reserved_special_token_13|>",
+    "<|reserved_special_token_14|>",
+    "<|reserved_special_token_15|>",
+    "<|reserved_special_token_16|>",
+    "<|reserved_special_token_17|>",
+    "<|reserved_special_token_18|>",
+    "<|reserved_special_token_19|>",
+    "<|reserved_special_token_20|>",
+    "<|reserved_special_token_21|>",
+    "<|reserved_special_token_22|>",
+    "<|reserved_special_token_23|>",
+    "<|reserved_special_token_24|>",
+    "<|reserved_special_token_25|>",
+    "<|reserved_special_token_26|>",
+    "<|reserved_special_token_27|>",
+    "<|reserved_special_token_28|>",
+    "<|reserved_special_token_29|>",
+    "<|reserved_special_token_30|>",
+    "<|reserved_special_token_31|>"
+  ],
+  "model_max_length": 4096,
+  "pad_token": "<|endoftext|>",
+  "padding_side": "right",
+  "tokenizer_class": "TokenizersBackend",
+  "truncation_side": "right",
+  "unk_token": "<unk>"
+}