Upload folder using huggingface_hub

README.md

@@ -40,6 +40,7 @@ Developed by **[Dvitva AI](https://dvitva.ai)**.
 | **Vocab size** | 86,075 (base + 4 chat tokens) |
 | **Tokenizer** | Morfessor + BPE (Telugu morpheme-aware) |
 | **Fine-tuning** | Full SFT on Telugu conversations |
+| **Best val loss** | 2.4830389234855454 |
 | **Developed by** | [Dvitva AI](https://dvitva.ai) |
 
 ## Chat Template
@@ -66,6 +67,8 @@ The model generates after `<|assistant|>` and stops at `<|end|>`.
 
 | Token | ID |
 |---|---|
+| `<bos>` | 2 |
+| `<eos>` | 3 |
 | `<|system|>` | 86071 |
 | `<|user|>` | 86072 |
 | `<|assistant|>` | 86073 |
@@ -126,7 +129,7 @@ print(tokenizer.decode(outputs[0], skip_special_tokens=False))
 
 ### Using the CLI chat script
 
-For the best interactive experience, use the `chat.py` CLI from the [training repo](https://github.com/dvitvaai/telugu-lm):
+For the best experience, use the included `chat.py` CLI:
 
 ```bash
 # Interactive multi-turn chat
@@ -156,12 +159,10 @@ This model uses a **Morfessor + BPE hybrid tokenizer** designed for Telugu:
 
 ```python
 import morfessor, re
-from huggingface_hub import hf_hub_download
 
-# Load Morfessor model from the repo
-morf_path = hf_hub_download(repo_id="dvitvaai/pothana-chat-300M", filename="morfessor_telugu.bin")
+# Load Morfessor model
 io = morfessor.MorfessorIO()
-morf_model = io.read_binary_model_file(morf_path)
+morf_model = io.read_binary_model_file("morfessor_telugu.bin")
 
 TELUGU_RE = re.compile(r"[\u0C00-\u0C7F]+")
 

config.json

@@ -21,10 +21,13 @@
   "tie_word_embeddings": true,
   "pad_token_id": 0,
   "bos_token_id": 2,
-  "eos_token_id": [3, 86074],
+  "eos_token_id": [
+    3,
+    86074
+  ],
   "attention_dropout": 0.0,
   "initializer_range": 0.02,
   "pretraining_tp": 1,
   "use_cache": true,
   "transformers_version": "4.40.0"
-}
+}

generation_config.json

@@ -1,7 +1,10 @@
 {
   "_from_model_config": true,
   "bos_token_id": 2,
-  "eos_token_id": [3, 86074],
+  "eos_token_id": [
+    3,
+    86074
+  ],
   "pad_token_id": 0,
   "do_sample": true,
   "temperature": 0.7,
@@ -10,4 +13,4 @@
   "max_new_tokens": 256,
   "repetition_penalty": 1.1,
   "transformers_version": "4.40.0"
-}
+}

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:236a8a7692f176c516db8a5c7448795000e1677de1c2798cb75c7d37aa6bee1f
+oid sha256:f2fb678f8659adecbe45390748938a5b3ec93c7bc5d5fa133fbab54723f8b168
 size 1380356280

special_tokens_map.json

@@ -3,5 +3,12 @@
   "eos_token": "<eos>",
   "unk_token": "<unk>",
   "pad_token": "<pad>",
-  "additional_special_tokens": ["<|system|>", "<|user|>", "<|assistant|>", "<|end|>"]
-}
+  "additional_special_tokens": [
+    "<|system|>",
+    "<|user|>",
+    "<|assistant|>",
+    "<|end|>",
+    "<bos>",
+    "<eos>"
+  ]
+}

tokenizer_class.py

@@ -8,8 +8,14 @@ class TeluguTokenizer(PreTrainedTokenizerFast):
     Tokens ending with @@ are continuation pieces that join to the next token.
     This class overrides decode() to strip @@ markers and join morphemes:
         "రెడ్డి@@ గారు" → "రెడ్డిగారు"
+
+    Also strips chat special tokens (<|system|>, <|user|>, <|assistant|>, <|end|>)
+    from decoded output for clean text.
     """
 
+    # Chat special tokens to strip from output
+    _CHAT_SPECIALS = ["<|system|>", "<|user|>", "<|assistant|>", "<|end|>"]
+
     def decode(self, token_ids, skip_special_tokens=False, **kwargs):
         text = super().decode(token_ids, skip_special_tokens=skip_special_tokens, **kwargs)
         # Strip @@ continuation markers:
@@ -17,4 +23,10 @@ class TeluguTokenizer(PreTrainedTokenizerFast):
         text = text.replace("@@ ", "")
         # Handle remaining @@ (before punctuation, end of string, etc.)
         text = text.replace("@@", "")
+        # Strip chat special tokens
+        for special in self._CHAT_SPECIALS:
+            text = text.replace(special, "")
+        # Clean up extra whitespace from removed tokens
+        import re
+        text = re.sub(r"  +", " ", text).strip()
         return text

tokenizer_config.json

@@ -15,11 +15,18 @@
   "add_eos_token": false,
   "clean_up_tokenization_spaces": false,
   "model_max_length": 2048,
-  "additional_special_tokens": ["<|system|>", "<|user|>", "<|assistant|>", "<|end|>"],
-  "chat_template": "{% for message in messages %}{% if loop.first %}<bos>{% endif %}{% if message['role'] == 'system' %}<|system|> {{ message['content'] }} <|end|>{% elif message['role'] == 'user' %}<|user|> {{ message['content'] }} <|end|>{% elif message['role'] == 'assistant' %}<|assistant|> {{ message['content'] }} <|end|>{% endif %}{% endfor %}{% if add_generation_prompt %}<|assistant|>{% endif %}",
   "extra_info": {
     "type": "morfessor_bpe_telugu",
     "separator": "@@",
     "note": "This tokenizer expects Morfessor-segmented text as input. For raw Telugu text, run Morfessor segmentation first using the included morfessor_telugu.bin model. Tokens ending with '@@' are continuation pieces that join to the next token. The decoder handles @@ removal automatically."
-  }
-}
+  },
+  "additional_special_tokens": [
+    "<|system|>",
+    "<|user|>",
+    "<|assistant|>",
+    "<|end|>",
+    "<bos>",
+    "<eos>"
+  ],
+  "chat_template": "{{ bos_token }}{% for message in messages %}{% if message['role'] == 'system' %}<|system|>{{ message['content'] }}<|end|>{% elif message['role'] == 'user' %}<|user|>{{ message['content'] }}<|end|>{% elif message['role'] == 'assistant' %}<|assistant|>{{ message['content'] }}<|end|>{% endif %}{% endfor %}{% if add_generation_prompt %}<|assistant|>{% endif %}"
+}