Update README.md

README.md

@@ -7,9 +7,9 @@ license_link: LICENSE
 # 💎 gemma Robotics & Routing Tokenizer
 - Gemma Pro Tokenizer . Fully compatible with gemma4 models.
 - Size {
- Vocab size: 128365
- pad_token_id: None
- eos_token_id: 128001
+ Vocab size: 262251
+ pad_token_id: 0
+ eos_token_id: 1
  }
 - Compatible with gemma 4 • Optimized for Code . Use resize function without adaptation, see examples below
 - It needs 100k example to fully adapt routing features for RAG Routing model . So check gemma modification tokenizer rules
@@ -197,12 +197,12 @@ Designed for Banking and Fintech Institutions
 - # Tesr Tokenizer size !
 (venv_ji) root@jirack2:# python -c '
 from transformers import AutoTokenizer
-tok = AutoTokenizer.from_pretrained("./jirack_code_tokenizer_fixed")
+tok = AutoTokenizer.from_pretrained("./GemmaRoboticsTokenizer")
 print("Vocab size:", len(tok))
 print("pad_token_id:", tok.pad_token_id)
 print("eos_token_id:", tok.eos_token_id)
 '
 
-Vocab size: 128365
-pad_token_id: None
-eos_token_id: 128001
+Vocab size: 262251
+pad_token_id: 0
+eos_token_id: 1