Update README.md
Browse files
README.md
CHANGED
|
@@ -2,4 +2,38 @@
|
|
| 2 |
library_name: transformers
|
| 3 |
license: mit
|
| 4 |
---
|
| 5 |
-
**Mean Perplexity: `1425.6071816176866`**
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2 |
library_name: transformers
|
| 3 |
license: mit
|
| 4 |
---
|
| 5 |
+
**Mean Perplexity: `1425.6071816176866`**
|
| 6 |
+
|
| 7 |
+
|
| 8 |
+
|
| 9 |
+
|
| 10 |
+
|
| 11 |
+
|
| 12 |
+
|
| 13 |
+
|
| 14 |
+
This is a filler text
|
| 15 |
+
|
| 16 |
+
|
| 17 |
+
| | Tokenizer | English | Hindi | Tamil | Bengali | Malayalam | Telugu | Gujarati | Punjabi | Code_Python | Code_Java | c++ | Math |
|
| 18 |
+
|---:|:---------------------------------------------------|----------:|--------:|--------:|----------:|------------:|---------:|-----------:|----------:|--------------:|------------:|------:|-------:|
|
| 19 |
+
| 0 | deepseek-ai/DeepSeek-R1 (128k) | 338874 | 22855 | 48957 | 39617 | 73928 | 40345 | 101020 | 79172 | 5231 | 2224 | 7055 | 5376 |
|
| 20 |
+
| 1 | unsloth/phi-4 (100k) | 308645 | 40456 | 59750 | 116122 | 149889 | 48689 | 118335 | 87413 | 4809 | 2110 | 6529 | 5573 |
|
| 21 |
+
| 2 | deepseek-ai/DeepSeek-R1-Distill-Llama-8B (128k) | 308512 | 21110 | 59625 | 115138 | 149883 | 48661 | 118061 | 86765 | 4809 | 2111 | 6530 | 5574 |
|
| 22 |
+
| 3 | unsloth/gemma-2-9b-it(256k) | 323335 | 15916 | 53913 | 53402 | 57219 | 47610 | 107925 | 87222 | 5948 | 2569 | 8639 | 5871 |
|
| 23 |
+
| 4 | Ornaments/72k-Bilingual-BBPE-TK-SPM (72k) | 366710 | 11447 | 61408 | 94191 | 97207 | 50229 | 117874 | 90045 | 8201 | 4000 | 13706 | 5585 |
|
| 24 |
+
| 5 | Ornaments/72k-Bilingual-BBPE-TK-SPM-Identity (72k) | 330830 | 10318 | 59089 | 93740 | 92655 | 44975 | 109411 | 87922 | 7819 | 3743 | 12953 | 5253 |
|
| 25 |
+
| 6 | Ornaments/72k-TK-BBPE-HF (72k) | 321274 | 10813 | 67585 | 159985 | 193813 | 55654 | 134397 | 97063 | 5225 | 2263 | 7090 | 5150 |
|
| 26 |
+
| 7 | nvidia/Nemotron-4-Mini-Hindi-4B-Instruct (256k) | 332271 | 14327 | 55473 | 36615 | 45783 | 48270 | 160115 | 117174 | 6186 | 2732 | 8861 | 6136 |
|
| 27 |
+
| 8 | sarvamai/OpenHathi-7B-Hi-v0.1-Base (48k) | 370133 | 15633 | 67845 | 120340 | 105953 | 68315 | 159122 | 113817 | 6595 | 2792 | 9233 | 6223 |
|
| 28 |
+
| 9 | sarvamai/sarvam-1 (68k) | 385386 | 11257 | 61396 | 27348 | 31822 | 51463 | 119666 | 103344 | 7331 | 3068 | 9724 | 6864 |
|
| 29 |
+
|
| 30 |
+
|
| 31 |
+
|
| 32 |
+
asasjop
|
| 33 |
+
dasklfjads
|
| 34 |
+
adsfjdlk
|
| 35 |
+
fadsjfl;
|
| 36 |
+
'
|
| 37 |
+
afjlkeadksfpoa
|
| 38 |
+
jdslkjfj[a0'f'kasd
|
| 39 |
+
mdlkjfae[fpowe[fkdl;]]
|