Hugging Face's logo

devildasdf
/
NEXORA

Safetensors
English
Hindi
nexora
research
training
agents
prototype
Model card Files
xet
 Community
NEXORA
File size: 2,372 Bytes
12496fc
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
{
  "byte": {
    "english": {
      "characters": 77,
      "bytes": 77,
      "tokens": 77,
      "characters_per_token": 1.0,
      "roundtrip": true
    },
    "hindi": {
      "characters": 72,
      "bytes": 188,
      "tokens": 188,
      "characters_per_token": 0.3829787234042553,
      "roundtrip": true
    },
    "hinglish": {
      "characters": 70,
      "bytes": 70,
      "tokens": 70,
      "characters_per_token": 1.0,
      "roundtrip": true
    },
    "python": {
      "characters": 70,
      "bytes": 70,
      "tokens": 70,
      "characters_per_token": 1.0,
      "roundtrip": true
    },
    "json": {
      "characters": 61,
      "bytes": 61,
      "tokens": 61,
      "characters_per_token": 1.0,
      "roundtrip": true
    },
    "math": {
      "characters": 45,
      "bytes": 57,
      "tokens": 57,
      "characters_per_token": 0.7894736842105263,
      "roundtrip": true
    },
    "shell": {
      "characters": 38,
      "bytes": 38,
      "tokens": 38,
      "characters_per_token": 1.0,
      "roundtrip": true
    },
    "xml_url": {
      "characters": 66,
      "bytes": 67,
      "tokens": 67,
      "characters_per_token": 0.9850746268656716,
      "roundtrip": true
    }
  },
  "qwen_bpe": {
    "english": {
      "tokens": 13,
      "characters_per_token": 5.923076923076923,
      "roundtrip": true
    },
    "hindi": {
      "tokens": 61,
      "characters_per_token": 1.180327868852459,
      "roundtrip": true
    },
    "hinglish": {
      "tokens": 22,
      "characters_per_token": 3.1818181818181817,
      "roundtrip": true
    },
    "python": {
      "tokens": 18,
      "characters_per_token": 3.888888888888889,
      "roundtrip": true
    },
    "json": {
      "tokens": 14,
      "characters_per_token": 4.357142857142857,
      "roundtrip": true
    },
    "math": {
      "tokens": 33,
      "characters_per_token": 1.3636363636363635,
      "roundtrip": true
    },
    "shell": {
      "tokens": 11,
      "characters_per_token": 3.4545454545454546,
      "roundtrip": true
    },
    "xml_url": {
      "tokens": 21,
      "characters_per_token": 3.142857142857143,
      "roundtrip": true
    }
  },
  "limitation": "Eight public examples, not a representative benchmark or vocabulary-size study"
}