Hugging Face's logo

AbstractPhil
/
geolip-bytelex

tokenization
byte-level
distillation
lexicon
Model card Files
xet
 Community
geolip-bytelex
File size: 2,407 Bytes
3d25c99
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
{
 "split_H_bits": 3.0,
 "corpus_bytes": 8192000,
 "rows": [
  {
   "tokenizer": "bert-wordpiece",
   "tokens": 30517,
   "word_frac": 0.469,
   "divides_frac": 0.27,
   "cohesive_frac": 0.73,
   "mean_internal_Hmax_bits": 2.587,
   "secs": 1.3
  },
  {
   "tokenizer": "byt5-control",
   "tokens": 256,
   "word_frac": 0.301,
   "divides_frac": 0.0,
   "cohesive_frac": 0.0,
   "mean_internal_Hmax_bits": 0.0,
   "secs": 0.0
  },
  {
   "tokenizer": "cl100k_base",
   "tokens": 100256,
   "word_frac": 0.32,
   "divides_frac": 0.287,
   "cohesive_frac": 0.713,
   "mean_internal_Hmax_bits": 2.544,
   "secs": 3.0
  },
  {
   "tokenizer": "deepseek-v3",
   "tokens": 127997,
   "word_frac": 0.216,
   "divides_frac": 0.317,
   "cohesive_frac": 0.683,
   "mean_internal_Hmax_bits": 2.606,
   "secs": 2.7
  },
  {
   "tokenizer": "gpt2",
   "tokens": 50256,
   "word_frac": 0.479,
   "divides_frac": 0.343,
   "cohesive_frac": 0.657,
   "mean_internal_Hmax_bits": 2.67,
   "secs": 0.9
  },
  {
   "tokenizer": "llama3",
   "tokens": 128000,
   "word_frac": 0.251,
   "divides_frac": 0.283,
   "cohesive_frac": 0.717,
   "mean_internal_Hmax_bits": 2.528,
   "secs": 2.6
  },
  {
   "tokenizer": "mistral-v03",
   "tokens": 31997,
   "word_frac": 0.413,
   "divides_frac": 0.292,
   "cohesive_frac": 0.708,
   "mean_internal_Hmax_bits": 2.577,
   "secs": 0.6
  },
  {
   "tokenizer": "o200k_base",
   "tokens": 199998,
   "word_frac": 0.167,
   "divides_frac": 0.288,
   "cohesive_frac": 0.712,
   "mean_internal_Hmax_bits": 2.522,
   "secs": 4.0
  },
  {
   "tokenizer": "qwen3-newest",
   "tokens": 248044,
   "word_frac": 0.128,
   "divides_frac": 0.299,
   "cohesive_frac": 0.701,
   "mean_internal_Hmax_bits": 2.557,
   "secs": 4.7
  },
  {
   "tokenizer": "smollm2",
   "tokens": 49135,
   "word_frac": 0.502,
   "divides_frac": 0.356,
   "cohesive_frac": 0.644,
   "mean_internal_Hmax_bits": 2.703,
   "secs": 0.8
  },
  {
   "tokenizer": "t5-unigram",
   "tokens": 31997,
   "word_frac": 0.482,
   "divides_frac": 0.348,
   "cohesive_frac": 0.652,
   "mean_internal_Hmax_bits": 2.695,
   "secs": 0.6
  },
  {
   "tokenizer": "xlmr",
   "tokens": 249997,
   "word_frac": 0.047,
   "divides_frac": 0.282,
   "cohesive_frac": 0.718,
   "mean_internal_Hmax_bits": 2.489,
   "secs": 4.8
  }
 ]
}