Hugging Face's logo

AbstractPhil
/
geolip-bytelex

tokenization
byte-level
distillation
lexicon
Model card Files
xet
 Community
geolip-bytelex
File size: 1,960 Bytes
f8b0b26
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
{
 "32000": {
  "bytes": 32138,
  "tokens_A_flan": 9607,
  "tokens_B_bert": 8271,
  "tok_per_100b_A": 29.89,
  "tok_per_100b_B": 25.74,
  "co_boundary_rate": 0.923,
  "matrix_nnz": 94,
  "cells_at_90": 59,
  "cells_at_99": 89,
  "cells_at_999": 94,
  "mass_total": 8148,
  "frac_11_cells": 0.8085,
  "frac_11_mass": 0.829,
  "erank_11": 42.91,
  "distinct_11_pairs": 76,
  "shape_hist_top": {
   "1x1": 76,
   "2x1": 13,
   "3x1": 4,
   "1x2": 1
  }
 },
 "64000": {
  "bytes": 64234,
  "tokens_A_flan": 19231,
  "tokens_B_bert": 16552,
  "tok_per_100b_A": 29.94,
  "tok_per_100b_B": 25.77,
  "co_boundary_rate": 0.9227,
  "matrix_nnz": 94,
  "cells_at_90": 59,
  "cells_at_99": 89,
  "cells_at_999": 94,
  "mass_total": 16305,
  "frac_11_cells": 0.8085,
  "frac_11_mass": 0.8307,
  "erank_11": 42.84,
  "distinct_11_pairs": 76,
  "shape_hist_top": {
   "1x1": 76,
   "2x1": 13,
   "3x1": 4,
   "1x2": 1
  }
 },
 "128000": {
  "bytes": 128208,
  "tokens_A_flan": 38309,
  "tokens_B_bert": 32916,
  "tok_per_100b_A": 29.88,
  "tok_per_100b_B": 25.67,
  "co_boundary_rate": 0.9236,
  "matrix_nnz": 94,
  "cells_at_90": 60,
  "cells_at_99": 90,
  "cells_at_999": 94,
  "mass_total": 32450,
  "frac_11_cells": 0.8085,
  "frac_11_mass": 0.8319,
  "erank_11": 43.14,
  "distinct_11_pairs": 76,
  "shape_hist_top": {
   "1x1": 76,
   "2x1": 13,
   "3x1": 4,
   "1x2": 1
  }
 },
 "256016": {
  "bytes": 256016,
  "tokens_A_flan": 79013,
  "tokens_B_bert": 68746,
  "tok_per_100b_A": 30.86,
  "tok_per_100b_B": 26.85,
  "co_boundary_rate": 0.8908,
  "matrix_nnz": 1362,
  "cells_at_90": 103,
  "cells_at_99": 868,
  "cells_at_999": 1297,
  "mass_total": 65362,
  "frac_11_cells": 0.13,
  "frac_11_mass": 0.7983,
  "erank_11": 53.7,
  "distinct_11_pairs": 177,
  "shape_hist_top": {
   "2x2": 718,
   "2x1": 379,
   "1x1": 177,
   "2x3": 59,
   "1x2": 25,
   "3x1": 4
  }
 }
}