Hugging Face's logo

SlayerLab
/
TokenizerTest-32m-B

Text Generation
PyTorch
Safetensors
Polish
polish
language-model
research
tokenizer-comparison
Model card Files
xet
 Community
TokenizerTest-32m-B
File size: 2,384 Bytes
98eae28
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
{
 "definition": "bits/byte = sum of NLL over text tokens / ln 2 / UTF-8 bytes; non-overlapping 1024-token windows; EOS targets excluded",
 "trainer_sha256": "0223c0830ed4ae323f6424038a2de7bfb6b425653f3591dd55844f95706f5d17",
 "tokenizer_sha256": "77eeebed509827f18b859ff159f29b2405d24d5b84381d6226f22e4fb6ca98d7",
 "sets": {
  "W": {
   "sha256": "234e7ec21bf2236bc74425591531f2c4772f8c49ff853d57f6fa16f33ee90ac7",
   "docs": 463,
   "text_bytes": 1430217,
   "tokens_text": 351268
  },
  "val": {
   "sha256": "748d13898e736666d96f1c9307222c52414a9b66b06bf212e7a07e765babc668",
   "docs": 2723,
   "text_bytes": 7232820,
   "tokens_text": 1748065
  }
 },
 "device": "cpu",
 "ckpts": [
  {
   "step": 10000,
   "ckpt_sha256": "13a9130e6ac322c3a342c6bbcdf0838bdaa82e7e6962c0b4ab4ee1bdb677d2c8",
   "W": 1.246255139078163,
   "val": 1.2008817462071877
  },
  {
   "step": 20000,
   "ckpt_sha256": "78d637de184f406d22621b4512fc79da9d2b47f9ba532525aba44199505bdd5f",
   "W": 1.189575099180923,
   "val": 1.152323463429622
  },
  {
   "step": 30000,
   "ckpt_sha256": "2ec241ef98aaf7d54a15398bbb3d64dd2f0d3d6e0d562cd5e2aa4b7f7a468ac3",
   "W": 1.163382329460513,
   "val": 1.1278191393784163
  },
  {
   "step": 40000,
   "ckpt_sha256": "0bf61a0a7595c56f22af420dcd7a956a087c49100be258660fbc5ab109934b1b",
   "W": 1.1434992106638442,
   "val": 1.1109230609739182
  },
  {
   "step": 50000,
   "ckpt_sha256": "2c8189ed98f6bd27064205725ac1430c815cb4a718c764a57178632ff676b590",
   "W": 1.1290988704930747,
   "val": 1.096905167732812
  },
  {
   "step": 60000,
   "ckpt_sha256": "a24e66893894405f763a83ea28eee3432e8ccb6911f15e59715119ae1705089f",
   "W": 1.1144814554280478,
   "val": 1.0850306176283422
  },
  {
   "step": 70000,
   "ckpt_sha256": "2cedd2505161e340b2ef8ad79199d431ae54aec6cecac6d131f7d76dddfcea34",
   "W": 1.1024713768041312,
   "val": 1.0752361156563053
  },
  {
   "step": 80000,
   "ckpt_sha256": "f074891643fe2efa2bcf2e8545cd6ade5e024baf29fe47f161fcc43ac092c76c",
   "W": 1.0943567744308031,
   "val": 1.0677255240917687
  },
  {
   "step": 90000,
   "ckpt_sha256": "16f2089bd49d108383660a206a93551edacaa69ebe76686b65b2e32f9df5104a",
   "W": 1.0892155925274738,
   "val": 1.0633125873779654
  },
  {
   "step": 95000,
   "ckpt_sha256": "538505bc15e4332231b3e2958b02b7567348b3d43b8974e57f48526266879ab9",
   "W": 1.0882527260062658,
   "val": 1.062229520611975
  }
 ]
}