File size: 37,581 Bytes
63be543
f5b81b7
63be543
 
 
f5b81b7
63be543
384c0cc
 
63be543
384c0cc
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
63be543
384c0cc
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
f5b81b7
384c0cc
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
63be543
 
f5b81b7
 
63be543
f5b81b7
 
 
 
fe60d85
 
 
f5b81b7
 
 
 
 
fe60d85
f5b81b7
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
fe60d85
 
 
 
 
 
 
 
 
 
c1cee49
384c0cc
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
c1cee49
f5b81b7
 
 
 
 
 
 
384c0cc
 
 
 
 
 
 
 
f5b81b7
 
384c0cc
 
faa51b0
 
384c0cc
 
 
f5b81b7
 
 
fe60d85
 
 
 
 
 
 
 
 
 
 
 
 
 
384c0cc
 
 
f5b81b7
 
c1cee49
384c0cc
c1cee49
 
384c0cc
 
 
 
 
 
 
c1cee49
 
384c0cc
 
 
c1cee49
 
f5b81b7
 
 
 
 
fe60d85
 
 
 
 
 
 
 
 
 
f5b81b7
 
 
 
 
 
 
384c0cc
 
 
fe60d85
 
384c0cc
 
 
f5b81b7
 
 
 
 
 
 
63be543
 
 
fe60d85
 
 
63be543
 
 
 
 
fe60d85
63be543
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
fe60d85
 
 
 
 
 
 
 
 
 
c1cee49
384c0cc
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
c1cee49
63be543
 
 
 
 
 
 
384c0cc
 
 
 
 
 
 
 
63be543
 
384c0cc
 
faa51b0
 
384c0cc
 
 
63be543
 
 
fe60d85
 
 
 
 
 
 
 
 
 
 
 
 
 
384c0cc
 
 
63be543
 
c1cee49
384c0cc
c1cee49
 
384c0cc
 
 
 
 
 
 
c1cee49
 
384c0cc
 
 
c1cee49
 
63be543
 
 
 
 
fe60d85
 
 
 
 
 
 
 
 
 
63be543
 
 
 
 
 
 
384c0cc
 
 
fe60d85
 
384c0cc
 
 
63be543
 
 
 
 
f5b81b7
63be543
f5b81b7
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
63be543
f67024c
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="utf-8">
<meta name="viewport" content="width=device-width, initial-scale=1">
<title>Mission – Slayer 1B</title>
<style>
  :root { --bg:#0b0d11; --card:#141821; --card2:#181d28; --line:#252b38; --line2:#323a4a; --fg:#eceef2; --soft:#c9ced8; --muted:#9aa3b2;
          --acc:#4fb477; --acc2:#e0565b; --acc3:#6ea6e6; --warn:#e8c37a; --r:16px; color-scheme:dark; }
  * { box-sizing:border-box; }
  html { scroll-behavior:smooth; -webkit-text-size-adjust:100%; }
  body { margin:0; color:var(--fg); font:16px/1.65 ui-sans-serif,system-ui,-apple-system,"Segoe UI",Roboto,"Helvetica Neue",Arial,sans-serif;
         background:radial-gradient(900px 480px at 12% -80px,rgba(79,180,119,.16),transparent 70%),radial-gradient(760px 420px at 92% -60px,rgba(110,166,230,.12),transparent 70%),var(--bg);
         background-repeat:no-repeat; text-rendering:optimizeLegibility; }
  ::selection { background:rgba(79,180,119,.35); }
  main { max-width:1040px; margin:0 auto; padding:28px 20px 80px; }
  header { padding:44px 0 8px; }
  header h1 { font-size:clamp(2.3rem,6vw,3.7rem); line-height:1.05; margin:0 0 12px; letter-spacing:-.035em; font-weight:800; text-wrap:balance;
              color:var(--fg); background:linear-gradient(100deg,#fff 0%,#d4f3e1 45%,var(--acc) 100%); -webkit-background-clip:text; background-clip:text; -webkit-text-fill-color:transparent; }
  header .by { color:var(--acc); font-size:.82rem; font-weight:600; letter-spacing:.14em; text-transform:uppercase; margin:0 0 22px; }
  .lead { font-size:1.16rem; line-height:1.7; max-width:64ch; color:var(--soft); }
  .pills { display:flex; flex-wrap:wrap; gap:10px; margin:26px 0 8px; }
  .pill { display:inline-block; background:rgba(20,24,33,.8); border:1px solid var(--line2); border-radius:999px; padding:6px 14px; font-size:.9rem; color:var(--muted); }
  .pill::before { content:""; display:inline-block; width:6px; height:6px; margin:0 8px 2px 0; vertical-align:middle; border-radius:50%; background:var(--acc); }
  .pill b { color:var(--fg); font-weight:700; font-variant-numeric:tabular-nums; }
  section { margin-top:72px; }
  h2 { display:flex; align-items:center; gap:12px; font-size:1.6rem; line-height:1.25; letter-spacing:-.02em; margin:0 0 20px; text-wrap:balance; scroll-margin-top:24px; }
  h2::before { content:""; width:5px; height:1.15em; border-radius:3px; background:linear-gradient(var(--acc),rgba(110,166,230,.9)); flex:none; }
  .cards { display:grid; grid-template-columns:repeat(auto-fit,minmax(250px,1fr)); gap:16px; }
  .card { background:linear-gradient(180deg,var(--card2),var(--card)); border:1px solid var(--line); border-radius:var(--r); padding:20px 22px; box-shadow:0 1px 0 rgba(255,255,255,.04) inset; }
  .card h3 { margin:0 0 4px; font-size:1.18rem; letter-spacing:-.01em; display:flex; flex-wrap:wrap; align-items:center; gap:4px 10px; }
  .card .meta { color:var(--muted); font-size:.93rem; margin-bottom:12px; }
  .card p { margin:10px 0; color:var(--soft); }
  .card a { font-weight:500; }
  .tag { display:inline-block; font-size:.7rem; font-weight:700; letter-spacing:.06em; text-transform:uppercase; padding:3px 9px; border-radius:999px; vertical-align:middle; line-height:1.4; white-space:nowrap; }
  .tag.const { background:rgba(79,180,119,.16); color:#8fe0ad; box-shadow:0 0 0 1px rgba(79,180,119,.35) inset; }
  .tag.live { background:rgba(232,195,122,.14); color:var(--warn); box-shadow:0 0 0 1px rgba(232,195,122,.35) inset; }
  .tag.dec { background:rgba(154,163,178,.14); color:var(--muted); box-shadow:0 0 0 1px rgba(154,163,178,.28) inset; }
  .tag.live::before { content:""; display:inline-block; width:6px; height:6px; margin:0 6px 1px 0; border-radius:50%; background:var(--warn); }
  .card-status { grid-column:1 / -1; border-left:3px solid var(--line2); }
  .card-status:has(.tag.const) { border-left-color:var(--acc); }
  .card-status:has(.tag.live) { border-left-color:var(--warn); }
  .wrap { overflow-x:auto; background:var(--card); border:1px solid var(--line); border-radius:var(--r); -webkit-overflow-scrolling:touch; }
  table { width:100%; border-collapse:collapse; min-width:520px; }
  th, td { padding:12px 16px; text-align:right; border-bottom:1px solid var(--line); font-variant-numeric:tabular-nums; white-space:nowrap; }
  th:first-child, td:first-child { text-align:left; position:sticky; left:0; background:var(--card); min-width:11.5rem; white-space:normal; z-index:1; }
  th { color:var(--muted); font-weight:600; font-size:.78rem; letter-spacing:.07em; text-transform:uppercase; background:rgba(255,255,255,.03); vertical-align:middle; }
  th:first-child { background:#191e29; }
  tr:last-child td { border-bottom:none; }
  tr:hover td { background:rgba(255,255,255,.025); }
  tr:hover td:first-child { background:#191e29; }
  td.best { color:var(--acc); font-weight:700; background:rgba(79,180,119,.09); }
  tr:hover td.best { background:rgba(79,180,119,.14); }
  section > p { max-width:78ch; }
  .note { color:var(--muted); font-size:.93rem; line-height:1.7; max-width:78ch; margin:12px 0 0; }
  code { font:.84em ui-monospace,SFMono-Regular,Menlo,Consolas,monospace; background:rgba(255,255,255,.06); padding:2px 6px; border-radius:6px; color:var(--soft); }
  .sha { overflow-wrap:anywhere; font-size:.78rem; }
  figure { margin:0; background:var(--bg); border-radius:var(--r); padding:0; overflow:hidden; box-shadow:0 0 0 1px var(--line2), 0 22px 56px -26px rgba(0,0,0,.85); }
  figure img { width:100%; height:auto; display:block; }
  figure + .note { margin-top:16px; }
  .note + figure { margin-top:28px; }
  ol.steps { list-style:none; padding:0; margin:0; counter-reset:step; display:grid; gap:12px; }
  ol.steps li { counter-increment:step; position:relative; margin:0; padding:18px 20px 18px 66px; background:linear-gradient(180deg,var(--card2),var(--card)); border:1px solid var(--line); border-radius:var(--r); color:var(--soft); }
  ol.steps li::before { content:counter(step); position:absolute; left:20px; top:18px; width:30px; height:30px; display:grid; place-items:center; border-radius:50%; background:rgba(79,180,119,.16); color:var(--acc); font-weight:800; font-size:.92rem; box-shadow:0 0 0 1px rgba(79,180,119,.35) inset; }
  ol.steps li b { color:var(--fg); }
  footer { margin-top:80px; color:var(--muted); font-size:.9rem; border-top:1px solid var(--line); padding-top:20px; }
  a { color:var(--acc3); text-decoration-thickness:1px; text-underline-offset:3px; text-decoration-color:rgba(110,166,230,.4); transition:color .15s,text-decoration-color .15s; }
  a:hover { color:#9cc3f0; text-decoration-color:currentColor; }
  a:focus-visible, button:focus-visible { outline:2px solid var(--acc); outline-offset:3px; border-radius:6px; }
  .lang { position:fixed; top:14px; right:16px; display:flex; gap:4px; background:rgba(20,24,33,.78); -webkit-backdrop-filter:blur(10px); backdrop-filter:blur(10px); border:1px solid var(--line2); border-radius:999px; padding:3px; z-index:10; }
  .lang button { background:none; border:0; color:var(--muted); font:600 .85rem system-ui,sans-serif; padding:5px 12px; border-radius:999px; cursor:pointer; transition:background .15s,color .15s; }
  .lang button:hover { color:var(--fg); }
  .lang button[aria-pressed="true"] { background:var(--acc); color:#07130c; }
  body[data-lang="en"] .l-pl, body[data-lang="pl"] .l-en { display:none; }
  @media (prefers-reduced-motion:no-preference) {
    .card { transition:transform .2s ease,border-color .2s ease,box-shadow .2s ease; }
    .card:hover { transform:translateY(-2px); border-color:var(--line2); box-shadow:0 14px 34px -18px rgba(0,0,0,.8); }
    .tag.live::before { animation:pulse 1.8s ease-in-out infinite; }
    @keyframes pulse { 0%,100% { opacity:1; transform:scale(1); } 50% { opacity:.35; transform:scale(.7); } }
  }
  @media (prefers-reduced-motion:reduce) { html { scroll-behavior:auto; } }
  @media (max-width:640px) {
    main { padding:12px 16px 64px; }
    .lang { position:static; margin:12px 16px 0 auto; width:max-content; }
    header { padding-top:24px; }
    section { margin-top:52px; }
    h2 { font-size:1.35rem; }
    th, td { padding:10px 12px; font-size:.92rem; }
    th:first-child, td:first-child { min-width:9.5rem; }
    .card { padding:18px; }
    ol.steps li { padding:16px 16px 16px 58px; }
    ol.steps li::before { left:16px; top:16px; }
    figure { padding:0; }
    figure:has(> img) { overflow-x:auto; -webkit-overflow-scrolling:touch; }
    figure:has(> img) img { width:760px; max-width:none; }
  }
</style>
</head>
<body data-lang="en">
<nav class="lang" aria-label="Language"><button type="button" data-set="en" aria-pressed="true">EN</button><button type="button" data-set="pl" aria-pressed="false">PL</button></nav>
<main>
<div class="l-en" lang="en">
<header>
  <h1>Mission – Slayer 1B</h1>
  <p class="by">Arkadiusz Słota – Fabryka AI (SlayerLab)</p>
  <p class="lead">We are building a Polish language model of ~1 billion parameters from scratch and improving it through
  checkpointed continuation. Released model weights are public; resumable checkpoint archives are published
  separately. Follow each archive link for its current files. Training progress and measured evaluation results are reported separately below.</p>
  <div class="pills">
    <span class="pill"><b>1,073,912,851</b> parameters</span>
    <span class="pill">tokenizer <b>32,768</b>, Polish</span>
    <span class="pill">context <b>4096</b> tokens</span>
    <span class="pill">data: ~82% Polish, ~18% English</span>
    <span class="pill">released weights: <b>Apache-2.0</b></span>
  </div>
</header>

<section>
  <h2>Versions</h2>
  <div class="cards">
    <div class="card">
      <h3>Slayer1B 1.00 <span class="tag dec">decayed</span></h3>
      <div class="meta">1.5 B training tokens</div>
      <a href="https://huggingface.co/SlayerLab/Slayer1B-1.00">model</a> ·
      <a href="https://huggingface.co/SlayerLab/Slayer1B-1.00-checkpoints">checkpoints</a> ·
      <a href="https://track.fabryka.ai/run/48744ed2-3155-4294-b899-a6fda1f851da">run</a>
    </div>
    <div class="card">
      <h3>Slayer1B 1.01 <span class="tag dec">decayed</span></h3>
      <div class="meta">3.0 B tokens · continued from 1.00</div>
      <a href="https://huggingface.co/SlayerLab/Slayer1B-1.01">model</a> ·
      <a href="https://huggingface.co/SlayerLab/Slayer1B-1.01-checkpoints">checkpoints</a> ·
      <a href="https://track.fabryka.ai/run/4934429a-d393-42e0-8aac-03eae6b71ff7">run</a>
    </div>
    <div class="card">
      <h3>Slayer1B 1.02 <span class="tag const">constant LR</span></h3>
      <div class="meta">6.0 B tokens · continued from 1.01</div>
      <a href="https://huggingface.co/SlayerLab/Slayer1B-1.02">model</a> ·
      <a href="https://huggingface.co/SlayerLab/Slayer1B-1.02-checkpoints">checkpoints</a> ·
      <a href="https://track.fabryka.ai/run/21ae1234-82c2-42b7-887b-cbcc69901ae1">run</a>
    </div>
    <div class="card card-status">
      <h3>Slayer1B 1.03 <span class="tag const">training complete</span></h3>
      <div class="meta">native step 7150 · training loss 2.3955</div>
      <p>Resume 6750 → 7150 completed on 4×L40S. The final 7150 model weights and lm-eval benchmark results are public.</p>
      <p class="note">Checkpoint SHA-256: <code class="sha">d7cad9673f3ee706f3022ad6a97fa7593604806d4d25ba936c27885a8f115d97</code></p>
      <a href="https://huggingface.co/SlayerLab/Slayer1B-1.03">model</a> ·
      <a href="https://track.fabryka.ai/run/7ca4f84f-3b07-4525-8cde-f3a936dc3425">completed training run</a> ·
      <a href="https://track.fabryka.ai/run/1426c0a8-2f07-5a03-80fc-92f7e922aa4b">checkpoint benchmarks</a> ·
      <a href="https://huggingface.co/SlayerLab/Slayer1B-1.03-checkpoints">native checkpoint archive — current files</a>
    </div>
    <div class="card card-status">
      <h3>Slayer1B 1.04 <span class="tag const">released</span></h3>
      <div class="meta">native step 10010 · 10.5 B training tokens</div>
      <p>Corpus continuation 7150 → 10010 on 4×L40S, constant learning rate. The run then moved to 8×L40S at this checkpoint
      after independent SHA-256 verification (three sources) and a proof that the global batch is reproduced.</p>
      <p class="note">Checkpoint SHA-256: <code class="sha">04438959d7b317d1a8d972b264e9610a06fd8addde7349c6e9f458ee56dc1e51</code></p>
      <a href="https://huggingface.co/SlayerLab/Slayer1B-1.04">model</a> ·
      <a href="https://huggingface.co/SlayerLab/Slayer1B-1.04-checkpoints">checkpoints</a> ·
      <a href="https://track.fabryka.ai/run/f056bb5c-f1da-56d5-9776-cc33063f10a5">run</a>
    </div>
    <div class="card card-status">
      <h3>Slayer1B 1.05 <span class="tag const">released</span></h3>
      <div class="meta">native step 12870 · 13.5 B training tokens</div>
      <p>Continuation 10010 → 12870 on 8×L40S, same model, data order and global batch, constant learning rate.</p>
      <p class="note">Checkpoint SHA-256: <code class="sha">435082ff36becfbb96803622139f38d2afa0ba1a9d2fc284bad96f5a8edc1181</code></p>
      <a href="https://huggingface.co/SlayerLab/Slayer1B-1.05">model</a> ·
      <a href="https://huggingface.co/SlayerLab/Slayer1B-1.05-checkpoints">checkpoints</a> ·
      <a href="https://track.fabryka.ai/run/f056bb5c-f1da-56d5-9776-cc33063f10a5">run</a>
    </div>
    <div class="card card-status">
      <h3>Slayer1B 1.06 <span class="tag const">released · full corpus</span></h3>
      <div class="meta">native step 13848 · 14.5 B training tokens · one full pass</div>
      <p>Continuation 12870 → 13848 on 8×L40S. It ends where the prepared corpus ends: every token of the corpus was seen
      exactly once (measured, not planned). Final training loss 2.276; no learning-rate decay yet.</p>
      <p class="note">Checkpoint SHA-256: <code class="sha">1fc7396ca1790805128244b2f827e02b9fe88e352534cd887d0c488f121d1e4d</code></p>
      <a href="https://huggingface.co/SlayerLab/Slayer1B-1.06">model</a> ·
      <a href="https://huggingface.co/SlayerLab/Slayer1B-1.06-checkpoints">checkpoints</a> ·
      <a href="https://track.fabryka.ai/run/f056bb5c-f1da-56d5-9776-cc33063f10a5">run</a>
    </div>
  </div>
</section>

<section>
  <h2>Results</h2>
  <div class="wrap">
  <table>
    <tr><th>test</th><th>1.00</th><th>1.01</th><th>1.02</th><th>1.03</th><th>1.04</th><th>1.05</th><th>1.06</th></tr>
    <tr><td>training tokens</td><td>1.5 B</td><td>3.0 B</td><td>6.0 B</td><td>7.5 B</td><td>10.5 B</td><td>13.5 B</td><td>14.5 B</td></tr>
    <tr><td>native step</td><td>1430</td><td>2860</td><td>5720</td><td>7150</td><td>10010</td><td>12870</td><td>13848</td></tr>
    <tr><td>ARC-Easy (EN), %</td><td>42.89</td><td>47.98</td><td>48.99</td><td>—</td><td>50.59</td><td>52.19</td><td class="best">52.99</td></tr>
    <tr><td>BLiMP (EN, grammar), %</td><td>71.51</td><td>73.63</td><td>74.71</td><td>—</td><td>74.17</td><td>75.53</td><td class="best">76.12</td></tr>
    <tr><td>WikiText-2, byte perplexity ↓</td><td>2.5857</td><td>2.4230</td><td>2.3735</td><td>—</td><td>2.3078</td><td class="best">2.2816</td><td>2.2849</td></tr>
    <tr><td>MultiBLiMP (PL, grammar), %</td><td>97.89</td><td>98.29</td><td>98.38</td><td>—</td><td class="best">98.87</td><td>98.47</td><td>98.41</td></tr>
    <tr><td>ARC-Easy-PL, %</td><td>36.91</td><td>40.36</td><td>41.84</td><td>—</td><td>44.02</td><td class="best">45.20</td><td>45.03</td></tr>
  </table>
  </div>
  <p class="note">One instrument for the versions that have values (1.00–1.02 and 1.04–1.06): leaderboard protocol of
  <a href="https://track.fabryka.ai/models">track.fabryka.ai/models</a>, zero-shot, on the released checkpoints. Column 1.03 is a dash on purpose: its run
  (<a href="https://track.fabryka.ai/run/7ca4f84f-3b07-4525-8cde-f3a936dc3425">1.03</a>) was measured with the older lm-eval task set
  (hellaswag, piqa, lambada, arc_challenge, boolq, mmlu — see the additional history section), so there are no values in this instrument;
  the dash is not a missing measurement. 1.00 and 1.01 ended with a learning-rate decay; 1.02 and 1.04–1.06 did not, so comparisons
  across that boundary are not clean. Neighbouring checkpoints differ by up to about a point on single tests, so small differences
  between 1.05 and 1.06 are within run noise.</p>
</section>

<section>
  <h2>Additional benchmark history</h2>
  <p><a href="https://track.fabryka.ai/run/1426c0a8-2f07-5a03-80fc-92f7e922aa4b">16 checkpoints from 1.00–1.03, through final 7150</a>
  have separate lm-eval 0.4.13, zero-shot, fp32 results: 13 historical points plus 6900, 7050 and 7150.
  These results use a different benchmark procedure from the table above.</p>
  <p class="note">Accuracy and normalized accuracy are distinct series, in fractions (0–1); LAMBADA perplexity is separate.
  The scan found n-gram overlap in shared/en for BoolQ, MMLU and ARC; its significance is still being assessed.
  Overlap is not by itself a measured contamination rate. Open Charts and filter by <code>eval/</code>.
  This is an imported result series, not a native automatic-benchmark job.</p>
  <p class="note">The three new points use the same runner and task procedure, but a different runtime:
  Torch 2.8.0+cu128, tokenizers 0.23.2 and NumPy 2.2.6, versus 2.14.1+cu126, 0.23.3 and 2.5.3 for the historical points.
  The 6750 → 6900 boundary is not an identical-environment checkpoint comparison.</p>
</section>

<section>
  <h2>Score curves: 1.00–1.06</h2>
  <figure><img src="scores-history-106.png" alt="Evaluation history of Slayer1B 1.00 to 1.06 versus training tokens"></figure>
  <p class="note">Measured values only, no fits, no trend lines. 1.00–1.02: leaderboard protocol, one checkpoint every 150 steps; shaded bands mark the learning-rate decay of 1.00 and 1.01. 1.03: training loss only — it was evaluated with an older lm-eval task set, so it has no points in the score panels. 1.04–1.06: one continuation from step 7150 at a constant learning rate; diamonds mark the releases 1.04 (10010), 1.05 (12870) and 1.06 (13848, end of the corpus). The in-run evaluator measured every 50 steps up to 9100, then control points 9400/9700/10000, then every 200 steps; breaks in the line are gaps in that schedule, not missing training. eff = (BLiMP + ARC + WikiScore) / 3, the board formula, which reproduces every published 1.00–1.02 value within 0.01. Data: <a href="scores-history-106.csv">scores-history-106.csv</a>.</p>
</section>

<section>
  <h2>Release points: 1.04 → 1.06</h2>
  <div class="wrap">
  <table>
    <tr><th>test</th><th>1.04 · 10010</th><th>1.05 · 12870</th><th>1.06 · 13848</th></tr>
    <tr><td>BLiMP (EN, grammar), %</td><td>74.17</td><td>75.53</td><td class="best">76.12</td></tr>
    <tr><td>ARC-Easy (EN), %</td><td>50.59</td><td>52.19</td><td class="best">52.99</td></tr>
    <tr><td>WikiText-2, byte perplexity ↓</td><td>2.3078</td><td class="best">2.2816</td><td>2.2849</td></tr>
    <tr><td>WikiText-2, token perplexity ↓</td><td>31.29</td><td class="best">29.86</td><td>30.03</td></tr>
    <tr><td>MultiBLiMP (PL, grammar), %</td><td class="best">98.87</td><td>98.47</td><td>98.41</td></tr>
    <tr><td>ARC-Easy-PL, %</td><td>44.02</td><td class="best">45.20</td><td>45.03</td></tr>
  </table>
  </div>
  <p class="note">Measured on the released checkpoints by the run's automatic evaluator (same instrument as the results table), read
  from the <a href="https://track.fabryka.ai/run/f056bb5c-f1da-56d5-9776-cc33063f10a5">public continuation run</a>. The whole continuation
  used a constant learning rate; a final decay has not been applied, so these are not end-of-training numbers for the model line.</p>
</section>

<section>
  <h2>How we do it</h2>
  <ol class="steps">
    <li><b>From scratch.</b> A transformer (20 layers, width 2048), our own Polish tokenizer, Polish data with an English
      minority (<a href="https://huggingface.co/datasets/SlayerLab/slayer-pl-8x3b">slayer-pl-8x3b</a>).</li>
    <li><b>Checkpointed continuation.</b> Earlier decayed versions branched from checkpoints before their final decay.
      Version names mark milestones in one global data trajectory, not mandatory process restarts.
      Retention maintenance resumed from verified checkpoint 7750 without resetting the data order.
      The final learning-rate decay is planned once, at the end.</li>
    <li><b>Source and model units.</b> The source corpus is counted as ~24 B <code>cl100k</code> tokens.
      Our v6/32k tokenizer produces separate <code>uint16</code> train/validation files. Actual model-token counts and
      stream boundaries come from pinned manifests. Next packs are prepared while the current stage trains.</li>
    <li><b>We verify everything.</b> Checkpoint SHA-256 records are preserved outside the training machine.
      Before local pruning, a full copy on a separate archive is verified and its receipt is persisted atomically.
      From version 1.01 on, a loss forecast is written down before each run and evaluated afterwards.</li>
  </ol>
  <p class="note">Technical report:
  <a href="https://github.com/slayerlabs/Fabryka-AI-Notes/tree/main/papers/slayerlabs/2026-10-23_slayer1b-continuation">Slayer1B: Continuing a Polish-Dominant Base Model</a>.</p>
</section>

<section>
  <h2>What's next</h2>
  <p><b>Corpus continuation 1.04–1.06 — complete.</b> The global trajectory ran from native step 7150 to 13848, the end of the
  prepared corpus: 14.5 B training tokens, one full pass, about 13.5 tokens per parameter. Releases 1.04 (10010), 1.05 (12870) and
  1.06 (13848) are public, each with its checkpoint archive. <a href="https://track.fabryka.ai/run/f056bb5c-f1da-56d5-9776-cc33063f10a5">Training run</a>.</p>
  <p class="note">The linked runs belong to the author's Track account. The completed 1.03 run includes its resumed steps;
  corpus continuation and benchmark history are separate records of the same model lineage.</p>
  <p><b>Beyond 13848 — roadmap candidates, not scheduled runs:</b> continuing to about 20 tokens per parameter with fresh data,
  a final learning-rate decay, and extending the context window in stages (first 32k tokens, up to 256k as the target), each stage
  checked with long-context tests. Each follow-up needs its own plan, cost estimate and go-ahead.</p>
  <p class="note">These are base models: they predict the continuation of a text, they were not tuned for chat or for
  following instructions, and they can be wrong.</p>
</section>

<footer>Arkadiusz Słota – Fabryka AI (SlayerLab) · models: <a href="https://huggingface.co/SlayerLab">huggingface.co/SlayerLab</a></footer>
</div>
<div class="l-pl" lang="pl">
<header>
  <h1>Misja – Slayer 1B</h1>
  <p class="by">Arkadiusz Słota – Fabryka AI (SlayerLab)</p>
  <p class="lead">Budujemy od zera polski model językowy wielkości ~1 mld parametrów i ulepszamy go przez kontynuację
  z punktów kontrolnych. Wagi wydanych modeli są publiczne; archiwa checkpointów do wznowienia publikujemy
  osobno. Linki archiwów pokazują ich aktualne pliki. Postęp treningu i zmierzone wyniki ewaluacji pokazujemy poniżej oddzielnie.</p>
  <div class="pills">
    <span class="pill"><b>1 073 912 851</b> parametrów</span>
    <span class="pill">tokenizer <b>32 768</b>, polski</span>
    <span class="pill">kontekst <b>4096</b> tokenów</span>
    <span class="pill">dane: ~82 % PL, ~18 % EN</span>
    <span class="pill">opublikowane wagi: <b>Apache-2.0</b></span>
  </div>
</header>

<section>
  <h2>Wersje</h2>
  <div class="cards">
    <div class="card">
      <h3>Slayer1B 1.00 <span class="tag dec">wygaszony</span></h3>
      <div class="meta">1,5 mld tokenów treningu</div>
      <a href="https://huggingface.co/SlayerLab/Slayer1B-1.00">model</a> ·
      <a href="https://huggingface.co/SlayerLab/Slayer1B-1.00-checkpoints">punkty kontrolne</a> ·
      <a href="https://track.fabryka.ai/run/48744ed2-3155-4294-b899-a6fda1f851da">przebieg</a>
    </div>
    <div class="card">
      <h3>Slayer1B 1.01 <span class="tag dec">wygaszony</span></h3>
      <div class="meta">3,0 mld tokenów · dalszy trening 1.00</div>
      <a href="https://huggingface.co/SlayerLab/Slayer1B-1.01">model</a> ·
      <a href="https://huggingface.co/SlayerLab/Slayer1B-1.01-checkpoints">punkty kontrolne</a> ·
      <a href="https://track.fabryka.ai/run/4934429a-d393-42e0-8aac-03eae6b71ff7">przebieg</a>
    </div>
    <div class="card">
      <h3>Slayer1B 1.02 <span class="tag const">stały LR</span></h3>
      <div class="meta">6,0 mld tokenów · dalszy trening 1.01</div>
      <a href="https://huggingface.co/SlayerLab/Slayer1B-1.02">model</a> ·
      <a href="https://huggingface.co/SlayerLab/Slayer1B-1.02-checkpoints">punkty kontrolne</a> ·
      <a href="https://track.fabryka.ai/run/21ae1234-82c2-42b7-887b-cbcc69901ae1">przebieg</a>
    </div>
    <div class="card card-status">
      <h3>Slayer1B 1.03 <span class="tag const">trening zakończony</span></h3>
      <div class="meta">krok native 7150 · strata treningowa 2,3955</div>
      <p>Wznowienie 6750 → 7150 zakończone na 4×L40S. Wagi końcowego modelu 7150 i wyniki benchmarków lm-eval są publiczne.</p>
      <p class="note">SHA-256 checkpointu: <code class="sha">d7cad9673f3ee706f3022ad6a97fa7593604806d4d25ba936c27885a8f115d97</code></p>
      <a href="https://huggingface.co/SlayerLab/Slayer1B-1.03">model</a> ·
      <a href="https://track.fabryka.ai/run/7ca4f84f-3b07-4525-8cde-f3a936dc3425">zakończony przebieg treningu</a> ·
      <a href="https://track.fabryka.ai/run/1426c0a8-2f07-5a03-80fc-92f7e922aa4b">benchmarki checkpointów</a> ·
      <a href="https://huggingface.co/SlayerLab/Slayer1B-1.03-checkpoints">archiwum checkpointów natywnych — aktualne pliki</a>
    </div>
    <div class="card card-status">
      <h3>Slayer1B 1.04 <span class="tag const">wydany</span></h3>
      <div class="meta">krok native 10010 · 10,5 mld tokenów treningu</div>
      <p>Kontynuacja korpusu 7150 → 10010 na 4×L40S, stałe tempo uczenia. Na tym checkpoincie bieg przeszedł na 8×L40S
      po niezależnej weryfikacji SHA-256 (trzy źródła) i dowodzie, że globalny batch jest odtworzony.</p>
      <p class="note">SHA-256 checkpointu: <code class="sha">04438959d7b317d1a8d972b264e9610a06fd8addde7349c6e9f458ee56dc1e51</code></p>
      <a href="https://huggingface.co/SlayerLab/Slayer1B-1.04">model</a> ·
      <a href="https://huggingface.co/SlayerLab/Slayer1B-1.04-checkpoints">punkty kontrolne</a> ·
      <a href="https://track.fabryka.ai/run/f056bb5c-f1da-56d5-9776-cc33063f10a5">przebieg</a>
    </div>
    <div class="card card-status">
      <h3>Slayer1B 1.05 <span class="tag const">wydany</span></h3>
      <div class="meta">krok native 12870 · 13,5 mld tokenów treningu</div>
      <p>Kontynuacja 10010 → 12870 na 8×L40S: ten sam model, kolejność danych i globalny batch, stałe tempo uczenia.</p>
      <p class="note">SHA-256 checkpointu: <code class="sha">435082ff36becfbb96803622139f38d2afa0ba1a9d2fc284bad96f5a8edc1181</code></p>
      <a href="https://huggingface.co/SlayerLab/Slayer1B-1.05">model</a> ·
      <a href="https://huggingface.co/SlayerLab/Slayer1B-1.05-checkpoints">punkty kontrolne</a> ·
      <a href="https://track.fabryka.ai/run/f056bb5c-f1da-56d5-9776-cc33063f10a5">przebieg</a>
    </div>
    <div class="card card-status">
      <h3>Slayer1B 1.06 <span class="tag const">wydany · cały korpus</span></h3>
      <div class="meta">krok native 13848 · 14,5 mld tokenów treningu · jedno pełne przejście</div>
      <p>Kontynuacja 12870 → 13848 na 8×L40S. Kończy się tam, gdzie kończy się przygotowany korpus: każdy jego token model
      zobaczył dokładnie raz (zmierzone, nie zaplanowane). Końcowa strata treningowa 2,276; bez wygaszania tempa uczenia.</p>
      <p class="note">SHA-256 checkpointu: <code class="sha">1fc7396ca1790805128244b2f827e02b9fe88e352534cd887d0c488f121d1e4d</code></p>
      <a href="https://huggingface.co/SlayerLab/Slayer1B-1.06">model</a> ·
      <a href="https://huggingface.co/SlayerLab/Slayer1B-1.06-checkpoints">punkty kontrolne</a> ·
      <a href="https://track.fabryka.ai/run/f056bb5c-f1da-56d5-9776-cc33063f10a5">przebieg</a>
    </div>
  </div>
</section>

<section>
  <h2>Wyniki</h2>
  <div class="wrap">
  <table>
    <tr><th>test</th><th>1.00</th><th>1.01</th><th>1.02</th><th>1.03</th><th>1.04</th><th>1.05</th><th>1.06</th></tr>
    <tr><td>tokeny treningu</td><td>1,5 mld</td><td>3,0 mld</td><td>6,0 mld</td><td>7,5 mld</td><td>10,5 mld</td><td>13,5 mld</td><td>14,5 mld</td></tr>
    <tr><td>krok native</td><td>1430</td><td>2860</td><td>5720</td><td>7150</td><td>10010</td><td>12870</td><td>13848</td></tr>
    <tr><td>ARC-Easy (EN), %</td><td>42,89</td><td>47,98</td><td>48,99</td><td>—</td><td>50,59</td><td>52,19</td><td class="best">52,99</td></tr>
    <tr><td>BLiMP (EN, gramatyka), %</td><td>71,51</td><td>73,63</td><td>74,71</td><td>—</td><td>74,17</td><td>75,53</td><td class="best">76,12</td></tr>
    <tr><td>WikiText-2, perpleksja na bajt ↓</td><td>2,5857</td><td>2,4230</td><td>2,3735</td><td>—</td><td>2,3078</td><td class="best">2,2816</td><td>2,2849</td></tr>
    <tr><td>MultiBLiMP (PL, gramatyka), %</td><td>97,89</td><td>98,29</td><td>98,38</td><td>—</td><td class="best">98,87</td><td>98,47</td><td>98,41</td></tr>
    <tr><td>ARC-Easy-PL, %</td><td>36,91</td><td>40,36</td><td>41,84</td><td>—</td><td>44,02</td><td class="best">45,20</td><td>45,03</td></tr>
  </table>
  </div>
  <p class="note">Jeden przyrząd dla wersji, które mają wartości (1.00–1.02 oraz 1.04–1.06): protokół tablicy
  <a href="https://track.fabryka.ai/models">track.fabryka.ai/models</a>, zero-shot, na wydanych checkpointach. Kolumna 1.03 ma kreskę celowo: jej run
  (<a href="https://track.fabryka.ai/run/7ca4f84f-3b07-4525-8cde-f3a936dc3425">1.03</a>) mierzono starszym zestawem lm-eval
  (hellaswag, piqa, lambada, arc_challenge, boolq, mmlu — patrz sekcja dodatkowej historii), więc w tym przyrządzie nie ma wartości;
  kreska nie oznacza brakującego pomiaru. 1.00 i 1.01 kończyły się wygaszaniem tempa uczenia, 1.02 i 1.04–1.06 nie — porównania
  przez tę granicę nie są „czyste”. Sąsiednie checkpointy różnią się na pojedynczych testach nawet o ok. punkt, więc małe różnice
  między 1.05 a 1.06 mieszczą się w szumie biegu.</p>
</section>

<section>
  <h2>Dodatkowa historia benchmarków</h2>
  <p><a href="https://track.fabryka.ai/run/1426c0a8-2f07-5a03-80fc-92f7e922aa4b">16 checkpointów 1.00–1.03, do końcowego 7150</a>
  ma osobne wyniki lm-eval 0.4.13, zero-shot, fp32: 13 historycznych punktów oraz 6900, 7050 i 7150.
  To inna procedura benchmarkowa niż w tabeli powyżej.</p>
  <p class="note">Trafność i trafność normalizowana to osobne serie, w ułamkach (0–1); perpleksja LAMBADA jest oddzielnie.
  Skan wykrył nakładkę n-gramową w shared/en dla BoolQ, MMLU i ARC; ocena jej znaczenia nadal trwa.
  Sama nakładka nie jest pomiarem odsetka rzeczywistego przecieku. W Charts wpisz w filtrze <code>eval/</code>.
  To seria zaimportowanych wyników, nie natywny job automatycznej ewaluacji.</p>
  <p class="note">Trzy nowe punkty mają ten sam runner i procedurę zadań, ale inne środowisko:
  Torch 2.8.0+cu128, tokenizers 0.23.2 i NumPy 2.2.6 zamiast 2.14.1+cu126, 0.23.3 i 2.5.3 dla punktów historycznych.
  Granica 6750 → 6900 nie jest porównaniem checkpointów w identycznym środowisku.</p>
</section>

<section>
  <h2>Krzywe wyników: 1.00–1.06</h2>
  <figure><img src="scores-history-106.png" alt="Historia wyników Slayer1B od 1.00 do 1.06 w funkcji liczby tokenów treningu"></figure>
  <p class="note">Same pomiary, bez dopasowań i bez linii trendu. 1.00–1.02: protokół tablicy, punkt co 150 kroków; pasma oznaczają wygaszanie LR w 1.00 i 1.01. 1.03: tylko strata treningowa — mierzony starszym zestawem lm-eval, więc w panelach wyników nie ma punktów. 1.04–1.06: jedna kontynuacja od kroku 7150 przy stałym LR; romby oznaczają wydania 1.04 (10010), 1.05 (12870) i 1.06 (13848, koniec korpusu). Ewaluator w biegu mierzył co 50 kroków do 9100, potem punkty kontrolne 9400/9700/10000, potem co 200 kroków; przerwy w linii to luki w tym harmonogramie, nie brakujący trening. eff = (BLiMP + ARC + WikiScore) / 3, formuła tablicy, która odtwarza każdą opublikowaną wartość 1.00–1.02 z dokładnością 0,01. Dane: <a href="scores-history-106.csv">scores-history-106.csv</a>.</p>
</section>

<section>
  <h2>Punkty wydań: 1.04 → 1.06</h2>
  <div class="wrap">
  <table>
    <tr><th>test</th><th>1.04 · 10010</th><th>1.05 · 12870</th><th>1.06 · 13848</th></tr>
    <tr><td>BLiMP (EN, gramatyka), %</td><td>74,17</td><td>75,53</td><td class="best">76,12</td></tr>
    <tr><td>ARC-Easy (EN), %</td><td>50,59</td><td>52,19</td><td class="best">52,99</td></tr>
    <tr><td>WikiText-2, perpleksja na bajt ↓</td><td>2,3078</td><td class="best">2,2816</td><td>2,2849</td></tr>
    <tr><td>WikiText-2, perpleksja na token ↓</td><td>31,29</td><td class="best">29,86</td><td>30,03</td></tr>
    <tr><td>MultiBLiMP (PL, gramatyka), %</td><td class="best">98,87</td><td>98,47</td><td>98,41</td></tr>
    <tr><td>ARC-Easy-PL, %</td><td>44,02</td><td class="best">45,20</td><td>45,03</td></tr>
  </table>
  </div>
  <p class="note">Zmierzone na wydanych checkpointach automatycznym ewaluatorem biegu (ten sam przyrząd co tabela wyników), odczytane
  z <a href="https://track.fabryka.ai/run/f056bb5c-f1da-56d5-9776-cc33063f10a5">publicznego biegu kontynuacji</a>. Cała kontynuacja
  szła przy stałym tempie uczenia; końcowego wygaszania nie zastosowano, więc to nie są końcowe liczby tej linii modelu.</p>
</section>

<section>
  <h2>Jak to robimy</h2>
  <ol class="steps">
    <li><b>Od zera.</b> Architektura typu transformer (20 warstw, szerokość 2048), własny polski tokenizer, polskie dane
      z mniejszością angielską (<a href="https://huggingface.co/datasets/SlayerLab/slayer-pl-8x3b">slayer-pl-8x3b</a>).</li>
    <li><b>Kontynuacja z checkpointów.</b> Wcześniejsze wygaszane wersje odgałęziały się sprzed końcowego wygaszania.
      Nazwy wersji oznaczają etapy jednej globalnej trajektorii danych, nie obowiązkowe restarty procesu.
      Konserwacja retencji wznowiła trening ze zweryfikowanego checkpointu 7750 bez zerowania kolejności danych.
      Końcowe wygaszanie tempa uczenia planujemy raz, na końcu.</li>
    <li><b>Jednostki źródła i modelu.</b> Około 24 mld tokenów źródła liczymy przez <code>cl100k</code>.
      Nasz tokenizer v6/32k tworzy osobne pliki train/val w <code>uint16</code>. Liczby tokenów modelu i granice
      strumieni pochodzą z przypiętych manifestów. Następne wsady przygotowujemy podczas bieżącego treningu.</li>
    <li><b>Wszystko sprawdzamy.</b> Sumy SHA-256 checkpointów zachowujemy poza maszyną treningową.
      Przed lokalnym usunięciem pełna kopia na osobnym archiwum jest weryfikowana, a jej receipt utrwalany atomowo.
      Od wersji 1.01 przed każdym biegiem zapisujemy prognozę straty i potem ją oceniamy.</li>
  </ol>
  <p class="note">Raport techniczny:
  <a href="https://github.com/slayerlabs/Fabryka-AI-Notes/tree/main/papers/slayerlabs/2026-10-23_slayer1b-continuation">Slayer1B: Continuing a Polish-Dominant Base Model</a>.</p>
</section>

<section>
  <h2>Co dalej</h2>
  <p><b>Kontynuacja korpusu 1.04–1.06 — zakończona.</b> Globalna trajektoria przeszła od kroku native 7150 do 13848, czyli do
  końca przygotowanego korpusu: 14,5 mld tokenów treningu, jedno pełne przejście, ok. 13,5 tokenu na parametr. Wydania 1.04 (10010),
  1.05 (12870) i 1.06 (13848) są publiczne, każde z archiwum checkpointów. <a href="https://track.fabryka.ai/run/f056bb5c-f1da-56d5-9776-cc33063f10a5">Bieg treningu</a>.</p>
  <p class="note">Powiązane przebiegi są na koncie autora w Tracku. Zakończony run 1.03 obejmuje wznowione kroki;
  dalszy trening i historia benchmarków to osobne zapisy tej samej linii modelu.</p>
  <p><b>Po 13848 — kandydaci roadmapy, nie zaplanowane biegi:</b> dalszy trening do ok. 20 tokenów na parametr na świeżych danych,
  końcowe wygaszanie tempa uczenia oraz etapami wydłużanie okna kontekstu (najpierw 32k tokenów, docelowo do 256k), każdy etap
  sprawdzany testami długiego kontekstu. Każdy z tych kroków wymaga własnego planu, wyceny i zgody.</p>
  <p class="note">To modele bazowe: przewidują dalszy ciąg tekstu, nie były dostrajane do rozmowy ani wykonywania poleceń
  i mogą się mylić.</p>
</section>

<footer>Arkadiusz Słota – Fabryka AI (SlayerLab) · modele: <a href="https://huggingface.co/SlayerLab">huggingface.co/SlayerLab</a></footer>
</div>
</main>
<script>
(function () {
  var q = new URLSearchParams(location.search).get("lang");
  var saved = null; try { saved = localStorage.getItem("slayer-lang"); } catch (e) {}
  function set(l) {
    if (l !== "pl") l = "en";
    document.body.setAttribute("data-lang", l);
    document.documentElement.lang = l;
    document.title = l === "pl" ? "Misja – Slayer 1B" : "Mission – Slayer 1B";
    document.querySelectorAll(".lang button").forEach(function (b) { b.setAttribute("aria-pressed", String(b.dataset.set === l)); });
    try { localStorage.setItem("slayer-lang", l); } catch (e) {}
  }
  document.querySelectorAll(".lang button").forEach(function (b) { b.addEventListener("click", function () { set(b.dataset.set); }); });
  set(q || saved || "en");
})();
</script>
</body>
</html>