Download index.html from SlayerLab/Slayer1B: direct link, hf CLI and curl.
- Browser
- Download file 37.6 kB
-
https://huggingface.co/spaces/SlayerLab/Slayer1B/resolve/main/index.html
- Command line
-
hf download hf://spaces/SlayerLab/Slayer1B/index.html
-
curl -L -o index.html https://huggingface.co/spaces/SlayerLab/Slayer1B/resolve/main/index.html
37.6 kB
| <html lang="en"> | |
| <head> | |
| <meta charset="utf-8"> | |
| <meta name="viewport" content="width=device-width, initial-scale=1"> | |
| <title>Mission – Slayer 1B</title> | |
| <style> | |
| :root { --bg:#0b0d11; --card:#141821; --card2:#181d28; --line:#252b38; --line2:#323a4a; --fg:#eceef2; --soft:#c9ced8; --muted:#9aa3b2; | |
| --acc:#4fb477; --acc2:#e0565b; --acc3:#6ea6e6; --warn:#e8c37a; --r:16px; color-scheme:dark; } | |
| * { box-sizing:border-box; } | |
| html { scroll-behavior:smooth; -webkit-text-size-adjust:100%; } | |
| body { margin:0; color:var(--fg); font:16px/1.65 ui-sans-serif,system-ui,-apple-system,"Segoe UI",Roboto,"Helvetica Neue",Arial,sans-serif; | |
| background:radial-gradient(900px 480px at 12% -80px,rgba(79,180,119,.16),transparent 70%),radial-gradient(760px 420px at 92% -60px,rgba(110,166,230,.12),transparent 70%),var(--bg); | |
| background-repeat:no-repeat; text-rendering:optimizeLegibility; } | |
| ::selection { background:rgba(79,180,119,.35); } | |
| main { max-width:1040px; margin:0 auto; padding:28px 20px 80px; } | |
| header { padding:44px 0 8px; } | |
| header h1 { font-size:clamp(2.3rem,6vw,3.7rem); line-height:1.05; margin:0 0 12px; letter-spacing:-.035em; font-weight:800; text-wrap:balance; | |
| color:var(--fg); background:linear-gradient(100deg,#fff 0%,#d4f3e1 45%,var(--acc) 100%); -webkit-background-clip:text; background-clip:text; -webkit-text-fill-color:transparent; } | |
| header .by { color:var(--acc); font-size:.82rem; font-weight:600; letter-spacing:.14em; text-transform:uppercase; margin:0 0 22px; } | |
| .lead { font-size:1.16rem; line-height:1.7; max-width:64ch; color:var(--soft); } | |
| .pills { display:flex; flex-wrap:wrap; gap:10px; margin:26px 0 8px; } | |
| .pill { display:inline-block; background:rgba(20,24,33,.8); border:1px solid var(--line2); border-radius:999px; padding:6px 14px; font-size:.9rem; color:var(--muted); } | |
| .pill::before { content:""; display:inline-block; width:6px; height:6px; margin:0 8px 2px 0; vertical-align:middle; border-radius:50%; background:var(--acc); } | |
| .pill b { color:var(--fg); font-weight:700; font-variant-numeric:tabular-nums; } | |
| section { margin-top:72px; } | |
| h2 { display:flex; align-items:center; gap:12px; font-size:1.6rem; line-height:1.25; letter-spacing:-.02em; margin:0 0 20px; text-wrap:balance; scroll-margin-top:24px; } | |
| h2::before { content:""; width:5px; height:1.15em; border-radius:3px; background:linear-gradient(var(--acc),rgba(110,166,230,.9)); flex:none; } | |
| .cards { display:grid; grid-template-columns:repeat(auto-fit,minmax(250px,1fr)); gap:16px; } | |
| .card { background:linear-gradient(180deg,var(--card2),var(--card)); border:1px solid var(--line); border-radius:var(--r); padding:20px 22px; box-shadow:0 1px 0 rgba(255,255,255,.04) inset; } | |
| .card h3 { margin:0 0 4px; font-size:1.18rem; letter-spacing:-.01em; display:flex; flex-wrap:wrap; align-items:center; gap:4px 10px; } | |
| .card .meta { color:var(--muted); font-size:.93rem; margin-bottom:12px; } | |
| .card p { margin:10px 0; color:var(--soft); } | |
| .card a { font-weight:500; } | |
| .tag { display:inline-block; font-size:.7rem; font-weight:700; letter-spacing:.06em; text-transform:uppercase; padding:3px 9px; border-radius:999px; vertical-align:middle; line-height:1.4; white-space:nowrap; } | |
| .tag.const { background:rgba(79,180,119,.16); color:#8fe0ad; box-shadow:0 0 0 1px rgba(79,180,119,.35) inset; } | |
| .tag.live { background:rgba(232,195,122,.14); color:var(--warn); box-shadow:0 0 0 1px rgba(232,195,122,.35) inset; } | |
| .tag.dec { background:rgba(154,163,178,.14); color:var(--muted); box-shadow:0 0 0 1px rgba(154,163,178,.28) inset; } | |
| .tag.live::before { content:""; display:inline-block; width:6px; height:6px; margin:0 6px 1px 0; border-radius:50%; background:var(--warn); } | |
| .card-status { grid-column:1 / -1; border-left:3px solid var(--line2); } | |
| .card-status:has(.tag.const) { border-left-color:var(--acc); } | |
| .card-status:has(.tag.live) { border-left-color:var(--warn); } | |
| .wrap { overflow-x:auto; background:var(--card); border:1px solid var(--line); border-radius:var(--r); -webkit-overflow-scrolling:touch; } | |
| table { width:100%; border-collapse:collapse; min-width:520px; } | |
| th, td { padding:12px 16px; text-align:right; border-bottom:1px solid var(--line); font-variant-numeric:tabular-nums; white-space:nowrap; } | |
| th:first-child, td:first-child { text-align:left; position:sticky; left:0; background:var(--card); min-width:11.5rem; white-space:normal; z-index:1; } | |
| th { color:var(--muted); font-weight:600; font-size:.78rem; letter-spacing:.07em; text-transform:uppercase; background:rgba(255,255,255,.03); vertical-align:middle; } | |
| th:first-child { background:#191e29; } | |
| tr:last-child td { border-bottom:none; } | |
| tr:hover td { background:rgba(255,255,255,.025); } | |
| tr:hover td:first-child { background:#191e29; } | |
| td.best { color:var(--acc); font-weight:700; background:rgba(79,180,119,.09); } | |
| tr:hover td.best { background:rgba(79,180,119,.14); } | |
| section > p { max-width:78ch; } | |
| .note { color:var(--muted); font-size:.93rem; line-height:1.7; max-width:78ch; margin:12px 0 0; } | |
| code { font:.84em ui-monospace,SFMono-Regular,Menlo,Consolas,monospace; background:rgba(255,255,255,.06); padding:2px 6px; border-radius:6px; color:var(--soft); } | |
| .sha { overflow-wrap:anywhere; font-size:.78rem; } | |
| figure { margin:0; background:var(--bg); border-radius:var(--r); padding:0; overflow:hidden; box-shadow:0 0 0 1px var(--line2), 0 22px 56px -26px rgba(0,0,0,.85); } | |
| figure img { width:100%; height:auto; display:block; } | |
| figure + .note { margin-top:16px; } | |
| .note + figure { margin-top:28px; } | |
| ol.steps { list-style:none; padding:0; margin:0; counter-reset:step; display:grid; gap:12px; } | |
| ol.steps li { counter-increment:step; position:relative; margin:0; padding:18px 20px 18px 66px; background:linear-gradient(180deg,var(--card2),var(--card)); border:1px solid var(--line); border-radius:var(--r); color:var(--soft); } | |
| ol.steps li::before { content:counter(step); position:absolute; left:20px; top:18px; width:30px; height:30px; display:grid; place-items:center; border-radius:50%; background:rgba(79,180,119,.16); color:var(--acc); font-weight:800; font-size:.92rem; box-shadow:0 0 0 1px rgba(79,180,119,.35) inset; } | |
| ol.steps li b { color:var(--fg); } | |
| footer { margin-top:80px; color:var(--muted); font-size:.9rem; border-top:1px solid var(--line); padding-top:20px; } | |
| a { color:var(--acc3); text-decoration-thickness:1px; text-underline-offset:3px; text-decoration-color:rgba(110,166,230,.4); transition:color .15s,text-decoration-color .15s; } | |
| a:hover { color:#9cc3f0; text-decoration-color:currentColor; } | |
| a:focus-visible, button:focus-visible { outline:2px solid var(--acc); outline-offset:3px; border-radius:6px; } | |
| .lang { position:fixed; top:14px; right:16px; display:flex; gap:4px; background:rgba(20,24,33,.78); -webkit-backdrop-filter:blur(10px); backdrop-filter:blur(10px); border:1px solid var(--line2); border-radius:999px; padding:3px; z-index:10; } | |
| .lang button { background:none; border:0; color:var(--muted); font:600 .85rem system-ui,sans-serif; padding:5px 12px; border-radius:999px; cursor:pointer; transition:background .15s,color .15s; } | |
| .lang button:hover { color:var(--fg); } | |
| .lang button[aria-pressed="true"] { background:var(--acc); color:#07130c; } | |
| body[data-lang="en"] .l-pl, body[data-lang="pl"] .l-en { display:none; } | |
| @media (prefers-reduced-motion:no-preference) { | |
| .card { transition:transform .2s ease,border-color .2s ease,box-shadow .2s ease; } | |
| .card:hover { transform:translateY(-2px); border-color:var(--line2); box-shadow:0 14px 34px -18px rgba(0,0,0,.8); } | |
| .tag.live::before { animation:pulse 1.8s ease-in-out infinite; } | |
| @keyframes pulse { 0%,100% { opacity:1; transform:scale(1); } 50% { opacity:.35; transform:scale(.7); } } | |
| } | |
| @media (prefers-reduced-motion:reduce) { html { scroll-behavior:auto; } } | |
| @media (max-width:640px) { | |
| main { padding:12px 16px 64px; } | |
| .lang { position:static; margin:12px 16px 0 auto; width:max-content; } | |
| header { padding-top:24px; } | |
| section { margin-top:52px; } | |
| h2 { font-size:1.35rem; } | |
| th, td { padding:10px 12px; font-size:.92rem; } | |
| th:first-child, td:first-child { min-width:9.5rem; } | |
| .card { padding:18px; } | |
| ol.steps li { padding:16px 16px 16px 58px; } | |
| ol.steps li::before { left:16px; top:16px; } | |
| figure { padding:0; } | |
| figure:has(> img) { overflow-x:auto; -webkit-overflow-scrolling:touch; } | |
| figure:has(> img) img { width:760px; max-width:none; } | |
| } | |
| </style> | |
| </head> | |
| <body data-lang="en"> | |
| <nav class="lang" aria-label="Language"><button type="button" data-set="en" aria-pressed="true">EN</button><button type="button" data-set="pl" aria-pressed="false">PL</button></nav> | |
| <main> | |
| <div class="l-en" lang="en"> | |
| <header> | |
| <h1>Mission – Slayer 1B</h1> | |
| <p class="by">Arkadiusz Słota – Fabryka AI (SlayerLab)</p> | |
| <p class="lead">We are building a Polish language model of ~1 billion parameters from scratch and improving it through | |
| checkpointed continuation. Released model weights are public; resumable checkpoint archives are published | |
| separately. Follow each archive link for its current files. Training progress and measured evaluation results are reported separately below.</p> | |
| <div class="pills"> | |
| <span class="pill"><b>1,073,912,851</b> parameters</span> | |
| <span class="pill">tokenizer <b>32,768</b>, Polish</span> | |
| <span class="pill">context <b>4096</b> tokens</span> | |
| <span class="pill">data: ~82% Polish, ~18% English</span> | |
| <span class="pill">released weights: <b>Apache-2.0</b></span> | |
| </div> | |
| </header> | |
| <section> | |
| <h2>Versions</h2> | |
| <div class="cards"> | |
| <div class="card"> | |
| <h3>Slayer1B 1.00 <span class="tag dec">decayed</span></h3> | |
| <div class="meta">1.5 B training tokens</div> | |
| <a href="https://huggingface.co/SlayerLab/Slayer1B-1.00">model</a> · | |
| <a href="https://huggingface.co/SlayerLab/Slayer1B-1.00-checkpoints">checkpoints</a> · | |
| <a href="https://track.fabryka.ai/run/48744ed2-3155-4294-b899-a6fda1f851da">run</a> | |
| </div> | |
| <div class="card"> | |
| <h3>Slayer1B 1.01 <span class="tag dec">decayed</span></h3> | |
| <div class="meta">3.0 B tokens · continued from 1.00</div> | |
| <a href="https://huggingface.co/SlayerLab/Slayer1B-1.01">model</a> · | |
| <a href="https://huggingface.co/SlayerLab/Slayer1B-1.01-checkpoints">checkpoints</a> · | |
| <a href="https://track.fabryka.ai/run/4934429a-d393-42e0-8aac-03eae6b71ff7">run</a> | |
| </div> | |
| <div class="card"> | |
| <h3>Slayer1B 1.02 <span class="tag const">constant LR</span></h3> | |
| <div class="meta">6.0 B tokens · continued from 1.01</div> | |
| <a href="https://huggingface.co/SlayerLab/Slayer1B-1.02">model</a> · | |
| <a href="https://huggingface.co/SlayerLab/Slayer1B-1.02-checkpoints">checkpoints</a> · | |
| <a href="https://track.fabryka.ai/run/21ae1234-82c2-42b7-887b-cbcc69901ae1">run</a> | |
| </div> | |
| <div class="card card-status"> | |
| <h3>Slayer1B 1.03 <span class="tag const">training complete</span></h3> | |
| <div class="meta">native step 7150 · training loss 2.3955</div> | |
| <p>Resume 6750 → 7150 completed on 4×L40S. The final 7150 model weights and lm-eval benchmark results are public.</p> | |
| <p class="note">Checkpoint SHA-256: <code class="sha">d7cad9673f3ee706f3022ad6a97fa7593604806d4d25ba936c27885a8f115d97</code></p> | |
| <a href="https://huggingface.co/SlayerLab/Slayer1B-1.03">model</a> · | |
| <a href="https://track.fabryka.ai/run/7ca4f84f-3b07-4525-8cde-f3a936dc3425">completed training run</a> · | |
| <a href="https://track.fabryka.ai/run/1426c0a8-2f07-5a03-80fc-92f7e922aa4b">checkpoint benchmarks</a> · | |
| <a href="https://huggingface.co/SlayerLab/Slayer1B-1.03-checkpoints">native checkpoint archive — current files</a> | |
| </div> | |
| <div class="card card-status"> | |
| <h3>Slayer1B 1.04 <span class="tag const">released</span></h3> | |
| <div class="meta">native step 10010 · 10.5 B training tokens</div> | |
| <p>Corpus continuation 7150 → 10010 on 4×L40S, constant learning rate. The run then moved to 8×L40S at this checkpoint | |
| after independent SHA-256 verification (three sources) and a proof that the global batch is reproduced.</p> | |
| <p class="note">Checkpoint SHA-256: <code class="sha">04438959d7b317d1a8d972b264e9610a06fd8addde7349c6e9f458ee56dc1e51</code></p> | |
| <a href="https://huggingface.co/SlayerLab/Slayer1B-1.04">model</a> · | |
| <a href="https://huggingface.co/SlayerLab/Slayer1B-1.04-checkpoints">checkpoints</a> · | |
| <a href="https://track.fabryka.ai/run/f056bb5c-f1da-56d5-9776-cc33063f10a5">run</a> | |
| </div> | |
| <div class="card card-status"> | |
| <h3>Slayer1B 1.05 <span class="tag const">released</span></h3> | |
| <div class="meta">native step 12870 · 13.5 B training tokens</div> | |
| <p>Continuation 10010 → 12870 on 8×L40S, same model, data order and global batch, constant learning rate.</p> | |
| <p class="note">Checkpoint SHA-256: <code class="sha">435082ff36becfbb96803622139f38d2afa0ba1a9d2fc284bad96f5a8edc1181</code></p> | |
| <a href="https://huggingface.co/SlayerLab/Slayer1B-1.05">model</a> · | |
| <a href="https://huggingface.co/SlayerLab/Slayer1B-1.05-checkpoints">checkpoints</a> · | |
| <a href="https://track.fabryka.ai/run/f056bb5c-f1da-56d5-9776-cc33063f10a5">run</a> | |
| </div> | |
| <div class="card card-status"> | |
| <h3>Slayer1B 1.06 <span class="tag const">released · full corpus</span></h3> | |
| <div class="meta">native step 13848 · 14.5 B training tokens · one full pass</div> | |
| <p>Continuation 12870 → 13848 on 8×L40S. It ends where the prepared corpus ends: every token of the corpus was seen | |
| exactly once (measured, not planned). Final training loss 2.276; no learning-rate decay yet.</p> | |
| <p class="note">Checkpoint SHA-256: <code class="sha">1fc7396ca1790805128244b2f827e02b9fe88e352534cd887d0c488f121d1e4d</code></p> | |
| <a href="https://huggingface.co/SlayerLab/Slayer1B-1.06">model</a> · | |
| <a href="https://huggingface.co/SlayerLab/Slayer1B-1.06-checkpoints">checkpoints</a> · | |
| <a href="https://track.fabryka.ai/run/f056bb5c-f1da-56d5-9776-cc33063f10a5">run</a> | |
| </div> | |
| </div> | |
| </section> | |
| <section> | |
| <h2>Results</h2> | |
| <div class="wrap"> | |
| <table> | |
| <tr><th>test</th><th>1.00</th><th>1.01</th><th>1.02</th><th>1.03</th><th>1.04</th><th>1.05</th><th>1.06</th></tr> | |
| <tr><td>training tokens</td><td>1.5 B</td><td>3.0 B</td><td>6.0 B</td><td>7.5 B</td><td>10.5 B</td><td>13.5 B</td><td>14.5 B</td></tr> | |
| <tr><td>native step</td><td>1430</td><td>2860</td><td>5720</td><td>7150</td><td>10010</td><td>12870</td><td>13848</td></tr> | |
| <tr><td>ARC-Easy (EN), %</td><td>42.89</td><td>47.98</td><td>48.99</td><td>—</td><td>50.59</td><td>52.19</td><td class="best">52.99</td></tr> | |
| <tr><td>BLiMP (EN, grammar), %</td><td>71.51</td><td>73.63</td><td>74.71</td><td>—</td><td>74.17</td><td>75.53</td><td class="best">76.12</td></tr> | |
| <tr><td>WikiText-2, byte perplexity ↓</td><td>2.5857</td><td>2.4230</td><td>2.3735</td><td>—</td><td>2.3078</td><td class="best">2.2816</td><td>2.2849</td></tr> | |
| <tr><td>MultiBLiMP (PL, grammar), %</td><td>97.89</td><td>98.29</td><td>98.38</td><td>—</td><td class="best">98.87</td><td>98.47</td><td>98.41</td></tr> | |
| <tr><td>ARC-Easy-PL, %</td><td>36.91</td><td>40.36</td><td>41.84</td><td>—</td><td>44.02</td><td class="best">45.20</td><td>45.03</td></tr> | |
| </table> | |
| </div> | |
| <p class="note">One instrument for the versions that have values (1.00–1.02 and 1.04–1.06): leaderboard protocol of | |
| <a href="https://track.fabryka.ai/models">track.fabryka.ai/models</a>, zero-shot, on the released checkpoints. Column 1.03 is a dash on purpose: its run | |
| (<a href="https://track.fabryka.ai/run/7ca4f84f-3b07-4525-8cde-f3a936dc3425">1.03</a>) was measured with the older lm-eval task set | |
| (hellaswag, piqa, lambada, arc_challenge, boolq, mmlu — see the additional history section), so there are no values in this instrument; | |
| the dash is not a missing measurement. 1.00 and 1.01 ended with a learning-rate decay; 1.02 and 1.04–1.06 did not, so comparisons | |
| across that boundary are not clean. Neighbouring checkpoints differ by up to about a point on single tests, so small differences | |
| between 1.05 and 1.06 are within run noise.</p> | |
| </section> | |
| <section> | |
| <h2>Additional benchmark history</h2> | |
| <p><a href="https://track.fabryka.ai/run/1426c0a8-2f07-5a03-80fc-92f7e922aa4b">16 checkpoints from 1.00–1.03, through final 7150</a> | |
| have separate lm-eval 0.4.13, zero-shot, fp32 results: 13 historical points plus 6900, 7050 and 7150. | |
| These results use a different benchmark procedure from the table above.</p> | |
| <p class="note">Accuracy and normalized accuracy are distinct series, in fractions (0–1); LAMBADA perplexity is separate. | |
| The scan found n-gram overlap in shared/en for BoolQ, MMLU and ARC; its significance is still being assessed. | |
| Overlap is not by itself a measured contamination rate. Open Charts and filter by <code>eval/</code>. | |
| This is an imported result series, not a native automatic-benchmark job.</p> | |
| <p class="note">The three new points use the same runner and task procedure, but a different runtime: | |
| Torch 2.8.0+cu128, tokenizers 0.23.2 and NumPy 2.2.6, versus 2.14.1+cu126, 0.23.3 and 2.5.3 for the historical points. | |
| The 6750 → 6900 boundary is not an identical-environment checkpoint comparison.</p> | |
| </section> | |
| <section> | |
| <h2>Score curves: 1.00–1.06</h2> | |
| <figure><img src="scores-history-106.png" alt="Evaluation history of Slayer1B 1.00 to 1.06 versus training tokens"></figure> | |
| <p class="note">Measured values only, no fits, no trend lines. 1.00–1.02: leaderboard protocol, one checkpoint every 150 steps; shaded bands mark the learning-rate decay of 1.00 and 1.01. 1.03: training loss only — it was evaluated with an older lm-eval task set, so it has no points in the score panels. 1.04–1.06: one continuation from step 7150 at a constant learning rate; diamonds mark the releases 1.04 (10010), 1.05 (12870) and 1.06 (13848, end of the corpus). The in-run evaluator measured every 50 steps up to 9100, then control points 9400/9700/10000, then every 200 steps; breaks in the line are gaps in that schedule, not missing training. eff = (BLiMP + ARC + WikiScore) / 3, the board formula, which reproduces every published 1.00–1.02 value within 0.01. Data: <a href="scores-history-106.csv">scores-history-106.csv</a>.</p> | |
| </section> | |
| <section> | |
| <h2>Release points: 1.04 → 1.06</h2> | |
| <div class="wrap"> | |
| <table> | |
| <tr><th>test</th><th>1.04 · 10010</th><th>1.05 · 12870</th><th>1.06 · 13848</th></tr> | |
| <tr><td>BLiMP (EN, grammar), %</td><td>74.17</td><td>75.53</td><td class="best">76.12</td></tr> | |
| <tr><td>ARC-Easy (EN), %</td><td>50.59</td><td>52.19</td><td class="best">52.99</td></tr> | |
| <tr><td>WikiText-2, byte perplexity ↓</td><td>2.3078</td><td class="best">2.2816</td><td>2.2849</td></tr> | |
| <tr><td>WikiText-2, token perplexity ↓</td><td>31.29</td><td class="best">29.86</td><td>30.03</td></tr> | |
| <tr><td>MultiBLiMP (PL, grammar), %</td><td class="best">98.87</td><td>98.47</td><td>98.41</td></tr> | |
| <tr><td>ARC-Easy-PL, %</td><td>44.02</td><td class="best">45.20</td><td>45.03</td></tr> | |
| </table> | |
| </div> | |
| <p class="note">Measured on the released checkpoints by the run's automatic evaluator (same instrument as the results table), read | |
| from the <a href="https://track.fabryka.ai/run/f056bb5c-f1da-56d5-9776-cc33063f10a5">public continuation run</a>. The whole continuation | |
| used a constant learning rate; a final decay has not been applied, so these are not end-of-training numbers for the model line.</p> | |
| </section> | |
| <section> | |
| <h2>How we do it</h2> | |
| <ol class="steps"> | |
| <li><b>From scratch.</b> A transformer (20 layers, width 2048), our own Polish tokenizer, Polish data with an English | |
| minority (<a href="https://huggingface.co/datasets/SlayerLab/slayer-pl-8x3b">slayer-pl-8x3b</a>).</li> | |
| <li><b>Checkpointed continuation.</b> Earlier decayed versions branched from checkpoints before their final decay. | |
| Version names mark milestones in one global data trajectory, not mandatory process restarts. | |
| Retention maintenance resumed from verified checkpoint 7750 without resetting the data order. | |
| The final learning-rate decay is planned once, at the end.</li> | |
| <li><b>Source and model units.</b> The source corpus is counted as ~24 B <code>cl100k</code> tokens. | |
| Our v6/32k tokenizer produces separate <code>uint16</code> train/validation files. Actual model-token counts and | |
| stream boundaries come from pinned manifests. Next packs are prepared while the current stage trains.</li> | |
| <li><b>We verify everything.</b> Checkpoint SHA-256 records are preserved outside the training machine. | |
| Before local pruning, a full copy on a separate archive is verified and its receipt is persisted atomically. | |
| From version 1.01 on, a loss forecast is written down before each run and evaluated afterwards.</li> | |
| </ol> | |
| <p class="note">Technical report: | |
| <a href="https://github.com/slayerlabs/Fabryka-AI-Notes/tree/main/papers/slayerlabs/2026-10-23_slayer1b-continuation">Slayer1B: Continuing a Polish-Dominant Base Model</a>.</p> | |
| </section> | |
| <section> | |
| <h2>What's next</h2> | |
| <p><b>Corpus continuation 1.04–1.06 — complete.</b> The global trajectory ran from native step 7150 to 13848, the end of the | |
| prepared corpus: 14.5 B training tokens, one full pass, about 13.5 tokens per parameter. Releases 1.04 (10010), 1.05 (12870) and | |
| 1.06 (13848) are public, each with its checkpoint archive. <a href="https://track.fabryka.ai/run/f056bb5c-f1da-56d5-9776-cc33063f10a5">Training run</a>.</p> | |
| <p class="note">The linked runs belong to the author's Track account. The completed 1.03 run includes its resumed steps; | |
| corpus continuation and benchmark history are separate records of the same model lineage.</p> | |
| <p><b>Beyond 13848 — roadmap candidates, not scheduled runs:</b> continuing to about 20 tokens per parameter with fresh data, | |
| a final learning-rate decay, and extending the context window in stages (first 32k tokens, up to 256k as the target), each stage | |
| checked with long-context tests. Each follow-up needs its own plan, cost estimate and go-ahead.</p> | |
| <p class="note">These are base models: they predict the continuation of a text, they were not tuned for chat or for | |
| following instructions, and they can be wrong.</p> | |
| </section> | |
| <footer>Arkadiusz Słota – Fabryka AI (SlayerLab) · models: <a href="https://huggingface.co/SlayerLab">huggingface.co/SlayerLab</a></footer> | |
| </div> | |
| <div class="l-pl" lang="pl"> | |
| <header> | |
| <h1>Misja – Slayer 1B</h1> | |
| <p class="by">Arkadiusz Słota – Fabryka AI (SlayerLab)</p> | |
| <p class="lead">Budujemy od zera polski model językowy wielkości ~1 mld parametrów i ulepszamy go przez kontynuację | |
| z punktów kontrolnych. Wagi wydanych modeli są publiczne; archiwa checkpointów do wznowienia publikujemy | |
| osobno. Linki archiwów pokazują ich aktualne pliki. Postęp treningu i zmierzone wyniki ewaluacji pokazujemy poniżej oddzielnie.</p> | |
| <div class="pills"> | |
| <span class="pill"><b>1 073 912 851</b> parametrów</span> | |
| <span class="pill">tokenizer <b>32 768</b>, polski</span> | |
| <span class="pill">kontekst <b>4096</b> tokenów</span> | |
| <span class="pill">dane: ~82 % PL, ~18 % EN</span> | |
| <span class="pill">opublikowane wagi: <b>Apache-2.0</b></span> | |
| </div> | |
| </header> | |
| <section> | |
| <h2>Wersje</h2> | |
| <div class="cards"> | |
| <div class="card"> | |
| <h3>Slayer1B 1.00 <span class="tag dec">wygaszony</span></h3> | |
| <div class="meta">1,5 mld tokenów treningu</div> | |
| <a href="https://huggingface.co/SlayerLab/Slayer1B-1.00">model</a> · | |
| <a href="https://huggingface.co/SlayerLab/Slayer1B-1.00-checkpoints">punkty kontrolne</a> · | |
| <a href="https://track.fabryka.ai/run/48744ed2-3155-4294-b899-a6fda1f851da">przebieg</a> | |
| </div> | |
| <div class="card"> | |
| <h3>Slayer1B 1.01 <span class="tag dec">wygaszony</span></h3> | |
| <div class="meta">3,0 mld tokenów · dalszy trening 1.00</div> | |
| <a href="https://huggingface.co/SlayerLab/Slayer1B-1.01">model</a> · | |
| <a href="https://huggingface.co/SlayerLab/Slayer1B-1.01-checkpoints">punkty kontrolne</a> · | |
| <a href="https://track.fabryka.ai/run/4934429a-d393-42e0-8aac-03eae6b71ff7">przebieg</a> | |
| </div> | |
| <div class="card"> | |
| <h3>Slayer1B 1.02 <span class="tag const">stały LR</span></h3> | |
| <div class="meta">6,0 mld tokenów · dalszy trening 1.01</div> | |
| <a href="https://huggingface.co/SlayerLab/Slayer1B-1.02">model</a> · | |
| <a href="https://huggingface.co/SlayerLab/Slayer1B-1.02-checkpoints">punkty kontrolne</a> · | |
| <a href="https://track.fabryka.ai/run/21ae1234-82c2-42b7-887b-cbcc69901ae1">przebieg</a> | |
| </div> | |
| <div class="card card-status"> | |
| <h3>Slayer1B 1.03 <span class="tag const">trening zakończony</span></h3> | |
| <div class="meta">krok native 7150 · strata treningowa 2,3955</div> | |
| <p>Wznowienie 6750 → 7150 zakończone na 4×L40S. Wagi końcowego modelu 7150 i wyniki benchmarków lm-eval są publiczne.</p> | |
| <p class="note">SHA-256 checkpointu: <code class="sha">d7cad9673f3ee706f3022ad6a97fa7593604806d4d25ba936c27885a8f115d97</code></p> | |
| <a href="https://huggingface.co/SlayerLab/Slayer1B-1.03">model</a> · | |
| <a href="https://track.fabryka.ai/run/7ca4f84f-3b07-4525-8cde-f3a936dc3425">zakończony przebieg treningu</a> · | |
| <a href="https://track.fabryka.ai/run/1426c0a8-2f07-5a03-80fc-92f7e922aa4b">benchmarki checkpointów</a> · | |
| <a href="https://huggingface.co/SlayerLab/Slayer1B-1.03-checkpoints">archiwum checkpointów natywnych — aktualne pliki</a> | |
| </div> | |
| <div class="card card-status"> | |
| <h3>Slayer1B 1.04 <span class="tag const">wydany</span></h3> | |
| <div class="meta">krok native 10010 · 10,5 mld tokenów treningu</div> | |
| <p>Kontynuacja korpusu 7150 → 10010 na 4×L40S, stałe tempo uczenia. Na tym checkpoincie bieg przeszedł na 8×L40S | |
| po niezależnej weryfikacji SHA-256 (trzy źródła) i dowodzie, że globalny batch jest odtworzony.</p> | |
| <p class="note">SHA-256 checkpointu: <code class="sha">04438959d7b317d1a8d972b264e9610a06fd8addde7349c6e9f458ee56dc1e51</code></p> | |
| <a href="https://huggingface.co/SlayerLab/Slayer1B-1.04">model</a> · | |
| <a href="https://huggingface.co/SlayerLab/Slayer1B-1.04-checkpoints">punkty kontrolne</a> · | |
| <a href="https://track.fabryka.ai/run/f056bb5c-f1da-56d5-9776-cc33063f10a5">przebieg</a> | |
| </div> | |
| <div class="card card-status"> | |
| <h3>Slayer1B 1.05 <span class="tag const">wydany</span></h3> | |
| <div class="meta">krok native 12870 · 13,5 mld tokenów treningu</div> | |
| <p>Kontynuacja 10010 → 12870 na 8×L40S: ten sam model, kolejność danych i globalny batch, stałe tempo uczenia.</p> | |
| <p class="note">SHA-256 checkpointu: <code class="sha">435082ff36becfbb96803622139f38d2afa0ba1a9d2fc284bad96f5a8edc1181</code></p> | |
| <a href="https://huggingface.co/SlayerLab/Slayer1B-1.05">model</a> · | |
| <a href="https://huggingface.co/SlayerLab/Slayer1B-1.05-checkpoints">punkty kontrolne</a> · | |
| <a href="https://track.fabryka.ai/run/f056bb5c-f1da-56d5-9776-cc33063f10a5">przebieg</a> | |
| </div> | |
| <div class="card card-status"> | |
| <h3>Slayer1B 1.06 <span class="tag const">wydany · cały korpus</span></h3> | |
| <div class="meta">krok native 13848 · 14,5 mld tokenów treningu · jedno pełne przejście</div> | |
| <p>Kontynuacja 12870 → 13848 na 8×L40S. Kończy się tam, gdzie kończy się przygotowany korpus: każdy jego token model | |
| zobaczył dokładnie raz (zmierzone, nie zaplanowane). Końcowa strata treningowa 2,276; bez wygaszania tempa uczenia.</p> | |
| <p class="note">SHA-256 checkpointu: <code class="sha">1fc7396ca1790805128244b2f827e02b9fe88e352534cd887d0c488f121d1e4d</code></p> | |
| <a href="https://huggingface.co/SlayerLab/Slayer1B-1.06">model</a> · | |
| <a href="https://huggingface.co/SlayerLab/Slayer1B-1.06-checkpoints">punkty kontrolne</a> · | |
| <a href="https://track.fabryka.ai/run/f056bb5c-f1da-56d5-9776-cc33063f10a5">przebieg</a> | |
| </div> | |
| </div> | |
| </section> | |
| <section> | |
| <h2>Wyniki</h2> | |
| <div class="wrap"> | |
| <table> | |
| <tr><th>test</th><th>1.00</th><th>1.01</th><th>1.02</th><th>1.03</th><th>1.04</th><th>1.05</th><th>1.06</th></tr> | |
| <tr><td>tokeny treningu</td><td>1,5 mld</td><td>3,0 mld</td><td>6,0 mld</td><td>7,5 mld</td><td>10,5 mld</td><td>13,5 mld</td><td>14,5 mld</td></tr> | |
| <tr><td>krok native</td><td>1430</td><td>2860</td><td>5720</td><td>7150</td><td>10010</td><td>12870</td><td>13848</td></tr> | |
| <tr><td>ARC-Easy (EN), %</td><td>42,89</td><td>47,98</td><td>48,99</td><td>—</td><td>50,59</td><td>52,19</td><td class="best">52,99</td></tr> | |
| <tr><td>BLiMP (EN, gramatyka), %</td><td>71,51</td><td>73,63</td><td>74,71</td><td>—</td><td>74,17</td><td>75,53</td><td class="best">76,12</td></tr> | |
| <tr><td>WikiText-2, perpleksja na bajt ↓</td><td>2,5857</td><td>2,4230</td><td>2,3735</td><td>—</td><td>2,3078</td><td class="best">2,2816</td><td>2,2849</td></tr> | |
| <tr><td>MultiBLiMP (PL, gramatyka), %</td><td>97,89</td><td>98,29</td><td>98,38</td><td>—</td><td class="best">98,87</td><td>98,47</td><td>98,41</td></tr> | |
| <tr><td>ARC-Easy-PL, %</td><td>36,91</td><td>40,36</td><td>41,84</td><td>—</td><td>44,02</td><td class="best">45,20</td><td>45,03</td></tr> | |
| </table> | |
| </div> | |
| <p class="note">Jeden przyrząd dla wersji, które mają wartości (1.00–1.02 oraz 1.04–1.06): protokół tablicy | |
| <a href="https://track.fabryka.ai/models">track.fabryka.ai/models</a>, zero-shot, na wydanych checkpointach. Kolumna 1.03 ma kreskę celowo: jej run | |
| (<a href="https://track.fabryka.ai/run/7ca4f84f-3b07-4525-8cde-f3a936dc3425">1.03</a>) mierzono starszym zestawem lm-eval | |
| (hellaswag, piqa, lambada, arc_challenge, boolq, mmlu — patrz sekcja dodatkowej historii), więc w tym przyrządzie nie ma wartości; | |
| kreska nie oznacza brakującego pomiaru. 1.00 i 1.01 kończyły się wygaszaniem tempa uczenia, 1.02 i 1.04–1.06 nie — porównania | |
| przez tę granicę nie są „czyste”. Sąsiednie checkpointy różnią się na pojedynczych testach nawet o ok. punkt, więc małe różnice | |
| między 1.05 a 1.06 mieszczą się w szumie biegu.</p> | |
| </section> | |
| <section> | |
| <h2>Dodatkowa historia benchmarków</h2> | |
| <p><a href="https://track.fabryka.ai/run/1426c0a8-2f07-5a03-80fc-92f7e922aa4b">16 checkpointów 1.00–1.03, do końcowego 7150</a> | |
| ma osobne wyniki lm-eval 0.4.13, zero-shot, fp32: 13 historycznych punktów oraz 6900, 7050 i 7150. | |
| To inna procedura benchmarkowa niż w tabeli powyżej.</p> | |
| <p class="note">Trafność i trafność normalizowana to osobne serie, w ułamkach (0–1); perpleksja LAMBADA jest oddzielnie. | |
| Skan wykrył nakładkę n-gramową w shared/en dla BoolQ, MMLU i ARC; ocena jej znaczenia nadal trwa. | |
| Sama nakładka nie jest pomiarem odsetka rzeczywistego przecieku. W Charts wpisz w filtrze <code>eval/</code>. | |
| To seria zaimportowanych wyników, nie natywny job automatycznej ewaluacji.</p> | |
| <p class="note">Trzy nowe punkty mają ten sam runner i procedurę zadań, ale inne środowisko: | |
| Torch 2.8.0+cu128, tokenizers 0.23.2 i NumPy 2.2.6 zamiast 2.14.1+cu126, 0.23.3 i 2.5.3 dla punktów historycznych. | |
| Granica 6750 → 6900 nie jest porównaniem checkpointów w identycznym środowisku.</p> | |
| </section> | |
| <section> | |
| <h2>Krzywe wyników: 1.00–1.06</h2> | |
| <figure><img src="scores-history-106.png" alt="Historia wyników Slayer1B od 1.00 do 1.06 w funkcji liczby tokenów treningu"></figure> | |
| <p class="note">Same pomiary, bez dopasowań i bez linii trendu. 1.00–1.02: protokół tablicy, punkt co 150 kroków; pasma oznaczają wygaszanie LR w 1.00 i 1.01. 1.03: tylko strata treningowa — mierzony starszym zestawem lm-eval, więc w panelach wyników nie ma punktów. 1.04–1.06: jedna kontynuacja od kroku 7150 przy stałym LR; romby oznaczają wydania 1.04 (10010), 1.05 (12870) i 1.06 (13848, koniec korpusu). Ewaluator w biegu mierzył co 50 kroków do 9100, potem punkty kontrolne 9400/9700/10000, potem co 200 kroków; przerwy w linii to luki w tym harmonogramie, nie brakujący trening. eff = (BLiMP + ARC + WikiScore) / 3, formuła tablicy, która odtwarza każdą opublikowaną wartość 1.00–1.02 z dokładnością 0,01. Dane: <a href="scores-history-106.csv">scores-history-106.csv</a>.</p> | |
| </section> | |
| <section> | |
| <h2>Punkty wydań: 1.04 → 1.06</h2> | |
| <div class="wrap"> | |
| <table> | |
| <tr><th>test</th><th>1.04 · 10010</th><th>1.05 · 12870</th><th>1.06 · 13848</th></tr> | |
| <tr><td>BLiMP (EN, gramatyka), %</td><td>74,17</td><td>75,53</td><td class="best">76,12</td></tr> | |
| <tr><td>ARC-Easy (EN), %</td><td>50,59</td><td>52,19</td><td class="best">52,99</td></tr> | |
| <tr><td>WikiText-2, perpleksja na bajt ↓</td><td>2,3078</td><td class="best">2,2816</td><td>2,2849</td></tr> | |
| <tr><td>WikiText-2, perpleksja na token ↓</td><td>31,29</td><td class="best">29,86</td><td>30,03</td></tr> | |
| <tr><td>MultiBLiMP (PL, gramatyka), %</td><td class="best">98,87</td><td>98,47</td><td>98,41</td></tr> | |
| <tr><td>ARC-Easy-PL, %</td><td>44,02</td><td class="best">45,20</td><td>45,03</td></tr> | |
| </table> | |
| </div> | |
| <p class="note">Zmierzone na wydanych checkpointach automatycznym ewaluatorem biegu (ten sam przyrząd co tabela wyników), odczytane | |
| z <a href="https://track.fabryka.ai/run/f056bb5c-f1da-56d5-9776-cc33063f10a5">publicznego biegu kontynuacji</a>. Cała kontynuacja | |
| szła przy stałym tempie uczenia; końcowego wygaszania nie zastosowano, więc to nie są końcowe liczby tej linii modelu.</p> | |
| </section> | |
| <section> | |
| <h2>Jak to robimy</h2> | |
| <ol class="steps"> | |
| <li><b>Od zera.</b> Architektura typu transformer (20 warstw, szerokość 2048), własny polski tokenizer, polskie dane | |
| z mniejszością angielską (<a href="https://huggingface.co/datasets/SlayerLab/slayer-pl-8x3b">slayer-pl-8x3b</a>).</li> | |
| <li><b>Kontynuacja z checkpointów.</b> Wcześniejsze wygaszane wersje odgałęziały się sprzed końcowego wygaszania. | |
| Nazwy wersji oznaczają etapy jednej globalnej trajektorii danych, nie obowiązkowe restarty procesu. | |
| Konserwacja retencji wznowiła trening ze zweryfikowanego checkpointu 7750 bez zerowania kolejności danych. | |
| Końcowe wygaszanie tempa uczenia planujemy raz, na końcu.</li> | |
| <li><b>Jednostki źródła i modelu.</b> Około 24 mld tokenów źródła liczymy przez <code>cl100k</code>. | |
| Nasz tokenizer v6/32k tworzy osobne pliki train/val w <code>uint16</code>. Liczby tokenów modelu i granice | |
| strumieni pochodzą z przypiętych manifestów. Następne wsady przygotowujemy podczas bieżącego treningu.</li> | |
| <li><b>Wszystko sprawdzamy.</b> Sumy SHA-256 checkpointów zachowujemy poza maszyną treningową. | |
| Przed lokalnym usunięciem pełna kopia na osobnym archiwum jest weryfikowana, a jej receipt utrwalany atomowo. | |
| Od wersji 1.01 przed każdym biegiem zapisujemy prognozę straty i potem ją oceniamy.</li> | |
| </ol> | |
| <p class="note">Raport techniczny: | |
| <a href="https://github.com/slayerlabs/Fabryka-AI-Notes/tree/main/papers/slayerlabs/2026-10-23_slayer1b-continuation">Slayer1B: Continuing a Polish-Dominant Base Model</a>.</p> | |
| </section> | |
| <section> | |
| <h2>Co dalej</h2> | |
| <p><b>Kontynuacja korpusu 1.04–1.06 — zakończona.</b> Globalna trajektoria przeszła od kroku native 7150 do 13848, czyli do | |
| końca przygotowanego korpusu: 14,5 mld tokenów treningu, jedno pełne przejście, ok. 13,5 tokenu na parametr. Wydania 1.04 (10010), | |
| 1.05 (12870) i 1.06 (13848) są publiczne, każde z archiwum checkpointów. <a href="https://track.fabryka.ai/run/f056bb5c-f1da-56d5-9776-cc33063f10a5">Bieg treningu</a>.</p> | |
| <p class="note">Powiązane przebiegi są na koncie autora w Tracku. Zakończony run 1.03 obejmuje wznowione kroki; | |
| dalszy trening i historia benchmarków to osobne zapisy tej samej linii modelu.</p> | |
| <p><b>Po 13848 — kandydaci roadmapy, nie zaplanowane biegi:</b> dalszy trening do ok. 20 tokenów na parametr na świeżych danych, | |
| końcowe wygaszanie tempa uczenia oraz etapami wydłużanie okna kontekstu (najpierw 32k tokenów, docelowo do 256k), każdy etap | |
| sprawdzany testami długiego kontekstu. Każdy z tych kroków wymaga własnego planu, wyceny i zgody.</p> | |
| <p class="note">To modele bazowe: przewidują dalszy ciąg tekstu, nie były dostrajane do rozmowy ani wykonywania poleceń | |
| i mogą się mylić.</p> | |
| </section> | |
| <footer>Arkadiusz Słota – Fabryka AI (SlayerLab) · modele: <a href="https://huggingface.co/SlayerLab">huggingface.co/SlayerLab</a></footer> | |
| </div> | |
| </main> | |
| <script> | |
| (function () { | |
| var q = new URLSearchParams(location.search).get("lang"); | |
| var saved = null; try { saved = localStorage.getItem("slayer-lang"); } catch (e) {} | |
| function set(l) { | |
| if (l !== "pl") l = "en"; | |
| document.body.setAttribute("data-lang", l); | |
| document.documentElement.lang = l; | |
| document.title = l === "pl" ? "Misja – Slayer 1B" : "Mission – Slayer 1B"; | |
| document.querySelectorAll(".lang button").forEach(function (b) { b.setAttribute("aria-pressed", String(b.dataset.set === l)); }); | |
| try { localStorage.setItem("slayer-lang", l); } catch (e) {} | |
| } | |
| document.querySelectorAll(".lang button").forEach(function (b) { b.addEventListener("click", function () { set(b.dataset.set); }); }); | |
| set(q || saved || "en"); | |
| })(); | |
| </script> | |
| </body> | |
| </html> | |