Document pretraining mix, founder authorship and version-specific Bielik references
Browse files- .gitattributes +1 -0
- BIELIK_COMPARISON.json +106 -0
- BIELIK_COMPARISON.md +27 -0
- DATA_CARD.md +3 -1
- EXTERNAL_REFERENCES.json +37 -0
- EXTERNAL_REFERENCES.md +17 -0
- README.md +15 -8
- TRAINING_MIX.json +9 -0
- TRAINING_MIX.md +34 -0
- plots/hellaswag-progress.png +2 -2
- plots/hellaswag-progress.svg +672 -330
- plots/plot_benchmarks.py +10 -4
- plots/plot_training_mix.py +31 -0
- plots/training-mix.pdf +0 -0
- plots/training-mix.png +3 -0
- plots/training-mix.svg +2775 -0
.gitattributes
CHANGED
|
@@ -45,3 +45,4 @@ plots/openbookqa.png filter=lfs diff=lfs merge=lfs -text
|
|
| 45 |
plots/piqa.png filter=lfs diff=lfs merge=lfs -text
|
| 46 |
plots/winogrande.png filter=lfs diff=lfs merge=lfs -text
|
| 47 |
plots/hellaswag-progress.png filter=lfs diff=lfs merge=lfs -text
|
|
|
|
|
|
| 45 |
plots/piqa.png filter=lfs diff=lfs merge=lfs -text
|
| 46 |
plots/winogrande.png filter=lfs diff=lfs merge=lfs -text
|
| 47 |
plots/hellaswag-progress.png filter=lfs diff=lfs merge=lfs -text
|
| 48 |
+
plots/training-mix.png filter=lfs diff=lfs merge=lfs -text
|
BIELIK_COMPARISON.json
ADDED
|
@@ -0,0 +1,106 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"reference_model": "speakleash/Bielik-11B-v3.1-Instruct",
|
| 3 |
+
"reference_revision": "d6e755d356ad418d0f0e30b8c428db267415ddb9",
|
| 4 |
+
"units": "percent; differences in percentage points",
|
| 5 |
+
"records": [
|
| 6 |
+
{
|
| 7 |
+
"benchmark": "arc_challenge",
|
| 8 |
+
"metric": "acc_norm",
|
| 9 |
+
"slayer_update": 45000,
|
| 10 |
+
"slayer_score": 34.8123,
|
| 11 |
+
"bielik_v31_score": null,
|
| 12 |
+
"gap_pp": null,
|
| 13 |
+
"status": "matching v3.1 result not located"
|
| 14 |
+
},
|
| 15 |
+
{
|
| 16 |
+
"benchmark": "hellaswag",
|
| 17 |
+
"metric": "acc_norm",
|
| 18 |
+
"slayer_update": 45000,
|
| 19 |
+
"slayer_score": 53.6547,
|
| 20 |
+
"bielik_v31_score": null,
|
| 21 |
+
"gap_pp": null,
|
| 22 |
+
"status": "matching v3.1 result not located"
|
| 23 |
+
},
|
| 24 |
+
{
|
| 25 |
+
"benchmark": "piqa",
|
| 26 |
+
"metric": "acc_norm",
|
| 27 |
+
"slayer_update": 45000,
|
| 28 |
+
"slayer_score": 70.29379999999999,
|
| 29 |
+
"bielik_v31_score": null,
|
| 30 |
+
"gap_pp": null,
|
| 31 |
+
"status": "matching v3.1 result not located"
|
| 32 |
+
},
|
| 33 |
+
{
|
| 34 |
+
"benchmark": "winogrande",
|
| 35 |
+
"metric": "acc",
|
| 36 |
+
"slayer_update": 45000,
|
| 37 |
+
"slayer_score": 59.037099999999995,
|
| 38 |
+
"bielik_v31_score": null,
|
| 39 |
+
"gap_pp": null,
|
| 40 |
+
"status": "matching v3.1 result not located"
|
| 41 |
+
},
|
| 42 |
+
{
|
| 43 |
+
"benchmark": "boolq",
|
| 44 |
+
"metric": "acc",
|
| 45 |
+
"slayer_update": 45000,
|
| 46 |
+
"slayer_score": 61.9266,
|
| 47 |
+
"bielik_v31_score": null,
|
| 48 |
+
"gap_pp": null,
|
| 49 |
+
"status": "matching v3.1 result not located"
|
| 50 |
+
},
|
| 51 |
+
{
|
| 52 |
+
"benchmark": "openbookqa",
|
| 53 |
+
"metric": "acc_norm",
|
| 54 |
+
"slayer_update": 45000,
|
| 55 |
+
"slayer_score": 36.0,
|
| 56 |
+
"bielik_v31_score": null,
|
| 57 |
+
"gap_pp": null,
|
| 58 |
+
"status": "matching v3.1 result not located"
|
| 59 |
+
},
|
| 60 |
+
{
|
| 61 |
+
"benchmark": "belebele_pol",
|
| 62 |
+
"metric": "acc",
|
| 63 |
+
"slayer_update": 45000,
|
| 64 |
+
"slayer_score": 25.333299999999998,
|
| 65 |
+
"bielik_v31_score": null,
|
| 66 |
+
"gap_pp": null,
|
| 67 |
+
"status": "matching v3.1 result not located"
|
| 68 |
+
},
|
| 69 |
+
{
|
| 70 |
+
"benchmark": "hellaswag_pl",
|
| 71 |
+
"metric": "acc_norm",
|
| 72 |
+
"slayer_update": 45000,
|
| 73 |
+
"slayer_score": 46.783500000000004,
|
| 74 |
+
"bielik_v31_score": null,
|
| 75 |
+
"gap_pp": null,
|
| 76 |
+
"status": "matching v3.1 result not located"
|
| 77 |
+
},
|
| 78 |
+
{
|
| 79 |
+
"benchmark": "mmlu_pl",
|
| 80 |
+
"metric": "acc",
|
| 81 |
+
"slayer_update": 45000,
|
| 82 |
+
"slayer_score": 25.7371,
|
| 83 |
+
"bielik_v31_score": null,
|
| 84 |
+
"gap_pp": null,
|
| 85 |
+
"status": "matching v3.1 result not located"
|
| 86 |
+
},
|
| 87 |
+
{
|
| 88 |
+
"benchmark": "gsm8k",
|
| 89 |
+
"metric": "exact_match,strict-match",
|
| 90 |
+
"slayer_update": 10000,
|
| 91 |
+
"slayer_score": 0.9855951478392722,
|
| 92 |
+
"bielik_v31_score": null,
|
| 93 |
+
"gap_pp": null,
|
| 94 |
+
"status": "matching v3.1 result not located"
|
| 95 |
+
},
|
| 96 |
+
{
|
| 97 |
+
"benchmark": "sst2",
|
| 98 |
+
"metric": "acc,none",
|
| 99 |
+
"slayer_update": 10000,
|
| 100 |
+
"slayer_score": 58.256880733944946,
|
| 101 |
+
"bielik_v31_score": null,
|
| 102 |
+
"gap_pp": null,
|
| 103 |
+
"status": "matching v3.1 result not located"
|
| 104 |
+
}
|
| 105 |
+
]
|
| 106 |
+
}
|
BIELIK_COMPARISON.md
ADDED
|
@@ -0,0 +1,27 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Slayer1B versus Bielik 11B v3.1: comparison coverage
|
| 2 |
+
|
| 3 |
+
The requested reference is **speakleash/Bielik-11B-v3.1-Instruct**. Its [official card](https://huggingface.co/speakleash/Bielik-11B-v3.1-Instruct), revision `d6e755d356ad418d0f0e30b8c428db267415ddb9`, supplies no benchmark score table for these tasks. A targeted public search on 16 November 2026 found no usable matching scores. This is a search result, not proof that no evaluation exists anywhere.
|
| 4 |
+
|
| 5 |
+
The [user-linked Space](https://huggingface.co/spaces/speakleash/european_leaderboard_bielik) instead lists **v2.3 Instruct**. The published v3 technical report concerns **v3 / v3.0 Instruct**, not v3.1. Neither can be relabeled as v3.1.
|
| 6 |
+
|
| 7 |
+
Gap is defined as **Bielik v3.1 score minus Slayer score**, in percentage points; positive values would mean Slayer trails. Missing scores and gaps are recorded as null, not zero. No numeric v3.1 gap can currently be calculated from these sources.
|
| 8 |
+
|
| 9 |
+
| Benchmark | Metric | Slayer checkpoint | Slayer (%) | Bielik v3.1 (%) | Gap (pp) |
|
| 10 |
+
|---|---|---|---:|---:|---:|
|
| 11 |
+
| ARC-Challenge | acc_norm | 45k | 34.81 | Not located | — |
|
| 12 |
+
| HellaSwag (English) | acc_norm | 45k | 53.65 | Not located | — |
|
| 13 |
+
| PIQA | acc_norm | 45k | 70.29 | Not located | — |
|
| 14 |
+
| WinoGrande | acc | 45k | 59.04 | Not located | — |
|
| 15 |
+
| BoolQ | acc | 45k | 61.93 | Not located | — |
|
| 16 |
+
| OpenBookQA | acc_norm | 45k | 36.00 | Not located | — |
|
| 17 |
+
| Belebele (Polish) | acc | 45k | 25.33 | Not located | — |
|
| 18 |
+
| HellaSwag (Polish) | acc_norm | 45k | 46.78 | Not located | — |
|
| 19 |
+
| MMLU-PL | acc | 45k | 25.74 | Not located | — |
|
| 20 |
+
| GSM8K | exact_match,strict-match | 10k (separate protocol) | 0.99 | Not located | — |
|
| 21 |
+
| SST2 | acc,none | 10k (separate protocol) | 58.26 | Not located | — |
|
| 22 |
+
|
| 23 |
+
## What is needed to calculate the gaps
|
| 24 |
+
|
| 25 |
+
Evaluate the pinned v3.1 checkpoint on the same item sets, task definitions, primary metrics, context/truncation rules and prompting setup as Slayer. For the nine multiple-choice tasks, use the documented zero-shot continuation-likelihood protocol; separately retain GSM8K 5-shot strict match and SST-2 zero-shot settings. Record whether the instruct model is scored without its chat template, and retain that setting for reproducibility. Generation-based API answers are not interchangeable with continuation log likelihood. Each new result needs evaluator/checkpoint pins, item counts and confidence intervals.
|
| 26 |
+
|
| 27 |
+
No Bielik model was downloaded or evaluated as part of this documentation change. Older published reference scores are preserved separately in [EXTERNAL_REFERENCES.md](EXTERNAL_REFERENCES.md).
|
DATA_CARD.md
CHANGED
|
@@ -12,7 +12,9 @@ tags:
|
|
| 12 |
|
| 13 |
# Slayer1B 1.09 — staged training data
|
| 14 |
|
| 15 |
-
**Arkadiusz Słota, Anna Sieczka, Piotr Styła / Fabryka AI (SlayerLab)**
|
|
|
|
|
|
|
| 16 |
|
| 17 |
**Historical CPT data snapshot, 10 November 2026, 19:23 UTC:** the continuation from Slayer1B 1.08 was paused by the cost guard after the on-node update-11,600 checkpoint. At that snapshot, weights-only and full recovery checkpoints at update 10,000 had been verified on the trusted local workstation; this does not assert a local update-11,600 copy. The c01–c06 queue was byte-approved and reproduced on the training host, providing data through update 30,000. Input capacity is distinct from consumed training exposure. This dated snapshot does not describe current hardware availability, a completed future corpus or achieved over-100B content-unique history; later execution status belongs to the linked CPT Track history and operator reports.
|
| 18 |
|
|
|
|
| 12 |
|
| 13 |
# Slayer1B 1.09 — staged training data
|
| 14 |
|
| 15 |
+
**Arkadiusz Słota, Anna Sieczka, Piotr Styła, Kacper Wikiel / Fabryka AI (SlayerLab)**
|
| 16 |
+
|
| 17 |
+
Kacper Wikiel: Fabryka AI founder and model advisor. Model training funded by Fabryka AI.
|
| 18 |
|
| 19 |
**Historical CPT data snapshot, 10 November 2026, 19:23 UTC:** the continuation from Slayer1B 1.08 was paused by the cost guard after the on-node update-11,600 checkpoint. At that snapshot, weights-only and full recovery checkpoints at update 10,000 had been verified on the trusted local workstation; this does not assert a local update-11,600 copy. The c01–c06 queue was byte-approved and reproduced on the training host, providing data through update 30,000. Input capacity is distinct from consumed training exposure. This dated snapshot does not describe current hardware availability, a completed future corpus or achieved over-100B content-unique history; later execution status belongs to the linked CPT Track history and operator reports.
|
| 20 |
|
EXTERNAL_REFERENCES.json
ADDED
|
@@ -0,0 +1,37 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"benchmark": "hellaswag",
|
| 3 |
+
"units": "percent",
|
| 4 |
+
"source_url": "https://arxiv.org/pdf/2601.11579",
|
| 5 |
+
"source_tables": [
|
| 6 |
+
10,
|
| 7 |
+
11
|
| 8 |
+
],
|
| 9 |
+
"source_pdf_sha256": "8344ca8320b4e5597662d7b3dfc9c3175fc206a382e5ee2566e230130647b98f",
|
| 10 |
+
"downloaded_pdf_url": "https://raw.githubusercontent.com/speakleash/bielik-papers/main/v3/Bielik_11B_v3.pdf",
|
| 11 |
+
"source_git_revision": "735bfee1125fe8b497ac2769de94822a11f77167",
|
| 12 |
+
"protocol_description": "Authors report Open LLM Leaderboard results. Standard v1 HellaSwag is 10-shot acc_norm; exact run settings not reproduced here. Slayer results are 0-shot. Contextual reference only; no matched-run difference or efficiency claim.",
|
| 13 |
+
"protocol_source": "https://huggingface.co/docs/leaderboards/open_llm_leaderboard/archive",
|
| 14 |
+
"records": [
|
| 15 |
+
{
|
| 16 |
+
"model": "Bielik-11B-v3",
|
| 17 |
+
"variant": "base",
|
| 18 |
+
"score": 81.38,
|
| 19 |
+
"table": 10
|
| 20 |
+
},
|
| 21 |
+
{
|
| 22 |
+
"model": "Bielik-11B-v3.0-Instruct",
|
| 23 |
+
"variant": "instruct",
|
| 24 |
+
"score": 81.96,
|
| 25 |
+
"table": 11
|
| 26 |
+
}
|
| 27 |
+
],
|
| 28 |
+
"other_version_reference": {
|
| 29 |
+
"model": "Bielik-11B-v2.3-Instruct",
|
| 30 |
+
"benchmark": "hellaswag_pl",
|
| 31 |
+
"score": 71.0,
|
| 32 |
+
"reported_precision": "rounded 0.71 in the published Polish leaderboard",
|
| 33 |
+
"source": "https://huggingface.co/spaces/speakleash/european_leaderboard_bielik",
|
| 34 |
+
"space_revision": "38f8f5f2c71125d9a4e125ec6953f9c5a273b6f3",
|
| 35 |
+
"protocol": "European LLM Leaderboard; exact run settings not established; do not substitute for v3.1 or English HellaSwag"
|
| 36 |
+
}
|
| 37 |
+
}
|
EXTERNAL_REFERENCES.md
ADDED
|
@@ -0,0 +1,17 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Published HellaSwag references
|
| 2 |
+
|
| 3 |
+
| Model | Variant | Published HellaSwag (%) | Source |
|
| 4 |
+
|---|---|---:|---|
|
| 5 |
+
| Bielik-11B-v3 | Base | 81.38 | Technical report, Table 10 |
|
| 6 |
+
| Bielik-11B-v3.0-Instruct | Instruction-tuned | 81.96 | Technical report, Table 11 |
|
| 7 |
+
| Slayer1B-1.09, update 45,000 | Base | 53.65 | EVALUATION.json, zero-shot acc_norm |
|
| 8 |
+
|
| 9 |
+
The Bielik figures are author-reported results from [Bielik 11B v3: Multilingual Large Language Model for European Languages](https://arxiv.org/pdf/2601.11579), section 5.6, tables 10 and 11. The same values appear in the [PDF linked from the official model card](https://github.com/speakleash/bielik-papers/blob/main/v3/Bielik_11B_v3.pdf).
|
| 10 |
+
|
| 11 |
+
The report identifies these as Open LLM Leaderboard results. The [official archived v1 protocol](https://huggingface.co/docs/leaderboards/open_llm_leaderboard/archive) specifies **10-shot HellaSwag acc_norm**. The exact Bielik evaluator revision, tokenizer behavior, and per-run settings have not been reproduced here. Slayer uses its separately documented **zero-shot** protocol, float32, with 10,042 validation items. The chart therefore uses Bielik base as an **external reference**, not a matched evaluation or a causal measure of size or training efficiency. It is an approximately 11B model, whereas Slayer is approximately 1.07B. The instruction-tuned score is listed separately rather than substituted for the base result.
|
| 12 |
+
|
| 13 |
+
No confidence intervals for Bielik are supplied by these report tables, so none are invented. Bielik is not inserted into Slayer's EVALUATION.json or model-index measurements. The source record is [EXTERNAL_REFERENCES.json](EXTERNAL_REFERENCES.json).
|
| 14 |
+
|
| 15 |
+
## User-linked European leaderboard
|
| 16 |
+
|
| 17 |
+
The [SpeakLeash Space](https://huggingface.co/spaces/speakleash/european_leaderboard_bielik) at revision `38f8f5f2c71125d9a4e125ec6953f9c5a273b6f3` embeds a published spreadsheet. Its Polish sheet lists **Bielik-11B-v2.3-Instruct**, HellaSwag **0.71**, equivalent to approximately **71%**. This is a rounded Polish-language result for **v2.3**, not v3.0/v3.1 or English HellaSwag. Slayer 1.09 at 45k has Polish HellaSwag acc_norm 46.78%; the displayed cross-source difference is roughly 24 percentage points, with unmatched evaluation settings. It is not used as the requested v3.1 gap.
|
README.md
CHANGED
|
@@ -131,13 +131,13 @@ model-index:
|
|
| 131 |
|
| 132 |
# Slayer1B 1.09
|
| 133 |
|
| 134 |
-
Slayer1B is a 1.07B-parameter Polish–English base
|
| 135 |
|
| 136 |
-
**Status:** latest published and evaluated checkpoint: **45k updates**; training ongoing, with a revisable target of approximately **80k**. Execution may be ahead
|
| 137 |
|
| 138 |
## Results
|
| 139 |
|
| 140 |
-
HellaSwag increases from **43.4%** on 1.08 to **53.7%** at 45k updates, a **10.2 percentage-point** gain.
|
| 141 |
|
| 142 |

|
| 143 |
|
|
@@ -149,11 +149,13 @@ HellaSwag increases from **43.4%** on 1.08 to **53.7%** at 45k updates, a **10.2
|
|
| 149 |
| WinoGrande | 53.7 | 59.0 |
|
| 150 |
| HellaSwag-PL | 43.9 | 46.8 |
|
| 151 |
|
| 152 |
-
|
|
|
|
|
|
|
| 153 |
|
| 154 |
## Usage
|
| 155 |
|
| 156 |
-
The [
|
| 157 |
|
| 158 |
```python
|
| 159 |
from huggingface_hub import hf_hub_download
|
|
@@ -189,12 +191,17 @@ Exact settings are in [TRAINING_CONFIGURATION.md](TRAINING_CONFIGURATION.md).
|
|
| 189 |
|
| 190 |
## Data
|
| 191 |
|
| 192 |
-
|
|
|
|
|
|
|
|
|
|
| 193 |
|
| 194 |
## Limitations and links
|
| 195 |
|
| 196 |
-
|
| 197 |
|
| 198 |
[Exact evaluation data](EVALUATION.json) · [Training history and detailed results](TRAINING_LOG.md) · [Live training log](https://track.fabryka.ai/run/71cf98ec-7c60-54a8-ac47-0ea7a23f573a) · [Checkpoint archive](https://huggingface.co/SlayerLab/Slayer1B-1.09-checkpoints) · [Plotting script](plots/plot_benchmarks.py)
|
| 199 |
|
| 200 |
-
Apache-2.0. Authors: Arkadiusz
|
|
|
|
|
|
|
|
|
| 131 |
|
| 132 |
# Slayer1B 1.09
|
| 133 |
|
| 134 |
+
Slayer1B is a 1.07B-parameter Polish–English base model from SlayerLab / Fabryka AI. Version 1.09 continues pretraining from [1.08](https://huggingface.co/SlayerLab/Slayer1B-1.08) for text completion.
|
| 135 |
|
| 136 |
+
**Status:** latest published and evaluated checkpoint: **45k updates**; training ongoing, with a revisable target of approximately **80k**. Execution may be ahead.
|
| 137 |
|
| 138 |
## Results
|
| 139 |
|
| 140 |
+
HellaSwag increases from **43.4%** on 1.08 to **53.7%** at 45k updates, a **10.2 percentage-point** gain. Slayer results use zero-shot, float32 evaluation.
|
| 141 |
|
| 142 |

|
| 143 |
|
|
|
|
| 149 |
| WinoGrande | 53.7 | 59.0 |
|
| 150 |
| HellaSwag-PL | 43.9 | 46.8 |
|
| 151 |
|
| 152 |
+
Percentages; WinoGrande uses accuracy, the others length-normalized accuracy. [All nine benchmark plots](plots/benchmark-overview.png) show sample counts and pointwise 95% confidence intervals. Random-choice references are 25% for four answers and 50% for two; BoolQ also shows its majority-class baseline. GSM8K and SST-2 use a [separate protocol and plot](plots/additional-benchmarks.png).
|
| 153 |
+
|
| 154 |
+
[Bielik 11B v3](https://arxiv.org/pdf/2601.11579) reports **81.4% base / 82.0% Instruct** on HellaSwag. Its Open LLM Leaderboard setup differs from our zero-shot protocol; the base-model line provides context, not a matched comparison. [Sources](EXTERNAL_REFERENCES.md). [Per-benchmark v3.1 comparison](BIELIK_COMPARISON.md): matching scores not located.
|
| 155 |
|
| 156 |
## Usage
|
| 157 |
|
| 158 |
+
The [archive](https://huggingface.co/SlayerLab/Slayer1B-1.09-checkpoints) contains native PyTorch weights. Install `torch`, `numpy`, `tokenizers`, and `huggingface_hub`; use Python 3.11 or later. The helper downloads 4.3 GB of weights.
|
| 159 |
|
| 160 |
```python
|
| 161 |
from huggingface_hub import hf_hub_download
|
|
|
|
| 191 |
|
| 192 |
## Data
|
| 193 |
|
| 194 |
+

|
| 195 |
+
|
| 196 |
+
|
| 197 |
+
The documented continuation mix through 45k is 20% Polish / 80% English source allocation: selected GoLLeM web, FinePDFs-Edu, and FineWeb-Edu. At 45k, estimated cumulative training exposure is **65.9B token occurrences**, including the inherited model and a known duplicate correction; this is not a globally unique-token count. See [DATA_CARD.md](DATA_CARD.md) for sources; [mix by stage](TRAINING_MIX.md).
|
| 198 |
|
| 199 |
## Limitations and links
|
| 200 |
|
| 201 |
+
The base model is not instruction-tuned. Polish Belebele and MMLU scores remain near the 25% chance reference. Training-lineage contamination checks are incomplete. Generated text can be incorrect, biased, or personal.
|
| 202 |
|
| 203 |
[Exact evaluation data](EVALUATION.json) · [Training history and detailed results](TRAINING_LOG.md) · [Live training log](https://track.fabryka.ai/run/71cf98ec-7c60-54a8-ac47-0ea7a23f573a) · [Checkpoint archive](https://huggingface.co/SlayerLab/Slayer1B-1.09-checkpoints) · [Plotting script](plots/plot_benchmarks.py)
|
| 204 |
|
| 205 |
+
Apache-2.0. Authors: Arkadiusz Słota, Anna Sieczka, Piotr Styła, Aleksander Ogrodzki, Paweł Puzio, Marcin Kuśmierski, Kacper Wikiel / Fabryka AI.
|
| 206 |
+
|
| 207 |
+
Kacper Wikiel: founder and model advisor. Funded by Fabryka AI.
|
TRAINING_MIX.json
ADDED
|
@@ -0,0 +1,9 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"scope": "1.09 continuation from 1.08; source-bucket allocation, not measured per-token language",
|
| 3 |
+
"latest_evaluated_update": 45000,
|
| 4 |
+
"segments": [
|
| 5 |
+
{"first_update": 1, "last_update": 75000, "polish_percent": 20, "english_percent": 80, "status": "documented allocation; consumption shown only through latest evaluated checkpoint"},
|
| 6 |
+
{"first_update": 75001, "last_update": 80000, "polish_percent": 30, "english_percent": 70, "status": "later approved allocation; inputs reported built, consumption not established"}
|
| 7 |
+
],
|
| 8 |
+
"sources": ["DATA_CARD.md", "TRAINING_LOG.md", "TRAINING_CONFIGURATION.md"]
|
| 9 |
+
}
|
TRAINING_MIX.md
ADDED
|
@@ -0,0 +1,34 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Training mix: 1.08 → 1.09
|
| 2 |
+
|
| 3 |
+

|
| 4 |
+
|
| 5 |
+
This chart describes **source-bucket allocation**, not language classification of every token, the cumulative mix inherited from 1.08, or proof that future updates have executed. The latest public evaluated checkpoint is update 45,000. The steps are local to the 1.09 continuation, starting from 1.08 weights.
|
| 6 |
+
|
| 7 |
+
| Continuation updates | Polish source allocation | English source allocation | Evidence |
|
| 8 |
+
|---|---:|---:|---|
|
| 9 |
+
| 1–75,000 | 20% | 80% | Documented main-base recipe for stages c01–c15 |
|
| 10 |
+
| 75,001–80,000 | 30% | 70% | Later approved allocation; c16–c17 reported built with measured 0.3000 Polish input shares; not evidence of consumption |
|
| 11 |
+
|
| 12 |
+
The 45,000-update checkpoint falls entirely in the first regime. It represents approximately **47.2B additional processed token occurrences**. Applying the nominal source allocation gives approximately **9.44B Polish-bucket / 37.7B English-bucket occurrences**; those are allocation-based estimates, not measured per-language consumption counts. Approximately 65.9B in the model card includes ancestral exposure and the documented duplicate correction; it is not the size of the new continuation alone.
|
| 13 |
+
|
| 14 |
+
## Documented source changes
|
| 15 |
+
|
| 16 |
+
| Continuation interval | Stage | Polish bucket | English bucket |
|
| 17 |
+
|---|---|---|---|
|
| 18 |
+
| Updates 1–5,000 | c01 | GoLLeM web, after ancestral exclusions | FineWeb-Edu |
|
| 19 |
+
| Updates 5,001–10,000 | c02 | GoLLeM web | FineWeb-Edu |
|
| 20 |
+
| Updates 10,001–25,000 | c03–c05 | FinePDFs-Edu PL | FineWeb-Edu |
|
| 21 |
+
| Updates 25,001–30,000 | c06 | FinePDFs-Edu PL + GoLLeM web; internal split not published | FineWeb-Edu |
|
| 22 |
+
| Updates 30,001–45,000 | c07–c09 | 20% Polish allocation documented; per-stage source composition not enumerated in the public table | 80% English allocation documented; no additional per-stage source breakdown published |
|
| 23 |
+
|
| 24 |
+
The initial input segment has explicit source counts: 4,278,190,159 FineWeb-Edu positions and 1,069,547,741 GoLLeM positions (about 80/20). Each stage processes less than its input capacity. Source counts therefore do not by themselves provide the exact counts selected by the runtime sampler.
|
| 25 |
+
|
| 26 |
+
The data card identifies PL Logika, English Technical, and 1.07 replay as inherited material / exclusion references, not fresh continuation inputs. Common Pile and FineMath entries are candidates, not a confirmed consumed mixture. The separate approximately 10B “gold” corpus is a future isolated option, not a documented part of the 45k continuation.
|
| 27 |
+
|
| 28 |
+
## Sources
|
| 29 |
+
|
| 30 |
+
- [DATA_CARD.md](DATA_CARD.md): “Historical byte-approved queue snapshot c01–c06”, “Byte-approved c01 reference”, and “Used source categories and later candidates”.
|
| 31 |
+
- [TRAINING_LOG.md](TRAINING_LOG.md): preserved “Current CPT configuration” table gives the c01–c15 versus c16–c17 allocation and the later build report.
|
| 32 |
+
- [TRAINING_CONFIGURATION.md](TRAINING_CONFIGURATION.md): main-base allocation and approval of the future change. Its earlier wording records approval before the later build report; neither establishes that update 75,001 has executed.
|
| 33 |
+
|
| 34 |
+
Chart values are in [TRAINING_MIX.json](TRAINING_MIX.json); reproduce with [plot_training_mix.py](plots/plot_training_mix.py).
|
plots/hellaswag-progress.png
CHANGED
|
Git LFS Details
|
|
Git LFS Details
|
plots/hellaswag-progress.svg
CHANGED
|
|
|
|
plots/plot_benchmarks.py
CHANGED
|
@@ -115,16 +115,22 @@ y = [item['acc_norm'] * 100 for _,item in points]
|
|
| 115 |
lo = [item['acc_norm_ci95'][0] * 100 for _,item in points]
|
| 116 |
hi = [item['acc_norm_ci95'][1] * 100 for _,item in points]
|
| 117 |
ax.fill_between(x, lo, hi, alpha=.15, color='#0891b2', label='Pointwise 95% CI')
|
| 118 |
-
ax.plot(x, y, 'o-', color='#0891b2', lw=2, label='1.09 checkpoints')
|
| 119 |
base = aggregate['models']['1.08']['tasks']['hellaswag']['acc_norm'] * 100
|
| 120 |
-
ax.axhline(base, ls='-.', color='#7c3aed', label=f'1.08: {base:.1f}%')
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 121 |
ax.axhline(25, ls='--', color='#64748b', label='Random chance: 25%')
|
| 122 |
-
ax.set(xlabel='1.09 continuation updates (thousands)', ylabel='HellaSwag acc_norm (%)', ylim=(23,
|
| 123 |
ax.set_title('Slayer1B · HellaSwag during continued pretraining', loc='left', weight='bold')
|
| 124 |
ax.grid(axis='y', alpha=.2)
|
| 125 |
ax.legend(loc='lower right')
|
| 126 |
ax.annotate(f'{y[-1]:.1f}%', (x[-1],y[-1]), xytext=(5,7), textcoords='offset points', color='#0891b2', weight='bold')
|
| 127 |
-
fig.text(.08,.02,'
|
| 128 |
fig.tight_layout(rect=(0,.05,1,1))
|
| 129 |
for ext in ('png','svg'):
|
| 130 |
fig.savefig(ROOT / f'hellaswag-progress.{ext}',dpi=200)
|
|
|
|
| 115 |
lo = [item['acc_norm_ci95'][0] * 100 for _,item in points]
|
| 116 |
hi = [item['acc_norm_ci95'][1] * 100 for _,item in points]
|
| 117 |
ax.fill_between(x, lo, hi, alpha=.15, color='#0891b2', label='Pointwise 95% CI')
|
| 118 |
+
ax.plot(x, y, 'o-', color='#0891b2', lw=2, label='1.09 checkpoints (0-shot)')
|
| 119 |
base = aggregate['models']['1.08']['tasks']['hellaswag']['acc_norm'] * 100
|
| 120 |
+
ax.axhline(base, ls='-.', color='#7c3aed', label=f'1.08: {base:.1f}% (0-shot)')
|
| 121 |
+
ref_path = ROOT / 'EXTERNAL_REFERENCES.json'
|
| 122 |
+
if not ref_path.exists():
|
| 123 |
+
ref_path = ROOT.parent / 'EXTERNAL_REFERENCES.json'
|
| 124 |
+
external = json.loads(ref_path.read_text())
|
| 125 |
+
reference_score = external['records'][0]['score']
|
| 126 |
+
ax.axhline(reference_score, ls=':', color='#b45309', lw=1.7, label=f'Bielik 11B v3 base: {reference_score:.1f}% (reported; different setup)')
|
| 127 |
ax.axhline(25, ls='--', color='#64748b', label='Random chance: 25%')
|
| 128 |
+
ax.set(xlabel='1.09 continuation updates (thousands)', ylabel='HellaSwag acc_norm (%)', ylim=(23,88), xticks=x)
|
| 129 |
ax.set_title('Slayer1B · HellaSwag during continued pretraining', loc='left', weight='bold')
|
| 130 |
ax.grid(axis='y', alpha=.2)
|
| 131 |
ax.legend(loc='lower right')
|
| 132 |
ax.annotate(f'{y[-1]:.1f}%', (x[-1],y[-1]), xytext=(5,7), textcoords='offset points', color='#0891b2', weight='bold')
|
| 133 |
+
fig.text(.08,.02,'Slayer: EVALUATION.json (0-shot). Bielik: technical report, Table 10 (Open LLM Leaderboard setup).\nDifferent protocols; reference only. Training-lineage contamination checks incomplete.',fontsize=9,color='#475569')
|
| 134 |
fig.tight_layout(rect=(0,.05,1,1))
|
| 135 |
for ext in ('png','svg'):
|
| 136 |
fig.savefig(ROOT / f'hellaswag-progress.{ext}',dpi=200)
|
plots/plot_training_mix.py
ADDED
|
@@ -0,0 +1,31 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Plot documented input allocation; do not imply consumption after the published checkpoint."""
|
| 2 |
+
from pathlib import Path
|
| 3 |
+
import json
|
| 4 |
+
import matplotlib
|
| 5 |
+
matplotlib.use('Agg')
|
| 6 |
+
import matplotlib.pyplot as plt
|
| 7 |
+
from matplotlib.patches import Patch
|
| 8 |
+
root=Path(__file__).resolve().parent
|
| 9 |
+
path=root/'TRAINING_MIX.json'
|
| 10 |
+
if not path.exists(): path=root.parent/'TRAINING_MIX.json'
|
| 11 |
+
x=json.loads(path.read_text())
|
| 12 |
+
a,b=x['segments']
|
| 13 |
+
assert all(s['polish_percent']+s['english_percent']==100 for s in x['segments'])
|
| 14 |
+
fig,ax=plt.subplots(figsize=(12,5.6))
|
| 15 |
+
ax.stackplot([0,75,80],[a['polish_percent'],b['polish_percent'],b['polish_percent']],[a['english_percent'],b['english_percent'],b['english_percent']],step='post',colors=['#7c3aed','#0891b2'])
|
| 16 |
+
ax.axvspan(45,80,facecolor='white',alpha=.48,hatch='///',edgecolor='#64748b',linewidth=0)
|
| 17 |
+
ax.axvline(45,color='#0f172a',lw=1.7,ls='--')
|
| 18 |
+
ax.axvline(75,color='#475569',lw=1,ls=':')
|
| 19 |
+
ax.text(21,10,'Polish 20%',color='white',ha='center',weight='bold',fontsize=13)
|
| 20 |
+
ax.text(21,60,'English 80%',color='white',ha='center',weight='bold',fontsize=13)
|
| 21 |
+
ax.annotate('Latest evaluated\ncheckpoint: 45k',(45,49),xytext=(40,49),ha='right',fontsize=10)
|
| 22 |
+
ax.text(61,61,'Beyond published 45k checkpoint\nAllocation shown; consumption unverified',ha='center',fontsize=10,color='#0f172a',bbox=dict(facecolor='white',alpha=.85,edgecolor='none'))
|
| 23 |
+
ax.annotate('30% PL / 70% EN\nfrom update 75,001',(77.5,30),xytext=(65,40),ha='center',fontsize=10,arrowprops=dict(arrowstyle='->',color='#334155'),bbox=dict(facecolor='white',alpha=.9,edgecolor='none'))
|
| 24 |
+
ax.set(xlim=(0,80),ylim=(0,100),xticks=[0,10,20,30,40,45,50,60,70,75,80],yticks=[0,20,30,50,70,80,100],xlabel='1.09 continuation updates (thousands)',ylabel='Input source allocation (%)')
|
| 25 |
+
ax.set_title('Slayer1B 1.08 → 1.09 · documented pretraining mix',loc='left',weight='bold',fontsize=16)
|
| 26 |
+
ax.spines[['top','right']].set_visible(False)
|
| 27 |
+
ax.grid(axis='y',alpha=.15)
|
| 28 |
+
fig.text(.08,.025,'Source: public training/data records. Language buckets describe new inputs; inherited 1.08 exposure is excluded.\nThe 75k change was approved and inputs reported built; this plot does not claim those updates executed.',fontsize=9,color='#475569')
|
| 29 |
+
fig.tight_layout(rect=(0,.09,1,1))
|
| 30 |
+
for ext in ('png','svg','pdf'): fig.savefig(root/f'training-mix.{ext}',dpi=200)
|
| 31 |
+
plt.close(fig)
|
plots/training-mix.pdf
ADDED
|
Binary file (31.6 kB). View file
|
|
|
plots/training-mix.png
ADDED
|
Git LFS Details
|
plots/training-mix.svg
ADDED
|
|