Benchmark-selected step 20000: Open SLM average 34.86%

README.md

@@ -21,8 +21,8 @@ Benchmark-selected checkpoint from a 7.85M-parameter
 Channel-Mixing Attention generalist language model using the unmodified native
 GPT-S 4,096-token tokenizer. It has no arithmetic-specific token splitting,
 place embeddings, role embeddings, or inference-time equation detection. It
-was selected at step 10,000 for an Open SLM Leaderboard-style average of
-34.51%. WikiText normalized BPB is reported separately
+was selected at step 20,000 for an Open SLM Leaderboard-style average of
+34.86%. WikiText normalized BPB is reported separately
 and is not used for selection.
 
 ## Loading
@@ -69,7 +69,7 @@ ordinary causal-language-model text, with no task-specific model features or
 tokenizer transformations. All four training sources are streamed natural-text
 corpora.
 
-## Zero-shot evaluation at step 10,000
+## Zero-shot evaluation at step 20,000
 
 The four lm-eval tasks use normalized accuracy when supplied by lm-eval
 0.4.12, with float32 weights and softmax. ArithMark
@@ -77,17 +77,17 @@ uses float32 weights and its official raw continuation log-likelihood-sum rule.
 
 | Benchmark | Accuracy |
 |---|---:|
-| HellaSwag | 27.43% |
-| ARC-Easy | 33.42% |
-| ARC-Challenge | 22.10% |
-| PIQA | 55.44% |
-| ArithMark-2 | 27.40% |
-| ARC mean | 27.76% |
-| Open SLM Leaderboard-style average | 34.51% |
+| HellaSwag | 27.54% |
+| ARC-Easy | 32.49% |
+| ARC-Challenge | 21.59% |
+| PIQA | 56.91% |
+| ArithMark-2 | 27.96% |
+| ARC mean | 27.04% |
+| Open SLM Leaderboard-style average | 34.86% |
 
 The average is `(HellaSwag + mean(ARC-Easy, ARC-Challenge) + PIQA +
 ArithMark-2) / 4`, matching the Open SLM Leaderboard formula.
 
-WikiText-103 validation at this step: loss 3.1590, perplexity
-23.55, normalized BPB 1.4283 over 358,911
+WikiText-103 validation at this step: loss 3.1099, perplexity
+22.42, normalized BPB 1.4061 over 358,911
 scored tokens and 1,145,226 normalized UTF-8 bytes.

benchmark_results.json

@@ -1,24 +1,24 @@
 {
-  "step": 10000,
+  "step": 20000,
   "parameters": 7849161,
   "lm_eval_version": "0.4.12",
   "num_fewshot": 0,
   "leaderboard_accuracy": {
-    "arc_easy": 0.3341750841750842,
-    "arc_challenge": 0.22098976109215018,
-    "hellaswag": 0.27434773949412466,
-    "piqa": 0.5544069640914037
+    "arc_easy": 0.32491582491582494,
+    "arc_challenge": 0.2158703071672355,
+    "hellaswag": 0.27544313881696875,
+    "piqa": 0.5690968443960827
   },
   "lm_eval_metric_policy": "acc_norm when available, otherwise acc",
   "selection_metric": "open_slm_leaderboard_average",
   "arithmark_2.0": {
-    "acc": 0.274,
-    "correct": 685,
+    "acc": 0.2796,
+    "correct": 699,
     "total": 2500,
     "by_operator_count": {
       "1": {
-        "acc": 0.2504,
-        "correct": 313,
+        "acc": 0.2664,
+        "correct": 333,
         "total": 1250
       },
       "2": {
@@ -27,19 +27,19 @@
         "total": 750
       },
       "3": {
-        "acc": 0.244,
-        "correct": 122,
+        "acc": 0.232,
+        "correct": 116,
         "total": 500
       }
     }
   },
-  "arc_average": 0.27758242263361715,
-  "open_slm_leaderboard_average": 0.34508428155478643,
+  "arc_average": 0.2703930660415302,
+  "open_slm_leaderboard_average": 0.34863326231364544,
   "average_formula": "(hellaswag + mean(arc_easy, arc_challenge) + piqa + arithmark_2.0) / 4",
   "validation": {
-    "loss": 3.1589507887713384,
-    "perplexity": 23.54587836439382,
-    "normalized_bpb": 1.4282787309946414,
+    "loss": 3.109883570577351,
+    "perplexity": 22.418434083453395,
+    "normalized_bpb": 1.4060936230832888,
     "tokens": 358911,
     "normalized_utf8_bytes": 1145226
   }

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:13f4aa1c153989df52fb9e892a9aeed84b227040a2a28308262d285e9f20ceb8
+oid sha256:7ca142e039d7fc013bf133eb6d9b55863153d2811f69562e6844879377842b21
 size 36330428