{ "_generated_by": "scripts/hf_release/generate_release_index.py", "_manifest_commit": "1a9a718", "_metric_rules": { "generation_spec": "only v2 files are read; v1 code_gen and gsm8k files are ignored", "stale_eval_guard": "a metric is written only if the eval file's metadata.step equals the released checkpoint's step", "blimp_scoring": "BOS-prefixed empty-context loglikelihood, comparable within a scale" }, "counts": { "models": 220, "with_all_metrics": 99 }, "models": { "full-128k-allmulti-equal-bpe": { "run": "full-128k-allmulti-equal-bpe", "works": [ "paper_a", "tokeval" ], "training_regime": "std-1B", "architecture": { "n_layer": 24, "n_embd": 1536, "n_head": 12, "n_kv_head": 12, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 128256, "total_parameters": 1272841994 }, "training": { "step": 8800, "total_batch_size": 1048576, "token_budget": 9227468800, "total_bytes_consumed": 63005448000, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "full-128k" }, "tokenizer_slug": "allmulti-equal-bpe", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 0, "metrics": { "val_bpb": 0.7176754255270483, "blimp_acc": 0.8189402985074629, "code_bpb_mean": 0.5210339837815584, "blimp_scoring": "bos_prefixed", "flores_mean_bpb": 2.651776787012788, "flores_n_languages": 215, "mbpp_pass1": 0.056, "humaneval_pass1": 0.06097560975609756, "generation_spec": { "version": "v2-2026-07-30", "bos_context": true, "eos_stop": true, "token_healing": true, "truncation": "left_to_max_length_minus_max_gen", "stop_truncation": "earliest_match_all_terms", "stop_match_decode": "specials_kept" }, "token_healing": true, "gsm8k_exact_match_strict": 0.045489006823351025, "gsm8k_exact_match_flexible": 0.05307050796057619 }, "metric_gaps": {}, "registry_note": "P2 GPT-4o regex; allmulti_equal (30 FineWeb-2 langs equal weight, max_files=50, repeat-sampled) data; norm=none; BPE; vocab 128,256; baseline of the tail-harm result; USED - S6d.4 equal-weighting ESTABLISHED result (rho=-0.595 vs log training weight, n=31 langs)", "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a139/cmeister/tokenizer-lm/checkpoints/full-128k-allmulti-equal-bpe", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "full-128k-apertus": { "run": "full-128k-apertus", "works": [ "tokeval", "tokeval_cross_scale" ], "training_regime": "std-1B", "architecture": { "n_layer": 24, "n_embd": 1536, "n_head": 12, "n_kv_head": 12, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "default 15.0, run predates the configurable field", "vocab_size": 131072, "total_parameters": 1285818122 }, "training": { "step": 8843, "total_batch_size": 1048576, "token_budget": 9272557568, "total_bytes_consumed": 61775995352, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "full-128k" }, "tokenizer_slug": "swiss-ai/Apertus-70B-2509", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 1, "metrics": { "val_bpb": 0.7196643583908005, "blimp_acc": 0.8191044776119404, "code_bpb_mean": 0.5261638466492652, "blimp_scoring": "bos_prefixed", "flores_mean_bpb": 2.7683366310551216, "flores_n_languages": 214, "mbpp_pass1": 0.086, "humaneval_pass1": 0.054878048780487805, "generation_spec": { "version": "v2-2026-07-30", "bos_context": true, "eos_stop": true, "token_healing": true, "truncation": "left_to_max_length_minus_max_gen", "stop_truncation": "earliest_match_all_terms", "stop_match_decode": "specials_kept" }, "token_healing": true, "gsm8k_exact_match_strict": 0.04094010614101592, "gsm8k_exact_match_flexible": 0.04624715693707354 }, "metric_gaps": {}, "registry_note": "Apertus/Mistral-Nemo (off-the-shelf); n/a (off-the-shelf, Apertus-70B-2509 pretraining mix) data; norm=n/a; off-the-shelf BPE; vocab 131,072; USED - panels.json main21 (camera-ready main panel)", "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a139/cmeister/tokenizer-lm/checkpoints/full-128k-apertus", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "full-128k-apertus-mathcode-scratch": { "run": "full-128k-apertus-mathcode-scratch", "works": [ "blog" ], "training_regime": "mathcode-20B-from-scratch", "architecture": { "n_layer": 24, "n_embd": 1536, "n_head": 12, "n_kv_head": 12, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 131072, "total_parameters": 1285818122 }, "training": { "step": 19073, "total_batch_size": 1048576, "token_budget": 19999490048, "total_bytes_consumed": 133241384072, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "full-128k-mathcode-scratch" }, "tokenizer_slug": "apertus", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 1, "metrics": { "val_bpb": 0.3368332422686206, "code_bpb_mean": 0.37824387350146926, "flores_mean_bpb": 3.4253459955956593, "flores_n_languages": 214, "mbpp_pass1": 0.0, "humaneval_pass1": 0.1524390243902439, "generation_spec": { "version": "v2-2026-07-30", "bos_context": true, "eos_stop": true, "token_healing": true, "truncation": "left_to_max_length_minus_max_gen", "stop_truncation": "earliest_match_all_terms", "stop_match_decode": "specials_kept" }, "token_healing": true, "gsm8k_exact_match_strict": 0.18423047763457165, "gsm8k_exact_match_flexible": 0.22896133434420016 }, "metric_gaps": { "blimp_acc": "no full-128k-apertus-mathcode-scratch_blimp_code_bpb.json" }, "registry_note": "#4, apertus pretok, slug `apertus`. Mistral-Nemo regex; see EXPERIMENTS_RESULTS.md \u00a73a for the RCA of the apertus-pretok MBPP code-gen breakage (real, reproducible effect, not an artifact).", "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a139/cmeister/tokenizer-lm/checkpoints/full-128k-apertus-mathcode-scratch", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "full-128k-apertus-seed42": { "run": "full-128k-apertus-seed42", "works": [ "paper_a", "tokeval_cross_scale" ], "training_regime": "std-1B", "architecture": { "n_layer": 24, "n_embd": 1536, "n_head": 12, "n_kv_head": 12, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 131072, "total_parameters": 1285818122 }, "training": { "step": 8843, "total_batch_size": 1048576, "token_budget": 9272557568, "total_bytes_consumed": 61775995352, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "full-128k" }, "tokenizer_slug": "swiss-ai/Apertus-70B-2509", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 1, "metrics": { "val_bpb": 0.7211964996436109, "blimp_acc": 0.8171194029850744, "code_bpb_mean": 0.5290844455840652, "blimp_scoring": "bos_prefixed", "flores_mean_bpb": 2.7557483320952856, "flores_n_languages": 215 }, "metric_gaps": { "mbpp_pass1": "no full-128k-apertus-seed42_code_v2.json", "humaneval_pass1": "no full-128k-apertus-seed42_code_v2.json", "gsm8k_exact_match_strict": "no full-128k-apertus-seed42_gsm8k_v2.json" }, "registry_note": null, "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a139/cmeister/tokenizer-lm/checkpoints/full-128k-apertus-seed42", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "full-128k-bpe-nfc-clean-balanced": { "run": "full-128k-bpe-nfc-clean-balanced", "works": [ "paper_a", "tokeval" ], "training_regime": "std-1B", "architecture": { "n_layer": 24, "n_embd": 1536, "n_head": 12, "n_kv_head": 12, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 128004, "total_parameters": 1271957258 }, "training": { "step": 8797, "total_batch_size": 1048576, "token_budget": 9224323072, "total_bytes_consumed": 65667053870, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "full-128k" }, "tokenizer_slug": "bpe-nfc-clean-balanced", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 128000, "metrics": { "val_bpb": 0.715856631626823, "blimp_acc": 0.8210597014925377, "code_bpb_mean": 0.5226398359086579, "blimp_scoring": "bos_prefixed", "flores_mean_bpb": 2.6477655259752817, "flores_n_languages": 215, "mbpp_pass1": 0.092, "humaneval_pass1": 0.08536585365853659, "generation_spec": { "version": "v2-2026-07-30", "bos_context": true, "eos_stop": true, "token_healing": true, "truncation": "left_to_max_length_minus_max_gen", "stop_truncation": "earliest_match_all_terms", "stop_match_decode": "specials_kept" }, "token_healing": true, "gsm8k_exact_match_strict": 0.03790750568612585, "gsm8k_exact_match_flexible": 0.04094010614101592 }, "metric_gaps": {}, "registry_note": "clean-multi regex (REGEX_CLEAN); balanced (D2) data; norm=NFC; BPE; vocab 128,004; non-parity BPE baseline for the pabpe-nfc-clean-* family; also the vocab-ladder v128k anchor; USED - balanced-1B bootstrap panel (S6.12) + paper panel + S6h code analysis (n=21 main21 general-metric rows use bpe-nfc-clean-balanced's twin? no - not in main21 itself); not in panels.json main21", "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a139/cmeister/tokenizer-lm/checkpoints/full-128k-bpe-nfc-clean-balanced", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "full-128k-bpe-nfc-clean-balanced-mathcode-scratch": { "run": "full-128k-bpe-nfc-clean-balanced-mathcode-scratch", "works": [ "blog", "tokeval" ], "training_regime": "mathcode-20B-from-scratch", "architecture": { "n_layer": 24, "n_embd": 1536, "n_head": 12, "n_kv_head": 12, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 128004, "total_parameters": 1271957258 }, "training": { "step": 19073, "total_batch_size": 1048576, "token_budget": 19999490048, "total_bytes_consumed": 142374413830, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "full-128k-mathcode-scratch" }, "tokenizer_slug": "bpe-nfc-clean-balanced", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 128000, "metrics": { "val_bpb": 0.33796765404762164, "code_bpb_mean": 0.3814762402622109, "flores_mean_bpb": 3.365416060327055, "flores_n_languages": 214, "mbpp_pass1": 0.218, "humaneval_pass1": 0.15853658536585366, "generation_spec": { "version": "v2-2026-07-30", "bos_context": true, "eos_stop": true, "token_healing": true, "truncation": "left_to_max_length_minus_max_gen", "stop_truncation": "earliest_match_all_terms", "stop_match_decode": "specials_kept" }, "token_healing": true, "gsm8k_exact_match_strict": 0.177407126611069, "gsm8k_exact_match_flexible": 0.20394238059135708 }, "metric_gaps": { "blimp_acc": "no full-128k-bpe-nfc-clean-balanced-mathcode-scratch_blimp_code_bpb.json" }, "registry_note": "#5, clean pretok, slug `bpe-nfc-clean-balanced`. Clean-regex BPE (NFC, balanced data).", "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a139/cmeister/tokenizer-lm/checkpoints/full-128k-bpe-nfc-clean-balanced-mathcode-scratch", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "full-128k-bpe-nfc-clean-balanced-v128k": { "run": "full-128k-bpe-nfc-clean-balanced-v128k", "works": [ "tokeval" ], "training_regime": "std-1B", "architecture": { "n_layer": 24, "n_embd": 1536, "n_head": 12, "n_kv_head": 12, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 128000, "total_parameters": 1271662346 }, "training": { "step": 8797, "total_batch_size": 1048576, "token_budget": 9224323072, "total_bytes_consumed": 70259721254, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "full-128k" }, "tokenizer_slug": "bpe-nfc-clean-balanced-v128k", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 0, "metrics": { "val_bpb": 0.716152463637094, "blimp_acc": 0.8208208955223881, "code_bpb_mean": 0.5244360004512393, "blimp_scoring": "bos_prefixed", "flores_mean_bpb": 2.644795981112231, "flores_n_languages": 215, "mbpp_pass1": 0.086, "humaneval_pass1": 0.09146341463414634, "generation_spec": { "version": "v2-2026-07-30", "bos_context": true, "eos_stop": true, "token_healing": true, "truncation": "left_to_max_length_minus_max_gen", "stop_truncation": "earliest_match_all_terms", "stop_match_decode": "specials_kept" }, "token_healing": true, "gsm8k_exact_match_strict": 0.032600454890068235, "gsm8k_exact_match_flexible": 0.03639120545868082 }, "metric_gaps": {}, "registry_note": "clean-multi regex (REGEX_CLEAN); balanced (D2) data; norm=NFC; BPE; vocab ~128k target (byte-matched: 8797 steps to match the v128k anchor's training bytes); byte-matched vocab-size ladder (equal training BYTES across the 5 points, unequal steps); the 'clean single-recipe ladder' S6f said had never been run; USED - panels.json vocab-ladder (camera-ready); write-up pending EXPERIMENTS_RESULTS.md (EXPERIMENTS_PLAN.md 'V1/vocab ladder', evals landed 2026-07-24/29)", "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a139/cmeister/tokenizer-lm/checkpoints/full-128k-bpe-nfc-clean-balanced-v128k", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "full-128k-bpe-nfc-clean-balanced-v160k": { "run": "full-128k-bpe-nfc-clean-balanced-v160k", "works": [ "tokeval" ], "training_regime": "std-1B", "architecture": { "n_layer": 24, "n_embd": 1536, "n_head": 12, "n_kv_head": 12, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 160000, "total_parameters": 1419118346 }, "training": { "step": 8667, "total_batch_size": 1048576, "token_budget": 9088008192, "total_bytes_consumed": 71426454399, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "full-128k" }, "tokenizer_slug": "bpe-nfc-clean-balanced-v160k", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 0, "metrics": { "val_bpb": 0.7142973980056193, "blimp_acc": 0.8268955223880593, "code_bpb_mean": 0.5210469295133383, "blimp_scoring": "bos_prefixed", "flores_mean_bpb": 2.6529233096337332, "flores_n_languages": 215, "mbpp_pass1": 0.064, "humaneval_pass1": 0.08536585365853659, "generation_spec": { "version": "v2-2026-07-30", "bos_context": true, "eos_stop": true, "token_healing": true, "truncation": "left_to_max_length_minus_max_gen", "stop_truncation": "earliest_match_all_terms", "stop_match_decode": "specials_kept" }, "token_healing": true, "gsm8k_exact_match_strict": 0.039423805913570885, "gsm8k_exact_match_flexible": 0.04094010614101592 }, "metric_gaps": {}, "registry_note": "clean-multi regex (REGEX_CLEAN); balanced (D2) data; norm=NFC; BPE; vocab ~160k target (byte-matched: 8667 steps to match the v128k anchor's training bytes); byte-matched vocab-size ladder (equal training BYTES across the 5 points, unequal steps); the 'clean single-recipe ladder' S6f said had never been run; USED - panels.json vocab-ladder (camera-ready); write-up pending EXPERIMENTS_RESULTS.md (EXPERIMENTS_PLAN.md 'V1/vocab ladder', evals landed 2026-07-24/29)", "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a139/cmeister/tokenizer-lm/checkpoints/full-128k-bpe-nfc-clean-balanced-v160k", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "full-128k-bpe-nfc-clean-balanced-v200k": { "run": "full-128k-bpe-nfc-clean-balanced-v200k", "works": [ "tokeval" ], "training_regime": "std-1B", "architecture": { "n_layer": 24, "n_embd": 1536, "n_head": 12, "n_kv_head": 12, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 200000, "total_parameters": 1603438346 }, "training": { "step": 8552, "total_batch_size": 1048576, "token_budget": 8967421952, "total_bytes_consumed": 72714808184, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "full-128k" }, "tokenizer_slug": "bpe-nfc-clean-balanced-v200k", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 0, "metrics": { "val_bpb": 0.7128314584843901, "blimp_acc": 0.815268656716418, "code_bpb_mean": 0.5192765421244038, "blimp_scoring": "bos_prefixed", "flores_mean_bpb": 2.669924969239849, "flores_n_languages": 215, "mbpp_pass1": 0.092, "humaneval_pass1": 0.09146341463414634, "generation_spec": { "version": "v2-2026-07-30", "bos_context": true, "eos_stop": true, "token_healing": true, "truncation": "left_to_max_length_minus_max_gen", "stop_truncation": "earliest_match_all_terms", "stop_match_decode": "specials_kept" }, "token_healing": true, "gsm8k_exact_match_strict": 0.03866565579984837, "gsm8k_exact_match_flexible": 0.04245640636846096 }, "metric_gaps": {}, "registry_note": "clean-multi regex (REGEX_CLEAN); balanced (D2) data; norm=NFC; BPE; vocab ~200k target (byte-matched: 8552 steps to match the v128k anchor's training bytes); byte-matched vocab-size ladder (equal training BYTES across the 5 points, unequal steps); the 'clean single-recipe ladder' S6f said had never been run; USED - panels.json vocab-ladder (camera-ready); write-up pending EXPERIMENTS_RESULTS.md (EXPERIMENTS_PLAN.md 'V1/vocab ladder', evals landed 2026-07-24/29)", "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a139/cmeister/tokenizer-lm/checkpoints/full-128k-bpe-nfc-clean-balanced-v200k", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "full-128k-bpe-nfc-clean-balanced-v64k": { "run": "full-128k-bpe-nfc-clean-balanced-v64k", "works": [ "tokeval" ], "training_regime": "std-1B", "architecture": { "n_layer": 24, "n_embd": 1536, "n_head": 12, "n_kv_head": 12, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 64000, "total_parameters": 976750346 }, "training": { "step": 9299, "total_batch_size": 1048576, "token_budget": 9750708224, "total_bytes_consumed": 66611963753, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "full-128k" }, "tokenizer_slug": "bpe-nfc-clean-balanced-v64k", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 0, "metrics": { "val_bpb": 0.7193331865186551, "blimp_acc": 0.8131194029850745, "code_bpb_mean": 0.5268749863541311, "blimp_scoring": "bos_prefixed", "flores_mean_bpb": 2.6168464994006033, "flores_n_languages": 215, "mbpp_pass1": 0.086, "humaneval_pass1": 0.07317073170731707, "generation_spec": { "version": "v2-2026-07-30", "bos_context": true, "eos_stop": true, "token_healing": true, "truncation": "left_to_max_length_minus_max_gen", "stop_truncation": "earliest_match_all_terms", "stop_match_decode": "specials_kept" }, "token_healing": true, "gsm8k_exact_match_strict": 0.05686125852918878, "gsm8k_exact_match_flexible": 0.058377558756633814 }, "metric_gaps": {}, "registry_note": "clean-multi regex (REGEX_CLEAN); balanced (D2) data; norm=NFC; BPE; vocab ~64k target (byte-matched: 9299 steps to match the v128k anchor's training bytes); byte-matched vocab-size ladder (equal training BYTES across the 5 points, unequal steps); the 'clean single-recipe ladder' S6f said had never been run; USED - panels.json vocab-ladder (camera-ready); write-up pending EXPERIMENTS_RESULTS.md (EXPERIMENTS_PLAN.md 'V1/vocab ladder', evals landed 2026-07-24/29)", "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a139/cmeister/tokenizer-lm/checkpoints/full-128k-bpe-nfc-clean-balanced-v64k", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "full-128k-bpe-nfc-clean-balanced-v96k": { "run": "full-128k-bpe-nfc-clean-balanced-v96k", "works": [ "tokeval" ], "training_regime": "std-1B", "architecture": { "n_layer": 24, "n_embd": 1536, "n_head": 12, "n_kv_head": 12, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 96000, "total_parameters": 1124206346 }, "training": { "step": 8986, "total_batch_size": 1048576, "token_budget": 9422503936, "total_bytes_consumed": 68734120678, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "full-128k" }, "tokenizer_slug": "bpe-nfc-clean-balanced-v96k", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 0, "metrics": { "val_bpb": 0.7186254102526758, "blimp_acc": 0.8216865671641791, "code_bpb_mean": 0.5264406583223474, "blimp_scoring": "bos_prefixed", "flores_mean_bpb": 2.644620729776012, "flores_n_languages": 215, "mbpp_pass1": 0.074, "humaneval_pass1": 0.09146341463414634, "generation_spec": { "version": "v2-2026-07-30", "bos_context": true, "eos_stop": true, "token_healing": true, "truncation": "left_to_max_length_minus_max_gen", "stop_truncation": "earliest_match_all_terms", "stop_match_decode": "specials_kept" }, "token_healing": true, "gsm8k_exact_match_strict": 0.03639120545868082, "gsm8k_exact_match_flexible": 0.045489006823351025 }, "metric_gaps": {}, "registry_note": "clean-multi regex (REGEX_CLEAN); balanced (D2) data; norm=NFC; BPE; vocab ~96k target (byte-matched: 8986 steps to match the v128k anchor's training bytes); byte-matched vocab-size ladder (equal training BYTES across the 5 points, unequal steps); the 'clean single-recipe ladder' S6f said had never been run; USED - panels.json vocab-ladder (camera-ready); write-up pending EXPERIMENTS_RESULTS.md (EXPERIMENTS_PLAN.md 'V1/vocab ladder', evals landed 2026-07-24/29)", "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a139/cmeister/tokenizer-lm/checkpoints/full-128k-bpe-nfc-clean-balanced-v96k", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "full-128k-bpe-nfc-clean-fineweb2full": { "run": "full-128k-bpe-nfc-clean-fineweb2full", "works": [ "paper_a" ], "training_regime": "std-1B", "architecture": { "n_layer": 24, "n_embd": 1536, "n_head": 12, "n_kv_head": 12, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 128000, "total_parameters": 1271662346 }, "training": { "step": 8796, "total_batch_size": 1048576, "token_budget": 9223274496, "total_bytes_consumed": 58068658752, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "full-128k" }, "tokenizer_slug": "bpe-nfc-clean-fineweb2full", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 127996, "metrics": { "val_bpb": 0.7249761642249047, "blimp_acc": 0.8208358208955225, "code_bpb_mean": 0.5276609173558784, "blimp_scoring": "bos_prefixed", "flores_mean_bpb": 2.998798131703424, "flores_n_languages": 215, "mbpp_pass1": 0.068, "humaneval_pass1": 0.07926829268292683, "generation_spec": { "version": "v2-2026-07-30", "bos_context": true, "eos_stop": true, "token_healing": true, "truncation": "left_to_max_length_minus_max_gen", "stop_truncation": "earliest_match_all_terms", "stop_match_decode": "specials_kept" }, "token_healing": true, "gsm8k_exact_match_strict": 0.02880970432145565, "gsm8k_exact_match_flexible": 0.03184230477634572 }, "metric_gaps": {}, "registry_note": "clean-multi regex; fineweb2full (full FineWeb-2 corpus) data; norm=NFC; BPE; vocab 128,000; standard-BPE non-parity baseline for the pabpe-nfc-clean-fineweb2full family; USED - S5b fineweb2full PA-BPE/BPE analysis + S6e.1 cohort update; fineweb2full data variant, excluded from the main paper panel per PAPER_PANEL.md rule", "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a139/cmeister/tokenizer-lm/checkpoints/full-128k-bpe-nfc-clean-fineweb2full", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "full-128k-bpe-nfc-clean-nomark-balanced": { "run": "full-128k-bpe-nfc-clean-nomark-balanced", "works": [ "paper_a" ], "training_regime": "std-1B", "architecture": { "n_layer": 24, "n_embd": 1536, "n_head": 12, "n_kv_head": 12, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 128000, "total_parameters": 1271662346 }, "training": { "step": 8797, "total_batch_size": 1048576, "token_budget": 9224323072, "total_bytes_consumed": 70205602110, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "full-128k" }, "tokenizer_slug": "bpe-nfc-clean-nomark-balanced", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 0, "metrics": { "val_bpb": 0.7163375459110471, "blimp_acc": 0.8215223880597016, "code_bpb_mean": 0.5240401515508685, "blimp_scoring": "bos_prefixed", "flores_mean_bpb": 2.6437112475716065, "flores_n_languages": 215, "mbpp_pass1": 0.09, "humaneval_pass1": 0.07926829268292683, "generation_spec": { "version": "v2-2026-07-30", "bos_context": true, "eos_stop": true, "token_healing": true, "truncation": "left_to_max_length_minus_max_gen", "stop_truncation": "earliest_match_all_terms", "stop_match_decode": "specials_kept" }, "token_healing": true, "gsm8k_exact_match_strict": 0.04245640636846096, "gsm8k_exact_match_flexible": 0.04473085670962851 }, "metric_gaps": {}, "registry_note": "clean-multi regex, combining-mark variant (REGEX_CLEAN_NOMARK: \\\\p{M} removed so combining marks split from base letters; documented in EXPERIMENTS_PLAN.md XL-pretok, located 2026-08-16); balanced (D2) data; norm=NFC; BPE; vocab 128,000 (verified from tokenizer.json 2026-08-16); combining-mark control run alongside the vocab-ladder cohort (CHRONOLOGICAL 2026-07-29: 'the eleventh' run, evals complete except MGSM, later fixed); USED - companion control for the vocab-ladder cohort (11th run in that batch, CHRONOLOGICAL 2026-07-29); not itself a panels.json entry; USED - crosslingual panel v1 rev5 Tier-1 member (user-approved inclusion 2026-08-16, configs/panels/crosslingual_panel_v1.json)", "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a139/cmeister/tokenizer-lm/checkpoints/full-128k-bpe-nfc-clean-nomark-balanced", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "full-128k-bpe-nfc-gpt4-fineweb2full": { "run": "full-128k-bpe-nfc-gpt4-fineweb2full", "works": [ "paper_a" ], "training_regime": "std-1B", "architecture": { "n_layer": 24, "n_embd": 1536, "n_head": 12, "n_kv_head": 12, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 128000, "total_parameters": 1271662346 }, "training": { "step": 8796, "total_batch_size": 1048576, "token_budget": 9223274496, "total_bytes_consumed": 57262408596, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "full-128k" }, "tokenizer_slug": "bpe-nfc-gpt4-fineweb2full", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 127996, "metrics": { "val_bpb": 0.7240783473399763, "blimp_acc": 0.823686567164179, "code_bpb_mean": 0.5254870925448322, "blimp_scoring": "bos_prefixed", "flores_mean_bpb": 2.9618560118364603, "flores_n_languages": 215, "mbpp_pass1": 0.08, "humaneval_pass1": 0.07317073170731707, "generation_spec": { "version": "v2-2026-07-30", "bos_context": true, "eos_stop": true, "token_healing": true, "truncation": "left_to_max_length_minus_max_gen", "stop_truncation": "earliest_match_all_terms", "stop_match_decode": "specials_kept" }, "token_healing": true, "gsm8k_exact_match_strict": 0.0356330553449583, "gsm8k_exact_match_flexible": 0.04245640636846096 }, "metric_gaps": {}, "registry_note": "GPT-4o regex; fineweb2full data; norm=NFC; BPE; vocab 128,000; standard-BPE non-parity baseline for the pabpe-nfc-gpt4-fineweb2full family; USED - S5b fineweb2full analysis + S6e.1; fineweb2full data variant, excluded from main paper panel", "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a139/cmeister/tokenizer-lm/checkpoints/full-128k-bpe-nfc-gpt4-fineweb2full", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "full-128k-bpe-nfc-plus2-balanced": { "run": "full-128k-bpe-nfc-plus2-balanced", "works": [ "paper_a", "tokeval" ], "training_regime": "std-1B", "architecture": { "n_layer": 24, "n_embd": 1536, "n_head": 12, "n_kv_head": 12, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 128256, "total_parameters": 1272841994 }, "training": { "step": 8800, "total_batch_size": 1048576, "token_budget": 9227468800, "total_bytes_consumed": 70016628000, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "full-128k" }, "tokenizer_slug": "bpe-nfc-plus2-balanced", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 0, "metrics": { "val_bpb": 0.7162315897661191, "blimp_acc": 0.8137164179104476, "code_bpb_mean": 0.5248969014124836, "blimp_scoring": "bos_prefixed", "flores_mean_bpb": 2.638139141518543, "flores_n_languages": 215, "mbpp_pass1": 0.11, "humaneval_pass1": 0.0975609756097561, "generation_spec": { "version": "v2-2026-07-30", "bos_context": true, "eos_stop": true, "token_healing": true, "truncation": "left_to_max_length_minus_max_gen", "stop_truncation": "earliest_match_all_terms", "stop_match_decode": "specials_kept" }, "token_healing": true, "gsm8k_exact_match_strict": 0.029567854435178165, "gsm8k_exact_match_flexible": 0.032600454890068235 }, "metric_gaps": {}, "registry_note": "clean-multi plus2 (REGEX_CLEAN_PLUS2: attaches both apostrophe forms, contraction arm removed); balanced (D2) data; norm=NFC; BPE; vocab 128,256; balanced-data control for the plus2 apostrophe-attachment pretok (ported from the Apertus-v2 fw2full family); USED - PAPER_PANEL.md paper-eligible addition (#5, landed after the S6.12 balanced-1B panel was built; paper_bootstrap_balanced_panel n=34); not in panels.json main21", "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a139/cmeister/tokenizer-lm/checkpoints/full-128k-bpe-nfc-plus2-balanced", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "full-128k-bpe-nfc-plus2-balanced-mathcode-scratch": { "run": "full-128k-bpe-nfc-plus2-balanced-mathcode-scratch", "works": [ "blog", "tokeval" ], "training_regime": "mathcode-20B-from-scratch", "architecture": { "n_layer": 24, "n_embd": 1536, "n_head": 12, "n_kv_head": 12, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 128256, "total_parameters": 1272841994 }, "training": { "step": 19073, "total_batch_size": 1048576, "token_budget": 19999490048, "total_bytes_consumed": 151753084755, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "full-128k-mathcode-scratch" }, "tokenizer_slug": "bpe-nfc-plus2-balanced", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 0, "metrics": { "val_bpb": 0.3384368951749275, "code_bpb_mean": 0.3785201421477085, "flores_mean_bpb": 3.372071226576324, "flores_n_languages": 214, "mbpp_pass1": 0.114, "humaneval_pass1": 0.1524390243902439, "generation_spec": { "version": "v2-2026-07-30", "bos_context": true, "eos_stop": true, "token_healing": true, "truncation": "left_to_max_length_minus_max_gen", "stop_truncation": "earliest_match_all_terms", "stop_match_decode": "specials_kept" }, "token_healing": true, "gsm8k_exact_match_strict": 0.21152388172858225, "gsm8k_exact_match_flexible": 0.24184988627748294 }, "metric_gaps": { "blimp_acc": "no full-128k-bpe-nfc-plus2-balanced-mathcode-scratch_blimp_code_bpb.json" }, "registry_note": "#27, clean pretok, slug `bpe-nfc-plus2-balanced`. Standard BPE, clean_multi regex + `plus2` per-language vocab-allocation modifier, `balanced` data, vocab 128,256. Paper-eligible; landed after the balanced-1B panel (\u00a76.12 track note). FINISHED (step 19073). MBPP 0.016 [0.006, 0.028] (seed 42; 3-seed family mean 0.1047, SD 0.0782, this row is the family minimum: see \u00a73a.5).", "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a139/cmeister/tokenizer-lm/checkpoints/full-128k-bpe-nfc-plus2-balanced-mathcode-scratch", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "full-128k-bpe-nfc-plus2-balanced-seed43-mathcode-scratch": { "run": "full-128k-bpe-nfc-plus2-balanced-seed43-mathcode-scratch", "works": [ "blog" ], "training_regime": "mathcode-20B-from-scratch", "architecture": { "n_layer": 24, "n_embd": 1536, "n_head": 12, "n_kv_head": 12, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 128256, "total_parameters": 1272841994 }, "training": { "step": 19073, "total_batch_size": 1048576, "token_budget": 19999490048, "total_bytes_consumed": 151753084755, "seed": 43, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "full-128k-mathcode-scratch" }, "tokenizer_slug": "bpe-nfc-plus2-balanced", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 0, "metrics": { "val_bpb": 0.3387550570042015, "flores_mean_bpb": 3.3593633851683737, "flores_n_languages": 215, "mbpp_pass1": 0.212, "humaneval_pass1": 0.17682926829268292, "generation_spec": { "version": "v2-2026-07-30", "bos_context": true, "eos_stop": true, "token_healing": true, "truncation": "left_to_max_length_minus_max_gen", "stop_truncation": "earliest_match_all_terms", "stop_match_decode": "specials_kept" }, "token_healing": true, "gsm8k_exact_match_strict": 0.20242608036391205, "gsm8k_exact_match_flexible": 0.23426838514025777 }, "metric_gaps": { "blimp_acc": "no full-128k-bpe-nfc-plus2-balanced-seed43-mathcode-scratch_blimp_code_bpb.json", "code_bpb_mean": "no full-128k-bpe-nfc-plus2-balanced-seed43-mathcode-scratch_blimp_code_bpb.json; no full-128k-bpe-nfc-plus2-balanced-seed43-mathcode-scratch_code_bpb.json" }, "registry_note": null, "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a139/cmeister/tokenizer-lm/checkpoints/full-128k-bpe-nfc-plus2-balanced-seed43-mathcode-scratch", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "full-128k-bpe-nfc-plus2-balanced-seed44-mathcode-scratch": { "run": "full-128k-bpe-nfc-plus2-balanced-seed44-mathcode-scratch", "works": [ "blog" ], "training_regime": "mathcode-20B-from-scratch", "architecture": { "n_layer": 24, "n_embd": 1536, "n_head": 12, "n_kv_head": 12, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 128256, "total_parameters": 1272841994 }, "training": { "step": 19073, "total_batch_size": 1048576, "token_budget": 19999490048, "total_bytes_consumed": 151753084755, "seed": 44, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "full-128k-mathcode-scratch" }, "tokenizer_slug": "bpe-nfc-plus2-balanced", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 0, "metrics": { "val_bpb": 0.3387273960099328, "flores_mean_bpb": 3.374212385169335, "flores_n_languages": 215, "mbpp_pass1": 0.228, "humaneval_pass1": 0.15853658536585366, "generation_spec": { "version": "v2-2026-07-30", "bos_context": true, "eos_stop": true, "token_healing": true, "truncation": "left_to_max_length_minus_max_gen", "stop_truncation": "earliest_match_all_terms", "stop_match_decode": "specials_kept" }, "token_healing": true, "gsm8k_exact_match_strict": 0.19863532979529946, "gsm8k_exact_match_flexible": 0.24336618650492797 }, "metric_gaps": { "blimp_acc": "no full-128k-bpe-nfc-plus2-balanced-seed44-mathcode-scratch_blimp_code_bpb.json", "code_bpb_mean": "no full-128k-bpe-nfc-plus2-balanced-seed44-mathcode-scratch_blimp_code_bpb.json; no full-128k-bpe-nfc-plus2-balanced-seed44-mathcode-scratch_code_bpb.json" }, "registry_note": null, "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a139/cmeister/tokenizer-lm/checkpoints/full-128k-bpe-nfc-plus2-balanced-seed44-mathcode-scratch", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "full-128k-bpe-nfc-plus3-balanced": { "run": "full-128k-bpe-nfc-plus3-balanced", "works": [ "paper_a", "tokeval" ], "training_regime": "std-1B", "architecture": { "n_layer": 24, "n_embd": 1536, "n_head": 12, "n_kv_head": 12, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 128256, "total_parameters": 1272841994 }, "training": { "step": 8800, "total_batch_size": 1048576, "token_budget": 9227468800, "total_bytes_consumed": 69921086400, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "full-128k" }, "tokenizer_slug": "bpe-nfc-plus3-balanced", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 0, "metrics": { "val_bpb": 0.7149187453363852, "blimp_acc": 0.8184626865671641, "code_bpb_mean": 0.523232159118093, "blimp_scoring": "bos_prefixed", "flores_mean_bpb": 2.6352858885574904, "flores_n_languages": 215, "mbpp_pass1": 0.066, "humaneval_pass1": 0.08536585365853659, "generation_spec": { "version": "v2-2026-07-30", "bos_context": true, "eos_stop": true, "token_healing": true, "truncation": "left_to_max_length_minus_max_gen", "stop_truncation": "earliest_match_all_terms", "stop_match_decode": "specials_kept" }, "token_healing": true, "gsm8k_exact_match_strict": 0.039423805913570885, "gsm8k_exact_match_flexible": 0.04094010614101592 }, "metric_gaps": {}, "registry_note": "clean-multi plus3 (REGEX_CLEAN_PLUS3: plus2 + guarded trailing-apostrophe attachment); balanced (D2) data; norm=NFC; BPE; vocab 128,256; balanced-data control for plus3; USED - PAPER_PANEL.md paper-eligible addition (#6); paper_bootstrap_balanced_panel n=34; not in panels.json main21", "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a139/cmeister/tokenizer-lm/checkpoints/full-128k-bpe-nfc-plus3-balanced", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "full-128k-bpe-nfc-plus3-balanced-mathcode-scratch": { "run": "full-128k-bpe-nfc-plus3-balanced-mathcode-scratch", "works": [ "blog", "tokeval" ], "training_regime": "mathcode-20B-from-scratch", "architecture": { "n_layer": 24, "n_embd": 1536, "n_head": 12, "n_kv_head": 12, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 128256, "total_parameters": 1272841994 }, "training": { "step": 19073, "total_batch_size": 1048576, "token_budget": 19999490048, "total_bytes_consumed": 151546009194, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "full-128k-mathcode-scratch" }, "tokenizer_slug": "bpe-nfc-plus3-balanced", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 0, "metrics": { "val_bpb": 0.33869831349958357, "code_bpb_mean": 0.3834472849799035, "flores_mean_bpb": 3.3760685904096084, "flores_n_languages": 214, "mbpp_pass1": 0.22, "humaneval_pass1": 0.17073170731707318, "generation_spec": { "version": "v2-2026-07-30", "bos_context": true, "eos_stop": true, "token_healing": true, "truncation": "left_to_max_length_minus_max_gen", "stop_truncation": "earliest_match_all_terms", "stop_match_decode": "specials_kept" }, "token_healing": true, "gsm8k_exact_match_strict": 0.19711902956785443, "gsm8k_exact_match_flexible": 0.24260803639120546 }, "metric_gaps": { "blimp_acc": "no full-128k-bpe-nfc-plus3-balanced-mathcode-scratch_blimp_code_bpb.json" }, "registry_note": "#28, clean pretok, slug `bpe-nfc-plus3-balanced`. Standard BPE, clean_multi regex + `plus3` allocation, `balanced`, vocab 128,256. Paper-eligible. FINISHED (step 19073). MBPP 0.154 [0.124, 0.186].", "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a139/cmeister/tokenizer-lm/checkpoints/full-128k-bpe-nfc-plus3-balanced-mathcode-scratch", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "full-128k-claude-allmulti-equal-bpe": { "run": "full-128k-claude-allmulti-equal-bpe", "works": [ "paper_a", "tokeval" ], "training_regime": "std-1B", "architecture": { "n_layer": 24, "n_embd": 1536, "n_head": 12, "n_kv_head": 12, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 128256, "total_parameters": 1272841994 }, "training": { "step": 8800, "total_batch_size": 1048576, "token_budget": 9227468800, "total_bytes_consumed": 64445796800, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "full-128k" }, "tokenizer_slug": "claude-allmulti-equal-bpe", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 0, "metrics": { "val_bpb": 0.7199554950159701, "blimp_acc": 0.8167611940298506, "code_bpb_mean": 0.5231038867012525, "blimp_scoring": "bos_prefixed", "flores_mean_bpb": 2.653515925549718, "flores_n_languages": 215 }, "metric_gaps": { "mbpp_pass1": "no full-128k-claude-allmulti-equal-bpe_code_v2.json", "humaneval_pass1": "no full-128k-claude-allmulti-equal-bpe_code_v2.json", "gsm8k_exact_match_strict": "no full-128k-claude-allmulti-equal-bpe_gsm8k_v2.json" }, "registry_note": "P3 Claude V1 regex; allmulti_equal data; norm=none; BPE; vocab 128,256; pretokenizer-generality arm, pairs with claude-balanced-bpe; USED - S6d.4 (rho=-0.558)", "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a139/cmeister/tokenizer-lm/checkpoints/full-128k-claude-allmulti-equal-bpe", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "full-128k-claude-balanced-bpe": { "run": "full-128k-claude-balanced-bpe", "works": [ "tokeval", "tokeval_cross_scale" ], "training_regime": "std-1B", "architecture": { "n_layer": 24, "n_embd": 1536, "n_head": 12, "n_kv_head": 12, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "default 15.0, run predates the configurable field", "vocab_size": 128256, "total_parameters": 1272841994 }, "training": { "step": 8800, "total_batch_size": 1048576, "token_budget": 9227468800, "total_bytes_consumed": 69807839200, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "full-128k" }, "tokenizer_slug": "claude-balanced-bpe", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 0, "metrics": { "val_bpb": 0.7139588270432892, "blimp_acc": 0.8212238805970153, "code_bpb_mean": 0.5184472344138235, "blimp_scoring": "bos_prefixed", "flores_mean_bpb": 2.642247695092226, "flores_n_languages": 214, "mbpp_pass1": 0.076, "humaneval_pass1": 0.07317073170731707, "generation_spec": { "version": "v2-2026-07-30", "bos_context": true, "eos_stop": true, "token_healing": true, "truncation": "left_to_max_length_minus_max_gen", "stop_truncation": "earliest_match_all_terms", "stop_match_decode": "specials_kept" }, "token_healing": true, "gsm8k_exact_match_strict": 0.03184230477634572, "gsm8k_exact_match_flexible": 0.03866565579984837 }, "metric_gaps": {}, "registry_note": "P3 Claude V1 regex (REGEX_CLAUDE); balanced (D2) data; norm=none; BPE; vocab 128,260; USED - panels.json main21 (camera-ready main panel)", "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a139/cmeister/tokenizer-lm/checkpoints/full-128k-claude-balanced-bpe", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "full-128k-claude-balanced-bpe-mathcode-scratch": { "run": "full-128k-claude-balanced-bpe-mathcode-scratch", "works": [ "blog", "tokeval" ], "training_regime": "mathcode-20B-from-scratch", "architecture": { "n_layer": 24, "n_embd": 1536, "n_head": 12, "n_kv_head": 12, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 128256, "total_parameters": 1272841994 }, "training": { "step": 19073, "total_batch_size": 1048576, "token_budget": 19999490048, "total_bytes_consumed": 151300558757, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "full-128k-mathcode-scratch" }, "tokenizer_slug": "claude-balanced-bpe", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 0, "metrics": { "val_bpb": 0.33648727932470396, "code_bpb_mean": 0.3861471966147764, "flores_mean_bpb": 3.3743421228681676, "flores_n_languages": 214, "mbpp_pass1": 0.25, "humaneval_pass1": 0.17073170731707318, "generation_spec": { "version": "v2-2026-07-30", "bos_context": true, "eos_stop": true, "token_healing": true, "truncation": "left_to_max_length_minus_max_gen", "stop_truncation": "earliest_match_all_terms", "stop_match_decode": "specials_kept" }, "token_healing": true, "gsm8k_exact_match_strict": 0.22744503411675512, "gsm8k_exact_match_flexible": 0.24715693707354056 }, "metric_gaps": { "blimp_acc": "no full-128k-claude-balanced-bpe-mathcode-scratch_blimp_code_bpb.json" }, "registry_note": "#29, claude pretok, slug `claude-balanced-bpe`. Original main-panel BPE, Claude regex, `balanced`, vocab 128,256. FINISHED (step 19073). Panel-best MBPP 0.220 [0.184, 0.256] (seed 42; 3-seed family mean 0.2040, SD 0.0139: see \u00a73a.5).", "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a139/cmeister/tokenizer-lm/checkpoints/full-128k-claude-balanced-bpe-mathcode-scratch", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "full-128k-claude-balanced-bpe-seed42": { "run": "full-128k-claude-balanced-bpe-seed42", "works": [ "paper_a", "tokeval_cross_scale" ], "training_regime": "std-1B", "architecture": { "n_layer": 24, "n_embd": 1536, "n_head": 12, "n_kv_head": 12, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 128256, "total_parameters": 1272841994 }, "training": { "step": 8800, "total_batch_size": 1048576, "token_budget": 9227468800, "total_bytes_consumed": 69807839200, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "full-128k" }, "tokenizer_slug": "claude-balanced-bpe", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 0, "metrics": { "val_bpb": 0.7155515391969438, "blimp_acc": 0.8163134328358209, "code_bpb_mean": 0.5202755220264306, "blimp_scoring": "bos_prefixed", "flores_mean_bpb": 2.641672000878933, "flores_n_languages": 215, "mbpp_pass1": 0.082, "humaneval_pass1": 0.07926829268292683, "generation_spec": { "version": "v2-2026-07-30", "bos_context": true, "eos_stop": true, "token_healing": true, "truncation": "left_to_max_length_minus_max_gen", "stop_truncation": "earliest_match_all_terms", "stop_match_decode": "specials_kept" }, "token_healing": true, "gsm8k_exact_match_strict": 0.03866565579984837, "gsm8k_exact_match_flexible": 0.04245640636846096 }, "metric_gaps": {}, "registry_note": null, "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a139/cmeister/tokenizer-lm/checkpoints/full-128k-claude-balanced-bpe-seed42", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "full-128k-claude-balanced-bpe-seed43-mathcode-scratch": { "run": "full-128k-claude-balanced-bpe-seed43-mathcode-scratch", "works": [ "blog" ], "training_regime": "mathcode-20B-from-scratch", "architecture": { "n_layer": 24, "n_embd": 1536, "n_head": 12, "n_kv_head": 12, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 128256, "total_parameters": 1272841994 }, "training": { "step": 19073, "total_batch_size": 1048576, "token_budget": 19999490048, "total_bytes_consumed": 151300558757, "seed": 43, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "full-128k-mathcode-scratch" }, "tokenizer_slug": "claude-balanced-bpe", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 0, "metrics": { "val_bpb": 0.337272087796867, "flores_mean_bpb": 3.3792211859742856, "flores_n_languages": 215, "mbpp_pass1": 0.234, "humaneval_pass1": 0.16463414634146342, "generation_spec": { "version": "v2-2026-07-30", "bos_context": true, "eos_stop": true, "token_healing": true, "truncation": "left_to_max_length_minus_max_gen", "stop_truncation": "earliest_match_all_terms", "stop_match_decode": "specials_kept" }, "token_healing": true, "gsm8k_exact_match_strict": 0.23199393479909022, "gsm8k_exact_match_flexible": 0.26838514025777105 }, "metric_gaps": { "blimp_acc": "no full-128k-claude-balanced-bpe-seed43-mathcode-scratch_blimp_code_bpb.json", "code_bpb_mean": "no full-128k-claude-balanced-bpe-seed43-mathcode-scratch_blimp_code_bpb.json; no full-128k-claude-balanced-bpe-seed43-mathcode-scratch_code_bpb.json" }, "registry_note": null, "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a139/cmeister/tokenizer-lm/checkpoints/full-128k-claude-balanced-bpe-seed43-mathcode-scratch", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "full-128k-claude-balanced-bpe-seed44-mathcode-scratch": { "run": "full-128k-claude-balanced-bpe-seed44-mathcode-scratch", "works": [ "blog" ], "training_regime": "mathcode-20B-from-scratch", "architecture": { "n_layer": 24, "n_embd": 1536, "n_head": 12, "n_kv_head": 12, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 128256, "total_parameters": 1272841994 }, "training": { "step": 19073, "total_batch_size": 1048576, "token_budget": 19999490048, "total_bytes_consumed": 151300558757, "seed": 44, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "full-128k-mathcode-scratch" }, "tokenizer_slug": "claude-balanced-bpe", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 0, "metrics": { "val_bpb": 0.33757502590578914, "flores_mean_bpb": 3.3411153775837144, "flores_n_languages": 215, "mbpp_pass1": 0.24, "humaneval_pass1": 0.17073170731707318, "generation_spec": { "version": "v2-2026-07-30", "bos_context": true, "eos_stop": true, "token_healing": true, "truncation": "left_to_max_length_minus_max_gen", "stop_truncation": "earliest_match_all_terms", "stop_match_decode": "specials_kept" }, "token_healing": true, "gsm8k_exact_match_strict": 0.21455648218347234, "gsm8k_exact_match_flexible": 0.244882486732373 }, "metric_gaps": { "blimp_acc": "no full-128k-claude-balanced-bpe-seed44-mathcode-scratch_blimp_code_bpb.json", "code_bpb_mean": "no full-128k-claude-balanced-bpe-seed44-mathcode-scratch_blimp_code_bpb.json; no full-128k-claude-balanced-bpe-seed44-mathcode-scratch_code_bpb.json" }, "registry_note": null, "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a139/cmeister/tokenizer-lm/checkpoints/full-128k-claude-balanced-bpe-seed44-mathcode-scratch", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "full-128k-claude-balanced-nfc-bpe": { "run": "full-128k-claude-balanced-nfc-bpe", "works": [ "tokeval", "tokeval_cross_scale" ], "training_regime": "std-1B", "architecture": { "n_layer": 24, "n_embd": 1536, "n_head": 12, "n_kv_head": 12, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "default 15.0, run predates the configurable field", "vocab_size": 128256, "total_parameters": 1272841994 }, "training": { "step": 8800, "total_batch_size": 1048576, "token_budget": 9227468800, "total_bytes_consumed": 69807698400, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "full-128k" }, "tokenizer_slug": "claude-balanced-nfc-bpe", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 0, "metrics": { "val_bpb": 0.7135749335882782, "blimp_acc": 0.8230895522388058, "code_bpb_mean": 0.517861350253653, "blimp_scoring": "bos_prefixed", "flores_mean_bpb": 2.637053701281863, "flores_n_languages": 214, "mbpp_pass1": 0.06, "humaneval_pass1": 0.08536585365853659, "generation_spec": { "version": "v2-2026-07-30", "bos_context": true, "eos_stop": true, "token_healing": true, "truncation": "left_to_max_length_minus_max_gen", "stop_truncation": "earliest_match_all_terms", "stop_match_decode": "specials_kept" }, "token_healing": true, "gsm8k_exact_match_strict": 0.03411675511751327, "gsm8k_exact_match_flexible": 0.03411675511751327 }, "metric_gaps": {}, "registry_note": "P3 Claude V1 regex; balanced (D2) data; norm=NFC; BPE; vocab 128,260; USED - panels.json main21 (camera-ready main panel)", "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a139/cmeister/tokenizer-lm/checkpoints/full-128k-claude-balanced-nfc-bpe", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "full-128k-claude-balanced-nfc-bpe-mathcode-scratch": { "run": "full-128k-claude-balanced-nfc-bpe-mathcode-scratch", "works": [ "blog", "tokeval" ], "training_regime": "mathcode-20B-from-scratch", "architecture": { "n_layer": 24, "n_embd": 1536, "n_head": 12, "n_kv_head": 12, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 128256, "total_parameters": 1272841994 }, "training": { "step": 19073, "total_batch_size": 1048576, "token_budget": 19999490048, "total_bytes_consumed": 151300253589, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "full-128k-mathcode-scratch" }, "tokenizer_slug": "claude-balanced-nfc-bpe", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 0, "metrics": { "val_bpb": 0.33714380335290617, "code_bpb_mean": 0.38451702385863634, "flores_mean_bpb": 3.3666954557164184, "flores_n_languages": 215, "mbpp_pass1": 0.224, "humaneval_pass1": 0.15853658536585366, "generation_spec": { "version": "v2-2026-07-30", "bos_context": true, "eos_stop": true, "token_healing": true, "truncation": "left_to_max_length_minus_max_gen", "stop_truncation": "earliest_match_all_terms", "stop_match_decode": "specials_kept" }, "token_healing": true, "gsm8k_exact_match_strict": 0.2304776345716452, "gsm8k_exact_match_flexible": 0.26838514025777105 }, "metric_gaps": { "blimp_acc": "no full-128k-claude-balanced-nfc-bpe-mathcode-scratch_blimp_code_bpb.json" }, "registry_note": "#50, claude pretok, slug `claude-balanced-nfc-bpe`. Main-grid Claude NFC cell. FINISHED at step 19073, 4-eval chain + code_bpb complete (verified 2026-08-09). In panel.", "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a139/cmeister/tokenizer-lm/checkpoints/full-128k-claude-balanced-nfc-bpe-mathcode-scratch", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "full-128k-claude-balanced-nfc-bpe-seed42": { "run": "full-128k-claude-balanced-nfc-bpe-seed42", "works": [ "paper_a", "tokeval_cross_scale" ], "training_regime": "std-1B", "architecture": { "n_layer": 24, "n_embd": 1536, "n_head": 12, "n_kv_head": 12, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 128256, "total_parameters": 1272841994 }, "training": { "step": 8800, "total_batch_size": 1048576, "token_budget": 9227468800, "total_bytes_consumed": 69807698400, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "full-128k" }, "tokenizer_slug": "claude-balanced-nfc-bpe", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 0, "metrics": { "val_bpb": 0.7152138612405923, "blimp_acc": 0.8188955223880598, "code_bpb_mean": 0.520628642499762, "blimp_scoring": "bos_prefixed", "flores_mean_bpb": 2.6415782571300936, "flores_n_languages": 215, "mbpp_pass1": 0.098, "humaneval_pass1": 0.08536585365853659, "generation_spec": { "version": "v2-2026-07-30", "bos_context": true, "eos_stop": true, "token_healing": true, "truncation": "left_to_max_length_minus_max_gen", "stop_truncation": "earliest_match_all_terms", "stop_match_decode": "specials_kept" }, "token_healing": true, "gsm8k_exact_match_strict": 0.043214556482183475, "gsm8k_exact_match_flexible": 0.05079605761940864 }, "metric_gaps": {}, "registry_note": null, "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a139/cmeister/tokenizer-lm/checkpoints/full-128k-claude-balanced-nfc-bpe-seed42", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "full-128k-claude-balanced-unigram": { "run": "full-128k-claude-balanced-unigram", "works": [ "tokeval", "tokeval_cross_scale" ], "training_regime": "std-1B", "architecture": { "n_layer": 24, "n_embd": 1536, "n_head": 12, "n_kv_head": 12, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "default 15.0, run predates the configurable field", "vocab_size": 128256, "total_parameters": 1272841994 }, "training": { "step": 8800, "total_batch_size": 1048576, "token_budget": 9227468800, "total_bytes_consumed": 59884660000, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "full-128k" }, "tokenizer_slug": "claude-balanced-unigram", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 0, "metrics": { "val_bpb": 0.7320967780165727, "blimp_acc": 0.8351641791044776, "code_bpb_mean": 0.5539673906500296, "blimp_scoring": "bos_prefixed", "flores_mean_bpb": 2.68308108385546, "flores_n_languages": 214, "mbpp_pass1": 0.052, "humaneval_pass1": 0.07317073170731707, "generation_spec": { "version": "v2-2026-07-30", "bos_context": true, "eos_stop": true, "token_healing": true, "truncation": "left_to_max_length_minus_max_gen", "stop_truncation": "earliest_match_all_terms", "stop_match_decode": "specials_kept" }, "token_healing": true, "gsm8k_exact_match_strict": 0.027293404094010616, "gsm8k_exact_match_flexible": 0.02880970432145565 }, "metric_gaps": {}, "registry_note": "P3 Claude V1 regex; balanced (D2) data; norm=none; UnigramLM (HF, default kwargs); vocab 128,260; USED - panels.json main21 (camera-ready main panel)", "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a139/cmeister/tokenizer-lm/checkpoints/full-128k-claude-balanced-unigram", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "full-128k-claude-balanced-unigram-mathcode-scratch": { "run": "full-128k-claude-balanced-unigram-mathcode-scratch", "works": [ "blog", "tokeval" ], "training_regime": "mathcode-20B-from-scratch", "architecture": { "n_layer": 24, "n_embd": 1536, "n_head": 12, "n_kv_head": 12, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 128256, "total_parameters": 1272841994 }, "training": { "step": 19073, "total_batch_size": 1048576, "token_budget": 19999490048, "total_bytes_consumed": 129793195475, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "full-128k-mathcode-scratch" }, "tokenizer_slug": "claude-balanced-unigram", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 0, "metrics": { "val_bpb": 0.3545650839336789, "code_bpb_mean": 0.4032088690906815, "flores_mean_bpb": 3.369774047069325, "flores_n_languages": 215, "mbpp_pass1": 0.222, "humaneval_pass1": 0.14634146341463414, "generation_spec": { "version": "v2-2026-07-30", "bos_context": true, "eos_stop": true, "token_healing": true, "truncation": "left_to_max_length_minus_max_gen", "stop_truncation": "earliest_match_all_terms", "stop_match_decode": "specials_kept" }, "token_healing": true, "gsm8k_exact_match_strict": 0.1645185746777862, "gsm8k_exact_match_flexible": 0.20773313115996966 }, "metric_gaps": { "blimp_acc": "no full-128k-claude-balanced-unigram-mathcode-scratch_blimp_code_bpb.json" }, "registry_note": "#54, claude pretok, slug `claude-balanced-unigram`. Main-grid UnigramLM cell. FINISHED at step 19073, 4-eval chain + code_bpb complete (verified 2026-08-09). In panel.", "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a139/cmeister/tokenizer-lm/checkpoints/full-128k-claude-balanced-unigram-mathcode-scratch", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "full-128k-claude-balanced-unigram-seed42": { "run": "full-128k-claude-balanced-unigram-seed42", "works": [ "paper_a", "tokeval_cross_scale" ], "training_regime": "std-1B", "architecture": { "n_layer": 24, "n_embd": 1536, "n_head": 12, "n_kv_head": 12, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 128256, "total_parameters": 1272841994 }, "training": { "step": 8800, "total_batch_size": 1048576, "token_budget": 9227468800, "total_bytes_consumed": 59884660000, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "full-128k" }, "tokenizer_slug": "claude-balanced-unigram", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 0, "metrics": { "val_bpb": 0.735912221128107, "blimp_acc": 0.840686567164179, "code_bpb_mean": 0.5577539471842974, "blimp_scoring": "bos_prefixed", "flores_mean_bpb": 2.6830926833304374, "flores_n_languages": 215, "mbpp_pass1": 0.058, "humaneval_pass1": 0.07317073170731707, "generation_spec": { "version": "v2-2026-07-30", "bos_context": true, "eos_stop": true, "token_healing": true, "truncation": "left_to_max_length_minus_max_gen", "stop_truncation": "earliest_match_all_terms", "stop_match_decode": "specials_kept" }, "token_healing": true, "gsm8k_exact_match_strict": 0.027293404094010616, "gsm8k_exact_match_flexible": 0.032600454890068235 }, "metric_gaps": {}, "registry_note": null, "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a139/cmeister/tokenizer-lm/checkpoints/full-128k-claude-balanced-unigram-seed42", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "full-128k-claude-english-bpe": { "run": "full-128k-claude-english-bpe", "works": [ "tokeval", "tokeval_cross_scale" ], "training_regime": "std-1B", "architecture": { "n_layer": 24, "n_embd": 1536, "n_head": 12, "n_kv_head": 12, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "default 15.0, run predates the configurable field", "vocab_size": 128256, "total_parameters": 1272841994 }, "training": { "step": 8800, "total_batch_size": 1048576, "token_budget": 9227468800, "total_bytes_consumed": 65874424000, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "full-128k" }, "tokenizer_slug": "claude-english-bpe", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 0, "metrics": { "val_bpb": 0.7312714204866293, "blimp_acc": 0.8178805970149255, "code_bpb_mean": 0.5454915866651372, "blimp_scoring": "bos_prefixed", "flores_mean_bpb": 2.6425029933386486, "flores_n_languages": 214, "mbpp_pass1": 0.068, "humaneval_pass1": 0.07926829268292683, "generation_spec": { "version": "v2-2026-07-30", "bos_context": true, "eos_stop": true, "token_healing": true, "truncation": "left_to_max_length_minus_max_gen", "stop_truncation": "earliest_match_all_terms", "stop_match_decode": "specials_kept" }, "token_healing": true, "gsm8k_exact_match_strict": 0.032600454890068235, "gsm8k_exact_match_flexible": 0.0356330553449583 }, "metric_gaps": {}, "registry_note": "P3 Claude V1 regex; english (D1) data; norm=none; BPE; vocab 128,260; affected by the initial_alphabet byte-drop defect (same class as gpt4o-english-bpe); USED - panels.json main21 (camera-ready main panel)", "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a139/cmeister/tokenizer-lm/checkpoints/full-128k-claude-english-bpe", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "full-128k-claude-english-bpe-mathcode-scratch": { "run": "full-128k-claude-english-bpe-mathcode-scratch", "works": [ "blog", "tokeval" ], "training_regime": "mathcode-20B-from-scratch", "architecture": { "n_layer": 24, "n_embd": 1536, "n_head": 12, "n_kv_head": 12, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 128256, "total_parameters": 1272841994 }, "training": { "step": 19073, "total_batch_size": 1048576, "token_budget": 19999490048, "total_bytes_consumed": 142775328290, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "full-128k-mathcode-scratch" }, "tokenizer_slug": "claude-english-bpe", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 0, "metrics": { "val_bpb": 0.34048861753468623, "code_bpb_mean": 0.36517357096573205, "flores_mean_bpb": 3.1992614684262377, "flores_n_languages": 215, "mbpp_pass1": 0.234, "humaneval_pass1": 0.16463414634146342, "generation_spec": { "version": "v2-2026-07-30", "bos_context": true, "eos_stop": true, "token_healing": true, "truncation": "left_to_max_length_minus_max_gen", "stop_truncation": "earliest_match_all_terms", "stop_match_decode": "specials_kept" }, "token_healing": true, "gsm8k_exact_match_strict": 0.221379833206975, "gsm8k_exact_match_flexible": 0.26156178923426837 }, "metric_gaps": { "blimp_acc": "no full-128k-claude-english-bpe-mathcode-scratch_blimp_code_bpb.json" }, "registry_note": "#51, claude pretok, slug `claude-english-bpe`. Main-grid Claude English-data cell. FINISHED at step 19073, 4-eval chain + code_bpb complete (verified 2026-08-09). In panel.", "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a139/cmeister/tokenizer-lm/checkpoints/full-128k-claude-english-bpe-mathcode-scratch", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "full-128k-claude-english-bpe-seed42": { "run": "full-128k-claude-english-bpe-seed42", "works": [ "paper_a", "tokeval_cross_scale" ], "training_regime": "std-1B", "architecture": { "n_layer": 24, "n_embd": 1536, "n_head": 12, "n_kv_head": 12, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 128256, "total_parameters": 1272841994 }, "training": { "step": 8800, "total_batch_size": 1048576, "token_budget": 9227468800, "total_bytes_consumed": 65874424000, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "full-128k" }, "tokenizer_slug": "claude-english-bpe", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 0, "metrics": { "val_bpb": 0.7329930803024799, "blimp_acc": 0.8168358208955225, "code_bpb_mean": 0.5426109795117547, "blimp_scoring": "bos_prefixed", "flores_mean_bpb": 2.6410893400233615, "flores_n_languages": 215, "mbpp_pass1": 0.072, "humaneval_pass1": 0.06707317073170732, "generation_spec": { "version": "v2-2026-07-30", "bos_context": true, "eos_stop": true, "token_healing": true, "truncation": "left_to_max_length_minus_max_gen", "stop_truncation": "earliest_match_all_terms", "stop_match_decode": "specials_kept" }, "token_healing": true, "gsm8k_exact_match_strict": 0.032600454890068235, "gsm8k_exact_match_flexible": 0.03790750568612585 }, "metric_gaps": {}, "registry_note": null, "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a139/cmeister/tokenizer-lm/checkpoints/full-128k-claude-english-bpe-seed42", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "full-128k-clauderev-balanced-bpe": { "run": "full-128k-clauderev-balanced-bpe", "works": [ "paper_a", "tokeval" ], "training_regime": "std-1B", "architecture": { "n_layer": 24, "n_embd": 1536, "n_head": 12, "n_kv_head": 12, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 128256, "total_parameters": 1272841994 }, "training": { "step": 8800, "total_batch_size": 1048576, "token_budget": 9227468800, "total_bytes_consumed": 71005158400, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "full-128k" }, "tokenizer_slug": "clauderev-balanced-bpe", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 0, "metrics": { "val_bpb": 0.7095599606329199, "blimp_acc": 0.8197910447761193, "code_bpb_mean": 0.520372828134578, "blimp_scoring": "bos_prefixed", "flores_mean_bpb": 2.6486381006261834, "flores_n_languages": 215, "mbpp_pass1": 0.06, "humaneval_pass1": 0.07317073170731707, "generation_spec": { "version": "v2-2026-07-30", "bos_context": true, "eos_stop": true, "token_healing": true, "truncation": "left_to_max_length_minus_max_gen", "stop_truncation": "earliest_match_all_terms", "stop_match_decode": "specials_kept" }, "token_healing": true, "gsm8k_exact_match_strict": 0.047763457164518575, "gsm8k_exact_match_flexible": 0.052312357846853674 }, "metric_gaps": {}, "registry_note": "REGEX_CLAUDE_REV (Claude V2: no camel-case split, no cross-script split, dedicated contraction clause, lookbehind-gated space-attach); balanced (D2) data; norm=NFD + delete CR/VT/FF/NEL + fold-whitespace-to-U+0020 (Sequence normalizer); BPE; vocab 128,256; Claude V2 spec (2026-07-05 build); BPB metrics were biased +6.40% by the NFD byte-denominator bug, FIXED 2026-07-13 (accuracy metrics MBPP/HumanEval/GSM8K/BLiMP/code_bpb unaffected); never enters a val_bpb/FLORES correlation per panels.json note; USED - panels.json pretok-controls (camera-ready)", "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a139/cmeister/tokenizer-lm/checkpoints/full-128k-clauderev-balanced-bpe", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "full-128k-clauderev-balanced-bpe-mathcode-scratch": { "run": "full-128k-clauderev-balanced-bpe-mathcode-scratch", "works": [ "blog" ], "training_regime": "mathcode-20B-from-scratch", "architecture": { "n_layer": 24, "n_embd": 1536, "n_head": 12, "n_kv_head": 12, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 128256, "total_parameters": 1272841994 }, "training": { "step": 19073, "total_batch_size": 1048576, "token_budget": 19999490048, "total_bytes_consumed": 153895612064, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "full-128k-mathcode-scratch" }, "tokenizer_slug": "clauderev-balanced-bpe", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 0, "metrics": { "val_bpb": 0.33694964163473035, "blimp_acc": 0.743537313432836, "code_bpb_mean": 0.3561613667750621, "blimp_scoring": "bos_prefixed", "flores_mean_bpb": 3.375535076836162, "flores_n_languages": 215, "mbpp_pass1": 0.238, "humaneval_pass1": 0.17073170731707318, "generation_spec": { "version": "v2-2026-07-30", "bos_context": true, "eos_stop": true, "token_healing": true, "truncation": "left_to_max_length_minus_max_gen", "stop_truncation": "earliest_match_all_terms", "stop_match_decode": "specials_kept" }, "token_healing": true, "gsm8k_exact_match_strict": 0.20773313115996966, "gsm8k_exact_match_flexible": 0.2517058377558757 }, "metric_gaps": {}, "registry_note": "#26, clauderev pretok, slug `clauderev-balanced-bpe`. Claude V2: revised-Claude pretok (no camel-case/script split, lookbehind-gated space-attach, contraction-suffix clause) + NFD+CR/VT/FF/NEL-delete+whitespace-fold normalization (std-1B twin note has full pretok detail). BPE, balanced data, vocab 128,256. FINISHED step 19073: MBPP 0.206 (3rd of 37), HumanEval 0.079, GSM8K 0.240, mc_total ~0.267. INVALIDATED 2026-07-13: FLORES/val BPB understated (NFD inflates the BPB byte denominator +6.40% overall, +126.9% Korean); MBPP/HumanEval/GSM8K/mc_*/code_bpb unaffected. See EXPERIMENTAL_SETUP.md 'BPB byte-denominator bias'.", "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a139/cmeister/tokenizer-lm/checkpoints/full-128k-clauderev-balanced-bpe-mathcode-scratch", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "full-128k-gpt2-balanced-bpe": { "run": "full-128k-gpt2-balanced-bpe", "works": [ "paper_a", "tokeval" ], "training_regime": "std-1B", "architecture": { "n_layer": 24, "n_embd": 1536, "n_head": 12, "n_kv_head": 12, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 128256, "total_parameters": 1272841994 }, "training": { "step": 8800, "total_batch_size": 1048576, "token_budget": 9227468800, "total_bytes_consumed": 69381127200, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "full-128k" }, "tokenizer_slug": "gpt2-balanced-bpe", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 0, "metrics": { "val_bpb": 0.712764934305536, "blimp_acc": 0.8157761194029848, "code_bpb_mean": 0.5154497166409894, "blimp_scoring": "bos_prefixed", "flores_mean_bpb": 2.640532228202252, "flores_n_languages": 214, "mbpp_pass1": 0.03, "humaneval_pass1": 0.07926829268292683, "generation_spec": { "version": "v2-2026-07-30", "bos_context": true, "eos_stop": true, "token_healing": true, "truncation": "left_to_max_length_minus_max_gen", "stop_truncation": "earliest_match_all_terms", "stop_match_decode": "specials_kept" }, "token_healing": true, "gsm8k_exact_match_strict": 0.027293404094010616, "gsm8k_exact_match_flexible": 0.03411675511751327 }, "metric_gaps": {}, "registry_note": "plain ByteLevel(use_regex=True) i.e. GPT-2 tiktoken regex; balanced (D2) data; norm=none; BPE; vocab 128,260; reference/default regex missing from the original panel; USED - panels.json pretok-controls (camera-ready)", "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a139/cmeister/tokenizer-lm/checkpoints/full-128k-gpt2-balanced-bpe", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "full-128k-gpt4o-allmulti-equal-nfc-bpe": { "run": "full-128k-gpt4o-allmulti-equal-nfc-bpe", "works": [ "paper_a", "tokeval" ], "training_regime": "std-1B", "architecture": { "n_layer": 24, "n_embd": 1536, "n_head": 12, "n_kv_head": 12, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 128256, "total_parameters": 1272841994 }, "training": { "step": 8800, "total_batch_size": 1048576, "token_budget": 9227468800, "total_bytes_consumed": 63006160800, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "full-128k" }, "tokenizer_slug": "gpt4o-allmulti-equal-nfc-bpe", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 0, "metrics": { "val_bpb": 0.7164581029723067, "blimp_acc": 0.8234776119402984, "code_bpb_mean": 0.5197707064555515, "blimp_scoring": "bos_prefixed", "flores_mean_bpb": 2.6459986350982936, "flores_n_languages": 215 }, "metric_gaps": { "mbpp_pass1": "no full-128k-gpt4o-allmulti-equal-nfc-bpe_code_v2.json", "humaneval_pass1": "no full-128k-gpt4o-allmulti-equal-nfc-bpe_code_v2.json", "gsm8k_exact_match_strict": "no full-128k-gpt4o-allmulti-equal-nfc-bpe_gsm8k_v2.json" }, "registry_note": "P2 GPT-4o regex; allmulti_equal data; norm=NFC; BPE; vocab 128,256; pure normalizer contrast against allmulti-equal-bpe; pairs with gpt4o-balanced-nfc-bpe; USED - S6d.4 (rho=-0.475)", "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a139/cmeister/tokenizer-lm/checkpoints/full-128k-gpt4o-allmulti-equal-nfc-bpe", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "full-128k-gpt4o-allmulti-equal-norepeat-bpe": { "run": "full-128k-gpt4o-allmulti-equal-norepeat-bpe", "works": [ "paper_a", "tokeval" ], "training_regime": "std-1B", "architecture": { "n_layer": 24, "n_embd": 1536, "n_head": 12, "n_kv_head": 12, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 128256, "total_parameters": 1272841994 }, "training": { "step": 8800, "total_batch_size": 1048576, "token_budget": 9227468800, "total_bytes_consumed": 63005448000, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "full-128k" }, "tokenizer_slug": "gpt4o-allmulti-equal-norepeat-bpe", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 0, "metrics": { "val_bpb": 0.7164844120425224, "blimp_acc": 0.8191791044776121, "code_bpb_mean": 0.519729913930031, "blimp_scoring": "bos_prefixed", "flores_mean_bpb": 2.647938997818136, "flores_n_languages": 215 }, "metric_gaps": { "mbpp_pass1": "no full-128k-gpt4o-allmulti-equal-norepeat-bpe_code_v2.json", "humaneval_pass1": "no full-128k-gpt4o-allmulti-equal-norepeat-bpe_code_v2.json", "gsm8k_exact_match_strict": "no full-128k-gpt4o-allmulti-equal-norepeat-bpe_gsm8k_v2.json" }, "registry_note": "P2 GPT-4o regex; allmulti_equal_norepeat (equal weights, max_files=50, NO repeat-sampling) data; norm=none; BPE; vocab 128,256; isolates repeat-sampling factor against allmulti-equal-bpe; USED - S6d.4 (rho=-0.581; repeat-sampling contributes at most ~1/4 of the tail penalty)", "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a139/cmeister/tokenizer-lm/checkpoints/full-128k-gpt4o-allmulti-equal-norepeat-bpe", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "full-128k-gpt4o-balanced-bpe": { "run": "full-128k-gpt4o-balanced-bpe", "works": [ "tokeval", "tokeval_cross_scale" ], "training_regime": "std-1B", "architecture": { "n_layer": 24, "n_embd": 1536, "n_head": 12, "n_kv_head": 12, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "default 15.0, run predates the configurable field", "vocab_size": 128256, "total_parameters": 1272841994 }, "training": { "step": 8800, "total_batch_size": 1048576, "token_budget": 9227468800, "total_bytes_consumed": 68117183200, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "full-128k" }, "tokenizer_slug": "gpt4o-balanced-bpe", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 0, "metrics": { "val_bpb": 0.7113825837842997, "blimp_acc": 0.816850746268657, "code_bpb_mean": 0.5183171305128864, "blimp_scoring": "bos_prefixed", "flores_mean_bpb": 2.6387506982410183, "flores_n_languages": 214, "mbpp_pass1": 0.068, "humaneval_pass1": 0.09146341463414634, "generation_spec": { "version": "v2-2026-07-30", "bos_context": true, "eos_stop": true, "token_healing": true, "truncation": "left_to_max_length_minus_max_gen", "stop_truncation": "earliest_match_all_terms", "stop_match_decode": "specials_kept" }, "token_healing": true, "gsm8k_exact_match_strict": 0.045489006823351025, "gsm8k_exact_match_flexible": 0.050037907505686124 }, "metric_gaps": {}, "registry_note": "P2 GPT-4o regex (REGEX_GPT4O); balanced (D2) data; norm=none; BPE; vocab 128,260; project baseline tokenizer (used as the fixed tokenizer for the softcap ablation and as the mathcode-scratch #1 baseline); USED - panels.json main21 (camera-ready main panel)", "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a139/cmeister/tokenizer-lm/checkpoints/full-128k-gpt4o-balanced-bpe", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "full-128k-gpt4o-balanced-bpe-seed42": { "run": "full-128k-gpt4o-balanced-bpe-seed42", "works": [ "paper_a", "tokeval_cross_scale" ], "training_regime": "std-1B", "architecture": { "n_layer": 24, "n_embd": 1536, "n_head": 12, "n_kv_head": 12, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 128256, "total_parameters": 1272841994 }, "training": { "step": 8800, "total_batch_size": 1048576, "token_budget": 9227468800, "total_bytes_consumed": 68117183200, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "full-128k" }, "tokenizer_slug": "gpt4o-balanced-bpe", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 0, "metrics": { "val_bpb": 0.7131214303656023, "blimp_acc": 0.8196268656716418, "code_bpb_mean": 0.5205519366009381, "blimp_scoring": "bos_prefixed", "flores_mean_bpb": 2.6409485662618386, "flores_n_languages": 215, "mbpp_pass1": 0.066, "humaneval_pass1": 0.06707317073170732, "generation_spec": { "version": "v2-2026-07-30", "bos_context": true, "eos_stop": true, "token_healing": true, "truncation": "left_to_max_length_minus_max_gen", "stop_truncation": "earliest_match_all_terms", "stop_match_decode": "specials_kept" }, "token_healing": true, "gsm8k_exact_match_strict": 0.04624715693707354, "gsm8k_exact_match_flexible": 0.052312357846853674 }, "metric_gaps": {}, "registry_note": null, "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a139/cmeister/tokenizer-lm/checkpoints/full-128k-gpt4o-balanced-bpe-seed42", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "full-128k-gpt4o-balanced-bpe-seed43-mathcode-scratch": { "run": "full-128k-gpt4o-balanced-bpe-seed43-mathcode-scratch", "works": [ "blog" ], "training_regime": "mathcode-20B-from-scratch", "architecture": { "n_layer": 24, "n_embd": 1536, "n_head": 12, "n_kv_head": 12, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 128256, "total_parameters": 1272841994 }, "training": { "step": 19073, "total_batch_size": 1048576, "token_budget": 19999490048, "total_bytes_consumed": 147636253997, "seed": 43, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "full-128k-mathcode-scratch" }, "tokenizer_slug": "gpt4o-balanced-bpe", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 0, "metrics": { "val_bpb": 0.3362506080752585, "flores_mean_bpb": 3.3809551460810874, "flores_n_languages": 215, "mbpp_pass1": 0.072, "humaneval_pass1": 0.15853658536585366, "generation_spec": { "version": "v2-2026-07-30", "bos_context": true, "eos_stop": true, "token_healing": true, "truncation": "left_to_max_length_minus_max_gen", "stop_truncation": "earliest_match_all_terms", "stop_match_decode": "specials_kept" }, "token_healing": true, "gsm8k_exact_match_strict": 0.24184988627748294, "gsm8k_exact_match_flexible": 0.27520849128127367 }, "metric_gaps": { "blimp_acc": "no full-128k-gpt4o-balanced-bpe-seed43-mathcode-scratch_blimp_code_bpb.json", "code_bpb_mean": "no full-128k-gpt4o-balanced-bpe-seed43-mathcode-scratch_blimp_code_bpb.json; no full-128k-gpt4o-balanced-bpe-seed43-mathcode-scratch_code_bpb.json" }, "registry_note": null, "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a139/cmeister/tokenizer-lm/checkpoints/full-128k-gpt4o-balanced-bpe-seed43-mathcode-scratch", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "full-128k-gpt4o-balanced-bpe-seed44-mathcode-scratch": { "run": "full-128k-gpt4o-balanced-bpe-seed44-mathcode-scratch", "works": [ "blog" ], "training_regime": "mathcode-20B-from-scratch", "architecture": { "n_layer": 24, "n_embd": 1536, "n_head": 12, "n_kv_head": 12, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 128256, "total_parameters": 1272841994 }, "training": { "step": 19073, "total_batch_size": 1048576, "token_budget": 19999490048, "total_bytes_consumed": 147636253997, "seed": 44, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "full-128k-mathcode-scratch" }, "tokenizer_slug": "gpt4o-balanced-bpe", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 0, "metrics": { "val_bpb": 0.33549283645238004, "flores_mean_bpb": 3.3683389882182975, "flores_n_languages": 215, "mbpp_pass1": 0.018, "humaneval_pass1": 0.16463414634146342, "generation_spec": { "version": "v2-2026-07-30", "bos_context": true, "eos_stop": true, "token_healing": true, "truncation": "left_to_max_length_minus_max_gen", "stop_truncation": "earliest_match_all_terms", "stop_match_decode": "specials_kept" }, "token_healing": true, "gsm8k_exact_match_strict": 0.23199393479909022, "gsm8k_exact_match_flexible": 0.2562547384382108 }, "metric_gaps": { "blimp_acc": "no full-128k-gpt4o-balanced-bpe-seed44-mathcode-scratch_blimp_code_bpb.json", "code_bpb_mean": "no full-128k-gpt4o-balanced-bpe-seed44-mathcode-scratch_blimp_code_bpb.json; no full-128k-gpt4o-balanced-bpe-seed44-mathcode-scratch_code_bpb.json" }, "registry_note": null, "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a139/cmeister/tokenizer-lm/checkpoints/full-128k-gpt4o-balanced-bpe-seed44-mathcode-scratch", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "full-128k-gpt4o-balanced-maxfiles50-bpe": { "run": "full-128k-gpt4o-balanced-maxfiles50-bpe", "works": [ "paper_a" ], "training_regime": "std-1B", "architecture": { "n_layer": 24, "n_embd": 1536, "n_head": 12, "n_kv_head": 12, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 128256, "total_parameters": 1272841994 }, "training": { "step": 8800, "total_batch_size": 1048576, "token_budget": 9227468800, "total_bytes_consumed": 68117183200, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "full-128k" }, "tokenizer_slug": "gpt4o-balanced-maxfiles50-bpe", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 0, "metrics": { "val_bpb": 0.7115753975968808, "blimp_acc": 0.8217014925373136, "code_bpb_mean": 0.5175390204895408, "blimp_scoring": "bos_prefixed", "flores_mean_bpb": 2.6288706791146534, "flores_n_languages": 215 }, "metric_gaps": { "mbpp_pass1": "no full-128k-gpt4o-balanced-maxfiles50-bpe_code_v2.json", "humaneval_pass1": "no full-128k-gpt4o-balanced-maxfiles50-bpe_code_v2.json", "gsm8k_exact_match_strict": "no full-128k-gpt4o-balanced-maxfiles50-bpe_gsm8k_v2.json" }, "registry_note": "P2 GPT-4o regex; balanced (proportional) weights, max_files raised to 50, no repeat data; norm=none; BPE; vocab 128,256; NULL CONTROL for the max_files cap; USED - S6d.4 null control (rho=+0.004, p=0.98 - exonerates max_files as the driver)", "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a139/cmeister/tokenizer-lm/checkpoints/full-128k-gpt4o-balanced-maxfiles50-bpe", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "full-128k-gpt4o-balanced-nfc-bpe": { "run": "full-128k-gpt4o-balanced-nfc-bpe", "works": [ "tokeval", "tokeval_cross_scale" ], "training_regime": "std-1B", "architecture": { "n_layer": 24, "n_embd": 1536, "n_head": 12, "n_kv_head": 12, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "default 15.0, run predates the configurable field", "vocab_size": 128256, "total_parameters": 1272841994 }, "training": { "step": 8800, "total_batch_size": 1048576, "token_budget": 9227468800, "total_bytes_consumed": 68118485600, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "full-128k" }, "tokenizer_slug": "gpt4o-balanced-nfc-bpe", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 0, "metrics": { "val_bpb": 0.7114993123173433, "blimp_acc": 0.8128955223880601, "code_bpb_mean": 0.5185776938972276, "blimp_scoring": "bos_prefixed", "flores_mean_bpb": 2.637031235082467, "flores_n_languages": 214, "mbpp_pass1": 0.068, "humaneval_pass1": 0.07926829268292683, "generation_spec": { "version": "v2-2026-07-30", "bos_context": true, "eos_stop": true, "token_healing": true, "truncation": "left_to_max_length_minus_max_gen", "stop_truncation": "earliest_match_all_terms", "stop_match_decode": "specials_kept" }, "token_healing": true, "gsm8k_exact_match_strict": 0.04852160727824109, "gsm8k_exact_match_flexible": 0.053828658074298714 }, "metric_gaps": {}, "registry_note": "P2 GPT-4o regex; balanced (D2) data; norm=NFC; BPE; vocab 128,260; USED - panels.json main21 (camera-ready main panel)", "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a139/cmeister/tokenizer-lm/checkpoints/full-128k-gpt4o-balanced-nfc-bpe", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "full-128k-gpt4o-balanced-nfc-bpe-mathcode-scratch": { "run": "full-128k-gpt4o-balanced-nfc-bpe-mathcode-scratch", "works": [ "blog", "tokeval" ], "training_regime": "mathcode-20B-from-scratch", "architecture": { "n_layer": 24, "n_embd": 1536, "n_head": 12, "n_kv_head": 12, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 128256, "total_parameters": 1272841994 }, "training": { "step": 19073, "total_batch_size": 1048576, "token_budget": 19999490048, "total_bytes_consumed": 147639076801, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "full-128k-mathcode-scratch" }, "tokenizer_slug": "gpt4o-balanced-nfc-bpe", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 0, "metrics": { "val_bpb": 0.3360314499008519, "code_bpb_mean": 0.3786237132757522, "flores_mean_bpb": 3.4023914971799996, "flores_n_languages": 214, "mbpp_pass1": 0.12, "humaneval_pass1": 0.1524390243902439, "generation_spec": { "version": "v2-2026-07-30", "bos_context": true, "eos_stop": true, "token_healing": true, "truncation": "left_to_max_length_minus_max_gen", "stop_truncation": "earliest_match_all_terms", "stop_match_decode": "specials_kept" }, "token_healing": true, "gsm8k_exact_match_strict": 0.20773313115996966, "gsm8k_exact_match_flexible": 0.2441243366186505 }, "metric_gaps": { "blimp_acc": "no full-128k-gpt4o-balanced-nfc-bpe-mathcode-scratch_blimp_code_bpb.json" }, "registry_note": "#2, gpt4o pretok, slug `gpt4o-balanced-nfc-bpe`. NFC pair to (1) for the digit-direction test.", "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a139/cmeister/tokenizer-lm/checkpoints/full-128k-gpt4o-balanced-nfc-bpe-mathcode-scratch", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "full-128k-gpt4o-balanced-nfc-bpe-seed42": { "run": "full-128k-gpt4o-balanced-nfc-bpe-seed42", "works": [ "paper_a", "tokeval_cross_scale" ], "training_regime": "std-1B", "architecture": { "n_layer": 24, "n_embd": 1536, "n_head": 12, "n_kv_head": 12, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 128256, "total_parameters": 1272841994 }, "training": { "step": 8800, "total_batch_size": 1048576, "token_budget": 9227468800, "total_bytes_consumed": 68118485600, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "full-128k" }, "tokenizer_slug": "gpt4o-balanced-nfc-bpe", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 0, "metrics": { "val_bpb": 0.7133388823384232, "blimp_acc": 0.8202089552238806, "code_bpb_mean": 0.520749213831268, "blimp_scoring": "bos_prefixed", "flores_mean_bpb": 2.6376940912300553, "flores_n_languages": 215, "mbpp_pass1": 0.05, "humaneval_pass1": 0.07926829268292683, "generation_spec": { "version": "v2-2026-07-30", "bos_context": true, "eos_stop": true, "token_healing": true, "truncation": "left_to_max_length_minus_max_gen", "stop_truncation": "earliest_match_all_terms", "stop_match_decode": "specials_kept" }, "token_healing": true, "gsm8k_exact_match_strict": 0.04700530705079606, "gsm8k_exact_match_flexible": 0.05307050796057619 }, "metric_gaps": {}, "registry_note": null, "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a139/cmeister/tokenizer-lm/checkpoints/full-128k-gpt4o-balanced-nfc-bpe-seed42", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "full-128k-gpt4o-balanced-unigram": { "run": "full-128k-gpt4o-balanced-unigram", "works": [ "tokeval", "tokeval_cross_scale" ], "training_regime": "std-1B", "architecture": { "n_layer": 24, "n_embd": 1536, "n_head": 12, "n_kv_head": 12, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "default 15.0, run predates the configurable field", "vocab_size": 128256, "total_parameters": 1272841994 }, "training": { "step": 8800, "total_batch_size": 1048576, "token_budget": 9227468800, "total_bytes_consumed": 59929337600, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "full-128k" }, "tokenizer_slug": "gpt4o-balanced-unigram", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 0, "metrics": { "val_bpb": 0.7314636077017965, "blimp_acc": 0.8325522388059705, "code_bpb_mean": 0.5537964877519875, "blimp_scoring": "bos_prefixed", "flores_mean_bpb": 2.6859967266679656, "flores_n_languages": 214, "mbpp_pass1": 0.066, "humaneval_pass1": 0.06707317073170732, "generation_spec": { "version": "v2-2026-07-30", "bos_context": true, "eos_stop": true, "token_healing": true, "truncation": "left_to_max_length_minus_max_gen", "stop_truncation": "earliest_match_all_terms", "stop_match_decode": "specials_kept" }, "token_healing": true, "gsm8k_exact_match_strict": 0.024260803639120546, "gsm8k_exact_match_flexible": 0.0310841546626232 }, "metric_gaps": {}, "registry_note": "P2 GPT-4o regex; balanced (D2) data; norm=none; UnigramLM (HF UnigramTrainer, default kwargs); vocab 128,260; USED - panels.json main21 (camera-ready main panel)", "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a139/cmeister/tokenizer-lm/checkpoints/full-128k-gpt4o-balanced-unigram", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "full-128k-gpt4o-balanced-unigram-mathcode-scratch": { "run": "full-128k-gpt4o-balanced-unigram-mathcode-scratch", "works": [ "blog", "tokeval" ], "training_regime": "mathcode-20B-from-scratch", "architecture": { "n_layer": 24, "n_embd": 1536, "n_head": 12, "n_kv_head": 12, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 128256, "total_parameters": 1272841994 }, "training": { "step": 19073, "total_batch_size": 1048576, "token_budget": 19999490048, "total_bytes_consumed": 129890029096, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "full-128k-mathcode-scratch" }, "tokenizer_slug": "gpt4o-balanced-unigram", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 0, "metrics": { "val_bpb": 0.3533980943471727, "code_bpb_mean": 0.3993466898373258, "flores_mean_bpb": 3.4055634092286033, "flores_n_languages": 215, "mbpp_pass1": 0.0, "humaneval_pass1": 0.13414634146341464, "generation_spec": { "version": "v2-2026-07-30", "bos_context": true, "eos_stop": true, "token_healing": true, "truncation": "left_to_max_length_minus_max_gen", "stop_truncation": "earliest_match_all_terms", "stop_match_decode": "specials_kept" }, "token_healing": true, "gsm8k_exact_match_strict": 0.1865049279757392, "gsm8k_exact_match_flexible": 0.23199393479909022 }, "metric_gaps": { "blimp_acc": "no full-128k-gpt4o-balanced-unigram-mathcode-scratch_blimp_code_bpb.json" }, "registry_note": "#53, gpt4o pretok, slug `gpt4o-balanced-unigram`. Main-grid UnigramLM cell (first UnigramLM run at 20B). FINISHED at step 19073, 4-eval chain + code_bpb complete (verified 2026-08-09). In panel.", "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a139/cmeister/tokenizer-lm/checkpoints/full-128k-gpt4o-balanced-unigram-mathcode-scratch", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "full-128k-gpt4o-balanced-unigram-seed42": { "run": "full-128k-gpt4o-balanced-unigram-seed42", "works": [ "paper_a", "tokeval_cross_scale" ], "training_regime": "std-1B", "architecture": { "n_layer": 24, "n_embd": 1536, "n_head": 12, "n_kv_head": 12, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 128256, "total_parameters": 1272841994 }, "training": { "step": 8800, "total_batch_size": 1048576, "token_budget": 9227468800, "total_bytes_consumed": 59929337600, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "full-128k" }, "tokenizer_slug": "gpt4o-balanced-unigram", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 0, "metrics": { "val_bpb": 0.7337511969511488, "blimp_acc": 0.8338208955223879, "code_bpb_mean": 0.5569454278048204, "blimp_scoring": "bos_prefixed", "flores_mean_bpb": 2.6873471717751567, "flores_n_languages": 215, "mbpp_pass1": 0.04, "humaneval_pass1": 0.07317073170731707, "generation_spec": { "version": "v2-2026-07-30", "bos_context": true, "eos_stop": true, "token_healing": true, "truncation": "left_to_max_length_minus_max_gen", "stop_truncation": "earliest_match_all_terms", "stop_match_decode": "specials_kept" }, "token_healing": true, "gsm8k_exact_match_strict": 0.032600454890068235, "gsm8k_exact_match_flexible": 0.03866565579984837 }, "metric_gaps": {}, "registry_note": null, "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a139/cmeister/tokenizer-lm/checkpoints/full-128k-gpt4o-balanced-unigram-seed42", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "full-128k-gpt4o-balanced-unigram-tuned": { "run": "full-128k-gpt4o-balanced-unigram-tuned", "works": [ "paper_a", "tokeval" ], "training_regime": "std-1B", "architecture": { "n_layer": 24, "n_embd": 1536, "n_head": 12, "n_kv_head": 12, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 128256, "total_parameters": 1272841994 }, "training": { "step": 8800, "total_batch_size": 1048576, "token_budget": 9227468800, "total_bytes_consumed": 68301156000, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "full-128k" }, "tokenizer_slug": "gpt4o-balanced-unigram-tuned", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 0, "metrics": { "val_bpb": 0.7332765694658377, "blimp_acc": 0.838014925373134, "code_bpb_mean": 0.5564997843345418, "blimp_scoring": "bos_prefixed", "flores_mean_bpb": 2.686368297404418, "flores_n_languages": 215, "mbpp_pass1": 0.042, "humaneval_pass1": 0.054878048780487805, "generation_spec": { "version": "v2-2026-07-30", "bos_context": true, "eos_stop": true, "token_healing": true, "truncation": "left_to_max_length_minus_max_gen", "stop_truncation": "earliest_match_all_terms", "stop_match_decode": "specials_kept" }, "token_healing": true, "gsm8k_exact_match_strict": 0.028051554207733132, "gsm8k_exact_match_flexible": 0.032600454890068235 }, "metric_gaps": {}, "registry_note": "P2 GPT-4o regex; balanced (D2) data; norm=none; UnigramLM (HF, UNIGRAM_TUNED_KWARGS: shrinking_factor=0.7, n_sub_iterations=3, max_piece_length=64, full byte initial_alphabet); vocab 128,260; tests whether 'Unigram loses' survives tuned HF UnigramTrainer hyperparameters; USED - balanced-1B bootstrap panel (S6.12, n=32/34) and paper_bootstrap_balanced_panel; not in panels.json main21 (superseded by default-kwargs gpt4o-balanced-unigram in the settled panel)", "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a139/cmeister/tokenizer-lm/checkpoints/full-128k-gpt4o-balanced-unigram-tuned", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "full-128k-gpt4o-code-bpe": { "run": "full-128k-gpt4o-code-bpe", "works": [ "tokeval", "tokeval_cross_scale" ], "training_regime": "std-1B", "architecture": { "n_layer": 24, "n_embd": 1536, "n_head": 12, "n_kv_head": 12, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "default 15.0, run predates the configurable field", "vocab_size": 128256, "total_parameters": 1272841994 }, "training": { "step": 8800, "total_batch_size": 1048576, "token_budget": 9227468800, "total_bytes_consumed": 59664431200, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "full-128k" }, "tokenizer_slug": "gpt4o-code-bpe", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 0, "metrics": { "val_bpb": 0.7238738916126876, "blimp_acc": 0.821, "code_bpb_mean": 0.5431368524901159, "blimp_scoring": "bos_prefixed", "flores_mean_bpb": 2.625442837285647, "flores_n_languages": 214, "mbpp_pass1": 0.054, "humaneval_pass1": 0.07317073170731707, "generation_spec": { "version": "v2-2026-07-30", "bos_context": true, "eos_stop": true, "token_healing": true, "truncation": "left_to_max_length_minus_max_gen", "stop_truncation": "earliest_match_all_terms", "stop_match_decode": "specials_kept" }, "token_healing": true, "gsm8k_exact_match_strict": 0.039423805913570885, "gsm8k_exact_match_flexible": 0.04473085670962851 }, "metric_gaps": {}, "registry_note": "P2 GPT-4o regex; code-heavy (D3) data; norm=none; BPE; vocab 128,260; USED - panels.json main21 (camera-ready main panel)", "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a139/cmeister/tokenizer-lm/checkpoints/full-128k-gpt4o-code-bpe", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "full-128k-gpt4o-code-bpe-mathcode-scratch": { "run": "full-128k-gpt4o-code-bpe-mathcode-scratch", "works": [ "blog", "tokeval" ], "training_regime": "mathcode-20B-from-scratch", "architecture": { "n_layer": 24, "n_embd": 1536, "n_head": 12, "n_kv_head": 12, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 128256, "total_parameters": 1272841994 }, "training": { "step": 19073, "total_batch_size": 1048576, "token_budget": 19999490048, "total_bytes_consumed": 129315874577, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "full-128k-mathcode-scratch" }, "tokenizer_slug": "gpt4o-code-bpe", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 0, "metrics": { "val_bpb": 0.33264187797540906, "code_bpb_mean": 0.3804701231471243, "flores_mean_bpb": 3.152441006253198, "flores_n_languages": 214, "mbpp_pass1": 0.016, "humaneval_pass1": 0.15853658536585366, "generation_spec": { "version": "v2-2026-07-30", "bos_context": true, "eos_stop": true, "token_healing": true, "truncation": "left_to_max_length_minus_max_gen", "stop_truncation": "earliest_match_all_terms", "stop_match_decode": "specials_kept" }, "token_healing": true, "gsm8k_exact_match_strict": 0.21607278241091737, "gsm8k_exact_match_flexible": 0.25549658832448824 }, "metric_gaps": { "blimp_acc": "no full-128k-gpt4o-code-bpe-mathcode-scratch_blimp_code_bpb.json" }, "registry_note": "#30, gpt4o pretok, slug `gpt4o-code-bpe`. GPT-4o regex, code-heavy tokenizer-training data, vocab 128,256. FINISHED (step 19073). MBPP 0.016 [0.006, 0.028].", "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a139/cmeister/tokenizer-lm/checkpoints/full-128k-gpt4o-code-bpe-mathcode-scratch", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "full-128k-gpt4o-code-bpe-seed42": { "run": "full-128k-gpt4o-code-bpe-seed42", "works": [ "paper_a", "tokeval_cross_scale" ], "training_regime": "std-1B", "architecture": { "n_layer": 24, "n_embd": 1536, "n_head": 12, "n_kv_head": 12, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 128256, "total_parameters": 1272841994 }, "training": { "step": 8800, "total_batch_size": 1048576, "token_budget": 9227468800, "total_bytes_consumed": 59664431200, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "full-128k" }, "tokenizer_slug": "gpt4o-code-bpe", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 0, "metrics": { "val_bpb": 0.7252439607412663, "blimp_acc": 0.8190149253731343, "code_bpb_mean": 0.5446616749526122, "blimp_scoring": "bos_prefixed", "flores_mean_bpb": 2.625681797157079, "flores_n_languages": 215, "mbpp_pass1": 0.08, "humaneval_pass1": 0.09146341463414634, "generation_spec": { "version": "v2-2026-07-30", "bos_context": true, "eos_stop": true, "token_healing": true, "truncation": "left_to_max_length_minus_max_gen", "stop_truncation": "earliest_match_all_terms", "stop_match_decode": "specials_kept" }, "token_healing": true, "gsm8k_exact_match_strict": 0.0310841546626232, "gsm8k_exact_match_flexible": 0.0356330553449583 }, "metric_gaps": {}, "registry_note": null, "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a139/cmeister/tokenizer-lm/checkpoints/full-128k-gpt4o-code-bpe-seed42", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "full-128k-gpt4o-codegrad00-bpe-mathcode-scratch": { "run": "full-128k-gpt4o-codegrad00-bpe-mathcode-scratch", "works": [ "blog" ], "training_regime": "mathcode-20B-from-scratch", "architecture": { "n_layer": 24, "n_embd": 1536, "n_head": 12, "n_kv_head": 12, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 128256, "total_parameters": 1272841994 }, "training": { "step": 19073, "total_batch_size": 1048576, "token_budget": 19999490048, "total_bytes_consumed": 153517795007, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "full-128k-mathcode-scratch" }, "tokenizer_slug": "gpt4o-codegrad00-bpe", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 0, "metrics": { "val_bpb": 0.339566757938572, "flores_mean_bpb": 3.357431043588078, "flores_n_languages": 215, "mbpp_pass1": 0.232, "humaneval_pass1": 0.16463414634146342, "generation_spec": { "version": "v2-2026-07-30", "bos_context": true, "eos_stop": true, "token_healing": true, "truncation": "left_to_max_length_minus_max_gen", "stop_truncation": "earliest_match_all_terms", "stop_match_decode": "specials_kept" }, "token_healing": true, "gsm8k_exact_match_strict": 0.20545868081880211, "gsm8k_exact_match_flexible": 0.23805913570887036 }, "metric_gaps": { "blimp_acc": "no full-128k-gpt4o-codegrad00-bpe-mathcode-scratch_blimp_code_bpb.json", "code_bpb_mean": "no full-128k-gpt4o-codegrad00-bpe-mathcode-scratch_blimp_code_bpb.json; no full-128k-gpt4o-codegrad00-bpe-mathcode-scratch_code_bpb.json" }, "registry_note": "#39, gpt4o pretok, slug `gpt4o-codegrad00-bpe`. AXES-2 code-fraction gradient, fusing regex (`REGEX_GPT4O`), code fraction 0%. Fused punct+newline vocab tokens: 482. FINISHED. MBPP 0.166 (generation_spec v1); v2 re-derivation (2026-08-04) 0.232, see \u00a73a.3. Not in panel.", "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a139/cmeister/tokenizer-lm/checkpoints/full-128k-gpt4o-codegrad00-bpe-mathcode-scratch", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "full-128k-gpt4o-codegrad05-bpe-mathcode-scratch": { "run": "full-128k-gpt4o-codegrad05-bpe-mathcode-scratch", "works": [ "blog" ], "training_regime": "mathcode-20B-from-scratch", "architecture": { "n_layer": 24, "n_embd": 1536, "n_head": 12, "n_kv_head": 12, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 128256, "total_parameters": 1272841994 }, "training": { "step": 19073, "total_batch_size": 1048576, "token_budget": 19999490048, "total_bytes_consumed": 151162393945, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "full-128k-mathcode-scratch" }, "tokenizer_slug": "gpt4o-codegrad05-bpe", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 0, "metrics": { "val_bpb": 0.3365866322863168, "flores_mean_bpb": 3.3960370708190832, "flores_n_languages": 215, "mbpp_pass1": 0.05, "humaneval_pass1": 0.1402439024390244, "generation_spec": { "version": "v2-2026-07-30", "bos_context": true, "eos_stop": true, "token_healing": true, "truncation": "left_to_max_length_minus_max_gen", "stop_truncation": "earliest_match_all_terms", "stop_match_decode": "specials_kept" }, "token_healing": true, "gsm8k_exact_match_strict": 0.21607278241091737, "gsm8k_exact_match_flexible": 0.24564063684609552 }, "metric_gaps": { "blimp_acc": "no full-128k-gpt4o-codegrad05-bpe-mathcode-scratch_blimp_code_bpb.json", "code_bpb_mean": "no full-128k-gpt4o-codegrad05-bpe-mathcode-scratch_blimp_code_bpb.json; no full-128k-gpt4o-codegrad05-bpe-mathcode-scratch_code_bpb.json" }, "registry_note": "#40, gpt4o pretok, slug `gpt4o-codegrad05-bpe`. AXES-2 code-fraction gradient, fusing regex (`REGEX_GPT4O`), code fraction 5%. Fused 749. FINISHED. MBPP 0.048 (v1); v2 0.050. Not in panel.", "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a139/cmeister/tokenizer-lm/checkpoints/full-128k-gpt4o-codegrad05-bpe-mathcode-scratch", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "full-128k-gpt4o-codegrad30-bpe-mathcode-scratch": { "run": "full-128k-gpt4o-codegrad30-bpe-mathcode-scratch", "works": [ "blog" ], "training_regime": "mathcode-20B-from-scratch", "architecture": { "n_layer": 24, "n_embd": 1536, "n_head": 12, "n_kv_head": 12, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 128256, "total_parameters": 1272841994 }, "training": { "step": 19073, "total_batch_size": 1048576, "token_budget": 19999490048, "total_bytes_consumed": 143650893428, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "full-128k-mathcode-scratch" }, "tokenizer_slug": "gpt4o-codegrad30-bpe", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 0, "metrics": { "val_bpb": 0.33496232135285, "flores_mean_bpb": 3.3437376646164667, "flores_n_languages": 215, "mbpp_pass1": 0.178, "humaneval_pass1": 0.15853658536585366, "generation_spec": { "version": "v2-2026-07-30", "bos_context": true, "eos_stop": true, "token_healing": true, "truncation": "left_to_max_length_minus_max_gen", "stop_truncation": "earliest_match_all_terms", "stop_match_decode": "specials_kept" }, "token_healing": true, "gsm8k_exact_match_strict": 0.20545868081880211, "gsm8k_exact_match_flexible": 0.25094768764215314 }, "metric_gaps": { "blimp_acc": "no full-128k-gpt4o-codegrad30-bpe-mathcode-scratch_blimp_code_bpb.json", "code_bpb_mean": "no full-128k-gpt4o-codegrad30-bpe-mathcode-scratch_blimp_code_bpb.json; no full-128k-gpt4o-codegrad30-bpe-mathcode-scratch_code_bpb.json" }, "registry_note": "#41, gpt4o pretok, slug `gpt4o-codegrad30-bpe`. AXES-2 code-fraction gradient, fusing regex (`REGEX_GPT4O`), code fraction 30%. Fused 1393. FINISHED. MBPP 0.048 (v1); v2 0.178. Not in panel.", "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a139/cmeister/tokenizer-lm/checkpoints/full-128k-gpt4o-codegrad30-bpe-mathcode-scratch", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "full-128k-gpt4o-codegrad50-bpe-mathcode-scratch": { "run": "full-128k-gpt4o-codegrad50-bpe-mathcode-scratch", "works": [ "blog" ], "training_regime": "mathcode-20B-from-scratch", "architecture": { "n_layer": 24, "n_embd": 1536, "n_head": 12, "n_kv_head": 12, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 128256, "total_parameters": 1272841994 }, "training": { "step": 19073, "total_batch_size": 1048576, "token_budget": 19999490048, "total_bytes_consumed": 138855769571, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "full-128k-mathcode-scratch" }, "tokenizer_slug": "gpt4o-codegrad50-bpe", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 0, "metrics": { "val_bpb": 0.33476615801535653, "flores_mean_bpb": 3.3840800213850546, "flores_n_languages": 215, "mbpp_pass1": 0.088, "humaneval_pass1": 0.18902439024390244, "generation_spec": { "version": "v2-2026-07-30", "bos_context": true, "eos_stop": true, "token_healing": true, "truncation": "left_to_max_length_minus_max_gen", "stop_truncation": "earliest_match_all_terms", "stop_match_decode": "specials_kept" }, "token_healing": true, "gsm8k_exact_match_strict": 0.21607278241091737, "gsm8k_exact_match_flexible": 0.265352539802881 }, "metric_gaps": { "blimp_acc": "no full-128k-gpt4o-codegrad50-bpe-mathcode-scratch_blimp_code_bpb.json", "code_bpb_mean": "no full-128k-gpt4o-codegrad50-bpe-mathcode-scratch_blimp_code_bpb.json; no full-128k-gpt4o-codegrad50-bpe-mathcode-scratch_code_bpb.json" }, "registry_note": "#42, gpt4o pretok, slug `gpt4o-codegrad50-bpe`. AXES-2 code-fraction gradient, fusing regex (`REGEX_GPT4O`), code fraction 50%. Fused 1831. FINISHED. MBPP 0.108 (v1); v2 0.088. Not in panel.", "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a139/cmeister/tokenizer-lm/checkpoints/full-128k-gpt4o-codegrad50-bpe-mathcode-scratch", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "full-128k-gpt4o-english-bpe": { "run": "full-128k-gpt4o-english-bpe", "works": [ "tokeval", "tokeval_cross_scale" ], "training_regime": "std-1B", "architecture": { "n_layer": 24, "n_embd": 1536, "n_head": 12, "n_kv_head": 12, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "default 15.0, run predates the configurable field", "vocab_size": 128256, "total_parameters": 1272841994 }, "training": { "step": 8800, "total_batch_size": 1048576, "token_budget": 9227468800, "total_bytes_consumed": 65128632800, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "full-128k" }, "tokenizer_slug": "gpt4o-english-bpe", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 0, "metrics": { "val_bpb": 0.735036299787199, "blimp_acc": 0.8158358208955222, "code_bpb_mean": 0.5583797506682657, "blimp_scoring": "bos_prefixed", "flores_mean_bpb": 2.6469080775193725, "flores_n_languages": 214, "mbpp_pass1": 0.074, "humaneval_pass1": 0.07317073170731707, "generation_spec": { "version": "v2-2026-07-30", "bos_context": true, "eos_stop": true, "token_healing": true, "truncation": "left_to_max_length_minus_max_gen", "stop_truncation": "earliest_match_all_terms", "stop_match_decode": "specials_kept" }, "token_healing": true, "gsm8k_exact_match_strict": 0.02577710386656558, "gsm8k_exact_match_flexible": 0.030326004548900682 }, "metric_gaps": {}, "registry_note": "P2 GPT-4o regex; english (D1) data; norm=none; BPE; vocab 128,260; affected by the initial_alphabet byte-drop defect (drops 32 control bytes incl. '{' and DEL); see gpt4o-english-fullbyte-bpe control; USED - panels.json main21 (camera-ready main panel)", "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a139/cmeister/tokenizer-lm/checkpoints/full-128k-gpt4o-english-bpe", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "full-128k-gpt4o-english-bpe-mathcode-scratch": { "run": "full-128k-gpt4o-english-bpe-mathcode-scratch", "works": [ "blog" ], "training_regime": "mathcode-20B-from-scratch", "architecture": { "n_layer": 24, "n_embd": 1536, "n_head": 12, "n_kv_head": 12, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 128256, "total_parameters": 1272841994 }, "training": { "step": 19073, "total_batch_size": 1048576, "token_budget": 19999490048, "total_bytes_consumed": 141158910613, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "full-128k-mathcode-scratch" }, "tokenizer_slug": "gpt4o-english-bpe", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 0, "metrics": { "val_bpb": 0.342942884295594, "code_bpb_mean": 0.37086217884139866, "flores_mean_bpb": 3.1757044421045038, "flores_n_languages": 214, "mbpp_pass1": 0.212, "humaneval_pass1": 0.17682926829268292, "generation_spec": { "version": "v2-2026-07-30", "bos_context": true, "eos_stop": true, "token_healing": true, "truncation": "left_to_max_length_minus_max_gen", "stop_truncation": "earliest_match_all_terms", "stop_match_decode": "specials_kept" }, "token_healing": true, "gsm8k_exact_match_strict": 0.20621683093252463, "gsm8k_exact_match_flexible": 0.23275208491281274 }, "metric_gaps": { "blimp_acc": "no full-128k-gpt4o-english-bpe-mathcode-scratch_blimp_code_bpb.json" }, "registry_note": "#31, gpt4o pretok, slug `gpt4o-english-bpe`. GPT-4o regex, English-only tokenizer-training data, vocab 128,256. FINISHED (step 19073). MBPP 0.210 [0.174, 0.246].", "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a139/cmeister/tokenizer-lm/checkpoints/full-128k-gpt4o-english-bpe-mathcode-scratch", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "full-128k-gpt4o-english-bpe-seed42": { "run": "full-128k-gpt4o-english-bpe-seed42", "works": [ "tokeval_cross_scale" ], "training_regime": "std-1B", "architecture": { "n_layer": 24, "n_embd": 1536, "n_head": 12, "n_kv_head": 12, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 128256, "total_parameters": 1272841994 }, "training": { "step": 8800, "total_batch_size": 1048576, "token_budget": 9227468800, "total_bytes_consumed": 65128632800, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "full-128k" }, "tokenizer_slug": "gpt4o-english-bpe", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 0, "metrics": { "val_bpb": 0.7374689939971976, "blimp_acc": 0.8065074626865676, "code_bpb_mean": 0.5566762465787464, "blimp_scoring": "bos_prefixed", "flores_mean_bpb": 2.6384577900595425, "flores_n_languages": 215, "mbpp_pass1": 0.06, "humaneval_pass1": 0.07317073170731707, "generation_spec": { "version": "v2-2026-07-30", "bos_context": true, "eos_stop": true, "token_healing": true, "truncation": "left_to_max_length_minus_max_gen", "stop_truncation": "earliest_match_all_terms", "stop_match_decode": "specials_kept" }, "token_healing": true, "gsm8k_exact_match_strict": 0.024260803639120546, "gsm8k_exact_match_flexible": 0.030326004548900682 }, "metric_gaps": {}, "registry_note": null, "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a139/cmeister/tokenizer-lm/checkpoints/full-128k-gpt4o-english-bpe-seed42", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "full-128k-gpt4o-english-fullbyte-bpe": { "run": "full-128k-gpt4o-english-fullbyte-bpe", "works": [ "paper_a", "tokeval" ], "training_regime": "std-1B", "architecture": { "n_layer": 24, "n_embd": 1536, "n_head": 12, "n_kv_head": 12, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 128256, "total_parameters": 1272841994 }, "training": { "step": 8800, "total_batch_size": 1048576, "token_budget": 9227468800, "total_bytes_consumed": 65111437600, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "full-128k" }, "tokenizer_slug": "gpt4o-english-fullbyte-bpe", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 0, "metrics": { "val_bpb": 0.7382166396904717, "blimp_acc": 0.8167910447761192, "code_bpb_mean": 0.5567630102431164, "blimp_scoring": "bos_prefixed", "flores_mean_bpb": 2.64415299419569, "flores_n_languages": 215, "mbpp_pass1": 0.056, "humaneval_pass1": 0.054878048780487805, "generation_spec": { "version": "v2-2026-07-30", "bos_context": true, "eos_stop": true, "token_healing": true, "truncation": "left_to_max_length_minus_max_gen", "stop_truncation": "earliest_match_all_terms", "stop_match_decode": "specials_kept" }, "token_healing": true, "gsm8k_exact_match_strict": 0.03335860500379075, "gsm8k_exact_match_flexible": 0.037149355572403335 }, "metric_gaps": {}, "registry_note": "P2 GPT-4o regex; english (D1) data; norm=none; BPE (trainer_kwargs full_byte_alphabet=True: initial_alphabet forced to the complete 256-entry ByteLevel alphabet); vocab 128,256; isolates the initial_alphabet byte-drop defect vs gpt4o-english-bpe (209/256 -> 256/256 bytes present; 47 fewer merges); USED - panels.json pretok-controls (camera-ready)", "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a139/cmeister/tokenizer-lm/checkpoints/full-128k-gpt4o-english-fullbyte-bpe", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "full-128k-gpt4o-english-fullbyte-bpe-mathcode-scratch": { "run": "full-128k-gpt4o-english-fullbyte-bpe-mathcode-scratch", "works": [ "blog", "tokeval" ], "training_regime": "mathcode-20B-from-scratch", "architecture": { "n_layer": 24, "n_embd": 1536, "n_head": 12, "n_kv_head": 12, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 128256, "total_parameters": 1272841994 }, "training": { "step": 19073, "total_batch_size": 1048576, "token_budget": 19999490048, "total_bytes_consumed": 141121641971, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "full-128k-mathcode-scratch" }, "tokenizer_slug": "gpt4o-english-fullbyte-bpe", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 0, "metrics": { "val_bpb": 0.34344991506677197, "code_bpb_mean": 0.3990911179451489, "flores_mean_bpb": 3.182336686454046, "flores_n_languages": 215, "mbpp_pass1": 0.064, "humaneval_pass1": 0.1524390243902439, "generation_spec": { "version": "v2-2026-07-30", "bos_context": true, "eos_stop": true, "token_healing": true, "truncation": "left_to_max_length_minus_max_gen", "stop_truncation": "earliest_match_all_terms", "stop_match_decode": "specials_kept" }, "token_healing": true, "gsm8k_exact_match_strict": 0.2175890826383624, "gsm8k_exact_match_flexible": 0.23351023502653526 }, "metric_gaps": { "blimp_acc": "no full-128k-gpt4o-english-fullbyte-bpe-mathcode-scratch_blimp_code_bpb.json" }, "registry_note": null, "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a139/cmeister/tokenizer-lm/checkpoints/full-128k-gpt4o-english-fullbyte-bpe-mathcode-scratch", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "full-128k-gpt4onl-balanced-bpe-mathcode-scratch": { "run": "full-128k-gpt4onl-balanced-bpe-mathcode-scratch", "works": [ "blog" ], "training_regime": "mathcode-20B-from-scratch", "architecture": { "n_layer": 24, "n_embd": 1536, "n_head": 12, "n_kv_head": 12, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 128256, "total_parameters": 1272841994 }, "training": { "step": 19073, "total_batch_size": 1048576, "token_budget": 19999490048, "total_bytes_consumed": 148079739393, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "full-128k-mathcode-scratch" }, "tokenizer_slug": "gpt4onl-balanced-bpe", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 0, "metrics": { "val_bpb": 0.337055151819168, "flores_mean_bpb": 3.3435001896408045, "flores_n_languages": 215, "mbpp_pass1": 0.224, "humaneval_pass1": 0.18902439024390244, "generation_spec": { "version": "v2-2026-07-30", "bos_context": true, "eos_stop": true, "token_healing": true, "truncation": "left_to_max_length_minus_max_gen", "stop_truncation": "earliest_match_all_terms", "stop_match_decode": "specials_kept" }, "token_healing": true, "gsm8k_exact_match_strict": 0.2259287338893101, "gsm8k_exact_match_flexible": 0.2585291887793783 }, "metric_gaps": { "blimp_acc": "no full-128k-gpt4onl-balanced-bpe-mathcode-scratch_blimp_code_bpb.json", "code_bpb_mean": "no full-128k-gpt4onl-balanced-bpe-mathcode-scratch_blimp_code_bpb.json; no full-128k-gpt4onl-balanced-bpe-mathcode-scratch_code_bpb.json" }, "registry_note": "#47, gpt4onl pretok, slug `gpt4onl-balanced-bpe`. AXES-2 single-mechanism pair: non-fusing regex on the balanced composition, fused 0. FINISHED. v1: MBPP 0.158, HumanEval 0.0183, GSM8K 0.242. Fusing counterpart is the unslugged baseline `full-128k-mathcode-scratch` (gpt4o-balanced-bpe, same recipe/step). Pair v1: MBPP 0.024 vs 0.158, HumanEval 0.0854 vs 0.0183 - benchmarks disagree on sign; both have clean prompt boundaries (\u00a73a.6), not the HumanEval artifact. v2 (2026-08-04): this run MBPP 0.224/HumanEval 0.1890; baseline MBPP 0.174/HumanEval 0.1646. Not in panel.", "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a139/cmeister/tokenizer-lm/checkpoints/full-128k-gpt4onl-balanced-bpe-mathcode-scratch", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "full-128k-gpt4onl-balanced-bpe-seed43-mathcode-scratch": { "run": "full-128k-gpt4onl-balanced-bpe-seed43-mathcode-scratch", "works": [ "blog" ], "training_regime": "mathcode-20B-from-scratch", "architecture": { "n_layer": 24, "n_embd": 1536, "n_head": 12, "n_kv_head": 12, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 128256, "total_parameters": 1272841994 }, "training": { "step": 19073, "total_batch_size": 1048576, "token_budget": 19999490048, "total_bytes_consumed": 148079739393, "seed": 43, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "full-128k-mathcode-scratch" }, "tokenizer_slug": "gpt4onl-balanced-bpe", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 0, "metrics": { "val_bpb": 0.3376554366247921, "flores_mean_bpb": 3.349377121305643, "flores_n_languages": 215, "mbpp_pass1": 0.246, "humaneval_pass1": 0.1951219512195122, "generation_spec": { "version": "v2-2026-07-30", "bos_context": true, "eos_stop": true, "token_healing": true, "truncation": "left_to_max_length_minus_max_gen", "stop_truncation": "earliest_match_all_terms", "stop_match_decode": "specials_kept" }, "token_healing": true, "gsm8k_exact_match_strict": 0.22971948445792267, "gsm8k_exact_match_flexible": 0.25549658832448824 }, "metric_gaps": { "blimp_acc": "no full-128k-gpt4onl-balanced-bpe-seed43-mathcode-scratch_blimp_code_bpb.json", "code_bpb_mean": "no full-128k-gpt4onl-balanced-bpe-seed43-mathcode-scratch_blimp_code_bpb.json; no full-128k-gpt4onl-balanced-bpe-seed43-mathcode-scratch_code_bpb.json" }, "registry_note": null, "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a139/cmeister/tokenizer-lm/checkpoints/full-128k-gpt4onl-balanced-bpe-seed43-mathcode-scratch", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "full-128k-gpt4onl-balanced-bpe-seed44-mathcode-scratch": { "run": "full-128k-gpt4onl-balanced-bpe-seed44-mathcode-scratch", "works": [ "blog" ], "training_regime": "mathcode-20B-from-scratch", "architecture": { "n_layer": 24, "n_embd": 1536, "n_head": 12, "n_kv_head": 12, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 128256, "total_parameters": 1272841994 }, "training": { "step": 19073, "total_batch_size": 1048576, "token_budget": 19999490048, "total_bytes_consumed": 148079739393, "seed": 44, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "full-128k-mathcode-scratch" }, "tokenizer_slug": "gpt4onl-balanced-bpe", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 0, "metrics": { "val_bpb": 0.33694131565842034, "flores_mean_bpb": 3.3315847422203544, "flores_n_languages": 215, "mbpp_pass1": 0.22, "humaneval_pass1": 0.15853658536585366, "generation_spec": { "version": "v2-2026-07-30", "bos_context": true, "eos_stop": true, "token_healing": true, "truncation": "left_to_max_length_minus_max_gen", "stop_truncation": "earliest_match_all_terms", "stop_match_decode": "specials_kept" }, "token_healing": true, "gsm8k_exact_match_strict": 0.20242608036391205, "gsm8k_exact_match_flexible": 0.2441243366186505 }, "metric_gaps": { "blimp_acc": "no full-128k-gpt4onl-balanced-bpe-seed44-mathcode-scratch_blimp_code_bpb.json", "code_bpb_mean": "no full-128k-gpt4onl-balanced-bpe-seed44-mathcode-scratch_blimp_code_bpb.json; no full-128k-gpt4onl-balanced-bpe-seed44-mathcode-scratch_code_bpb.json" }, "registry_note": null, "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a139/cmeister/tokenizer-lm/checkpoints/full-128k-gpt4onl-balanced-bpe-seed44-mathcode-scratch", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "full-128k-gpt4onl-codegrad00-bpe-mathcode-scratch": { "run": "full-128k-gpt4onl-codegrad00-bpe-mathcode-scratch", "works": [ "blog" ], "training_regime": "mathcode-20B-from-scratch", "architecture": { "n_layer": 24, "n_embd": 1536, "n_head": 12, "n_kv_head": 12, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 128256, "total_parameters": 1272841994 }, "training": { "step": 19073, "total_batch_size": 1048576, "token_budget": 19999490048, "total_bytes_consumed": 153764637773, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "full-128k-mathcode-scratch" }, "tokenizer_slug": "gpt4onl-codegrad00-bpe", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 0, "metrics": { "val_bpb": 0.34202783106124746, "flores_mean_bpb": 3.351781915121498, "flores_n_languages": 215, "mbpp_pass1": 0.248, "humaneval_pass1": 0.17073170731707318, "generation_spec": { "version": "v2-2026-07-30", "bos_context": true, "eos_stop": true, "token_healing": true, "truncation": "left_to_max_length_minus_max_gen", "stop_truncation": "earliest_match_all_terms", "stop_match_decode": "specials_kept" }, "token_healing": true, "gsm8k_exact_match_strict": 0.21076573161485973, "gsm8k_exact_match_flexible": 0.24336618650492797 }, "metric_gaps": { "blimp_acc": "no full-128k-gpt4onl-codegrad00-bpe-mathcode-scratch_blimp_code_bpb.json", "code_bpb_mean": "no full-128k-gpt4onl-codegrad00-bpe-mathcode-scratch_blimp_code_bpb.json; no full-128k-gpt4onl-codegrad00-bpe-mathcode-scratch_code_bpb.json" }, "registry_note": "#43, gpt4onl pretok, slug `gpt4onl-codegrad00-bpe`. AXES-2 code-fraction gradient, non-fusing regex (REGEX_GPT4O_NONL: punct arm loses trailing [\\r\\n/]*, else byte-identical to REGEX_GPT4O), tokenizer-corpus code fraction 0%. Fused 0 by construction. FINISHED. MBPP 0.232 (generation_spec v1), the highest of any run in the 20B mathcode family (v1-scored, like-for-like with the panel). v2 re-derivation (2026-08-04) 0.248, see \u00a73a.3. Not in panel.", "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a139/cmeister/tokenizer-lm/checkpoints/full-128k-gpt4onl-codegrad00-bpe-mathcode-scratch", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "full-128k-gpt4onl-codegrad05-bpe-mathcode-scratch": { "run": "full-128k-gpt4onl-codegrad05-bpe-mathcode-scratch", "works": [ "blog" ], "training_regime": "mathcode-20B-from-scratch", "architecture": { "n_layer": 24, "n_embd": 1536, "n_head": 12, "n_kv_head": 12, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 128256, "total_parameters": 1272841994 }, "training": { "step": 19073, "total_batch_size": 1048576, "token_budget": 19999490048, "total_bytes_consumed": 151521519462, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "full-128k-mathcode-scratch" }, "tokenizer_slug": "gpt4onl-codegrad05-bpe", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 0, "metrics": { "val_bpb": 0.3385269818145197, "flores_mean_bpb": 3.3565886166875782, "flores_n_languages": 215, "mbpp_pass1": 0.206, "humaneval_pass1": 0.16463414634146342, "generation_spec": { "version": "v2-2026-07-30", "bos_context": true, "eos_stop": true, "token_healing": true, "truncation": "left_to_max_length_minus_max_gen", "stop_truncation": "earliest_match_all_terms", "stop_match_decode": "specials_kept" }, "token_healing": true, "gsm8k_exact_match_strict": 0.21228203184230476, "gsm8k_exact_match_flexible": 0.24639878695981804 }, "metric_gaps": { "blimp_acc": "no full-128k-gpt4onl-codegrad05-bpe-mathcode-scratch_blimp_code_bpb.json", "code_bpb_mean": "no full-128k-gpt4onl-codegrad05-bpe-mathcode-scratch_blimp_code_bpb.json; no full-128k-gpt4onl-codegrad05-bpe-mathcode-scratch_code_bpb.json" }, "registry_note": "#44, gpt4onl pretok, slug `gpt4onl-codegrad05-bpe`. AXES-2 code-fraction gradient, non-fusing regex (`REGEX_GPT4O_NONL`), code fraction 5%. Fused 0. FINISHED. MBPP 0.204 (v1); v2 0.206. Not in panel.", "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a139/cmeister/tokenizer-lm/checkpoints/full-128k-gpt4onl-codegrad05-bpe-mathcode-scratch", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "full-128k-gpt4onl-codegrad30-bpe-mathcode-scratch": { "run": "full-128k-gpt4onl-codegrad30-bpe-mathcode-scratch", "works": [ "blog" ], "training_regime": "mathcode-20B-from-scratch", "architecture": { "n_layer": 24, "n_embd": 1536, "n_head": 12, "n_kv_head": 12, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 128256, "total_parameters": 1272841994 }, "training": { "step": 19073, "total_batch_size": 1048576, "token_budget": 19999490048, "total_bytes_consumed": 144184498749, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "full-128k-mathcode-scratch" }, "tokenizer_slug": "gpt4onl-codegrad30-bpe", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 0, "metrics": { "val_bpb": 0.3366932052292442, "flores_mean_bpb": 3.3328335389250148, "flores_n_languages": 215, "mbpp_pass1": 0.224, "humaneval_pass1": 0.17682926829268292, "generation_spec": { "version": "v2-2026-07-30", "bos_context": true, "eos_stop": true, "token_healing": true, "truncation": "left_to_max_length_minus_max_gen", "stop_truncation": "earliest_match_all_terms", "stop_match_decode": "specials_kept" }, "token_healing": true, "gsm8k_exact_match_strict": 0.2001516300227445, "gsm8k_exact_match_flexible": 0.2312357846853677 }, "metric_gaps": { "blimp_acc": "no full-128k-gpt4onl-codegrad30-bpe-mathcode-scratch_blimp_code_bpb.json", "code_bpb_mean": "no full-128k-gpt4onl-codegrad30-bpe-mathcode-scratch_blimp_code_bpb.json; no full-128k-gpt4onl-codegrad30-bpe-mathcode-scratch_code_bpb.json" }, "registry_note": "#45, gpt4onl pretok, slug `gpt4onl-codegrad30-bpe`. AXES-2 code-fraction gradient, non-fusing regex (`REGEX_GPT4O_NONL`), code fraction 30%. Fused 0. FINISHED. MBPP 0.178 (v1); v2 0.224. Not in panel.", "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a139/cmeister/tokenizer-lm/checkpoints/full-128k-gpt4onl-codegrad30-bpe-mathcode-scratch", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "full-128k-gpt4onl-codegrad50-bpe-mathcode-scratch": { "run": "full-128k-gpt4onl-codegrad50-bpe-mathcode-scratch", "works": [ "blog" ], "training_regime": "mathcode-20B-from-scratch", "architecture": { "n_layer": 24, "n_embd": 1536, "n_head": 12, "n_kv_head": 12, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 128256, "total_parameters": 1272841994 }, "training": { "step": 19073, "total_batch_size": 1048576, "token_budget": 19999490048, "total_bytes_consumed": 139282241851, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "full-128k-mathcode-scratch" }, "tokenizer_slug": "gpt4onl-codegrad50-bpe", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 0, "metrics": { "val_bpb": 0.3365895514079721, "flores_mean_bpb": 3.3430497401898602, "flores_n_languages": 215, "mbpp_pass1": 0.212, "humaneval_pass1": 0.1951219512195122, "generation_spec": { "version": "v2-2026-07-30", "bos_context": true, "eos_stop": true, "token_healing": true, "truncation": "left_to_max_length_minus_max_gen", "stop_truncation": "earliest_match_all_terms", "stop_match_decode": "specials_kept" }, "token_healing": true, "gsm8k_exact_match_strict": 0.2350265352539803, "gsm8k_exact_match_flexible": 0.26004548900682334 }, "metric_gaps": { "blimp_acc": "no full-128k-gpt4onl-codegrad50-bpe-mathcode-scratch_blimp_code_bpb.json", "code_bpb_mean": "no full-128k-gpt4onl-codegrad50-bpe-mathcode-scratch_blimp_code_bpb.json; no full-128k-gpt4onl-codegrad50-bpe-mathcode-scratch_code_bpb.json" }, "registry_note": "#46, gpt4onl pretok, slug `gpt4onl-codegrad50-bpe`. AXES-2 code-fraction gradient, non-fusing regex (`REGEX_GPT4O_NONL`), code fraction 50%. Fused 0. FINISHED. MBPP 0.174 (v1); v2 0.212. Not in panel.", "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a139/cmeister/tokenizer-lm/checkpoints/full-128k-gpt4onl-codegrad50-bpe-mathcode-scratch", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "full-128k-highmid-bpe": { "run": "full-128k-highmid-bpe", "works": [ "paper_a", "tokeval" ], "training_regime": "std-1B", "architecture": { "n_layer": 24, "n_embd": 1536, "n_head": 12, "n_kv_head": 12, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 128256, "total_parameters": 1272841994 }, "training": { "step": 8800, "total_batch_size": 1048576, "token_budget": 9227468800, "total_bytes_consumed": 69171696000, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "full-128k" }, "tokenizer_slug": "highmid-bpe", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 0, "metrics": { "val_bpb": 0.7137029022782296, "blimp_acc": 0.8148805970149254, "code_bpb_mean": 0.5213740689827874, "blimp_scoring": "bos_prefixed", "flores_mean_bpb": 2.6374848259239103, "flores_n_languages": 215, "mbpp_pass1": 0.096, "humaneval_pass1": 0.06707317073170732, "generation_spec": { "version": "v2-2026-07-30", "bos_context": true, "eos_stop": true, "token_healing": true, "truncation": "left_to_max_length_minus_max_gen", "stop_truncation": "earliest_match_all_terms", "stop_match_decode": "specials_kept" }, "token_healing": true, "gsm8k_exact_match_strict": 0.04245640636846096, "gsm8k_exact_match_flexible": 0.058377558756633814 }, "metric_gaps": {}, "registry_note": "P2 GPT-4o regex; highmid (21 high+mid-resource langs + English + math/code) data; norm=none; BPE; vocab 128,260; intermediate-language-coverage probe (tokenizer sees 21 langs); USED - S6d.3 coverage ladder; superseded within panels.json main21 by highmid-unigram", "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a139/cmeister/tokenizer-lm/checkpoints/full-128k-highmid-bpe", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "full-128k-highmid-unigram": { "run": "full-128k-highmid-unigram", "works": [ "paper_a", "tokeval" ], "training_regime": "std-1B", "architecture": { "n_layer": 24, "n_embd": 1536, "n_head": 12, "n_kv_head": 12, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 128256, "total_parameters": 1272841994 }, "training": { "step": 8800, "total_batch_size": 1048576, "token_budget": 9227468800, "total_bytes_consumed": 61515159200, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "full-128k" }, "tokenizer_slug": "highmid-unigram", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 0, "metrics": { "val_bpb": 0.7345206818002139, "blimp_acc": 0.8309552238805971, "code_bpb_mean": 0.5576028102921461, "blimp_scoring": "bos_prefixed", "flores_mean_bpb": 2.6747609744397476, "flores_n_languages": 215, "mbpp_pass1": 0.058, "humaneval_pass1": 0.07926829268292683, "generation_spec": { "version": "v2-2026-07-30", "bos_context": true, "eos_stop": true, "token_healing": true, "truncation": "left_to_max_length_minus_max_gen", "stop_truncation": "earliest_match_all_terms", "stop_match_decode": "specials_kept" }, "token_healing": true, "gsm8k_exact_match_strict": 0.02880970432145565, "gsm8k_exact_match_flexible": 0.032600454890068235 }, "metric_gaps": {}, "registry_note": "P2 GPT-4o regex; highmid (21 langs) data; norm=none; UnigramLM (HF, default kwargs); vocab 128,260; Unigram counterpart to highmid-bpe; USED - panels.json main21 (camera-ready main panel)", "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a139/cmeister/tokenizer-lm/checkpoints/full-128k-highmid-unigram", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "full-128k-highres-bpe": { "run": "full-128k-highres-bpe", "works": [ "paper_a", "tokeval" ], "training_regime": "std-1B", "architecture": { "n_layer": 24, "n_embd": 1536, "n_head": 12, "n_kv_head": 12, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 128256, "total_parameters": 1272841994 }, "training": { "step": 8800, "total_batch_size": 1048576, "token_budget": 9227468800, "total_bytes_consumed": 76247124800, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "full-128k" }, "tokenizer_slug": "highres-bpe", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 0, "metrics": { "val_bpb": 0.7127224232645878, "blimp_acc": 0.8137164179104476, "code_bpb_mean": 0.5216821970561181, "blimp_scoring": "bos_prefixed", "flores_mean_bpb": 2.617718386084036, "flores_n_languages": 215, "mbpp_pass1": 0.046, "humaneval_pass1": 0.054878048780487805, "generation_spec": { "version": "v2-2026-07-30", "bos_context": true, "eos_stop": true, "token_healing": true, "truncation": "left_to_max_length_minus_max_gen", "stop_truncation": "earliest_match_all_terms", "stop_match_decode": "specials_kept" }, "token_healing": true, "gsm8k_exact_match_strict": 0.034874905231235785, "gsm8k_exact_match_flexible": 0.04245640636846096 }, "metric_gaps": {}, "registry_note": "P2 GPT-4o regex; highres (6 high-resource langs + English + math/code) data; norm=none; BPE; vocab 128,260; intermediate-language-coverage probe (tokenizer sees 6 langs, LM trains on all 30); USED - S6d.3 coverage ladder (english-only/highres/highmid/balanced GPT-4o-BPE comparison); superseded within panels.json main21 by highres-unigram", "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a139/cmeister/tokenizer-lm/checkpoints/full-128k-highres-bpe", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "full-128k-highres-unigram": { "run": "full-128k-highres-unigram", "works": [ "paper_a", "tokeval" ], "training_regime": "std-1B", "architecture": { "n_layer": 24, "n_embd": 1536, "n_head": 12, "n_kv_head": 12, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 128256, "total_parameters": 1272841994 }, "training": { "step": 8800, "total_batch_size": 1048576, "token_budget": 9227468800, "total_bytes_consumed": 67182544000, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "full-128k" }, "tokenizer_slug": "highres-unigram", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 0, "metrics": { "val_bpb": 0.7305499098474624, "blimp_acc": 0.8239552238805969, "code_bpb_mean": 0.5556786069357298, "blimp_scoring": "bos_prefixed", "flores_mean_bpb": 2.666006768996987, "flores_n_languages": 215, "mbpp_pass1": 0.072, "humaneval_pass1": 0.07926829268292683, "generation_spec": { "version": "v2-2026-07-30", "bos_context": true, "eos_stop": true, "token_healing": true, "truncation": "left_to_max_length_minus_max_gen", "stop_truncation": "earliest_match_all_terms", "stop_match_decode": "specials_kept" }, "token_healing": true, "gsm8k_exact_match_strict": 0.026535253980288095, "gsm8k_exact_match_flexible": 0.030326004548900682 }, "metric_gaps": {}, "registry_note": "P2 GPT-4o regex; highres (6 langs) data; norm=none; UnigramLM (HF, default kwargs); vocab 128,260; Unigram counterpart to highres-bpe, completes the {BPE,Unigram}x{highres,highmid,balanced} grid; USED - panels.json main21 (camera-ready main panel)", "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a139/cmeister/tokenizer-lm/checkpoints/full-128k-highres-unigram", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "full-128k-llama3": { "run": "full-128k-llama3", "works": [ "tokeval", "tokeval_cross_scale" ], "training_regime": "std-1B", "architecture": { "n_layer": 24, "n_embd": 1536, "n_head": 12, "n_kv_head": 12, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "default 15.0, run predates the configurable field", "vocab_size": 128256, "total_parameters": 1272841994 }, "training": { "step": 8800, "total_batch_size": 1048576, "token_budget": 9227468800, "total_bytes_consumed": 59564639200, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "full-128k" }, "tokenizer_slug": "NousResearch/Meta-Llama-3-8B", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 128000, "metrics": { "val_bpb": 0.718139704096831, "blimp_acc": 0.8201940298507464, "code_bpb_mean": 0.5477978759768082, "blimp_scoring": "bos_prefixed", "flores_mean_bpb": 2.661359414824764, "flores_n_languages": 214, "mbpp_pass1": 0.052, "humaneval_pass1": 0.06707317073170732, "generation_spec": { "version": "v2-2026-07-30", "bos_context": true, "eos_stop": true, "token_healing": true, "truncation": "left_to_max_length_minus_max_gen", "stop_truncation": "earliest_match_all_terms", "stop_match_decode": "specials_kept" }, "token_healing": true, "gsm8k_exact_match_strict": 0.03639120545868082, "gsm8k_exact_match_flexible": 0.04245640636846096 }, "metric_gaps": {}, "registry_note": "LLaMA-3 tiktoken (off-the-shelf); n/a (off-the-shelf, Meta-Llama-3-8B pretraining mix) data; norm=n/a; off-the-shelf BPE; vocab 128,256; USED - panels.json main21 (camera-ready main panel)", "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a139/cmeister/tokenizer-lm/checkpoints/full-128k-llama3", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "full-128k-llama3-mathcode-scratch": { "run": "full-128k-llama3-mathcode-scratch", "works": [ "blog" ], "training_regime": "mathcode-20B-from-scratch", "architecture": { "n_layer": 24, "n_embd": 1536, "n_head": 12, "n_kv_head": 12, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 128256, "total_parameters": 1272841994 }, "training": { "step": 19073, "total_batch_size": 1048576, "token_budget": 19999490048, "total_bytes_consumed": 129099586757, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "full-128k-mathcode-scratch" }, "tokenizer_slug": "llama3", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 128000, "metrics": { "val_bpb": 0.3334353889147577, "code_bpb_mean": 0.3778736514478963, "flores_mean_bpb": 3.3787162974554765, "flores_n_languages": 214, "mbpp_pass1": 0.082, "humaneval_pass1": 0.17073170731707318, "generation_spec": { "version": "v2-2026-07-30", "bos_context": true, "eos_stop": true, "token_healing": true, "truncation": "left_to_max_length_minus_max_gen", "stop_truncation": "earliest_match_all_terms", "stop_match_decode": "specials_kept" }, "token_healing": true, "gsm8k_exact_match_strict": 0.23275208491281274, "gsm8k_exact_match_flexible": 0.2721758908263836 }, "metric_gaps": { "blimp_acc": "no full-128k-llama3-mathcode-scratch_blimp_code_bpb.json" }, "registry_note": "#32, llama3 pretok, slug `llama3`. Off-the-shelf Meta-Llama-3 tokenizer, vocab 128,256. FINISHED (step 19073). MBPP 0.166 [0.134, 0.198].", "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a139/cmeister/tokenizer-lm/checkpoints/full-128k-llama3-mathcode-scratch", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "full-128k-llama3-seed42": { "run": "full-128k-llama3-seed42", "works": [ "paper_a", "tokeval_cross_scale" ], "training_regime": "std-1B", "architecture": { "n_layer": 24, "n_embd": 1536, "n_head": 12, "n_kv_head": 12, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 128256, "total_parameters": 1272841994 }, "training": { "step": 8800, "total_batch_size": 1048576, "token_budget": 9227468800, "total_bytes_consumed": 59564639200, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "full-128k" }, "tokenizer_slug": "NousResearch/Meta-Llama-3-8B", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 128000, "metrics": { "val_bpb": 0.7197870317421321, "blimp_acc": 0.8257014925373133, "code_bpb_mean": 0.5493603027008278, "blimp_scoring": "bos_prefixed", "flores_mean_bpb": 2.6619825098575554, "flores_n_languages": 215, "mbpp_pass1": 0.082, "humaneval_pass1": 0.07317073170731707, "generation_spec": { "version": "v2-2026-07-30", "bos_context": true, "eos_stop": true, "token_healing": true, "truncation": "left_to_max_length_minus_max_gen", "stop_truncation": "earliest_match_all_terms", "stop_match_decode": "specials_kept" }, "token_healing": true, "gsm8k_exact_match_strict": 0.045489006823351025, "gsm8k_exact_match_flexible": 0.05155420773313116 }, "metric_gaps": {}, "registry_note": null, "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a139/cmeister/tokenizer-lm/checkpoints/full-128k-llama3-seed42", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "full-128k-mathcode-scratch": { "run": "full-128k-mathcode-scratch", "works": [ "blog", "tokeval" ], "training_regime": "mathcode-20B-from-scratch", "architecture": { "n_layer": 24, "n_embd": 1536, "n_head": 12, "n_kv_head": 12, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "default 15.0, run predates the configurable field", "vocab_size": 128256, "total_parameters": 1272841994 }, "training": { "step": 19073, "total_batch_size": 1048576, "token_budget": 19999490048, "total_bytes_consumed": 147636253997, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "full-128k-mathcode-scratch" }, "tokenizer_slug": "gpt4o-balanced-bpe", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 0, "metrics": { "val_bpb": 0.33515701265710024, "blimp_acc": 0.5596268656716418, "code_bpb_mean": 0.37733489353750416, "blimp_scoring": null, "flores_mean_bpb": 3.2665762921686197, "flores_n_languages": 214, "mbpp_pass1": 0.174, "humaneval_pass1": 0.16463414634146342, "generation_spec": { "version": "v2-2026-07-30", "bos_context": true, "eos_stop": true, "token_healing": true, "truncation": "left_to_max_length_minus_max_gen", "stop_truncation": "earliest_match_all_terms", "stop_match_decode": "specials_kept" }, "token_healing": true, "gsm8k_exact_match_strict": 0.19787717968157695, "gsm8k_exact_match_flexible": 0.22896133434420016 }, "metric_gaps": {}, "registry_note": "#1, gpt4o pretok, slug `gpt4o-balanced-bpe`. Original baseline (unprefixed).", "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a139/cmeister/tokenizer-lm/checkpoints/full-128k-mathcode-scratch", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "full-128k-pabpe-nfc-apertus-fineweb2full-capped-tuned-hybridwindow": { "run": "full-128k-pabpe-nfc-apertus-fineweb2full-capped-tuned-hybridwindow", "works": [ "paper_a_apertus_track" ], "training_regime": "std-1B", "architecture": { "n_layer": 24, "n_embd": 1536, "n_head": 12, "n_kv_head": 12, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 127835, "total_parameters": 1271072522 }, "training": { "step": 8794, "total_batch_size": 1048576, "token_budget": 9221177344, "total_bytes_consumed": 61590124482, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "full-128k" }, "tokenizer_slug": "pabpe-nfc-apertus-fineweb2full-capped-tuned-hybridwindow", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 127831, "metrics": { "val_bpb": 0.728561092002857, "blimp_acc": 0.8191194029850744, "code_bpb_mean": 0.531266649961596, "blimp_scoring": "bos_prefixed", "flores_mean_bpb": 2.9432145980087117, "flores_n_languages": 214, "mbpp_pass1": 0.088, "humaneval_pass1": 0.07317073170731707, "generation_spec": { "version": "v2-2026-07-30", "bos_context": true, "eos_stop": true, "token_healing": true, "truncation": "left_to_max_length_minus_max_gen", "stop_truncation": "earliest_match_all_terms", "stop_match_decode": "specials_kept" }, "token_healing": true, "gsm8k_exact_match_strict": 0.03790750568612585, "gsm8k_exact_match_flexible": 0.039423805913570885 }, "metric_gaps": {}, "registry_note": "PA-BPE hybrid global+window, apertus/Mistral-Nemo pretok regex; fineweb2full, capped + tuned data; norm=NFC; PA-BPE (hybrid global+window); vocab 127,835; 2x2 candidate; the apertus-pretok arm later diagnosed with the MBPP-collapse RCA (fuses code line-endings into superwords) at 20B scale; USED - Apertus-2 development track candidate; excluded from main paper panel", "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a139/cmeister/tokenizer-lm/checkpoints/full-128k-pabpe-nfc-apertus-fineweb2full-capped-tuned-hybridwindow", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "full-128k-pabpe-nfc-apertus-fineweb2full-capped-tuned-hybridwindow-mathcode-scratch": { "run": "full-128k-pabpe-nfc-apertus-fineweb2full-capped-tuned-hybridwindow-mathcode-scratch", "works": [ "blog" ], "training_regime": "mathcode-20B-from-scratch", "architecture": { "n_layer": 24, "n_embd": 1536, "n_head": 12, "n_kv_head": 12, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 127835, "total_parameters": 1271072522 }, "training": { "step": 19073, "total_batch_size": 1048576, "token_budget": 19999490048, "total_bytes_consumed": 133580673669, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "full-128k-mathcode-scratch" }, "tokenizer_slug": "pabpe-nfc-apertus-fineweb2full-capped-tuned-hybridwindow", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 127831, "metrics": { "val_bpb": 0.3444556663811808, "flores_mean_bpb": 3.485736844563002, "flores_n_languages": 214, "mbpp_pass1": 0.182, "humaneval_pass1": 0.1524390243902439, "generation_spec": { "version": "v2-2026-07-30", "bos_context": true, "eos_stop": true, "token_healing": true, "truncation": "left_to_max_length_minus_max_gen", "stop_truncation": "earliest_match_all_terms", "stop_match_decode": "specials_kept" }, "token_healing": true, "gsm8k_exact_match_strict": 0.21152388172858225, "gsm8k_exact_match_flexible": 0.23730098559514784 }, "metric_gaps": { "blimp_acc": "no full-128k-pabpe-nfc-apertus-fineweb2full-capped-tuned-hybridwindow-mathcode-scratch_blimp_code_bpb.json", "code_bpb_mean": "no full-128k-pabpe-nfc-apertus-fineweb2full-capped-tuned-hybridwindow-mathcode-scratch_blimp_code_bpb.json; no full-128k-pabpe-nfc-apertus-fineweb2full-capped-tuned-hybridwindow-mathcode-scratch_code_bpb.json" }, "registry_note": "#8, apertus pretok, slug `pabpe-nfc-apertus-fineweb2full-capped-tuned-hybridwindow`. PA-BPE candidate on apertus pretok.", "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a139/cmeister/tokenizer-lm/checkpoints/full-128k-pabpe-nfc-apertus-fineweb2full-capped-tuned-hybridwindow-mathcode-scratch", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "full-128k-pabpe-nfc-clean-balanced": { "run": "full-128k-pabpe-nfc-clean-balanced", "works": [ "paper_a", "tokeval" ], "training_regime": "std-1B", "architecture": { "n_layer": 24, "n_embd": 1536, "n_head": 12, "n_kv_head": 12, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 127836, "total_parameters": 1271072522 }, "training": { "step": 8794, "total_batch_size": 1048576, "token_budget": 9221177344, "total_bytes_consumed": 72926909582, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "full-128k" }, "tokenizer_slug": "pabpe-nfc-clean-balanced", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 127832, "metrics": { "val_bpb": 0.7234690685527866, "blimp_acc": 0.8185820895522389, "code_bpb_mean": 0.527456023839792, "blimp_scoring": "bos_prefixed", "flores_mean_bpb": 2.652974037092512, "flores_n_languages": 215, "mbpp_pass1": 0.082, "humaneval_pass1": 0.08536585365853659, "generation_spec": { "version": "v2-2026-07-30", "bos_context": true, "eos_stop": true, "token_healing": true, "truncation": "left_to_max_length_minus_max_gen", "stop_truncation": "earliest_match_all_terms", "stop_match_decode": "specials_kept" }, "token_healing": true, "gsm8k_exact_match_strict": 0.03411675511751327, "gsm8k_exact_match_flexible": 0.03639120545868082 }, "metric_gaps": {}, "registry_note": "PA-BPE (parity, base) with clean-multi regex; balanced (D2) data; norm=NFC; PA-BPE (base); vocab 127,836; same data as pabpe-nfc-gpt4-balanced (2a), different pretok regex (controlled pretok ablation); bug-fixed 2026-05-16 replacement (2b) for retired pabpe-balanced; USED - balanced-1B bootstrap panel (S6.12) + paper panel; not in panels.json main21", "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a139/cmeister/tokenizer-lm/checkpoints/full-128k-pabpe-nfc-clean-balanced", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "full-128k-pabpe-nfc-clean-balanced-hybridwindow-mathcode-scratch": { "run": "full-128k-pabpe-nfc-clean-balanced-hybridwindow-mathcode-scratch", "works": [ "blog" ], "training_regime": "mathcode-20B-from-scratch", "architecture": { "n_layer": 24, "n_embd": 1536, "n_head": 12, "n_kv_head": 12, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 127836, "total_parameters": 1271072522 }, "training": { "step": 19073, "total_batch_size": 1048576, "token_budget": 19999490048, "total_bytes_consumed": 152278889219, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "full-128k-mathcode-scratch" }, "tokenizer_slug": "pabpe-nfc-clean-balanced-hybridwindow", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 127832, "metrics": { "val_bpb": 0.3407285888388783, "flores_mean_bpb": 3.3507237031110604, "flores_n_languages": 214, "mbpp_pass1": 0.232, "humaneval_pass1": 0.1524390243902439, "generation_spec": { "version": "v2-2026-07-30", "bos_context": true, "eos_stop": true, "token_healing": true, "truncation": "left_to_max_length_minus_max_gen", "stop_truncation": "earliest_match_all_terms", "stop_match_decode": "specials_kept" }, "token_healing": true, "gsm8k_exact_match_strict": 0.21304018195602728, "gsm8k_exact_match_flexible": 0.2312357846853677 }, "metric_gaps": { "blimp_acc": "no full-128k-pabpe-nfc-clean-balanced-hybridwindow-mathcode-scratch_blimp_code_bpb.json", "code_bpb_mean": "no full-128k-pabpe-nfc-clean-balanced-hybridwindow-mathcode-scratch_blimp_code_bpb.json; no full-128k-pabpe-nfc-clean-balanced-hybridwindow-mathcode-scratch_code_bpb.json" }, "registry_note": "#6, clean pretok, slug `pabpe-nfc-clean-balanced-hybridwindow`. PA-BPE on clean pretok (balanced, HW).", "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a139/cmeister/tokenizer-lm/checkpoints/full-128k-pabpe-nfc-clean-balanced-hybridwindow-mathcode-scratch", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "full-128k-pabpe-nfc-clean-fineweb2full": { "run": "full-128k-pabpe-nfc-clean-fineweb2full", "works": [ "paper_a" ], "training_regime": "std-1B", "architecture": { "n_layer": 24, "n_embd": 1536, "n_head": 12, "n_kv_head": 12, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 127835, "total_parameters": 1271072522 }, "training": { "step": 8794, "total_batch_size": 1048576, "token_budget": 9221177344, "total_bytes_consumed": 72208518928, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "full-128k" }, "tokenizer_slug": "pabpe-nfc-clean-fineweb2full", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 127831, "metrics": { "val_bpb": 0.7485355018717734, "blimp_acc": 0.8243134328358208, "code_bpb_mean": 0.5609107321473775, "blimp_scoring": "bos_prefixed", "flores_mean_bpb": 2.841240013347876, "flores_n_languages": 215, "mbpp_pass1": 0.056, "humaneval_pass1": 0.054878048780487805, "generation_spec": { "version": "v2-2026-07-30", "bos_context": true, "eos_stop": true, "token_healing": true, "truncation": "left_to_max_length_minus_max_gen", "stop_truncation": "earliest_match_all_terms", "stop_match_decode": "specials_kept" }, "token_healing": true, "gsm8k_exact_match_strict": 0.022744503411675512, "gsm8k_exact_match_flexible": 0.025018953752843062 }, "metric_gaps": {}, "registry_note": "PA-BPE (parity, base) with clean-multi regex; fineweb2full data; norm=NFC; PA-BPE (base); vocab 127,835; USED - S5b fineweb2full analysis (best all-FLORES BPB 2.848 but worst val_bpb/BLiMP/code_bpb of the six); fineweb2full data variant, excluded from main paper panel", "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a139/cmeister/tokenizer-lm/checkpoints/full-128k-pabpe-nfc-clean-fineweb2full", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "full-128k-pabpe-nfc-clean-fineweb2full-capped-tuned-hybridwindow": { "run": "full-128k-pabpe-nfc-clean-fineweb2full-capped-tuned-hybridwindow", "works": [ "paper_a_apertus_track" ], "training_regime": "std-1B", "architecture": { "n_layer": 24, "n_embd": 1536, "n_head": 12, "n_kv_head": 12, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 127835, "total_parameters": 1271072522 }, "training": { "step": 8794, "total_batch_size": 1048576, "token_budget": 9221177344, "total_bytes_consumed": 61636354540, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "full-128k" }, "tokenizer_slug": "pabpe-nfc-clean-fineweb2full-capped-tuned-hybridwindow", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 127831, "metrics": { "val_bpb": 0.7288989708540804, "blimp_acc": 0.815955223880597, "code_bpb_mean": 0.5329979934179307, "blimp_scoring": "bos_prefixed", "flores_mean_bpb": 2.9645154627226296, "flores_n_languages": 214, "mbpp_pass1": 0.078, "humaneval_pass1": 0.07926829268292683, "generation_spec": { "version": "v2-2026-07-30", "bos_context": true, "eos_stop": true, "token_healing": true, "truncation": "left_to_max_length_minus_max_gen", "stop_truncation": "earliest_match_all_terms", "stop_match_decode": "specials_kept" }, "token_healing": true, "gsm8k_exact_match_strict": 0.032600454890068235, "gsm8k_exact_match_flexible": 0.03790750568612585 }, "metric_gaps": {}, "registry_note": "PA-BPE hybrid global+window, clean-multi regex; fineweb2full, capped ({1,16} repeat-run cap) + quota-tuned ('tuned'/v5 hand-tuned per-family weighting) data; norm=NFC; PA-BPE (hybrid global+window); vocab 127,835; one of 4 Apertus-2-track candidates in the original 2x2 (pretok x {apertus,clean}) design; USED - Apertus-2 development track candidate; excluded from main paper panel (fineweb2full data variant, PAPER_PANEL.md rule)", "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a139/cmeister/tokenizer-lm/checkpoints/full-128k-pabpe-nfc-clean-fineweb2full-capped-tuned-hybridwindow", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "full-128k-pabpe-nfc-clean-fineweb2full-capped-tuned-hybridwindow-mathcode-scratch": { "run": "full-128k-pabpe-nfc-clean-fineweb2full-capped-tuned-hybridwindow-mathcode-scratch", "works": [ "blog" ], "training_regime": "mathcode-20B-from-scratch", "architecture": { "n_layer": 24, "n_embd": 1536, "n_head": 12, "n_kv_head": 12, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 127835, "total_parameters": 1271072522 }, "training": { "step": 19073, "total_batch_size": 1048576, "token_budget": 19999490048, "total_bytes_consumed": 133680940430, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "full-128k-mathcode-scratch" }, "tokenizer_slug": "pabpe-nfc-clean-fineweb2full-capped-tuned-hybridwindow", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 127831, "metrics": { "val_bpb": 0.34503526842456766, "flores_mean_bpb": 3.4963244629478334, "flores_n_languages": 214, "mbpp_pass1": 0.234, "humaneval_pass1": 0.1524390243902439, "generation_spec": { "version": "v2-2026-07-30", "bos_context": true, "eos_stop": true, "token_healing": true, "truncation": "left_to_max_length_minus_max_gen", "stop_truncation": "earliest_match_all_terms", "stop_match_decode": "specials_kept" }, "token_healing": true, "gsm8k_exact_match_strict": 0.17589082638362397, "gsm8k_exact_match_flexible": 0.2047005307050796 }, "metric_gaps": { "blimp_acc": "no full-128k-pabpe-nfc-clean-fineweb2full-capped-tuned-hybridwindow-mathcode-scratch_blimp_code_bpb.json", "code_bpb_mean": "no full-128k-pabpe-nfc-clean-fineweb2full-capped-tuned-hybridwindow-mathcode-scratch_blimp_code_bpb.json; no full-128k-pabpe-nfc-clean-fineweb2full-capped-tuned-hybridwindow-mathcode-scratch_code_bpb.json" }, "registry_note": "#7, clean pretok, slug `pabpe-nfc-clean-fineweb2full-capped-tuned-hybridwindow`. PA-BPE candidate (fw2full, capped-tuned).", "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a139/cmeister/tokenizer-lm/checkpoints/full-128k-pabpe-nfc-clean-fineweb2full-capped-tuned-hybridwindow-mathcode-scratch", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "full-128k-pabpe-nfc-clean-fineweb2full-consv2-engfull-eu3-gm110k": { "run": "full-128k-pabpe-nfc-clean-fineweb2full-consv2-engfull-eu3-gm110k", "works": [ "paper_a_apertus_track" ], "training_regime": "std-1B", "architecture": { "n_layer": 24, "n_embd": 1536, "n_head": 12, "n_kv_head": 12, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 131072, "total_parameters": 1285818122 }, "training": { "step": 8843, "total_batch_size": 1048576, "token_budget": 9272557568, "total_bytes_consumed": 60615581520, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "full-128k" }, "tokenizer_slug": "pabpe-nfc-clean-fineweb2full-consv2-engfull-eu3-gm110k", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 1, "metrics": { "val_bpb": 0.7253181858942936, "blimp_acc": 0.8196119402985071, "code_bpb_mean": 0.5292204446848089, "blimp_scoring": "bos_prefixed", "flores_mean_bpb": 2.982115075242416, "flores_n_languages": 214, "mbpp_pass1": 0.078, "humaneval_pass1": 0.09146341463414634, "generation_spec": { "version": "v2-2026-07-30", "bos_context": true, "eos_stop": true, "token_healing": true, "truncation": "left_to_max_length_minus_max_gen", "stop_truncation": "earliest_match_all_terms", "stop_match_decode": "specials_kept" }, "token_healing": true, "gsm8k_exact_match_strict": 0.032600454890068235, "gsm8k_exact_match_flexible": 0.037149355572403335 }, "metric_gaps": {}, "registry_note": "PA-BPE hybrid global+window (gm110k), clean-multi regex; fineweb2full, consv2 engfull_eu3 (full English FineWeb-1 sample + European boost + Arabic fix) data; norm=NFC; PA-BPE (hybrid global+window); vocab 131,072 + 124 specials; shipped Apertus-v2 candidate 'preliminary_enh' (English-preserving); roster #22; USED - Apertus-2 track shipped candidate; S6f Contrast B (131k arm); excluded from main paper panel", "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a139/cmeister/tokenizer-lm/checkpoints/full-128k-pabpe-nfc-clean-fineweb2full-consv2-engfull-eu3-gm110k", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "full-128k-pabpe-nfc-clean-fineweb2full-consv2-engfull-eu3-gm110k-mathcode-scratch": { "run": "full-128k-pabpe-nfc-clean-fineweb2full-consv2-engfull-eu3-gm110k-mathcode-scratch", "works": [ "blog" ], "training_regime": "mathcode-20B-from-scratch", "architecture": { "n_layer": 24, "n_embd": 1536, "n_head": 12, "n_kv_head": 12, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 131072, "total_parameters": 1285818122 }, "training": { "step": 19073, "total_batch_size": 1048576, "token_budget": 19999490048, "total_bytes_consumed": 130738548720, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "full-128k-mathcode-scratch" }, "tokenizer_slug": "pabpe-nfc-clean-fineweb2full-consv2-engfull-eu3-gm110k", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 1, "metrics": { "val_bpb": 0.34234477191826124, "flores_mean_bpb": 3.5552682701866014, "flores_n_languages": 214, "mbpp_pass1": 0.224, "humaneval_pass1": 0.14634146341463414, "generation_spec": { "version": "v2-2026-07-30", "bos_context": true, "eos_stop": true, "token_healing": true, "truncation": "left_to_max_length_minus_max_gen", "stop_truncation": "earliest_match_all_terms", "stop_match_decode": "specials_kept" }, "token_healing": true, "gsm8k_exact_match_strict": 0.21228203184230476, "gsm8k_exact_match_flexible": 0.2350265352539803 }, "metric_gaps": { "blimp_acc": "no full-128k-pabpe-nfc-clean-fineweb2full-consv2-engfull-eu3-gm110k-mathcode-scratch_blimp_code_bpb.json", "code_bpb_mean": "no full-128k-pabpe-nfc-clean-fineweb2full-consv2-engfull-eu3-gm110k-mathcode-scratch_blimp_code_bpb.json; no full-128k-pabpe-nfc-clean-fineweb2full-consv2-engfull-eu3-gm110k-mathcode-scratch_code_bpb.json" }, "registry_note": "#22, clean pretok, slug `pabpe-nfc-clean-fineweb2full-consv2-engfull-eu3-gm110k`. **Apertus-v2 candidate** `preliminary_enh` (English-preserving: English-boosted + moderate EU + Arabic fix). consv2 `engfull_eu3` grouped config, gm110k, vocab 131072 + 124 specials, TemplateProcessing post-proc (suppressed in nanochat path). Source `\u2026_consv2_engfull_eu3_gm110k_v131k_sp124_eng5g`. Std-1B 2571385+, mathcode 2571425+. Queued 2026-06-19, `account=infra01`.", "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a139/cmeister/tokenizer-lm/checkpoints/full-128k-pabpe-nfc-clean-fineweb2full-consv2-engfull-eu3-gm110k-mathcode-scratch", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "full-128k-pabpe-nfc-clean-fineweb2full-consv2-eudata-gm110k": { "run": "full-128k-pabpe-nfc-clean-fineweb2full-consv2-eudata-gm110k", "works": [ "paper_a_apertus_track" ], "training_regime": "std-1B", "architecture": { "n_layer": 24, "n_embd": 1536, "n_head": 12, "n_kv_head": 12, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 131072, "total_parameters": 1285818122 }, "training": { "step": 8843, "total_batch_size": 1048576, "token_budget": 9272557568, "total_bytes_consumed": 60085213752, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "full-128k" }, "tokenizer_slug": "pabpe-nfc-clean-fineweb2full-consv2-eudata-gm110k", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 1, "metrics": { "val_bpb": 0.7273522461094571, "blimp_acc": 0.8184179104477616, "code_bpb_mean": 0.5323586574775419, "blimp_scoring": "bos_prefixed", "flores_mean_bpb": 2.958214536949256, "flores_n_languages": 214, "mbpp_pass1": 0.062, "humaneval_pass1": 0.04878048780487805, "generation_spec": { "version": "v2-2026-07-30", "bos_context": true, "eos_stop": true, "token_healing": true, "truncation": "left_to_max_length_minus_max_gen", "stop_truncation": "earliest_match_all_terms", "stop_match_decode": "specials_kept" }, "token_healing": true, "gsm8k_exact_match_strict": 0.029567854435178165, "gsm8k_exact_match_flexible": 0.03335860500379075 }, "metric_gaps": {}, "registry_note": "PA-BPE hybrid global+window (gm110k), clean-multi regex; fineweb2full, consv2 + eudata data; norm=NFC; PA-BPE (hybrid global+window); vocab ~131,196; same eudata config, higher global-merges warmup; USED - S6f Contrast B (131k arm) + Apertus-2 development track; excluded from main paper panel", "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a139/cmeister/tokenizer-lm/checkpoints/full-128k-pabpe-nfc-clean-fineweb2full-consv2-eudata-gm110k", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "full-128k-pabpe-nfc-clean-fineweb2full-consv2-eudata-gm90k": { "run": "full-128k-pabpe-nfc-clean-fineweb2full-consv2-eudata-gm90k", "works": [ "paper_a_apertus_track" ], "training_regime": "std-1B", "architecture": { "n_layer": 24, "n_embd": 1536, "n_head": 12, "n_kv_head": 12, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 131072, "total_parameters": 1285818122 }, "training": { "step": 8843, "total_batch_size": 1048576, "token_budget": 9272557568, "total_bytes_consumed": 61317149768, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "full-128k" }, "tokenizer_slug": "pabpe-nfc-clean-fineweb2full-consv2-eudata-gm90k", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 1, "metrics": { "val_bpb": 0.7276288753194399, "blimp_acc": 0.8188059701492535, "code_bpb_mean": 0.5322564133385435, "blimp_scoring": "bos_prefixed", "flores_mean_bpb": 2.9892469043285885, "flores_n_languages": 214, "mbpp_pass1": 0.088, "humaneval_pass1": 0.054878048780487805, "generation_spec": { "version": "v2-2026-07-30", "bos_context": true, "eos_stop": true, "token_healing": true, "truncation": "left_to_max_length_minus_max_gen", "stop_truncation": "earliest_match_all_terms", "stop_match_decode": "specials_kept" }, "token_healing": true, "gsm8k_exact_match_strict": 0.03184230477634572, "gsm8k_exact_match_flexible": 0.037149355572403335 }, "metric_gaps": {}, "registry_note": "PA-BPE hybrid global+window (gm90k), clean-multi regex, plus2/repcap8/capped/tuned base; fineweb2full, consv2 quota formula + eudata (European-data-quota grouped config) data; norm=NFC; PA-BPE (hybrid global+window); vocab ~131,196 (131,072 target + 124 specials); earlier eudata sweep point, superseded by the shipped consv2 candidates; USED - S6f Contrast B (PA-BPE consv2 family, 131k arm, n=12) + Apertus-2 development track; excluded from main paper panel", "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a139/cmeister/tokenizer-lm/checkpoints/full-128k-pabpe-nfc-clean-fineweb2full-consv2-eudata-gm90k", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "full-128k-pabpe-nfc-clean-fineweb2full-consv2-eudata4-gm100k": { "run": "full-128k-pabpe-nfc-clean-fineweb2full-consv2-eudata4-gm100k", "works": [ "paper_a_apertus_track" ], "training_regime": "std-1B", "architecture": { "n_layer": 24, "n_embd": 1536, "n_head": 12, "n_kv_head": 12, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 131072, "total_parameters": 1285818122 }, "training": { "step": 8843, "total_batch_size": 1048576, "token_budget": 9272557568, "total_bytes_consumed": 60642676472, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "full-128k" }, "tokenizer_slug": "pabpe-nfc-clean-fineweb2full-consv2-eudata4-gm100k", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 1, "metrics": { "val_bpb": 0.7261196904187828, "blimp_acc": 0.8146865671641791, "code_bpb_mean": 0.5310156660973219, "blimp_scoring": "bos_prefixed", "flores_mean_bpb": 2.9639366438466745, "flores_n_languages": 214, "mbpp_pass1": 0.078, "humaneval_pass1": 0.06707317073170732, "generation_spec": { "version": "v2-2026-07-30", "bos_context": true, "eos_stop": true, "token_healing": true, "truncation": "left_to_max_length_minus_max_gen", "stop_truncation": "earliest_match_all_terms", "stop_match_decode": "specials_kept" }, "token_healing": true, "gsm8k_exact_match_strict": 0.0310841546626232, "gsm8k_exact_match_flexible": 0.034874905231235785 }, "metric_gaps": {}, "registry_note": "PA-BPE hybrid global+window (gm100k), clean-multi regex; fineweb2full, consv2 + eudata4 variant data; norm=NFC; PA-BPE (hybrid global+window); vocab ~131,196; USED - S6f Contrast B (131k arm) + Apertus-2 development track; excluded from main paper panel", "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a139/cmeister/tokenizer-lm/checkpoints/full-128k-pabpe-nfc-clean-fineweb2full-consv2-eudata4-gm100k", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "full-128k-pabpe-nfc-clean-fineweb2full-consv2-eusino-v2c-frde-kr120-gm130k-v200k": { "run": "full-128k-pabpe-nfc-clean-fineweb2full-consv2-eusino-v2c-frde-kr120-gm130k-v200k", "works": [ "paper_a_apertus_track" ], "training_regime": "std-1B", "architecture": { "n_layer": 24, "n_embd": 1536, "n_head": 12, "n_kv_head": 12, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 200000, "total_parameters": 1603438346 }, "training": { "step": 9903, "total_batch_size": 1048576, "token_budget": 10384048128, "total_bytes_consumed": 74440593522, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "full-128k" }, "tokenizer_slug": "pabpe-nfc-clean-fineweb2full-consv2-eusino-v2c-frde-kr120-gm130k-v200k", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 1, "metrics": { "val_bpb": 0.7201628703050716, "blimp_acc": 0.8205522388059703, "code_bpb_mean": 0.523890471891906, "blimp_scoring": "bos_prefixed", "flores_mean_bpb": 3.069992442307267, "flores_n_languages": 214, "mbpp_pass1": 0.068, "humaneval_pass1": 0.06097560975609756, "generation_spec": { "version": "v2-2026-07-30", "bos_context": true, "eos_stop": true, "token_healing": true, "truncation": "left_to_max_length_minus_max_gen", "stop_truncation": "earliest_match_all_terms", "stop_match_decode": "specials_kept" }, "token_healing": true, "gsm8k_exact_match_strict": 0.0356330553449583, "gsm8k_exact_match_flexible": 0.04094010614101592 }, "metric_gaps": {}, "registry_note": "PA-BPE hybrid global+window (gm130k), clean-multi regex; fineweb2full, consv2 eusino_v2c + French/German boost + kr120 (+120MB Korean) data; norm=NFC; PA-BPE (hybrid global+window); vocab 200,064; the shipped preliminary_mul_200k recipe; roster #38; USED - S6f Contrast B (200k arm, n=2) + Apertus-2 track shipped candidate; excluded from main paper panel", "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a139/cmeister/tokenizer-lm/checkpoints/full-128k-pabpe-nfc-clean-fineweb2full-consv2-eusino-v2c-frde-kr120-gm130k-v200k", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "full-128k-pabpe-nfc-clean-fineweb2full-consv2-eusino-v2c-frde-kr120-gm130k-v200k-mathcode-scratch": { "run": "full-128k-pabpe-nfc-clean-fineweb2full-consv2-eusino-v2c-frde-kr120-gm130k-v200k-mathcode-scratch", "works": [ "blog" ], "training_regime": "mathcode-20B-from-scratch", "architecture": { "n_layer": 24, "n_embd": 1536, "n_head": 12, "n_kv_head": 12, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 200000, "total_parameters": 1603438346 }, "training": { "step": 19073, "total_batch_size": 1048576, "token_budget": 19999490048, "total_bytes_consumed": 143371245102, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "full-128k-mathcode-scratch" }, "tokenizer_slug": "pabpe-nfc-clean-fineweb2full-consv2-eusino-v2c-frde-kr120-gm130k-v200k", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 1, "metrics": { "val_bpb": 0.3414332492357774, "flores_mean_bpb": 3.584565348888925, "flores_n_languages": 214, "mbpp_pass1": 0.228, "humaneval_pass1": 0.17073170731707318, "generation_spec": { "version": "v2-2026-07-30", "bos_context": true, "eos_stop": true, "token_healing": true, "truncation": "left_to_max_length_minus_max_gen", "stop_truncation": "earliest_match_all_terms", "stop_match_decode": "specials_kept" }, "token_healing": true, "gsm8k_exact_match_strict": 0.2100075815011372, "gsm8k_exact_match_flexible": 0.2441243366186505 }, "metric_gaps": { "blimp_acc": "no full-128k-pabpe-nfc-clean-fineweb2full-consv2-eusino-v2c-frde-kr120-gm130k-v200k-mathcode-scratch_blimp_code_bpb.json", "code_bpb_mean": "no full-128k-pabpe-nfc-clean-fineweb2full-consv2-eusino-v2c-frde-kr120-gm130k-v200k-mathcode-scratch_blimp_code_bpb.json; no full-128k-pabpe-nfc-clean-fineweb2full-consv2-eusino-v2c-frde-kr120-gm130k-v200k-mathcode-scratch_code_bpb.json" }, "registry_note": "#38, clean pretok, slug `pabpe-nfc-clean-fineweb2full-consv2-eusino-v2c-frde-kr120-gm130k-v200k`. PA-BPE Apertus-v2 candidate, consv2 `eusino_v2c` + frde + kr120, gm130k, vocab 200,000 (md5 3cdc3bdd). Distinct tokenizer from roster #25 (`\u2026-eusino-v2c-gm120k-v200k`, gm120k, md5 a5bdcfae). FINISHED (step 19073). MBPP 0.206 [0.172, 0.242].", "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a139/cmeister/tokenizer-lm/checkpoints/full-128k-pabpe-nfc-clean-fineweb2full-consv2-eusino-v2c-frde-kr120-gm130k-v200k-mathcode-scratch", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "full-128k-pabpe-nfc-clean-fineweb2full-consv2-eusino-v2c-gm120k-v200k": { "run": "full-128k-pabpe-nfc-clean-fineweb2full-consv2-eusino-v2c-gm120k-v200k", "works": [ "paper_a_apertus_track" ], "training_regime": "std-1B", "architecture": { "n_layer": 24, "n_embd": 1536, "n_head": 12, "n_kv_head": 12, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 200000, "total_parameters": 1603438346 }, "training": { "step": 9903, "total_batch_size": 1048576, "token_budget": 10384048128, "total_bytes_consumed": 74997795720, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "full-128k" }, "tokenizer_slug": "pabpe-nfc-clean-fineweb2full-consv2-eusino-v2c-gm120k-v200k", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 1, "metrics": { "val_bpb": 0.7201429837377527, "blimp_acc": 0.8141194029850746, "code_bpb_mean": 0.5238153919083227, "blimp_scoring": "bos_prefixed", "flores_mean_bpb": 3.1039007372839995, "flores_n_languages": 215, "mbpp_pass1": 0.076, "humaneval_pass1": 0.07926829268292683, "generation_spec": { "version": "v2-2026-07-30", "bos_context": true, "eos_stop": true, "token_healing": true, "truncation": "left_to_max_length_minus_max_gen", "stop_truncation": "earliest_match_all_terms", "stop_match_decode": "specials_kept" }, "token_healing": true, "gsm8k_exact_match_strict": 0.0401819560272934, "gsm8k_exact_match_flexible": 0.043214556482183475 }, "metric_gaps": {}, "registry_note": "PA-BPE hybrid global+window (gm120k), clean-multi regex; fineweb2full, consv2 eusino_v2c (European+Sinotibetan rebalance) data; norm=NFC; PA-BPE (hybrid global+window); vocab 200,000; the 200k predecessor LM evaluated for shipped candidate 'preliminary_mul_200k'; roster #25; USED - S6f Contrast B (200k arm, n=2) + Apertus-2 track shipped candidate; excluded from main paper panel", "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a139/cmeister/tokenizer-lm/checkpoints/full-128k-pabpe-nfc-clean-fineweb2full-consv2-eusino-v2c-gm120k-v200k", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "full-128k-pabpe-nfc-clean-fineweb2full-consv2-eusino-v2c-gm120k-v200k-mathcode-scratch": { "run": "full-128k-pabpe-nfc-clean-fineweb2full-consv2-eusino-v2c-gm120k-v200k-mathcode-scratch", "works": [ "blog" ], "training_regime": "mathcode-20B-from-scratch", "architecture": { "n_layer": 24, "n_embd": 1536, "n_head": 12, "n_kv_head": 12, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 200000, "total_parameters": 1603438346 }, "training": { "step": 19073, "total_batch_size": 1048576, "token_budget": 19999490048, "total_bytes_consumed": 144444406520, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "full-128k-mathcode-scratch" }, "tokenizer_slug": "pabpe-nfc-clean-fineweb2full-consv2-eusino-v2c-gm120k-v200k", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 1, "metrics": { "val_bpb": 0.34148051897222004, "flores_mean_bpb": 3.5718106322903322, "flores_n_languages": 214, "mbpp_pass1": 0.18, "humaneval_pass1": 0.16463414634146342, "generation_spec": { "version": "v2-2026-07-30", "bos_context": true, "eos_stop": true, "token_healing": true, "truncation": "left_to_max_length_minus_max_gen", "stop_truncation": "earliest_match_all_terms", "stop_match_decode": "specials_kept" }, "token_healing": true, "gsm8k_exact_match_strict": 0.18877937831690675, "gsm8k_exact_match_flexible": 0.22289613343442002 }, "metric_gaps": { "blimp_acc": "no full-128k-pabpe-nfc-clean-fineweb2full-consv2-eusino-v2c-gm120k-v200k-mathcode-scratch_blimp_code_bpb.json", "code_bpb_mean": "no full-128k-pabpe-nfc-clean-fineweb2full-consv2-eusino-v2c-gm120k-v200k-mathcode-scratch_blimp_code_bpb.json; no full-128k-pabpe-nfc-clean-fineweb2full-consv2-eusino-v2c-gm120k-v200k-mathcode-scratch_code_bpb.json" }, "registry_note": "#25, clean pretok, slug `pabpe-nfc-clean-fineweb2full-consv2-eusino-v2c-gm120k-v200k`. **Apertus-v2 candidate** `preliminary_mul_200k` (200k all-rounder). consv2 `eusino_v2c`, gm120k, **vocab 200000** (larger embedding/unembedding than the 131k panel \u2014 fair size-match is o200k, not the other candidates), 124 specials, post-proc. Source `\u2026_consv2_eusino_v2c_gm120k_v200k_sp124_eng5g`. Std-1B 2571415+, mathcode 2571480+. Queued 2026-06-19, `account=infra01`.", "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a139/cmeister/tokenizer-lm/checkpoints/full-128k-pabpe-nfc-clean-fineweb2full-consv2-eusino-v2c-gm120k-v200k-mathcode-scratch", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "full-128k-pabpe-nfc-clean-fineweb2full-consv2-frde2-gm110k": { "run": "full-128k-pabpe-nfc-clean-fineweb2full-consv2-frde2-gm110k", "works": [ "paper_a_apertus_track" ], "training_regime": "std-1B", "architecture": { "n_layer": 24, "n_embd": 1536, "n_head": 12, "n_kv_head": 12, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 131072, "total_parameters": 1285818122 }, "training": { "step": 8843, "total_batch_size": 1048576, "token_budget": 9272557568, "total_bytes_consumed": 60733582512, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "full-128k" }, "tokenizer_slug": "pabpe-nfc-clean-fineweb2full-consv2-frde2-gm110k", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 1, "metrics": { "val_bpb": 0.7253267169497185, "blimp_acc": 0.8195373134328359, "code_bpb_mean": 0.5321604019509747, "blimp_scoring": "bos_prefixed", "flores_mean_bpb": 2.9792521465200528, "flores_n_languages": 214, "mbpp_pass1": 0.072, "humaneval_pass1": 0.07926829268292683, "generation_spec": { "version": "v2-2026-07-30", "bos_context": true, "eos_stop": true, "token_healing": true, "truncation": "left_to_max_length_minus_max_gen", "stop_truncation": "earliest_match_all_terms", "stop_match_decode": "specials_kept" }, "token_healing": true, "gsm8k_exact_match_strict": 0.032600454890068235, "gsm8k_exact_match_flexible": 0.039423805913570885 }, "metric_gaps": {}, "registry_note": "PA-BPE hybrid global+window (gm110k), clean-multi regex; fineweb2full, consv2 frde2 (more French/German, less Chinese) data; norm=NFC; PA-BPE (hybrid global+window); vocab 131,072 + 124 specials; shipped Apertus-v2 candidate 'preliminary_euh' (EU-dense, least fair of the four); roster #23; USED - Apertus-2 track shipped candidate; S6f Contrast B (131k arm); excluded from main paper panel", "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a139/cmeister/tokenizer-lm/checkpoints/full-128k-pabpe-nfc-clean-fineweb2full-consv2-frde2-gm110k", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "full-128k-pabpe-nfc-clean-fineweb2full-consv2-frde2-gm110k-mathcode-scratch": { "run": "full-128k-pabpe-nfc-clean-fineweb2full-consv2-frde2-gm110k-mathcode-scratch", "works": [ "blog" ], "training_regime": "mathcode-20B-from-scratch", "architecture": { "n_layer": 24, "n_embd": 1536, "n_head": 12, "n_kv_head": 12, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 131072, "total_parameters": 1285818122 }, "training": { "step": 19073, "total_batch_size": 1048576, "token_budget": 19999490048, "total_bytes_consumed": 130993058832, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "full-128k-mathcode-scratch" }, "tokenizer_slug": "pabpe-nfc-clean-fineweb2full-consv2-frde2-gm110k", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 1, "metrics": { "val_bpb": 0.34326210358173215, "flores_mean_bpb": 3.5365349823121583, "flores_n_languages": 214, "mbpp_pass1": 0.182, "humaneval_pass1": 0.16463414634146342, "generation_spec": { "version": "v2-2026-07-30", "bos_context": true, "eos_stop": true, "token_healing": true, "truncation": "left_to_max_length_minus_max_gen", "stop_truncation": "earliest_match_all_terms", "stop_match_decode": "specials_kept" }, "token_healing": true, "gsm8k_exact_match_strict": 0.19787717968157695, "gsm8k_exact_match_flexible": 0.22896133434420016 }, "metric_gaps": { "blimp_acc": "no full-128k-pabpe-nfc-clean-fineweb2full-consv2-frde2-gm110k-mathcode-scratch_blimp_code_bpb.json", "code_bpb_mean": "no full-128k-pabpe-nfc-clean-fineweb2full-consv2-frde2-gm110k-mathcode-scratch_blimp_code_bpb.json; no full-128k-pabpe-nfc-clean-fineweb2full-consv2-frde2-gm110k-mathcode-scratch_code_bpb.json" }, "registry_note": "#23, clean pretok, slug `pabpe-nfc-clean-fineweb2full-consv2-frde2-gm110k`. **Apertus-v2 candidate** `preliminary_euh` (EU-dense Fr/De-boosted, Chinese cut; least fair of the four). consv2 `frde2` grouped config, gm110k, vocab 131072 + 124 specials, post-proc. Source `\u2026_consv2_frde2_gm110k_v131k_sp124_eng5g`. Std-1B 2571395+, mathcode 2571444+. Queued 2026-06-19, `account=infra01`.", "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a139/cmeister/tokenizer-lm/checkpoints/full-128k-pabpe-nfc-clean-fineweb2full-consv2-frde2-gm110k-mathcode-scratch", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "full-128k-pabpe-nfc-clean-fineweb2full-consv2-plus3-repcap8": { "run": "full-128k-pabpe-nfc-clean-fineweb2full-consv2-plus3-repcap8", "works": [ "paper_a_apertus_track" ], "training_regime": "std-1B", "architecture": { "n_layer": 24, "n_embd": 1536, "n_head": 12, "n_kv_head": 12, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 131017, "total_parameters": 1285818122 }, "training": { "step": 8843, "total_batch_size": 1048576, "token_budget": 9272557568, "total_bytes_consumed": 63678628703, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "full-128k" }, "tokenizer_slug": "pabpe-nfc-clean-fineweb2full-consv2-plus3-repcap8", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 1, "metrics": { "val_bpb": 0.7277498476114497, "blimp_acc": 0.8207761194029849, "code_bpb_mean": 0.5317336699347034, "blimp_scoring": "bos_prefixed", "flores_mean_bpb": 2.968980373205703, "flores_n_languages": 214, "mbpp_pass1": 0.088, "humaneval_pass1": 0.054878048780487805, "generation_spec": { "version": "v2-2026-07-30", "bos_context": true, "eos_stop": true, "token_healing": true, "truncation": "left_to_max_length_minus_max_gen", "stop_truncation": "earliest_match_all_terms", "stop_match_decode": "specials_kept" }, "token_healing": true, "gsm8k_exact_match_strict": 0.03411675511751327, "gsm8k_exact_match_flexible": 0.04169825625473844 }, "metric_gaps": {}, "registry_note": "PA-BPE hybrid global+window, clean-multi plus3 regex + repcap8 (8+ char run cap); fineweb2full, consv2 baseline (reparam ratios) data; norm=NFC; PA-BPE (hybrid global+window); vocab 131,017 (55-token ParityBpe shortfall vs 131,072 target); shipped Apertus-v2 candidate 'preliminary_mul' (balanced baseline, fairest of the four); roster #24; USED - Apertus-2 track shipped candidate; S6f Contrast B (131k arm); excluded from main paper panel", "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a139/cmeister/tokenizer-lm/checkpoints/full-128k-pabpe-nfc-clean-fineweb2full-consv2-plus3-repcap8", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "full-128k-pabpe-nfc-clean-fineweb2full-consv2-plus3-repcap8-mathcode-scratch": { "run": "full-128k-pabpe-nfc-clean-fineweb2full-consv2-plus3-repcap8-mathcode-scratch", "works": [ "blog" ], "training_regime": "mathcode-20B-from-scratch", "architecture": { "n_layer": 24, "n_embd": 1536, "n_head": 12, "n_kv_head": 12, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 131017, "total_parameters": 1285818122 }, "training": { "step": 19073, "total_batch_size": 1048576, "token_budget": 19999490048, "total_bytes_consumed": 137345073533, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "full-128k-mathcode-scratch" }, "tokenizer_slug": "pabpe-nfc-clean-fineweb2full-consv2-plus3-repcap8", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 1, "metrics": { "val_bpb": 0.34396997225653375, "flores_mean_bpb": 3.528042232456428, "flores_n_languages": 214, "mbpp_pass1": 0.218, "humaneval_pass1": 0.18902439024390244, "generation_spec": { "version": "v2-2026-07-30", "bos_context": true, "eos_stop": true, "token_healing": true, "truncation": "left_to_max_length_minus_max_gen", "stop_truncation": "earliest_match_all_terms", "stop_match_decode": "specials_kept" }, "token_healing": true, "gsm8k_exact_match_strict": 0.17968157695223655, "gsm8k_exact_match_flexible": 0.21304018195602728 }, "metric_gaps": { "blimp_acc": "no full-128k-pabpe-nfc-clean-fineweb2full-consv2-plus3-repcap8-mathcode-scratch_blimp_code_bpb.json", "code_bpb_mean": "no full-128k-pabpe-nfc-clean-fineweb2full-consv2-plus3-repcap8-mathcode-scratch_blimp_code_bpb.json; no full-128k-pabpe-nfc-clean-fineweb2full-consv2-plus3-repcap8-mathcode-scratch_code_bpb.json" }, "registry_note": "#24, clean pretok, slug `pabpe-nfc-clean-fineweb2full-consv2-plus3-repcap8`. **Apertus-v2 candidate** `preliminary_mul` (balanced consv2 baseline, fairest of the four, no EU boost). `plus3`+`repcap8` consv2, vocab **131017** (55-token ParityBpe shortfall), no post-processor. Functionally distinct from roster #20 repcap8 (verified). Source `nfc_clean_multi_plus3_repcap8_capped_hybrid_window_tuned_consv2_v131k_sp124_eng5g`. Std-1B 2571405+, mathcode 2571462+. Queued 2026-06-19, `account=infra01`.", "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a139/cmeister/tokenizer-lm/checkpoints/full-128k-pabpe-nfc-clean-fineweb2full-consv2-plus3-repcap8-mathcode-scratch", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "full-128k-pabpe-nfc-clean-fineweb2full-consv2-prelim-mul-v131072": { "run": "full-128k-pabpe-nfc-clean-fineweb2full-consv2-prelim-mul-v131072", "works": [ "paper_a_apertus_track" ], "training_regime": "std-1B", "architecture": { "n_layer": 24, "n_embd": 1536, "n_head": 12, "n_kv_head": 12, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 131072, "total_parameters": 1285818122 }, "training": { "step": 8843, "total_batch_size": 1048576, "token_budget": 9272557568, "total_bytes_consumed": 63946138296, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "full-128k" }, "tokenizer_slug": "pabpe-nfc-clean-fineweb2full-consv2-prelim-mul-v131072", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 1, "metrics": { "val_bpb": 0.7277884917270039, "blimp_acc": 0.8144328358208955, "code_bpb_mean": 0.5309802681278109, "blimp_scoring": "bos_prefixed", "flores_mean_bpb": 2.965308043536212, "flores_n_languages": 214, "mbpp_pass1": 0.088, "humaneval_pass1": 0.07926829268292683, "generation_spec": { "version": "v2-2026-07-30", "bos_context": true, "eos_stop": true, "token_healing": true, "truncation": "left_to_max_length_minus_max_gen", "stop_truncation": "earliest_match_all_terms", "stop_match_decode": "specials_kept" }, "token_healing": true, "gsm8k_exact_match_strict": 0.029567854435178165, "gsm8k_exact_match_flexible": 0.037149355572403335 }, "metric_gaps": {}, "registry_note": "PA-BPE hybrid global+window, clean-multi plus3 + repcap8; fineweb2full, consv2 reparam (preliminary_mul) data; norm=NFC; PA-BPE (hybrid global+window); vocab 131,072; the preliminary_mul build registered under an explicit slug (distinct md5 from the roster #24 twin); USED - S6h code analysis is not applicable (20B-only); Apertus-2 track; excluded from main paper panel", "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a139/cmeister/tokenizer-lm/checkpoints/full-128k-pabpe-nfc-clean-fineweb2full-consv2-prelim-mul-v131072", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "full-128k-pabpe-nfc-clean-fineweb2full-consv2-prelim-mul-v131072-mathcode-scratch": { "run": "full-128k-pabpe-nfc-clean-fineweb2full-consv2-prelim-mul-v131072-mathcode-scratch", "works": [ "blog" ], "training_regime": "mathcode-20B-from-scratch", "architecture": { "n_layer": 24, "n_embd": 1536, "n_head": 12, "n_kv_head": 12, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 131072, "total_parameters": 1285818122 }, "training": { "step": 19073, "total_batch_size": 1048576, "token_budget": 19999490048, "total_bytes_consumed": 137922050856, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "full-128k-mathcode-scratch" }, "tokenizer_slug": "pabpe-nfc-clean-fineweb2full-consv2-prelim-mul-v131072", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 1, "metrics": { "val_bpb": 0.3439376331489919, "flores_mean_bpb": 3.504792999457629, "flores_n_languages": 214, "mbpp_pass1": 0.212, "humaneval_pass1": 0.15853658536585366, "generation_spec": { "version": "v2-2026-07-30", "bos_context": true, "eos_stop": true, "token_healing": true, "truncation": "left_to_max_length_minus_max_gen", "stop_truncation": "earliest_match_all_terms", "stop_match_decode": "specials_kept" }, "token_healing": true, "gsm8k_exact_match_strict": 0.1865049279757392, "gsm8k_exact_match_flexible": 0.2259287338893101 }, "metric_gaps": { "blimp_acc": "no full-128k-pabpe-nfc-clean-fineweb2full-consv2-prelim-mul-v131072-mathcode-scratch_blimp_code_bpb.json", "code_bpb_mean": "no full-128k-pabpe-nfc-clean-fineweb2full-consv2-prelim-mul-v131072-mathcode-scratch_blimp_code_bpb.json; no full-128k-pabpe-nfc-clean-fineweb2full-consv2-prelim-mul-v131072-mathcode-scratch_code_bpb.json" }, "registry_note": "#37, clean pretok, slug `pabpe-nfc-clean-fineweb2full-consv2-prelim-mul-v131072`. PA-BPE Apertus-v2 `preliminary_mul` variant, consv2, vocab 131,072 (md5 ead50f20). Distinct tokenizer from roster #24 (`\u2026-consv2-plus3-repcap8`, vocab 131,017, md5 879fd884). FINISHED (step 19073). MBPP 0.170 [0.138, 0.204].", "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a139/cmeister/tokenizer-lm/checkpoints/full-128k-pabpe-nfc-clean-fineweb2full-consv2-prelim-mul-v131072-mathcode-scratch", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "full-128k-pabpe-nfc-clean-fineweb2full-hybridwindow": { "run": "full-128k-pabpe-nfc-clean-fineweb2full-hybridwindow", "works": [ "paper_a" ], "training_regime": "std-1B", "architecture": { "n_layer": 24, "n_embd": 1536, "n_head": 12, "n_kv_head": 12, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 127835, "total_parameters": 1271072522 }, "training": { "step": 8794, "total_batch_size": 1048576, "token_budget": 9221177344, "total_bytes_consumed": 63127641060, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "full-128k" }, "tokenizer_slug": "pabpe-nfc-clean-fineweb2full-hybridwindow", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 127831, "metrics": { "val_bpb": 0.7277506388972832, "blimp_acc": 0.8183880597014924, "code_bpb_mean": 0.5288339782707951, "blimp_scoring": "bos_prefixed", "flores_mean_bpb": 2.9580493024618453, "flores_n_languages": 215, "mbpp_pass1": 0.072, "humaneval_pass1": 0.07317073170731707, "generation_spec": { "version": "v2-2026-07-30", "bos_context": true, "eos_stop": true, "token_healing": true, "truncation": "left_to_max_length_minus_max_gen", "stop_truncation": "earliest_match_all_terms", "stop_match_decode": "specials_kept" }, "token_healing": true, "gsm8k_exact_match_strict": 0.028051554207733132, "gsm8k_exact_match_flexible": 0.039423805913570885 }, "metric_gaps": {}, "registry_note": "PA-BPE hybrid global+window (gm=64000,W=100,a=2.0) with clean-multi regex; fineweb2full data; norm=NFC; PA-BPE (hybrid global+window); vocab 127,835; hybrid-window pulls val_bpb/BLiMP/code_bpb back toward BPE-like at the cost of most of the all-FLORES gain; USED - S5b fineweb2full analysis; fineweb2full data variant, excluded from main paper panel", "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a139/cmeister/tokenizer-lm/checkpoints/full-128k-pabpe-nfc-clean-fineweb2full-hybridwindow", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "full-128k-pabpe-nfc-clean-fineweb2full-plus2-capped-tuned-hybridwindow": { "run": "full-128k-pabpe-nfc-clean-fineweb2full-plus2-capped-tuned-hybridwindow", "works": [ "paper_a_apertus_track" ], "training_regime": "std-1B", "architecture": { "n_layer": 24, "n_embd": 1536, "n_head": 12, "n_kv_head": 12, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 127835, "total_parameters": 1271072522 }, "training": { "step": 8794, "total_batch_size": 1048576, "token_budget": 9221177344, "total_bytes_consumed": 61811355140, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "full-128k" }, "tokenizer_slug": "pabpe-nfc-clean-fineweb2full-plus2-capped-tuned-hybridwindow", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 127831, "metrics": { "val_bpb": 0.7294609874470572, "blimp_acc": 0.819074626865672, "code_bpb_mean": 0.5337855917608801, "blimp_scoring": "bos_prefixed", "flores_mean_bpb": 2.952720239196703, "flores_n_languages": 214, "mbpp_pass1": 0.086, "humaneval_pass1": 0.06707317073170732, "generation_spec": { "version": "v2-2026-07-30", "bos_context": true, "eos_stop": true, "token_healing": true, "truncation": "left_to_max_length_minus_max_gen", "stop_truncation": "earliest_match_all_terms", "stop_match_decode": "specials_kept" }, "token_healing": true, "gsm8k_exact_match_strict": 0.0310841546626232, "gsm8k_exact_match_flexible": 0.034874905231235785 }, "metric_gaps": {}, "registry_note": "PA-BPE hybrid global+window, clean-multi plus2 regex; fineweb2full, capped + tuned (v5) data; norm=NFC; PA-BPE (hybrid global+window); vocab ~131,196; plus2 tuned-data baseline; USED - S6f Contrast B (131k arm) + Apertus-2 track; excluded from main paper panel", "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a139/cmeister/tokenizer-lm/checkpoints/full-128k-pabpe-nfc-clean-fineweb2full-plus2-capped-tuned-hybridwindow", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "full-128k-pabpe-nfc-clean-fineweb2full-plus2-capped-tuned-hybridwindow-consv2-tailcuts-gm90k-v130-sp-eng5g": { "run": "full-128k-pabpe-nfc-clean-fineweb2full-plus2-capped-tuned-hybridwindow-consv2-tailcuts-gm90k-v130-sp-eng5g", "works": [ "paper_a_apertus_track" ], "training_regime": "std-1B", "architecture": { "n_layer": 24, "n_embd": 1536, "n_head": 12, "n_kv_head": 12, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 130929, "total_parameters": 1285228298 }, "training": { "step": 8841, "total_batch_size": 1048576, "token_budget": 9270460416, "total_bytes_consumed": 61636322286, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "full-128k" }, "tokenizer_slug": "pabpe-nfc-clean-fineweb2full-plus2-capped-tuned-hybridwindow-consv2-tailcuts-gm90k-v130-sp-eng5g", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 1, "metrics": { "val_bpb": 0.7264930536694052, "blimp_acc": 0.8192985074626866, "code_bpb_mean": 0.5280546913441093, "blimp_scoring": "bos_prefixed", "flores_mean_bpb": 2.9887710137252537, "flores_n_languages": 214, "mbpp_pass1": 0.086, "humaneval_pass1": 0.08536585365853659, "generation_spec": { "version": "v2-2026-07-30", "bos_context": true, "eos_stop": true, "token_healing": true, "truncation": "left_to_max_length_minus_max_gen", "stop_truncation": "earliest_match_all_terms", "stop_match_decode": "specials_kept" }, "token_healing": true, "gsm8k_exact_match_strict": 0.037149355572403335, "gsm8k_exact_match_flexible": 0.04094010614101592 }, "metric_gaps": {}, "registry_note": "PA-BPE hybrid global+window (gm90k), clean-multi plus2 regex; fineweb2full, consv2 + tailcuts (6 families demoted to ratio 1.0) + eng5g (+5GB English) data; norm=NFC; PA-BPE (hybrid global+window); vocab ~131,196 (~131k + full Apertus chat-template specials, ~99 tokens); Apertus-2 development ablation 'PA-Clean-plus2-A8'; USED - S6f Contrast B (131k arm) + Apertus-2 track ablation; excluded from main paper panel", "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a139/cmeister/tokenizer-lm/checkpoints/full-128k-pabpe-nfc-clean-fineweb2full-plus2-capped-tuned-hybridwindow-consv2-tailcuts-gm90k-v130-sp-eng5g", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "full-128k-pabpe-nfc-clean-fineweb2full-plus2-capped-tuned-hybridwindow-consv2-tailcuts-gm90k-v130-sp-eng5g-mathcode-scratch": { "run": "full-128k-pabpe-nfc-clean-fineweb2full-plus2-capped-tuned-hybridwindow-consv2-tailcuts-gm90k-v130-sp-eng5g-mathcode-scratch", "works": [ "blog" ], "training_regime": "mathcode-20B-from-scratch", "architecture": { "n_layer": 24, "n_embd": 1536, "n_head": 12, "n_kv_head": 12, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 130929, "total_parameters": 1285228298 }, "training": { "step": 19073, "total_batch_size": 1048576, "token_budget": 19999490048, "total_bytes_consumed": 132970204158, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "full-128k-mathcode-scratch" }, "tokenizer_slug": "pabpe-nfc-clean-fineweb2full-plus2-capped-tuned-hybridwindow-consv2-tailcuts-gm90k-v130-sp-eng5g", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 1, "metrics": { "val_bpb": 0.34134989396481596, "flores_mean_bpb": 3.516050924775895, "flores_n_languages": 214, "mbpp_pass1": 0.236, "humaneval_pass1": 0.1402439024390244, "generation_spec": { "version": "v2-2026-07-30", "bos_context": true, "eos_stop": true, "token_healing": true, "truncation": "left_to_max_length_minus_max_gen", "stop_truncation": "earliest_match_all_terms", "stop_match_decode": "specials_kept" }, "token_healing": true, "gsm8k_exact_match_strict": 0.18726307808946172, "gsm8k_exact_match_flexible": 0.21228203184230476 }, "metric_gaps": { "blimp_acc": "no full-128k-pabpe-nfc-clean-fineweb2full-plus2-capped-tuned-hybridwindow-consv2-tailcuts-gm90k-v130-sp-eng5g-mathcode-scratch_blimp_code_bpb.json", "code_bpb_mean": "no full-128k-pabpe-nfc-clean-fineweb2full-plus2-capped-tuned-hybridwindow-consv2-tailcuts-gm90k-v130-sp-eng5g-mathcode-scratch_blimp_code_bpb.json; no full-128k-pabpe-nfc-clean-fineweb2full-plus2-capped-tuned-hybridwindow-consv2-tailcuts-gm90k-v130-sp-eng5g-mathcode-scratch_code_bpb.json" }, "registry_note": "#18, clean pretok, slug `pabpe-nfc-clean-fineweb2full-plus2-capped-tuned-hybridwindow-consv2-tailcuts-gm90k-v130-sp-eng5g`. PA-BPE `plus2` + same `consv2-tailcuts-eng5g` data + specials config, global-merges=90k.", "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a139/cmeister/tokenizer-lm/checkpoints/full-128k-pabpe-nfc-clean-fineweb2full-plus2-capped-tuned-hybridwindow-consv2-tailcuts-gm90k-v130-sp-eng5g-mathcode-scratch", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "full-128k-pabpe-nfc-clean-fineweb2full-plus2-capped-tuned-hybridwindow-mathcode-scratch": { "run": "full-128k-pabpe-nfc-clean-fineweb2full-plus2-capped-tuned-hybridwindow-mathcode-scratch", "works": [ "blog" ], "training_regime": "mathcode-20B-from-scratch", "architecture": { "n_layer": 24, "n_embd": 1536, "n_head": 12, "n_kv_head": 12, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 127835, "total_parameters": 1271072522 }, "training": { "step": 19073, "total_batch_size": 1048576, "token_budget": 19999490048, "total_bytes_consumed": 134060493130, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "full-128k-mathcode-scratch" }, "tokenizer_slug": "pabpe-nfc-clean-fineweb2full-plus2-capped-tuned-hybridwindow", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 127831, "metrics": { "val_bpb": 0.34495804994861307, "flores_mean_bpb": 3.4562531366015037, "flores_n_languages": 214, "mbpp_pass1": 0.198, "humaneval_pass1": 0.15853658536585366, "generation_spec": { "version": "v2-2026-07-30", "bos_context": true, "eos_stop": true, "token_healing": true, "truncation": "left_to_max_length_minus_max_gen", "stop_truncation": "earliest_match_all_terms", "stop_match_decode": "specials_kept" }, "token_healing": true, "gsm8k_exact_match_strict": 0.18877937831690675, "gsm8k_exact_match_flexible": 0.22820318423047764 }, "metric_gaps": { "blimp_acc": "no full-128k-pabpe-nfc-clean-fineweb2full-plus2-capped-tuned-hybridwindow-mathcode-scratch_blimp_code_bpb.json", "code_bpb_mean": "no full-128k-pabpe-nfc-clean-fineweb2full-plus2-capped-tuned-hybridwindow-mathcode-scratch_blimp_code_bpb.json; no full-128k-pabpe-nfc-clean-fineweb2full-plus2-capped-tuned-hybridwindow-mathcode-scratch_code_bpb.json" }, "registry_note": "#14, clean pretok, slug `pabpe-nfc-clean-fineweb2full-plus2-capped-tuned-hybridwindow`. PA-BPE with the `plus2` per-language vocab-allocation modifier. Finished 2026-06-03; mc_total 0.311 (highest in panel).", "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a139/cmeister/tokenizer-lm/checkpoints/full-128k-pabpe-nfc-clean-fineweb2full-plus2-capped-tuned-hybridwindow-mathcode-scratch", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "full-128k-pabpe-nfc-clean-fineweb2full-plus3-capped-tuned-hybridwindow-consv2": { "run": "full-128k-pabpe-nfc-clean-fineweb2full-plus3-capped-tuned-hybridwindow-consv2", "works": [ "paper_a_apertus_track" ], "training_regime": "std-1B", "architecture": { "n_layer": 24, "n_embd": 1536, "n_head": 12, "n_kv_head": 12, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 127835, "total_parameters": 1271072522 }, "training": { "step": 8794, "total_batch_size": 1048576, "token_budget": 9221177344, "total_bytes_consumed": 63071957452, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "full-128k" }, "tokenizer_slug": "pabpe-nfc-clean-fineweb2full-plus3-capped-tuned-hybridwindow-consv2", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 127831, "metrics": { "val_bpb": 0.7294688936122221, "blimp_acc": 0.8239552238805968, "code_bpb_mean": 0.5336782775861199, "blimp_scoring": "bos_prefixed", "flores_mean_bpb": 2.979246304587864, "flores_n_languages": 214, "mbpp_pass1": 0.078, "humaneval_pass1": 0.06097560975609756, "generation_spec": { "version": "v2-2026-07-30", "bos_context": true, "eos_stop": true, "token_healing": true, "truncation": "left_to_max_length_minus_max_gen", "stop_truncation": "earliest_match_all_terms", "stop_match_decode": "specials_kept" }, "token_healing": true, "gsm8k_exact_match_strict": 0.017437452615617893, "gsm8k_exact_match_flexible": 0.024260803639120546 }, "metric_gaps": {}, "registry_note": "PA-BPE hybrid global+window, clean-multi plus3 regex; fineweb2full, consv2 (no tailcuts) data; norm=NFC; PA-BPE (hybrid global+window); vocab ~131,196; plus3 consv2 baseline; USED - S6f Contrast B (131k arm) + Apertus-2 track; excluded from main paper panel", "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a139/cmeister/tokenizer-lm/checkpoints/full-128k-pabpe-nfc-clean-fineweb2full-plus3-capped-tuned-hybridwindow-consv2", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "full-128k-pabpe-nfc-clean-fineweb2full-plus3-capped-tuned-hybridwindow-consv2-mathcode-scratch": { "run": "full-128k-pabpe-nfc-clean-fineweb2full-plus3-capped-tuned-hybridwindow-consv2-mathcode-scratch", "works": [ "blog" ], "training_regime": "mathcode-20B-from-scratch", "architecture": { "n_layer": 24, "n_embd": 1536, "n_head": 12, "n_kv_head": 12, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 127835, "total_parameters": 1271072522 }, "training": { "step": 19073, "total_batch_size": 1048576, "token_budget": 19999490048, "total_bytes_consumed": 136794569534, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "full-128k-mathcode-scratch" }, "tokenizer_slug": "pabpe-nfc-clean-fineweb2full-plus3-capped-tuned-hybridwindow-consv2", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 127831, "metrics": { "val_bpb": 0.34411453574874007, "flores_mean_bpb": 3.51622118667608, "flores_n_languages": 214, "mbpp_pass1": 0.22, "humaneval_pass1": 0.17682926829268292, "generation_spec": { "version": "v2-2026-07-30", "bos_context": true, "eos_stop": true, "token_healing": true, "truncation": "left_to_max_length_minus_max_gen", "stop_truncation": "earliest_match_all_terms", "stop_match_decode": "specials_kept" }, "token_healing": true, "gsm8k_exact_match_strict": 0.19332827899924185, "gsm8k_exact_match_flexible": 0.221379833206975 }, "metric_gaps": { "blimp_acc": "no full-128k-pabpe-nfc-clean-fineweb2full-plus3-capped-tuned-hybridwindow-consv2-mathcode-scratch_blimp_code_bpb.json", "code_bpb_mean": "no full-128k-pabpe-nfc-clean-fineweb2full-plus3-capped-tuned-hybridwindow-consv2-mathcode-scratch_blimp_code_bpb.json; no full-128k-pabpe-nfc-clean-fineweb2full-plus3-capped-tuned-hybridwindow-consv2-mathcode-scratch_code_bpb.json" }, "registry_note": "#15, clean pretok, slug `pabpe-nfc-clean-fineweb2full-plus3-capped-tuned-hybridwindow-consv2`. PA-BPE `plus3` + `consv2` data config. Finished 2026-06-05; mc_total 0.271 [0.249, 0.294] **below plus2's 0.311 [0.288, 0.335]** (mc_math subset Wilson CIs separate: 0.280 vs 0.354). MBPP / HumanEval / GSM8K all CI-overlap with plus2 \u2014 plus2 stays the better-performing variant.", "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a139/cmeister/tokenizer-lm/checkpoints/full-128k-pabpe-nfc-clean-fineweb2full-plus3-capped-tuned-hybridwindow-consv2-mathcode-scratch", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "full-128k-pabpe-nfc-clean-fineweb2full-plus3-capped-tuned-hybridwindow-consv2-tailcuts-gm70k-v130-sp-eng5g": { "run": "full-128k-pabpe-nfc-clean-fineweb2full-plus3-capped-tuned-hybridwindow-consv2-tailcuts-gm70k-v130-sp-eng5g", "works": [ "paper_a_apertus_track" ], "training_regime": "std-1B", "architecture": { "n_layer": 24, "n_embd": 1536, "n_head": 12, "n_kv_head": 12, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 130929, "total_parameters": 1285228298 }, "training": { "step": 8841, "total_batch_size": 1048576, "token_budget": 9270460416, "total_bytes_consumed": 63690802707, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "full-128k" }, "tokenizer_slug": "pabpe-nfc-clean-fineweb2full-plus3-capped-tuned-hybridwindow-consv2-tailcuts-gm70k-v130-sp-eng5g", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 1, "metrics": { "val_bpb": 0.7275032359038739, "blimp_acc": 0.8209402985074628, "code_bpb_mean": 0.5291931590166571, "blimp_scoring": "bos_prefixed", "flores_mean_bpb": 2.9754391952607704, "flores_n_languages": 214, "mbpp_pass1": 0.09, "humaneval_pass1": 0.06097560975609756, "generation_spec": { "version": "v2-2026-07-30", "bos_context": true, "eos_stop": true, "token_healing": true, "truncation": "left_to_max_length_minus_max_gen", "stop_truncation": "earliest_match_all_terms", "stop_match_decode": "specials_kept" }, "token_healing": true, "gsm8k_exact_match_strict": 0.03639120545868082, "gsm8k_exact_match_flexible": 0.04473085670962851 }, "metric_gaps": {}, "registry_note": "PA-BPE hybrid global+window (gm70k), clean-multi plus3 regex; fineweb2full, consv2 + tailcuts + eng5g data; norm=NFC; PA-BPE (hybrid global+window); vocab ~131,196; Apertus-2 development ablation 'PA-Clean-plus3-A6'; USED - S6f Contrast B (131k arm) + Apertus-2 track ablation; excluded from main paper panel", "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a139/cmeister/tokenizer-lm/checkpoints/full-128k-pabpe-nfc-clean-fineweb2full-plus3-capped-tuned-hybridwindow-consv2-tailcuts-gm70k-v130-sp-eng5g", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "full-128k-pabpe-nfc-clean-fineweb2full-plus3-capped-tuned-hybridwindow-consv2-tailcuts-gm70k-v130-sp-eng5g-mathcode-scratch": { "run": "full-128k-pabpe-nfc-clean-fineweb2full-plus3-capped-tuned-hybridwindow-consv2-tailcuts-gm70k-v130-sp-eng5g-mathcode-scratch", "works": [ "blog" ], "training_regime": "mathcode-20B-from-scratch", "architecture": { "n_layer": 24, "n_embd": 1536, "n_head": 12, "n_kv_head": 12, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 130929, "total_parameters": 1285228298 }, "training": { "step": 19073, "total_batch_size": 1048576, "token_budget": 19999490048, "total_bytes_consumed": 137402406971, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "full-128k-mathcode-scratch" }, "tokenizer_slug": "pabpe-nfc-clean-fineweb2full-plus3-capped-tuned-hybridwindow-consv2-tailcuts-gm70k-v130-sp-eng5g", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 1, "metrics": { "val_bpb": 0.34293975437865554, "flores_mean_bpb": 3.500058100497096, "flores_n_languages": 214, "mbpp_pass1": 0.214, "humaneval_pass1": 0.13414634146341464, "generation_spec": { "version": "v2-2026-07-30", "bos_context": true, "eos_stop": true, "token_healing": true, "truncation": "left_to_max_length_minus_max_gen", "stop_truncation": "earliest_match_all_terms", "stop_match_decode": "specials_kept" }, "token_healing": true, "gsm8k_exact_match_strict": 0.20090978013646701, "gsm8k_exact_match_flexible": 0.22971948445792267 }, "metric_gaps": { "blimp_acc": "no full-128k-pabpe-nfc-clean-fineweb2full-plus3-capped-tuned-hybridwindow-consv2-tailcuts-gm70k-v130-sp-eng5g-mathcode-scratch_blimp_code_bpb.json", "code_bpb_mean": "no full-128k-pabpe-nfc-clean-fineweb2full-plus3-capped-tuned-hybridwindow-consv2-tailcuts-gm70k-v130-sp-eng5g-mathcode-scratch_blimp_code_bpb.json; no full-128k-pabpe-nfc-clean-fineweb2full-plus3-capped-tuned-hybridwindow-consv2-tailcuts-gm70k-v130-sp-eng5g-mathcode-scratch_code_bpb.json" }, "registry_note": "#17, clean pretok, slug `pabpe-nfc-clean-fineweb2full-plus3-capped-tuned-hybridwindow-consv2-tailcuts-gm70k-v130-sp-eng5g`. PA-BPE `plus3` + `consv2-tailcuts-eng5g` data config (parity_aware_config_grouped_fineweb2full_quota_tuned_consv2_tailcuts_eng5g.json), global-merges=70k, vocab 130929 with full Apertus chat-template specials (99 added tokens) baked in upstream.", "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a139/cmeister/tokenizer-lm/checkpoints/full-128k-pabpe-nfc-clean-fineweb2full-plus3-capped-tuned-hybridwindow-consv2-tailcuts-gm70k-v130-sp-eng5g-mathcode-scratch", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "full-128k-pabpe-nfc-clean-fineweb2full-plus3-capped-tuned-hybridwindow-consv2-tailcuts-gm90k-v130-sp-eng5g": { "run": "full-128k-pabpe-nfc-clean-fineweb2full-plus3-capped-tuned-hybridwindow-consv2-tailcuts-gm90k-v130-sp-eng5g", "works": [ "paper_a_apertus_track" ], "training_regime": "std-1B", "architecture": { "n_layer": 24, "n_embd": 1536, "n_head": 12, "n_kv_head": 12, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 130929, "total_parameters": 1285228298 }, "training": { "step": 8841, "total_batch_size": 1048576, "token_budget": 9270460416, "total_bytes_consumed": 61617057747, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "full-128k" }, "tokenizer_slug": "pabpe-nfc-clean-fineweb2full-plus3-capped-tuned-hybridwindow-consv2-tailcuts-gm90k-v130-sp-eng5g", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 1, "metrics": { "val_bpb": 0.726475413557967, "blimp_acc": 0.8127164179104476, "code_bpb_mean": 0.5286779776134367, "blimp_scoring": "bos_prefixed", "flores_mean_bpb": 2.9638969184385138, "flores_n_languages": 214, "mbpp_pass1": 0.098, "humaneval_pass1": 0.09146341463414634, "generation_spec": { "version": "v2-2026-07-30", "bos_context": true, "eos_stop": true, "token_healing": true, "truncation": "left_to_max_length_minus_max_gen", "stop_truncation": "earliest_match_all_terms", "stop_match_decode": "specials_kept" }, "token_healing": true, "gsm8k_exact_match_strict": 0.034874905231235785, "gsm8k_exact_match_flexible": 0.039423805913570885 }, "metric_gaps": {}, "registry_note": "PA-BPE hybrid global+window (gm90k), clean-multi plus3 regex; fineweb2full, consv2 + tailcuts + eng5g data; norm=NFC; PA-BPE (hybrid global+window); vocab ~131,196; Apertus-2 development ablation 'PA-Clean-plus3-A8'; USED - S6f Contrast B (131k arm) + Apertus-2 track ablation; excluded from main paper panel", "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a139/cmeister/tokenizer-lm/checkpoints/full-128k-pabpe-nfc-clean-fineweb2full-plus3-capped-tuned-hybridwindow-consv2-tailcuts-gm90k-v130-sp-eng5g", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "full-128k-pabpe-nfc-clean-fineweb2full-plus3-capped-tuned-hybridwindow-consv2-tailcuts-gm90k-v130-sp-eng5g-mathcode-scratch": { "run": "full-128k-pabpe-nfc-clean-fineweb2full-plus3-capped-tuned-hybridwindow-consv2-tailcuts-gm90k-v130-sp-eng5g-mathcode-scratch", "works": [ "blog" ], "training_regime": "mathcode-20B-from-scratch", "architecture": { "n_layer": 24, "n_embd": 1536, "n_head": 12, "n_kv_head": 12, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 130929, "total_parameters": 1285228298 }, "training": { "step": 19073, "total_batch_size": 1048576, "token_budget": 19999490048, "total_bytes_consumed": 132928644091, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "full-128k-mathcode-scratch" }, "tokenizer_slug": "pabpe-nfc-clean-fineweb2full-plus3-capped-tuned-hybridwindow-consv2-tailcuts-gm90k-v130-sp-eng5g", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 1, "metrics": { "val_bpb": 0.34171313250169755, "flores_mean_bpb": 3.5478004395829843, "flores_n_languages": 214, "mbpp_pass1": 0.24, "humaneval_pass1": 0.1524390243902439, "generation_spec": { "version": "v2-2026-07-30", "bos_context": true, "eos_stop": true, "token_healing": true, "truncation": "left_to_max_length_minus_max_gen", "stop_truncation": "earliest_match_all_terms", "stop_match_decode": "specials_kept" }, "token_healing": true, "gsm8k_exact_match_strict": 0.20242608036391205, "gsm8k_exact_match_flexible": 0.24336618650492797 }, "metric_gaps": { "blimp_acc": "no full-128k-pabpe-nfc-clean-fineweb2full-plus3-capped-tuned-hybridwindow-consv2-tailcuts-gm90k-v130-sp-eng5g-mathcode-scratch_blimp_code_bpb.json", "code_bpb_mean": "no full-128k-pabpe-nfc-clean-fineweb2full-plus3-capped-tuned-hybridwindow-consv2-tailcuts-gm90k-v130-sp-eng5g-mathcode-scratch_blimp_code_bpb.json; no full-128k-pabpe-nfc-clean-fineweb2full-plus3-capped-tuned-hybridwindow-consv2-tailcuts-gm90k-v130-sp-eng5g-mathcode-scratch_code_bpb.json" }, "registry_note": "#19, clean pretok, slug `pabpe-nfc-clean-fineweb2full-plus3-capped-tuned-hybridwindow-consv2-tailcuts-gm90k-v130-sp-eng5g`. PA-BPE `plus3` + same config as #18 but `plus3` pretok, global-merges=90k.", "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a139/cmeister/tokenizer-lm/checkpoints/full-128k-pabpe-nfc-clean-fineweb2full-plus3-capped-tuned-hybridwindow-consv2-tailcuts-gm90k-v130-sp-eng5g-mathcode-scratch", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "full-128k-pabpe-nfc-clean-fw2full-plus3-cap-hw-consv2-gm90k-tailcuts-eng5g-repcap8": { "run": "full-128k-pabpe-nfc-clean-fw2full-plus3-cap-hw-consv2-gm90k-tailcuts-eng5g-repcap8", "works": [ "paper_a_apertus_track" ], "training_regime": "std-1B", "architecture": { "n_layer": 24, "n_embd": 1536, "n_head": 12, "n_kv_head": 12, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 131017, "total_parameters": 1285818122 }, "training": { "step": 8843, "total_batch_size": 1048576, "token_budget": 9272557568, "total_bytes_consumed": 61615433001, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "full-128k" }, "tokenizer_slug": "pabpe-nfc-clean-fw2full-plus3-cap-hw-consv2-gm90k-tailcuts-eng5g-repcap8", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 1, "metrics": { "val_bpb": 0.7262090143619204, "blimp_acc": 0.8244328358208957, "code_bpb_mean": 0.5266014430572129, "blimp_scoring": "bos_prefixed", "flores_mean_bpb": 3.008393821448739, "flores_n_languages": 214, "mbpp_pass1": 0.098, "humaneval_pass1": 0.08536585365853659, "generation_spec": { "version": "v2-2026-07-30", "bos_context": true, "eos_stop": true, "token_healing": true, "truncation": "left_to_max_length_minus_max_gen", "stop_truncation": "earliest_match_all_terms", "stop_match_decode": "specials_kept" }, "token_healing": true, "gsm8k_exact_match_strict": 0.03184230477634572, "gsm8k_exact_match_flexible": 0.0356330553449583 }, "metric_gaps": {}, "registry_note": "PA-BPE hybrid global+window (gm90k), clean-multi plus3 regex + repcap8; fineweb2full, consv2 + tailcuts + eng5g data; norm=NFC; PA-BPE (hybrid global+window); vocab ~131,141 (131,017 + 124 specials; Apertus precedent for >128k vocab); Apertus-2 development ablation 'PA-Clean-plus3-repcap8fr-A8'; USED - S6f Contrast B (131k arm) + Apertus-2 track ablation; excluded from main paper panel", "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a139/cmeister/tokenizer-lm/checkpoints/full-128k-pabpe-nfc-clean-fw2full-plus3-cap-hw-consv2-gm90k-tailcuts-eng5g-repcap8", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "full-128k-pabpe-nfc-clean-fw2full-plus3-cap-hw-consv2-gm90k-tailcuts-eng5g-repcap8-mathcode-scratch": { "run": "full-128k-pabpe-nfc-clean-fw2full-plus3-cap-hw-consv2-gm90k-tailcuts-eng5g-repcap8-mathcode-scratch", "works": [ "blog" ], "training_regime": "mathcode-20B-from-scratch", "architecture": { "n_layer": 24, "n_embd": 1536, "n_head": 12, "n_kv_head": 12, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 131017, "total_parameters": 1285818122 }, "training": { "step": 19073, "total_batch_size": 1048576, "token_budget": 19999490048, "total_bytes_consumed": 132895075611, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "full-128k-mathcode-scratch" }, "tokenizer_slug": "pabpe-nfc-clean-fw2full-plus3-cap-hw-consv2-gm90k-tailcuts-eng5g-repcap8", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 1, "metrics": { "val_bpb": 0.34175041096316866, "flores_mean_bpb": 3.5627034742441057, "flores_n_languages": 214, "mbpp_pass1": 0.192, "humaneval_pass1": 0.18292682926829268, "generation_spec": { "version": "v2-2026-07-30", "bos_context": true, "eos_stop": true, "token_healing": true, "truncation": "left_to_max_length_minus_max_gen", "stop_truncation": "earliest_match_all_terms", "stop_match_decode": "specials_kept" }, "token_healing": true, "gsm8k_exact_match_strict": 0.2001516300227445, "gsm8k_exact_match_flexible": 0.22971948445792267 }, "metric_gaps": { "blimp_acc": "no full-128k-pabpe-nfc-clean-fw2full-plus3-cap-hw-consv2-gm90k-tailcuts-eng5g-repcap8-mathcode-scratch_blimp_code_bpb.json", "code_bpb_mean": "no full-128k-pabpe-nfc-clean-fw2full-plus3-cap-hw-consv2-gm90k-tailcuts-eng5g-repcap8-mathcode-scratch_blimp_code_bpb.json; no full-128k-pabpe-nfc-clean-fw2full-plus3-cap-hw-consv2-gm90k-tailcuts-eng5g-repcap8-mathcode-scratch_code_bpb.json" }, "registry_note": "#20, clean pretok, slug `pabpe-nfc-clean-fw2full-plus3-cap-hw-consv2-gm90k-tailcuts-eng5g-repcap8`. PA-BPE plus3+consv2+tailcuts+eng5g+gm90k+repcap8+fixed-regex variant. Vocab 131,017 + 124 specials (chat/reasoning/reserve slots); apertus precedent for >128k vocab. Source `nfc_clean_multi_plus3_repcap8_capped_hybrid_window_tuned_consv2_tailcuts_gm90k_v131k_sp124_eng5g_fr`. Distinct from roster #19 (`...gm90k-v130-sp-eng5g`) by md5 (5fafe099 vs 081e4ec3): differs by repcap8 + fixed-regex pretok. FINISHED step 19073 (resumed after an InvalidQOS stall at step 16000).", "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a139/cmeister/tokenizer-lm/checkpoints/full-128k-pabpe-nfc-clean-fw2full-plus3-cap-hw-consv2-gm90k-tailcuts-eng5g-repcap8-mathcode-scratch", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "full-128k-pabpe-nfc-gpt4-balanced": { "run": "full-128k-pabpe-nfc-gpt4-balanced", "works": [ "paper_a", "tokeval", "tokeval_cross_scale" ], "training_regime": "std-1B", "architecture": { "n_layer": 24, "n_embd": 1536, "n_head": 12, "n_kv_head": 12, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 127826, "total_parameters": 1271072522 }, "training": { "step": 8794, "total_batch_size": 1048576, "token_budget": 9221177344, "total_bytes_consumed": 76284810542, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "full-128k" }, "tokenizer_slug": "pabpe-nfc-gpt4-balanced", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 127822, "metrics": { "val_bpb": 0.7254768948770942, "blimp_acc": 0.816761194029851, "code_bpb_mean": 0.5335020022769154, "blimp_scoring": "bos_prefixed", "flores_mean_bpb": 2.661832658506986, "flores_n_languages": 215, "mbpp_pass1": 0.058, "humaneval_pass1": 0.06707317073170732, "generation_spec": { "version": "v2-2026-07-30", "bos_context": true, "eos_stop": true, "token_healing": true, "truncation": "left_to_max_length_minus_max_gen", "stop_truncation": "earliest_match_all_terms", "stop_match_decode": "specials_kept" }, "token_healing": true, "gsm8k_exact_match_strict": 0.029567854435178165, "gsm8k_exact_match_flexible": 0.029567854435178165 }, "metric_gaps": {}, "registry_note": "PA-BPE (parity, grouped=25 script-family units) with GPT-4o regex; balanced (D2) data; norm=NFC; PA-BPE (ParityBpeTrainer); vocab 127,826; bug-fixed 2026-05-16 re-export of the retired pabpe-balanced (2a); replaces the newline-stripping-bug original; USED - panels.json main21 (camera-ready main panel)", "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a139/cmeister/tokenizer-lm/checkpoints/full-128k-pabpe-nfc-gpt4-balanced", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "full-128k-pabpe-nfc-gpt4-balanced-hybridwindow": { "run": "full-128k-pabpe-nfc-gpt4-balanced-hybridwindow", "works": [ "paper_a", "tokeval" ], "training_regime": "std-1B", "architecture": { "n_layer": 24, "n_embd": 1536, "n_head": 12, "n_kv_head": 12, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 127826, "total_parameters": 1271072522 }, "training": { "step": 8794, "total_batch_size": 1048576, "token_budget": 9221177344, "total_bytes_consumed": 69596058966, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "full-128k" }, "tokenizer_slug": "pabpe-nfc-gpt4-balanced-hybridwindow", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 127822, "metrics": { "val_bpb": 0.7193782291843958, "blimp_acc": 0.816134328358209, "code_bpb_mean": 0.5239938767725334, "blimp_scoring": "bos_prefixed", "flores_mean_bpb": 2.655629831496573, "flores_n_languages": 215, "mbpp_pass1": 0.06, "humaneval_pass1": 0.0975609756097561, "generation_spec": { "version": "v2-2026-07-30", "bos_context": true, "eos_stop": true, "token_healing": true, "truncation": "left_to_max_length_minus_max_gen", "stop_truncation": "earliest_match_all_terms", "stop_match_decode": "specials_kept" }, "token_healing": true, "gsm8k_exact_match_strict": 0.037149355572403335, "gsm8k_exact_match_flexible": 0.0401819560272934 }, "metric_gaps": {}, "registry_note": "PA-BPE hybrid global+window (global_merges=64000, window=100, alpha=2.0, signal=ratios) with GPT-4o regex; balanced (D2) data; norm=NFC; PA-BPE (hybrid global+window); vocab 127,826; tests whether the hybrid-window scheme improves over base PA-BPE at fixed data/pretok; USED - balanced-1B bootstrap panel (S6.12) + paper panel; not in panels.json main21", "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a139/cmeister/tokenizer-lm/checkpoints/full-128k-pabpe-nfc-gpt4-balanced-hybridwindow", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "full-128k-pabpe-nfc-gpt4-balanced-mathcode-scratch": { "run": "full-128k-pabpe-nfc-gpt4-balanced-mathcode-scratch", "works": [ "blog", "tokeval" ], "training_regime": "mathcode-20B-from-scratch", "architecture": { "n_layer": 24, "n_embd": 1536, "n_head": 12, "n_kv_head": 12, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 127826, "total_parameters": 1271072522 }, "training": { "step": 19073, "total_batch_size": 1048576, "token_budget": 19999490048, "total_bytes_consumed": 165451465939, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "full-128k-mathcode-scratch" }, "tokenizer_slug": "pabpe-nfc-gpt4-balanced", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 127822, "metrics": { "val_bpb": 0.3468567492776554, "code_bpb_mean": 0.38777905081348607, "flores_mean_bpb": 3.3580068430546723, "flores_n_languages": 215, "mbpp_pass1": 0.13, "humaneval_pass1": 0.16463414634146342, "generation_spec": { "version": "v2-2026-07-30", "bos_context": true, "eos_stop": true, "token_healing": true, "truncation": "left_to_max_length_minus_max_gen", "stop_truncation": "earliest_match_all_terms", "stop_match_decode": "specials_kept" }, "token_healing": true, "gsm8k_exact_match_strict": 0.20318423047763456, "gsm8k_exact_match_flexible": 0.23805913570887036 }, "metric_gaps": { "blimp_acc": "no full-128k-pabpe-nfc-gpt4-balanced-mathcode-scratch_blimp_code_bpb.json" }, "registry_note": "#56, gpt4 pretok, slug `pabpe-nfc-gpt4-balanced`. Main-grid PA-BPE cell (first PA-BPE run at 20B). FINISHED at step 19073, 4-eval chain + code_bpb complete (verified 2026-08-09). In panel.", "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a139/cmeister/tokenizer-lm/checkpoints/full-128k-pabpe-nfc-gpt4-balanced-mathcode-scratch", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "full-128k-pabpe-nfc-gpt4-fineweb2full": { "run": "full-128k-pabpe-nfc-gpt4-fineweb2full", "works": [ "paper_a" ], "training_regime": "std-1B", "architecture": { "n_layer": 24, "n_embd": 1536, "n_head": 12, "n_kv_head": 12, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 127825, "total_parameters": 1271072522 }, "training": { "step": 8794, "total_batch_size": 1048576, "token_budget": 9221177344, "total_bytes_consumed": 72845529906, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "full-128k" }, "tokenizer_slug": "pabpe-nfc-gpt4-fineweb2full", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 127821, "metrics": { "val_bpb": 0.7447323227163575, "blimp_acc": 0.8236119402985074, "code_bpb_mean": 0.5606503370154372, "blimp_scoring": "bos_prefixed", "flores_mean_bpb": 2.822055797135025, "flores_n_languages": 215, "mbpp_pass1": 0.068, "humaneval_pass1": 0.08536585365853659, "generation_spec": { "version": "v2-2026-07-30", "bos_context": true, "eos_stop": true, "token_healing": true, "truncation": "left_to_max_length_minus_max_gen", "stop_truncation": "earliest_match_all_terms", "stop_match_decode": "specials_kept" }, "token_healing": true, "gsm8k_exact_match_strict": 0.03411675511751327, "gsm8k_exact_match_flexible": 0.04094010614101592 }, "metric_gaps": {}, "registry_note": "PA-BPE (parity, base) with GPT-4o regex; fineweb2full data; norm=NFC; PA-BPE (base); vocab 127,825; USED - S5b fineweb2full analysis; fineweb2full data variant, excluded from main paper panel", "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a139/cmeister/tokenizer-lm/checkpoints/full-128k-pabpe-nfc-gpt4-fineweb2full", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "full-128k-pabpe-nfc-gpt4-fineweb2full-hybridwindow": { "run": "full-128k-pabpe-nfc-gpt4-fineweb2full-hybridwindow", "works": [ "paper_a" ], "training_regime": "std-1B", "architecture": { "n_layer": 24, "n_embd": 1536, "n_head": 12, "n_kv_head": 12, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 127825, "total_parameters": 1271072522 }, "training": { "step": 8794, "total_batch_size": 1048576, "token_budget": 9221177344, "total_bytes_consumed": 62964864120, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "full-128k" }, "tokenizer_slug": "pabpe-nfc-gpt4-fineweb2full-hybridwindow", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 127821, "metrics": { "val_bpb": 0.7277658585120369, "blimp_acc": 0.827, "code_bpb_mean": 0.5309002808854102, "blimp_scoring": "bos_prefixed", "flores_mean_bpb": 2.954243294480877, "flores_n_languages": 215, "mbpp_pass1": 0.074, "humaneval_pass1": 0.06707317073170732, "generation_spec": { "version": "v2-2026-07-30", "bos_context": true, "eos_stop": true, "token_healing": true, "truncation": "left_to_max_length_minus_max_gen", "stop_truncation": "earliest_match_all_terms", "stop_match_decode": "specials_kept" }, "token_healing": true, "gsm8k_exact_match_strict": 0.039423805913570885, "gsm8k_exact_match_flexible": 0.04852160727824109 }, "metric_gaps": {}, "registry_note": "PA-BPE hybrid global+window with GPT-4o regex; fineweb2full data; norm=NFC; PA-BPE (hybrid global+window); vocab 127,825; USED - S5b fineweb2full analysis; fineweb2full data variant, excluded from main paper panel", "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a139/cmeister/tokenizer-lm/checkpoints/full-128k-pabpe-nfc-gpt4-fineweb2full-hybridwindow", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "full-128k-punct-allmulti-equal-bpe": { "run": "full-128k-punct-allmulti-equal-bpe", "works": [ "paper_a", "tokeval" ], "training_regime": "std-1B", "architecture": { "n_layer": 24, "n_embd": 1536, "n_head": 12, "n_kv_head": 12, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 128256, "total_parameters": 1272841994 }, "training": { "step": 8800, "total_batch_size": 1048576, "token_budget": 9227468800, "total_bytes_consumed": 61181692000, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "full-128k" }, "tokenizer_slug": "punct-allmulti-equal-bpe", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 0, "metrics": { "val_bpb": 0.7222909830438362, "blimp_acc": 0.8184626865671643, "code_bpb_mean": 0.5298100623186873, "blimp_scoring": "bos_prefixed", "flores_mean_bpb": 2.6482277858490395, "flores_n_languages": 215 }, "metric_gaps": { "mbpp_pass1": "no full-128k-punct-allmulti-equal-bpe_code_v2.json", "humaneval_pass1": "no full-128k-punct-allmulti-equal-bpe_code_v2.json", "gsm8k_exact_match_strict": "no full-128k-punct-allmulti-equal-bpe_gsm8k_v2.json" }, "registry_note": "P1 Punctuation+ByteLevel; allmulti_equal data; norm=none; BPE; vocab 128,256; pairs with punct-balanced-bpe; USED - S6d.4 (rho=-0.375, weakest of the four - punct also compresses the tail least)", "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a139/cmeister/tokenizer-lm/checkpoints/full-128k-punct-allmulti-equal-bpe", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "full-128k-punct-balanced-bpe": { "run": "full-128k-punct-balanced-bpe", "works": [ "tokeval", "tokeval_cross_scale" ], "training_regime": "std-1B", "architecture": { "n_layer": 24, "n_embd": 1536, "n_head": 12, "n_kv_head": 12, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "default 15.0, run predates the configurable field", "vocab_size": 128256, "total_parameters": 1272841994 }, "training": { "step": 8800, "total_batch_size": 1048576, "token_budget": 9227468800, "total_bytes_consumed": 70269874400, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "full-128k" }, "tokenizer_slug": "punct-balanced-bpe", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 0, "metrics": { "val_bpb": 0.7170390552723569, "blimp_acc": 0.8140149253731344, "code_bpb_mean": 0.5272228231489715, "blimp_scoring": "bos_prefixed", "flores_mean_bpb": 2.647426551753968, "flores_n_languages": 214, "mbpp_pass1": 0.094, "humaneval_pass1": 0.07317073170731707, "generation_spec": { "version": "v2-2026-07-30", "bos_context": true, "eos_stop": true, "token_healing": true, "truncation": "left_to_max_length_minus_max_gen", "stop_truncation": "earliest_match_all_terms", "stop_match_decode": "specials_kept" }, "token_healing": true, "gsm8k_exact_match_strict": 0.030326004548900682, "gsm8k_exact_match_flexible": 0.03411675511751327 }, "metric_gaps": {}, "registry_note": "P1 Punctuation+ByteLevel; balanced (D2) data; norm=none; BPE; vocab 128,260; USED - panels.json main21 (camera-ready main panel)", "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a139/cmeister/tokenizer-lm/checkpoints/full-128k-punct-balanced-bpe", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "full-128k-punct-balanced-bpe-mathcode-scratch": { "run": "full-128k-punct-balanced-bpe-mathcode-scratch", "works": [ "blog", "tokeval" ], "training_regime": "mathcode-20B-from-scratch", "architecture": { "n_layer": 24, "n_embd": 1536, "n_head": 12, "n_kv_head": 12, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 128256, "total_parameters": 1272841994 }, "training": { "step": 19073, "total_batch_size": 1048576, "token_budget": 19999490048, "total_bytes_consumed": 152301967549, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "full-128k-mathcode-scratch" }, "tokenizer_slug": "punct-balanced-bpe", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 0, "metrics": { "val_bpb": 0.3404114563994627, "blimp_acc": 0.5531044776119404, "code_bpb_mean": 0.38008554959295715, "blimp_scoring": null, "flores_mean_bpb": 3.332889085550648, "flores_n_languages": 214, "mbpp_pass1": 0.23, "humaneval_pass1": 0.17073170731707318, "generation_spec": { "version": "v2-2026-07-30", "bos_context": true, "eos_stop": true, "token_healing": true, "truncation": "left_to_max_length_minus_max_gen", "stop_truncation": "earliest_match_all_terms", "stop_match_decode": "specials_kept" }, "token_healing": true, "gsm8k_exact_match_strict": 0.18877937831690675, "gsm8k_exact_match_flexible": 0.25018953752843065 }, "metric_gaps": {}, "registry_note": "#13, punct pretok, slug `punct-balanced-bpe`. Punctuation-pretok BPE.", "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a139/cmeister/tokenizer-lm/checkpoints/full-128k-punct-balanced-bpe-mathcode-scratch", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "full-128k-punct-balanced-bpe-seed42": { "run": "full-128k-punct-balanced-bpe-seed42", "works": [ "paper_a", "tokeval_cross_scale" ], "training_regime": "std-1B", "architecture": { "n_layer": 24, "n_embd": 1536, "n_head": 12, "n_kv_head": 12, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 128256, "total_parameters": 1272841994 }, "training": { "step": 8800, "total_batch_size": 1048576, "token_budget": 9227468800, "total_bytes_consumed": 70269874400, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "full-128k" }, "tokenizer_slug": "punct-balanced-bpe", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 0, "metrics": { "val_bpb": 0.7185115767080312, "blimp_acc": 0.8170746268656716, "code_bpb_mean": 0.5302227287291184, "blimp_scoring": "bos_prefixed", "flores_mean_bpb": 2.6468641008007117, "flores_n_languages": 215, "mbpp_pass1": 0.076, "humaneval_pass1": 0.06097560975609756, "generation_spec": { "version": "v2-2026-07-30", "bos_context": true, "eos_stop": true, "token_healing": true, "truncation": "left_to_max_length_minus_max_gen", "stop_truncation": "earliest_match_all_terms", "stop_match_decode": "specials_kept" }, "token_healing": true, "gsm8k_exact_match_strict": 0.03866565579984837, "gsm8k_exact_match_flexible": 0.04245640636846096 }, "metric_gaps": {}, "registry_note": null, "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a139/cmeister/tokenizer-lm/checkpoints/full-128k-punct-balanced-bpe-seed42", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "full-128k-punct-balanced-bpe-seed42-mathcode-scratch": { "run": "full-128k-punct-balanced-bpe-seed42-mathcode-scratch", "works": [ "blog" ], "training_regime": "mathcode-20B-from-scratch", "architecture": { "n_layer": 24, "n_embd": 1536, "n_head": 12, "n_kv_head": 12, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 128256, "total_parameters": 1272841994 }, "training": { "step": 19073, "total_batch_size": 1048576, "token_budget": 19999490048, "total_bytes_consumed": 152301967549, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "full-128k-mathcode-scratch" }, "tokenizer_slug": "punct-balanced-bpe", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 0, "metrics": { "val_bpb": 0.34034366356012596, "flores_mean_bpb": 3.3227860170269508, "flores_n_languages": 215, "mbpp_pass1": 0.192, "humaneval_pass1": 0.17073170731707318, "generation_spec": { "version": "v2-2026-07-30", "bos_context": true, "eos_stop": true, "token_healing": true, "truncation": "left_to_max_length_minus_max_gen", "stop_truncation": "earliest_match_all_terms", "stop_match_decode": "specials_kept" }, "token_healing": true }, "metric_gaps": { "blimp_acc": "no full-128k-punct-balanced-bpe-seed42-mathcode-scratch_blimp_code_bpb.json", "code_bpb_mean": "no full-128k-punct-balanced-bpe-seed42-mathcode-scratch_blimp_code_bpb.json; no full-128k-punct-balanced-bpe-seed42-mathcode-scratch_code_bpb.json", "gsm8k_exact_match_strict": "no full-128k-punct-balanced-bpe-seed42-mathcode-scratch_gsm8k_v2.json" }, "registry_note": null, "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a139/cmeister/tokenizer-lm/checkpoints/full-128k-punct-balanced-bpe-seed42-mathcode-scratch", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "full-128k-punct-balanced-bpe-seed43-mathcode-scratch": { "run": "full-128k-punct-balanced-bpe-seed43-mathcode-scratch", "works": [ "blog" ], "training_regime": "mathcode-20B-from-scratch", "architecture": { "n_layer": 24, "n_embd": 1536, "n_head": 12, "n_kv_head": 12, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 128256, "total_parameters": 1272841994 }, "training": { "step": 19073, "total_batch_size": 1048576, "token_budget": 19999490048, "total_bytes_consumed": 152301967549, "seed": 43, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "full-128k-mathcode-scratch" }, "tokenizer_slug": "punct-balanced-bpe", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 0, "metrics": { "val_bpb": 0.34137498932537774, "flores_mean_bpb": 3.356340922797924, "flores_n_languages": 215, "mbpp_pass1": 0.008, "humaneval_pass1": 0.16463414634146342, "generation_spec": { "version": "v2-2026-07-30", "bos_context": true, "eos_stop": true, "token_healing": true, "truncation": "left_to_max_length_minus_max_gen", "stop_truncation": "earliest_match_all_terms", "stop_match_decode": "specials_kept" }, "token_healing": true, "gsm8k_exact_match_strict": 0.18953752843062927, "gsm8k_exact_match_flexible": 0.2304776345716452 }, "metric_gaps": { "blimp_acc": "no full-128k-punct-balanced-bpe-seed43-mathcode-scratch_blimp_code_bpb.json", "code_bpb_mean": "no full-128k-punct-balanced-bpe-seed43-mathcode-scratch_blimp_code_bpb.json; no full-128k-punct-balanced-bpe-seed43-mathcode-scratch_code_bpb.json" }, "registry_note": null, "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a139/cmeister/tokenizer-lm/checkpoints/full-128k-punct-balanced-bpe-seed43-mathcode-scratch", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "full-128k-punct-balanced-bpe-seed44-mathcode-scratch": { "run": "full-128k-punct-balanced-bpe-seed44-mathcode-scratch", "works": [ "blog" ], "training_regime": "mathcode-20B-from-scratch", "architecture": { "n_layer": 24, "n_embd": 1536, "n_head": 12, "n_kv_head": 12, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 128256, "total_parameters": 1272841994 }, "training": { "step": 19073, "total_batch_size": 1048576, "token_budget": 19999490048, "total_bytes_consumed": 152301967549, "seed": 44, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "full-128k-mathcode-scratch" }, "tokenizer_slug": "punct-balanced-bpe", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 0, "metrics": { "val_bpb": 0.3403527543062969, "flores_mean_bpb": 3.3464235720721844, "flores_n_languages": 215, "mbpp_pass1": 0.084, "humaneval_pass1": 0.18292682926829268, "generation_spec": { "version": "v2-2026-07-30", "bos_context": true, "eos_stop": true, "token_healing": true, "truncation": "left_to_max_length_minus_max_gen", "stop_truncation": "earliest_match_all_terms", "stop_match_decode": "specials_kept" }, "token_healing": true, "gsm8k_exact_match_strict": 0.20394238059135708, "gsm8k_exact_match_flexible": 0.24791508718726307 }, "metric_gaps": { "blimp_acc": "no full-128k-punct-balanced-bpe-seed44-mathcode-scratch_blimp_code_bpb.json", "code_bpb_mean": "no full-128k-punct-balanced-bpe-seed44-mathcode-scratch_blimp_code_bpb.json; no full-128k-punct-balanced-bpe-seed44-mathcode-scratch_code_bpb.json" }, "registry_note": null, "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a139/cmeister/tokenizer-lm/checkpoints/full-128k-punct-balanced-bpe-seed44-mathcode-scratch", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "full-128k-punct-balancedhalf-superbpe-t64k-copy": { "run": "full-128k-punct-balancedhalf-superbpe-t64k-copy", "works": [ "paper_a", "tokeval" ], "training_regime": "std-1B", "architecture": { "n_layer": 24, "n_embd": 1536, "n_head": 12, "n_kv_head": 12, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 128004, "total_parameters": 1271957258 }, "training": { "step": 8797, "total_batch_size": 1048576, "token_budget": 9224323072, "total_bytes_consumed": 68386628826, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "full-128k" }, "tokenizer_slug": "punct-balancedhalf-superbpe-t64k-copy", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 128000, "metrics": { "val_bpb": 0.7258874528699852, "blimp_acc": 0.807865671641791, "code_bpb_mean": 0.5273893624392161, "blimp_scoring": "bos_prefixed", "flores_mean_bpb": 2.6298317755538805, "flores_n_languages": 215, "mbpp_pass1": 0.044, "humaneval_pass1": 0.04878048780487805, "generation_spec": { "version": "v2-2026-07-30", "bos_context": true, "eos_stop": true, "token_healing": true, "truncation": "left_to_max_length_minus_max_gen", "stop_truncation": "earliest_match_all_terms", "stop_match_decode": "specials_kept" }, "token_healing": true, "gsm8k_exact_match_strict": 0.024260803639120546, "gsm8k_exact_match_flexible": 0.029567854435178165 }, "metric_gaps": {}, "registry_note": "SuperBPE stage-1 Punct(Isolated)+ByteLevel base (64k transition) / stage-2 minimal regex (allows superwords to span punctuation); balanced-half (D2 halved) data; norm=none (inherited from punct base); SuperBPE; vocab 128,004; the only SuperBPE variant EXEMPT from the S9.5 combining-mark bug (its stage-2 regex has no [^\\s\\p{L}\\p{N}] alternative); USED - balanced-1B bootstrap panel (S6.12) + paper panel; not in panels.json main21", "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a139/cmeister/tokenizer-lm/checkpoints/full-128k-punct-balancedhalf-superbpe-t64k-copy", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "full-128k-punct-english-bpe": { "run": "full-128k-punct-english-bpe", "works": [ "tokeval", "tokeval_cross_scale" ], "training_regime": "std-1B", "architecture": { "n_layer": 24, "n_embd": 1536, "n_head": 12, "n_kv_head": 12, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "default 15.0, run predates the configurable field", "vocab_size": 128256, "total_parameters": 1272841994 }, "training": { "step": 8800, "total_batch_size": 1048576, "token_budget": 9227468800, "total_bytes_consumed": 65285475200, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "full-128k" }, "tokenizer_slug": "punct-english-bpe", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 0, "metrics": { "val_bpb": 0.7342410854286585, "blimp_acc": 0.8088805970149258, "code_bpb_mean": 0.5581842356992628, "blimp_scoring": "bos_prefixed", "flores_mean_bpb": 2.637858622428686, "flores_n_languages": 214, "mbpp_pass1": 0.056, "humaneval_pass1": 0.07926829268292683, "generation_spec": { "version": "v2-2026-07-30", "bos_context": true, "eos_stop": true, "token_healing": true, "truncation": "left_to_max_length_minus_max_gen", "stop_truncation": "earliest_match_all_terms", "stop_match_decode": "specials_kept" }, "token_healing": true, "gsm8k_exact_match_strict": 0.018953752843062926, "gsm8k_exact_match_flexible": 0.028051554207733132 }, "metric_gaps": {}, "registry_note": "P1 Punctuation+ByteLevel; english (D1) data; norm=none; BPE; vocab 128,260; USED - panels.json main21 (camera-ready main panel)", "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a139/cmeister/tokenizer-lm/checkpoints/full-128k-punct-english-bpe", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "full-128k-punct-english-bpe-mathcode-scratch": { "run": "full-128k-punct-english-bpe-mathcode-scratch", "works": [ "blog", "tokeval" ], "training_regime": "mathcode-20B-from-scratch", "architecture": { "n_layer": 24, "n_embd": 1536, "n_head": 12, "n_kv_head": 12, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 128256, "total_parameters": 1272841994 }, "training": { "step": 19073, "total_batch_size": 1048576, "token_budget": 19999490048, "total_bytes_consumed": 141498848692, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "full-128k-mathcode-scratch" }, "tokenizer_slug": "punct-english-bpe", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 0, "metrics": { "val_bpb": 0.3434937319193573, "code_bpb_mean": 0.37235060078538484, "flores_mean_bpb": 3.1770676728331426, "flores_n_languages": 215, "mbpp_pass1": 0.228, "humaneval_pass1": 0.16463414634146342, "generation_spec": { "version": "v2-2026-07-30", "bos_context": true, "eos_stop": true, "token_healing": true, "truncation": "left_to_max_length_minus_max_gen", "stop_truncation": "earliest_match_all_terms", "stop_match_decode": "specials_kept" }, "token_healing": true, "gsm8k_exact_match_strict": 0.20849128127369218, "gsm8k_exact_match_flexible": 0.22365428354814254 }, "metric_gaps": { "blimp_acc": "no full-128k-punct-english-bpe-mathcode-scratch_blimp_code_bpb.json" }, "registry_note": "#49, punct pretok, slug `punct-english-bpe`. Main-grid English-data BPE cell. FINISHED at step 19073, 4-eval chain + code_bpb complete (verified 2026-08-09). In panel.", "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a139/cmeister/tokenizer-lm/checkpoints/full-128k-punct-english-bpe-mathcode-scratch", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "full-128k-punct-english-bpe-seed42": { "run": "full-128k-punct-english-bpe-seed42", "works": [ "paper_a", "tokeval_cross_scale" ], "training_regime": "std-1B", "architecture": { "n_layer": 24, "n_embd": 1536, "n_head": 12, "n_kv_head": 12, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 128256, "total_parameters": 1272841994 }, "training": { "step": 8800, "total_batch_size": 1048576, "token_budget": 9227468800, "total_bytes_consumed": 65285475200, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "full-128k" }, "tokenizer_slug": "punct-english-bpe", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 0, "metrics": { "val_bpb": 0.7359940385712558, "blimp_acc": 0.8072686567164181, "code_bpb_mean": 0.5550694968769654, "blimp_scoring": "bos_prefixed", "flores_mean_bpb": 2.636114260178155, "flores_n_languages": 215, "mbpp_pass1": 0.05, "humaneval_pass1": 0.054878048780487805, "generation_spec": { "version": "v2-2026-07-30", "bos_context": true, "eos_stop": true, "token_healing": true, "truncation": "left_to_max_length_minus_max_gen", "stop_truncation": "earliest_match_all_terms", "stop_match_decode": "specials_kept" }, "token_healing": true, "gsm8k_exact_match_strict": 0.03411675511751327, "gsm8k_exact_match_flexible": 0.04397270659590599 }, "metric_gaps": {}, "registry_note": null, "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a139/cmeister/tokenizer-lm/checkpoints/full-128k-punct-english-bpe-seed42", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "full-128k-rightalign-balanced-bpe": { "run": "full-128k-rightalign-balanced-bpe", "works": [ "tokeval", "tokeval_cross_scale" ], "training_regime": "std-1B", "architecture": { "n_layer": 24, "n_embd": 1536, "n_head": 12, "n_kv_head": 12, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "default 15.0, run predates the configurable field", "vocab_size": 128256, "total_parameters": 1272841994 }, "training": { "step": 8800, "total_batch_size": 1048576, "token_budget": 9227468800, "total_bytes_consumed": 68117764000, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "full-128k" }, "tokenizer_slug": "rightalign-balanced-bpe", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 0, "metrics": { "val_bpb": 0.7120323882905231, "blimp_acc": 0.815567164179104, "code_bpb_mean": 0.5194966805778097, "blimp_scoring": "bos_prefixed", "flores_mean_bpb": 2.6439208845854703, "flores_n_languages": 214, "mbpp_pass1": 0.082, "humaneval_pass1": 0.07926829268292683, "generation_spec": { "version": "v2-2026-07-30", "bos_context": true, "eos_stop": true, "token_healing": true, "truncation": "left_to_max_length_minus_max_gen", "stop_truncation": "earliest_match_all_terms", "stop_match_decode": "specials_kept" }, "token_healing": true, "gsm8k_exact_match_strict": 0.04094010614101592, "gsm8k_exact_match_flexible": 0.047763457164518575 }, "metric_gaps": {}, "registry_note": "P4 right-aligned digits (REGEX_GPT4O_RIGHTALIGN); balanced (D2) data; norm=none; BPE; vocab 128,260; USED - panels.json main21 (camera-ready main panel)", "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a139/cmeister/tokenizer-lm/checkpoints/full-128k-rightalign-balanced-bpe", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "full-128k-rightalign-balanced-bpe-mathcode-scratch": { "run": "full-128k-rightalign-balanced-bpe-mathcode-scratch", "works": [ "blog", "tokeval" ], "training_regime": "mathcode-20B-from-scratch", "architecture": { "n_layer": 24, "n_embd": 1536, "n_head": 12, "n_kv_head": 12, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 128256, "total_parameters": 1272841994 }, "training": { "step": 19073, "total_batch_size": 1048576, "token_budget": 19999490048, "total_bytes_consumed": 147637512815, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "full-128k-mathcode-scratch" }, "tokenizer_slug": "rightalign-balanced-bpe", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 0, "metrics": { "val_bpb": 0.3352981899578387, "code_bpb_mean": 0.379809350915327, "flores_mean_bpb": 3.383188683529629, "flores_n_languages": 215, "mbpp_pass1": 0.072, "humaneval_pass1": 0.1524390243902439, "generation_spec": { "version": "v2-2026-07-30", "bos_context": true, "eos_stop": true, "token_healing": true, "truncation": "left_to_max_length_minus_max_gen", "stop_truncation": "earliest_match_all_terms", "stop_match_decode": "specials_kept" }, "token_healing": true, "gsm8k_exact_match_strict": 0.23351023502653526, "gsm8k_exact_match_flexible": 0.265352539802881 }, "metric_gaps": { "blimp_acc": "no full-128k-rightalign-balanced-bpe-mathcode-scratch_blimp_code_bpb.json" }, "registry_note": "#52, rightalign pretok, slug `rightalign-balanced-bpe`. Main-grid RightAlign balanced BPE cell. FINISHED at step 19073, 4-eval chain + code_bpb complete (verified 2026-08-09). In panel.", "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a139/cmeister/tokenizer-lm/checkpoints/full-128k-rightalign-balanced-bpe-mathcode-scratch", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "full-128k-rightalign-balanced-bpe-seed42": { "run": "full-128k-rightalign-balanced-bpe-seed42", "works": [ "paper_a", "tokeval_cross_scale" ], "training_regime": "std-1B", "architecture": { "n_layer": 24, "n_embd": 1536, "n_head": 12, "n_kv_head": 12, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 128256, "total_parameters": 1272841994 }, "training": { "step": 8800, "total_batch_size": 1048576, "token_budget": 9227468800, "total_bytes_consumed": 68117764000, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "full-128k" }, "tokenizer_slug": "rightalign-balanced-bpe", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 0, "metrics": { "val_bpb": 0.7137766875032537, "blimp_acc": 0.8131492537313432, "code_bpb_mean": 0.5223377672129773, "blimp_scoring": "bos_prefixed", "flores_mean_bpb": 2.6409677778705793, "flores_n_languages": 215, "mbpp_pass1": 0.058, "humaneval_pass1": 0.07926829268292683, "generation_spec": { "version": "v2-2026-07-30", "bos_context": true, "eos_stop": true, "token_healing": true, "truncation": "left_to_max_length_minus_max_gen", "stop_truncation": "earliest_match_all_terms", "stop_match_decode": "specials_kept" }, "token_healing": true, "gsm8k_exact_match_strict": 0.037149355572403335, "gsm8k_exact_match_flexible": 0.04624715693707354 }, "metric_gaps": {}, "registry_note": null, "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a139/cmeister/tokenizer-lm/checkpoints/full-128k-rightalign-balanced-bpe-seed42", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "full-128k-rightalign-balanced-nfc-bpe": { "run": "full-128k-rightalign-balanced-nfc-bpe", "works": [ "tokeval", "tokeval_cross_scale" ], "training_regime": "std-1B", "architecture": { "n_layer": 24, "n_embd": 1536, "n_head": 12, "n_kv_head": 12, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "default 15.0, run predates the configurable field", "vocab_size": 128256, "total_parameters": 1272841994 }, "training": { "step": 8800, "total_batch_size": 1048576, "token_budget": 9227468800, "total_bytes_consumed": 68119057600, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "full-128k" }, "tokenizer_slug": "rightalign-balanced-nfc-bpe", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 0, "metrics": { "val_bpb": 0.7119887597970906, "blimp_acc": 0.8136567164179105, "code_bpb_mean": 0.5193163415702758, "blimp_scoring": "bos_prefixed", "flores_mean_bpb": 2.6383702373254945, "flores_n_languages": 214, "mbpp_pass1": 0.068, "humaneval_pass1": 0.07926829268292683, "generation_spec": { "version": "v2-2026-07-30", "bos_context": true, "eos_stop": true, "token_healing": true, "truncation": "left_to_max_length_minus_max_gen", "stop_truncation": "earliest_match_all_terms", "stop_match_decode": "specials_kept" }, "token_healing": true, "gsm8k_exact_match_strict": 0.039423805913570885, "gsm8k_exact_match_flexible": 0.04397270659590599 }, "metric_gaps": {}, "registry_note": "P4 right-aligned digits; balanced (D2) data; norm=NFC; BPE; vocab 128,260; USED - panels.json main21 (camera-ready main panel)", "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a139/cmeister/tokenizer-lm/checkpoints/full-128k-rightalign-balanced-nfc-bpe", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "full-128k-rightalign-balanced-nfc-bpe-mathcode-scratch": { "run": "full-128k-rightalign-balanced-nfc-bpe-mathcode-scratch", "works": [ "blog", "tokeval" ], "training_regime": "mathcode-20B-from-scratch", "architecture": { "n_layer": 24, "n_embd": 1536, "n_head": 12, "n_kv_head": 12, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 128256, "total_parameters": 1272841994 }, "training": { "step": 19073, "total_batch_size": 1048576, "token_budget": 19999490048, "total_bytes_consumed": 147640316546, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "full-128k-mathcode-scratch" }, "tokenizer_slug": "rightalign-balanced-nfc-bpe", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 0, "metrics": { "val_bpb": 0.3356379974663634, "code_bpb_mean": 0.38086114026234213, "flores_mean_bpb": 3.377811214706142, "flores_n_languages": 214, "mbpp_pass1": 0.104, "humaneval_pass1": 0.17073170731707318, "generation_spec": { "version": "v2-2026-07-30", "bos_context": true, "eos_stop": true, "token_healing": true, "truncation": "left_to_max_length_minus_max_gen", "stop_truncation": "earliest_match_all_terms", "stop_match_decode": "specials_kept" }, "token_healing": true, "gsm8k_exact_match_strict": 0.22365428354814254, "gsm8k_exact_match_flexible": 0.26156178923426837 }, "metric_gaps": { "blimp_acc": "no full-128k-rightalign-balanced-nfc-bpe-mathcode-scratch_blimp_code_bpb.json" }, "registry_note": "#3, rightalign pretok, slug `rightalign-balanced-nfc-bpe`. Digit-direction matched pair vs (2).", "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a139/cmeister/tokenizer-lm/checkpoints/full-128k-rightalign-balanced-nfc-bpe-mathcode-scratch", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "full-128k-rightalign-balanced-nfc-bpe-seed42": { "run": "full-128k-rightalign-balanced-nfc-bpe-seed42", "works": [ "paper_a", "tokeval_cross_scale" ], "training_regime": "std-1B", "architecture": { "n_layer": 24, "n_embd": 1536, "n_head": 12, "n_kv_head": 12, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 128256, "total_parameters": 1272841994 }, "training": { "step": 8800, "total_batch_size": 1048576, "token_budget": 9227468800, "total_bytes_consumed": 68119057600, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "full-128k" }, "tokenizer_slug": "rightalign-balanced-nfc-bpe", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 0, "metrics": { "val_bpb": 0.7138028167765688, "blimp_acc": 0.8157462686567161, "code_bpb_mean": 0.522530388505992, "blimp_scoring": "bos_prefixed", "flores_mean_bpb": 2.6416283064104826, "flores_n_languages": 215, "mbpp_pass1": 0.082, "humaneval_pass1": 0.09146341463414634, "generation_spec": { "version": "v2-2026-07-30", "bos_context": true, "eos_stop": true, "token_healing": true, "truncation": "left_to_max_length_minus_max_gen", "stop_truncation": "earliest_match_all_terms", "stop_match_decode": "specials_kept" }, "token_healing": true, "gsm8k_exact_match_strict": 0.04245640636846096, "gsm8k_exact_match_flexible": 0.050037907505686124 }, "metric_gaps": {}, "registry_note": null, "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a139/cmeister/tokenizer-lm/checkpoints/full-128k-rightalign-balanced-nfc-bpe-seed42", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "full-128k-rightalign-balanced-unigram": { "run": "full-128k-rightalign-balanced-unigram", "works": [ "tokeval", "tokeval_cross_scale" ], "training_regime": "std-1B", "architecture": { "n_layer": 24, "n_embd": 1536, "n_head": 12, "n_kv_head": 12, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "default 15.0, run predates the configurable field", "vocab_size": 128256, "total_parameters": 1272841994 }, "training": { "step": 8800, "total_batch_size": 1048576, "token_budget": 9227468800, "total_bytes_consumed": 59928184800, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "full-128k" }, "tokenizer_slug": "rightalign-balanced-unigram", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 0, "metrics": { "val_bpb": 0.7310138736467131, "blimp_acc": 0.8291940298507464, "code_bpb_mean": 0.5538140300742438, "blimp_scoring": "bos_prefixed", "flores_mean_bpb": 2.6891200256408188, "flores_n_languages": 214, "mbpp_pass1": 0.048, "humaneval_pass1": 0.07317073170731707, "generation_spec": { "version": "v2-2026-07-30", "bos_context": true, "eos_stop": true, "token_healing": true, "truncation": "left_to_max_length_minus_max_gen", "stop_truncation": "earliest_match_all_terms", "stop_match_decode": "specials_kept" }, "token_healing": true, "gsm8k_exact_match_strict": 0.0356330553449583, "gsm8k_exact_match_flexible": 0.043214556482183475 }, "metric_gaps": {}, "registry_note": "P4 right-aligned digits; balanced (D2) data; norm=none; UnigramLM (HF, default kwargs); vocab 128,260; USED - panels.json main21 (camera-ready main panel)", "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a139/cmeister/tokenizer-lm/checkpoints/full-128k-rightalign-balanced-unigram", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "full-128k-rightalign-balanced-unigram-mathcode-scratch": { "run": "full-128k-rightalign-balanced-unigram-mathcode-scratch", "works": [ "blog", "tokeval" ], "training_regime": "mathcode-20B-from-scratch", "architecture": { "n_layer": 24, "n_embd": 1536, "n_head": 12, "n_kv_head": 12, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 128256, "total_parameters": 1272841994 }, "training": { "step": 19073, "total_batch_size": 1048576, "token_budget": 19999490048, "total_bytes_consumed": 129887530533, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "full-128k-mathcode-scratch" }, "tokenizer_slug": "rightalign-balanced-unigram", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 0, "metrics": { "val_bpb": 0.35354189301466415, "code_bpb_mean": 0.39891069834510856, "flores_mean_bpb": 3.403116477153282, "flores_n_languages": 215, "mbpp_pass1": 0.0, "humaneval_pass1": 0.1402439024390244, "generation_spec": { "version": "v2-2026-07-30", "bos_context": true, "eos_stop": true, "token_healing": true, "truncation": "left_to_max_length_minus_max_gen", "stop_truncation": "earliest_match_all_terms", "stop_match_decode": "specials_kept" }, "token_healing": true, "gsm8k_exact_match_strict": 0.17513267626990145, "gsm8k_exact_match_flexible": 0.22971948445792267 }, "metric_gaps": { "blimp_acc": "no full-128k-rightalign-balanced-unigram-mathcode-scratch_blimp_code_bpb.json" }, "registry_note": "#55, rightalign pretok, slug `rightalign-balanced-unigram`. Main-grid UnigramLM cell. FINISHED at step 19073, 4-eval chain + code_bpb complete (verified 2026-08-09). In panel.", "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a139/cmeister/tokenizer-lm/checkpoints/full-128k-rightalign-balanced-unigram-mathcode-scratch", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "full-128k-rightalign-balanced-unigram-seed42": { "run": "full-128k-rightalign-balanced-unigram-seed42", "works": [ "paper_a", "tokeval_cross_scale" ], "training_regime": "std-1B", "architecture": { "n_layer": 24, "n_embd": 1536, "n_head": 12, "n_kv_head": 12, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 128256, "total_parameters": 1272841994 }, "training": { "step": 8800, "total_batch_size": 1048576, "token_budget": 9227468800, "total_bytes_consumed": 59928184800, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "full-128k" }, "tokenizer_slug": "rightalign-balanced-unigram", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 0, "metrics": { "val_bpb": 0.7337618761598241, "blimp_acc": 0.8277164179104478, "code_bpb_mean": 0.5566215811979148, "blimp_scoring": "bos_prefixed", "flores_mean_bpb": 2.6841635204854692, "flores_n_languages": 215, "mbpp_pass1": 0.044, "humaneval_pass1": 0.06707317073170732, "generation_spec": { "version": "v2-2026-07-30", "bos_context": true, "eos_stop": true, "token_healing": true, "truncation": "left_to_max_length_minus_max_gen", "stop_truncation": "earliest_match_all_terms", "stop_match_decode": "specials_kept" }, "token_healing": true, "gsm8k_exact_match_strict": 0.03184230477634572, "gsm8k_exact_match_flexible": 0.03411675511751327 }, "metric_gaps": {}, "registry_note": null, "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a139/cmeister/tokenizer-lm/checkpoints/full-128k-rightalign-balanced-unigram-seed42", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "full-128k-scripttok-bpe-scriptenc_gpt4o_cb": { "run": "full-128k-scripttok-bpe-scriptenc_gpt4o_cb", "works": [ "paper_a", "tokeval" ], "training_regime": "std-1B", "architecture": { "n_layer": 24, "n_embd": 1536, "n_head": 12, "n_kv_head": 12, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 128256, "total_parameters": 1272841994 }, "training": { "step": 8800, "total_batch_size": 1048576, "token_budget": 9227468800, "total_bytes_consumed": 67708388000, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "full-128k" }, "tokenizer_slug": "scripttok-bpe-scriptenc_gpt4o_cb", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 128255, "metrics": { "val_bpb": 0.7128471807824242, "blimp_acc": 0.8218507462686566, "code_bpb_mean": 0.5185598065707061, "blimp_scoring": "bos_prefixed", "flores_mean_bpb": 2.6572370027769137, "flores_n_languages": 215, "mbpp_pass1": 0.084, "humaneval_pass1": 0.08536585365853659, "generation_spec": { "version": "v2-2026-07-30", "bos_context": true, "eos_stop": true, "token_healing": true, "truncation": "left_to_max_length_minus_max_gen", "stop_truncation": "earliest_match_all_terms", "stop_match_decode": "specials_kept" }, "token_healing": true, "gsm8k_exact_match_strict": 0.04852160727824109, "gsm8k_exact_match_flexible": 0.05079605761940864 }, "metric_gaps": {}, "registry_note": "script_tok BPE, scriptenc_gpt4o_cb SCRIPT-encoding pretok; balanced (D2, script_tok's own fit sampler) data; norm=SCRIPT-encoding normalization; BPE (script_tok); vocab 128,256; requires pre-tokenized .npz shards; USED - panels.json structure-pretok (camera-ready)", "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a139/cmeister/tokenizer-lm/checkpoints/full-128k-scripttok-bpe-scriptenc_gpt4o_cb", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "full-128k-scripttok-bpe-scriptenc_gpt4o_cb-mathcode-scratch": { "run": "full-128k-scripttok-bpe-scriptenc_gpt4o_cb-mathcode-scratch", "works": [ "blog", "tokeval" ], "training_regime": "mathcode-20B-from-scratch", "architecture": { "n_layer": 24, "n_embd": 1536, "n_head": 12, "n_kv_head": 12, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 128256, "total_parameters": 1272841994 }, "training": { "step": 19073, "total_batch_size": 1048576, "token_budget": 19999490048, "total_bytes_consumed": 146750236855, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "full-128k-mathcode-scratch" }, "tokenizer_slug": "scripttok-bpe-scriptenc_gpt4o_cb", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 128255, "metrics": { "val_bpb": 0.33612096563369104, "code_bpb_mean": 0.37812117138811496, "flores_mean_bpb": 3.38748131750607, "flores_n_languages": 215, "mbpp_pass1": 0.136, "humaneval_pass1": 0.17073170731707318, "generation_spec": { "version": "v2-2026-07-30", "bos_context": true, "eos_stop": true, "token_healing": true, "truncation": "left_to_max_length_minus_max_gen", "stop_truncation": "earliest_match_all_terms", "stop_match_decode": "specials_kept" }, "token_healing": true, "gsm8k_exact_match_strict": 0.2047005307050796, "gsm8k_exact_match_flexible": 0.23730098559514784 }, "metric_gaps": { "blimp_acc": "no full-128k-scripttok-bpe-scriptenc_gpt4o_cb-mathcode-scratch_blimp_code_bpb.json" }, "registry_note": null, "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a139/cmeister/tokenizer-lm/checkpoints/full-128k-scripttok-bpe-scriptenc_gpt4o_cb-mathcode-scratch", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "full-128k-scripttok-mingram-scriptenc_cb": { "run": "full-128k-scripttok-mingram-scriptenc_cb", "works": [ "paper_a", "tokeval" ], "training_regime": "std-1B", "architecture": { "n_layer": 24, "n_embd": 1536, "n_head": 12, "n_kv_head": 12, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 128256, "total_parameters": 1272841994 }, "training": { "step": 8800, "total_batch_size": 1048576, "token_budget": 9227468800, "total_bytes_consumed": 70630463200, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "full-128k" }, "tokenizer_slug": "scripttok-mingram-scriptenc_cb", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 128255, "metrics": { "val_bpb": 0.7124263503328025, "blimp_acc": 0.8125074626865672, "code_bpb_mean": 0.5125317350292097, "blimp_scoring": "bos_prefixed", "flores_mean_bpb": 2.6849206877351186, "flores_n_languages": 215, "mbpp_pass1": 0.084, "humaneval_pass1": 0.08536585365853659, "generation_spec": { "version": "v2-2026-07-30", "bos_context": true, "eos_stop": true, "token_healing": true, "truncation": "left_to_max_length_minus_max_gen", "stop_truncation": "earliest_match_all_terms", "stop_match_decode": "specials_kept" }, "token_healing": true, "gsm8k_exact_match_strict": 0.04473085670962851, "gsm8k_exact_match_flexible": 0.04927975739196361 }, "metric_gaps": {}, "registry_note": "script_tok MinGram, scriptenc_cb SCRIPT-encoding pretok (-n 126339), PUA/unassigned code points stripped; balanced (D2, script_tok's own fit sampler/composition/seed) data; norm=SCRIPT-encoding normalization (designed, applied identically train/eval); MinGram; vocab 128,256 (1,916 atomic + 126,339 learned + synthesized BOS); requires pre-tokenized .npz shards; forward-fusing (newline+following indentation fuse into one atomic pretoken, later diagnosed as the MBPP/HumanEval-collapse mechanism at 20B, S3a.1/3a.6/S6h); USED - panels.json main21 (camera-ready main panel)", "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a139/cmeister/tokenizer-lm/checkpoints/full-128k-scripttok-mingram-scriptenc_cb", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "full-128k-scripttok-mingram-scriptenc_cb-mathcode-scratch": { "run": "full-128k-scripttok-mingram-scriptenc_cb-mathcode-scratch", "works": [ "blog", "tokeval" ], "training_regime": "mathcode-20B-from-scratch", "architecture": { "n_layer": 24, "n_embd": 1536, "n_head": 12, "n_kv_head": 12, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 128256, "total_parameters": 1272841994 }, "training": { "step": 19073, "total_batch_size": 1048576, "token_budget": 19999490048, "total_bytes_consumed": 153083502797, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "full-128k-mathcode-scratch" }, "tokenizer_slug": "scripttok-mingram-scriptenc_cb", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 128255, "metrics": { "val_bpb": 0.33485716807277754, "code_bpb_mean": 0.37927454694833107, "flores_mean_bpb": 3.4036921894353944, "flores_n_languages": 215, "mbpp_pass1": 0.016, "humaneval_pass1": 0.17682926829268292, "generation_spec": { "version": "v2-2026-07-30", "bos_context": true, "eos_stop": true, "token_healing": true, "truncation": "left_to_max_length_minus_max_gen", "stop_truncation": "earliest_match_all_terms", "stop_match_decode": "specials_kept" }, "token_healing": true, "gsm8k_exact_match_strict": 0.19939347990902198, "gsm8k_exact_match_flexible": 0.24260803639120546 }, "metric_gaps": { "blimp_acc": "no full-128k-scripttok-mingram-scriptenc_cb-mathcode-scratch_blimp_code_bpb.json" }, "registry_note": "#57, mingram pretok, slug `scripttok-mingram-scriptenc_cb`. MinGram/SCRIPT-encoding fusing variant, the twin roster #48 (`scriptenc_cb_nl`) compares against. Row added retroactively 2026-08-10 (documentation gap: run predates #48). FINISHED; v2 evals: MBPP 0.016, HumanEval 0.1768, GSM8K 0.1994, code BPB 0.3793. Correlation-panel member; joined tab:mc20b-results as the 17th custom row 2026-08-10 (user full-coherence ruling, superseding an earlier off-roster classification).", "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a139/cmeister/tokenizer-lm/checkpoints/full-128k-scripttok-mingram-scriptenc_cb-mathcode-scratch", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "full-128k-scripttok-mingram-scriptenc_cb_nl": { "run": "full-128k-scripttok-mingram-scriptenc_cb_nl", "works": [ "paper_a", "tokeval" ], "training_regime": "std-1B", "architecture": { "n_layer": 24, "n_embd": 1536, "n_head": 12, "n_kv_head": 12, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 128256, "total_parameters": 1272841994 }, "training": { "step": 8800, "total_batch_size": 1048576, "token_budget": 9227468800, "total_bytes_consumed": 70578596000, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "full-128k" }, "tokenizer_slug": "scripttok-mingram-scriptenc_cb_nl", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 128255, "metrics": { "val_bpb": 0.7134216868447818, "blimp_acc": 0.8145074626865673, "code_bpb_mean": 0.5161016806694816, "blimp_scoring": "bos_prefixed", "flores_mean_bpb": 2.670663746073611, "flores_n_languages": 215 }, "metric_gaps": { "mbpp_pass1": "no full-128k-scripttok-mingram-scriptenc_cb_nl_code_v2.json", "humaneval_pass1": "no full-128k-scripttok-mingram-scriptenc_cb_nl_code_v2.json", "gsm8k_exact_match_strict": "no full-128k-scripttok-mingram-scriptenc_cb_nl_gsm8k_v2.json" }, "registry_note": "script_tok MinGram, scriptenc_cb_nl (split_line_breaks=True: newline isolated from indentation, additive opt-in); balanced (D2, balanced_tlm10g_cb_nl corpus); SCRIPT-encoding normalization; MinGram; vocab 128,256 (1,916 atomic+126,339 learned+BOS); line-break-splitting ablation vs scripttok-mingram-scriptenc_cb (newline-fusion mechanism). USED - S3a.6/S3a.7/S6h newline-fusion analysis (std-1B twin of 20B pair, finished 2026-08-08); design-variant ablation, not a panels.json member.", "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a139/cmeister/tokenizer-lm/checkpoints/full-128k-scripttok-mingram-scriptenc_cb_nl", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "full-128k-scripttok-mingram-scriptenc_cb_nl-mathcode-scratch": { "run": "full-128k-scripttok-mingram-scriptenc_cb_nl-mathcode-scratch", "works": [ "blog", "tokeval" ], "training_regime": "mathcode-20B-from-scratch", "architecture": { "n_layer": 24, "n_embd": 1536, "n_head": 12, "n_kv_head": 12, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 128256, "total_parameters": 1272841994 }, "training": { "step": 19073, "total_batch_size": 1048576, "token_budget": 19999490048, "total_bytes_consumed": 152971086535, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "full-128k-mathcode-scratch" }, "tokenizer_slug": "scripttok-mingram-scriptenc_cb_nl", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 128255, "metrics": { "val_bpb": 0.336002609619353, "code_bpb_mean": 0.38298362754325693, "flores_mean_bpb": 3.4486723016230436, "flores_n_languages": 215, "mbpp_pass1": 0.216, "humaneval_pass1": 0.1524390243902439, "generation_spec": { "version": "v2-2026-07-30", "bos_context": true, "eos_stop": true, "token_healing": true, "truncation": "left_to_max_length_minus_max_gen", "stop_truncation": "earliest_match_all_terms", "stop_match_decode": "specials_kept" }, "token_healing": true, "gsm8k_exact_match_strict": 0.2350265352539803, "gsm8k_exact_match_flexible": 0.2721758908263836 }, "metric_gaps": { "blimp_acc": "no full-128k-scripttok-mingram-scriptenc_cb_nl-mathcode-scratch_blimp_code_bpb.json" }, "registry_note": "#48, mingram pretok, slug `scripttok-mingram-scriptenc_cb_nl`. MinGram/SCRIPT-encoding line-break-splitting variant (`split_line_breaks`, opt-in additive), the non-GPT4o-family test of the same mechanism. FINISHED. **MBPP 0.008 \u2192 0.214** vs its fusing twin (roster row for `scriptenc_cb`), HumanEval 0.000 \u2192 0.0610, GSM8K 0.216 \u2192 0.244. Pre-registered prediction (MBPP into 0.148-0.220) CONFIRMED. Evals under the 2780479-2780482 chain. Not in panel.", "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a139/cmeister/tokenizer-lm/checkpoints/full-128k-scripttok-mingram-scriptenc_cb_nl-mathcode-scratch", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "full-128k-superbpe-apertus-fineweb2full-capped-hybridwindow": { "run": "full-128k-superbpe-apertus-fineweb2full-capped-hybridwindow", "works": [ "paper_a_apertus_track" ], "training_regime": "std-1B", "architecture": { "n_layer": 24, "n_embd": 1536, "n_head": 12, "n_kv_head": 12, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 128004, "total_parameters": 1271957258 }, "training": { "step": 8797, "total_batch_size": 1048576, "token_budget": 9224323072, "total_bytes_consumed": 58123160129, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "full-128k" }, "tokenizer_slug": "superbpe-apertus-fineweb2full-capped-hybridwindow", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 128000, "metrics": { "val_bpb": 0.7330722568700316, "blimp_acc": 0.8146716417910445, "code_bpb_mean": 0.5405010740637953, "blimp_scoring": "bos_prefixed", "flores_mean_bpb": 3.0808713247974078, "flores_n_languages": 214, "mbpp_pass1": 0.074, "humaneval_pass1": 0.09146341463414634, "generation_spec": { "version": "v2-2026-07-30", "bos_context": true, "eos_stop": true, "token_healing": true, "truncation": "left_to_max_length_minus_max_gen", "stop_truncation": "earliest_match_all_terms", "stop_match_decode": "specials_kept" }, "token_healing": true, "gsm8k_exact_match_strict": 0.0310841546626232, "gsm8k_exact_match_flexible": 0.03639120545868082 }, "metric_gaps": {}, "registry_note": "SuperBPE on the apertus-pretok capped hybrid-window base; fineweb2full, capped data; norm=NFC; SuperBPE; vocab 128,004; 2x2 candidate; USED - Apertus-2 development track candidate; excluded from main paper panel", "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a139/cmeister/tokenizer-lm/checkpoints/full-128k-superbpe-apertus-fineweb2full-capped-hybridwindow", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "full-128k-superbpe-apertus-fineweb2full-capped-hybridwindow-mathcode-scratch": { "run": "full-128k-superbpe-apertus-fineweb2full-capped-hybridwindow-mathcode-scratch", "works": [ "blog" ], "training_regime": "mathcode-20B-from-scratch", "architecture": { "n_layer": 24, "n_embd": 1536, "n_head": 12, "n_kv_head": 12, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 128004, "total_parameters": 1271957258 }, "training": { "step": 19073, "total_batch_size": 1048576, "token_budget": 19999490048, "total_bytes_consumed": 126018305461, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "full-128k-mathcode-scratch" }, "tokenizer_slug": "superbpe-apertus-fineweb2full-capped-hybridwindow", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 128000, "metrics": { "val_bpb": 0.34260583858556143, "flores_mean_bpb": 3.663785438014377, "flores_n_languages": 214, "mbpp_pass1": 0.012, "humaneval_pass1": 0.15853658536585366, "generation_spec": { "version": "v2-2026-07-30", "bos_context": true, "eos_stop": true, "token_healing": true, "truncation": "left_to_max_length_minus_max_gen", "stop_truncation": "earliest_match_all_terms", "stop_match_decode": "specials_kept" }, "token_healing": true, "gsm8k_exact_match_strict": 0.19257012888551933, "gsm8k_exact_match_flexible": 0.22744503411675512 }, "metric_gaps": { "blimp_acc": "no full-128k-superbpe-apertus-fineweb2full-capped-hybridwindow-mathcode-scratch_blimp_code_bpb.json", "code_bpb_mean": "no full-128k-superbpe-apertus-fineweb2full-capped-hybridwindow-mathcode-scratch_blimp_code_bpb.json; no full-128k-superbpe-apertus-fineweb2full-capped-hybridwindow-mathcode-scratch_code_bpb.json" }, "registry_note": "#11, apertus pretok, slug `superbpe-apertus-fineweb2full-capped-hybridwindow`. SuperBPE candidate on apertus pretok.", "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a139/cmeister/tokenizer-lm/checkpoints/full-128k-superbpe-apertus-fineweb2full-capped-hybridwindow-mathcode-scratch", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "full-128k-superbpe-clean-c2-balanced": { "run": "full-128k-superbpe-clean-c2-balanced", "works": [ "paper_a", "tokeval" ], "training_regime": "std-1B", "architecture": { "n_layer": 24, "n_embd": 1536, "n_head": 12, "n_kv_head": 12, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 128004, "total_parameters": 1271957258 }, "training": { "step": 8797, "total_batch_size": 1048576, "token_budget": 9224323072, "total_bytes_consumed": 72553591786, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "full-128k" }, "tokenizer_slug": "superbpe-clean-c2-balanced", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 128000, "metrics": { "val_bpb": 0.7285011810271208, "blimp_acc": 0.8114328358208953, "code_bpb_mean": 0.5259040594970449, "blimp_scoring": "bos_prefixed", "flores_mean_bpb": 2.649908395389621, "flores_n_languages": 215, "mbpp_pass1": 0.084, "humaneval_pass1": 0.11585365853658537, "generation_spec": { "version": "v2-2026-07-30", "bos_context": true, "eos_stop": true, "token_healing": true, "truncation": "left_to_max_length_minus_max_gen", "stop_truncation": "earliest_match_all_terms", "stop_match_decode": "specials_kept" }, "token_healing": true, "gsm8k_exact_match_strict": 0.030326004548900682, "gsm8k_exact_match_flexible": 0.03639120545868082 }, "metric_gaps": {}, "registry_note": "SuperBPE stage-1 clean-multi (PA-BPE clean base) / stage-2 clean_multi C2 reduced regex; balanced (D2) data; norm=NFC (inherited from PA-BPE clean base); SuperBPE; vocab 128,004; Phase-5 clean_multi_c2_t90k stage-2 preset; USED - balanced-1B bootstrap panel (S6.12) + paper panel; not in panels.json main21", "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a139/cmeister/tokenizer-lm/checkpoints/full-128k-superbpe-clean-c2-balanced", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "full-128k-superbpe-clean-c2-balanced-mathcode-scratch": { "run": "full-128k-superbpe-clean-c2-balanced-mathcode-scratch", "works": [ "blog", "tokeval" ], "training_regime": "mathcode-20B-from-scratch", "architecture": { "n_layer": 24, "n_embd": 1536, "n_head": 12, "n_kv_head": 12, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 128004, "total_parameters": 1271957258 }, "training": { "step": 19073, "total_batch_size": 1048576, "token_budget": 19999490048, "total_bytes_consumed": 157305292274, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "full-128k-mathcode-scratch" }, "tokenizer_slug": "superbpe-clean-c2-balanced", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 128000, "metrics": { "val_bpb": 0.34319379203223876, "code_bpb_mean": 0.39056603890195457, "flores_mean_bpb": 3.351038195492764, "flores_n_languages": 214, "mbpp_pass1": 0.202, "humaneval_pass1": 0.13414634146341464, "generation_spec": { "version": "v2-2026-07-30", "bos_context": true, "eos_stop": true, "token_healing": true, "truncation": "left_to_max_length_minus_max_gen", "stop_truncation": "earliest_match_all_terms", "stop_match_decode": "specials_kept" }, "token_healing": true, "gsm8k_exact_match_strict": 0.24109173616376042, "gsm8k_exact_match_flexible": 0.244882486732373 }, "metric_gaps": { "blimp_acc": "no full-128k-superbpe-clean-c2-balanced-mathcode-scratch_blimp_code_bpb.json" }, "registry_note": "#33, clean pretok, slug `superbpe-clean-c2-balanced`. SuperBPE, clean_multi C2 stage-2 reduced regex on the PA-BPE clean base, `balanced`, vocab 128,004. FINISHED (step 19073). MBPP 0.176 [0.144, 0.210].", "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a139/cmeister/tokenizer-lm/checkpoints/full-128k-superbpe-clean-c2-balanced-mathcode-scratch", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "full-128k-superbpe-clean-c3-balanced": { "run": "full-128k-superbpe-clean-c3-balanced", "works": [ "paper_a", "tokeval" ], "training_regime": "std-1B", "architecture": { "n_layer": 24, "n_embd": 1536, "n_head": 12, "n_kv_head": 12, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 128004, "total_parameters": 1271957258 }, "training": { "step": 8797, "total_batch_size": 1048576, "token_budget": 9224323072, "total_bytes_consumed": 70479910164, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "full-128k" }, "tokenizer_slug": "superbpe-clean-c3-balanced", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 128000, "metrics": { "val_bpb": 0.7295809612395211, "blimp_acc": 0.8030746268656719, "code_bpb_mean": 0.5305765831960391, "blimp_scoring": "bos_prefixed", "flores_mean_bpb": 2.64350465932792, "flores_n_languages": 215, "mbpp_pass1": 0.044, "humaneval_pass1": 0.09146341463414634, "generation_spec": { "version": "v2-2026-07-30", "bos_context": true, "eos_stop": true, "token_healing": true, "truncation": "left_to_max_length_minus_max_gen", "stop_truncation": "earliest_match_all_terms", "stop_match_decode": "specials_kept" }, "token_healing": true, "gsm8k_exact_match_strict": 0.027293404094010616, "gsm8k_exact_match_flexible": 0.03184230477634572 }, "metric_gaps": {}, "registry_note": "SuperBPE stage-1 clean-multi / stage-2 clean_multi C3 reduced regex; balanced (D2) data; norm=NFC; SuperBPE; vocab 128,004; Phase-5 clean_multi_c3_t90k stage-2 preset; USED - balanced-1B bootstrap panel (S6.12) + paper panel; not in panels.json main21", "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a139/cmeister/tokenizer-lm/checkpoints/full-128k-superbpe-clean-c3-balanced", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "full-128k-superbpe-clean-c3-balanced-mathcode-scratch": { "run": "full-128k-superbpe-clean-c3-balanced-mathcode-scratch", "works": [ "blog", "tokeval" ], "training_regime": "mathcode-20B-from-scratch", "architecture": { "n_layer": 24, "n_embd": 1536, "n_head": 12, "n_kv_head": 12, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 128004, "total_parameters": 1271957258 }, "training": { "step": 19073, "total_batch_size": 1048576, "token_budget": 19999490048, "total_bytes_consumed": 152809290276, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "full-128k-mathcode-scratch" }, "tokenizer_slug": "superbpe-clean-c3-balanced", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 128000, "metrics": { "val_bpb": 0.344799716331132, "code_bpb_mean": 0.38962529072156843, "flores_mean_bpb": 3.3606069071813196, "flores_n_languages": 214, "mbpp_pass1": 0.19, "humaneval_pass1": 0.1402439024390244, "generation_spec": { "version": "v2-2026-07-30", "bos_context": true, "eos_stop": true, "token_healing": true, "truncation": "left_to_max_length_minus_max_gen", "stop_truncation": "earliest_match_all_terms", "stop_match_decode": "specials_kept" }, "token_healing": true, "gsm8k_exact_match_strict": 0.17058377558756635, "gsm8k_exact_match_flexible": 0.20242608036391205 }, "metric_gaps": { "blimp_acc": "no full-128k-superbpe-clean-c3-balanced-mathcode-scratch_blimp_code_bpb.json" }, "registry_note": "#34, clean pretok, slug `superbpe-clean-c3-balanced`. SuperBPE, clean_multi C3 stage-2 reduced regex, `balanced`, vocab 128,004. FINISHED (step 19073). MBPP 0.200 [0.166, 0.236].", "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a139/cmeister/tokenizer-lm/checkpoints/full-128k-superbpe-clean-c3-balanced-mathcode-scratch", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "full-128k-superbpe-clean-fineweb2full-capped-hybridwindow": { "run": "full-128k-superbpe-clean-fineweb2full-capped-hybridwindow", "works": [ "paper_a_apertus_track" ], "training_regime": "std-1B", "architecture": { "n_layer": 24, "n_embd": 1536, "n_head": 12, "n_kv_head": 12, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 128004, "total_parameters": 1271957258 }, "training": { "step": 8797, "total_batch_size": 1048576, "token_budget": 9224323072, "total_bytes_consumed": 59036341511, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "full-128k" }, "tokenizer_slug": "superbpe-clean-fineweb2full-capped-hybridwindow", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 128000, "metrics": { "val_bpb": 0.7322630359017345, "blimp_acc": 0.8139253731343283, "code_bpb_mean": 0.5362691238736629, "blimp_scoring": "bos_prefixed", "flores_mean_bpb": 3.038298249585932, "flores_n_languages": 214, "mbpp_pass1": 0.09, "humaneval_pass1": 0.08536585365853659, "generation_spec": { "version": "v2-2026-07-30", "bos_context": true, "eos_stop": true, "token_healing": true, "truncation": "left_to_max_length_minus_max_gen", "stop_truncation": "earliest_match_all_terms", "stop_match_decode": "specials_kept" }, "token_healing": true, "gsm8k_exact_match_strict": 0.0310841546626232, "gsm8k_exact_match_flexible": 0.0356330553449583 }, "metric_gaps": {}, "registry_note": "SuperBPE on the clean-multi-pretok capped hybrid-window base; fineweb2full, capped data; norm=NFC; SuperBPE; vocab 128,004; 2x2 candidate (t90k/v128k); USED - Apertus-2 development track candidate + S6h code analysis roster (20B twin only, see MC table); excluded from main paper panel", "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a139/cmeister/tokenizer-lm/checkpoints/full-128k-superbpe-clean-fineweb2full-capped-hybridwindow", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "full-128k-superbpe-clean-fineweb2full-capped-hybridwindow-mathcode-scratch": { "run": "full-128k-superbpe-clean-fineweb2full-capped-hybridwindow-mathcode-scratch", "works": [ "blog" ], "training_regime": "mathcode-20B-from-scratch", "architecture": { "n_layer": 24, "n_embd": 1536, "n_head": 12, "n_kv_head": 12, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 128004, "total_parameters": 1271957258 }, "training": { "step": 19073, "total_batch_size": 1048576, "token_budget": 19999490048, "total_bytes_consumed": 127998197299, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "full-128k-mathcode-scratch" }, "tokenizer_slug": "superbpe-clean-fineweb2full-capped-hybridwindow", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 128000, "metrics": { "val_bpb": 0.3423861965022762, "flores_mean_bpb": 3.640373138824759, "flores_n_languages": 214, "mbpp_pass1": 0.208, "humaneval_pass1": 0.13414634146341464, "generation_spec": { "version": "v2-2026-07-30", "bos_context": true, "eos_stop": true, "token_healing": true, "truncation": "left_to_max_length_minus_max_gen", "stop_truncation": "earliest_match_all_terms", "stop_match_decode": "specials_kept" }, "token_healing": true, "gsm8k_exact_match_strict": 0.2047005307050796, "gsm8k_exact_match_flexible": 0.2175890826383624 }, "metric_gaps": { "blimp_acc": "no full-128k-superbpe-clean-fineweb2full-capped-hybridwindow-mathcode-scratch_blimp_code_bpb.json", "code_bpb_mean": "no full-128k-superbpe-clean-fineweb2full-capped-hybridwindow-mathcode-scratch_blimp_code_bpb.json; no full-128k-superbpe-clean-fineweb2full-capped-hybridwindow-mathcode-scratch_code_bpb.json" }, "registry_note": "#9, clean pretok, slug `superbpe-clean-fineweb2full-capped-hybridwindow`. SuperBPE candidate (clean, t90k/v128k).", "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a139/cmeister/tokenizer-lm/checkpoints/full-128k-superbpe-clean-fineweb2full-capped-hybridwindow-mathcode-scratch", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "full-128k-superbpe-clean-fineweb2full-capped-hybridwindow-t110k-v130k": { "run": "full-128k-superbpe-clean-fineweb2full-capped-hybridwindow-t110k-v130k", "works": [ "paper_a_apertus_track" ], "training_regime": "std-1B", "architecture": { "n_layer": 24, "n_embd": 1536, "n_head": 12, "n_kv_head": 12, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 130004, "total_parameters": 1281099530 }, "training": { "step": 8827, "total_batch_size": 1048576, "token_budget": 9255780352, "total_bytes_consumed": 60452486276, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "full-128k" }, "tokenizer_slug": "superbpe-clean-fineweb2full-capped-hybridwindow-t110k-v130k", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 130000, "metrics": { "val_bpb": 0.7318322821516513, "blimp_acc": 0.8206417910447763, "code_bpb_mean": 0.5338679088662601, "blimp_scoring": "bos_prefixed", "flores_mean_bpb": 2.9932878985914204, "flores_n_languages": 214, "mbpp_pass1": 0.106, "humaneval_pass1": 0.08536585365853659, "generation_spec": { "version": "v2-2026-07-30", "bos_context": true, "eos_stop": true, "token_healing": true, "truncation": "left_to_max_length_minus_max_gen", "stop_truncation": "earliest_match_all_terms", "stop_match_decode": "specials_kept" }, "token_healing": true, "gsm8k_exact_match_strict": 0.032600454890068235, "gsm8k_exact_match_flexible": 0.039423805913570885 }, "metric_gaps": {}, "registry_note": "SuperBPE, clean-multi pretok base; fineweb2full, capped data; norm=NFC; SuperBPE; vocab ~130,000; SuperBPE stage-2 transition-point ablation (t110k vs the 2x2 candidate's t90k); USED - Apertus-2 development track ablation; excluded from main paper panel", "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a139/cmeister/tokenizer-lm/checkpoints/full-128k-superbpe-clean-fineweb2full-capped-hybridwindow-t110k-v130k", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "full-128k-superbpe-clean-fineweb2full-capped-hybridwindow-t110k-v130k-mathcode-scratch": { "run": "full-128k-superbpe-clean-fineweb2full-capped-hybridwindow-t110k-v130k-mathcode-scratch", "works": [ "blog" ], "training_regime": "mathcode-20B-from-scratch", "architecture": { "n_layer": 24, "n_embd": 1536, "n_head": 12, "n_kv_head": 12, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 130004, "total_parameters": 1281099530 }, "training": { "step": 19073, "total_batch_size": 1048576, "token_budget": 19999490048, "total_bytes_consumed": 130623118924, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "full-128k-mathcode-scratch" }, "tokenizer_slug": "superbpe-clean-fineweb2full-capped-hybridwindow-t110k-v130k", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 130000, "metrics": { "val_bpb": 0.34328670524163585, "flores_mean_bpb": 3.62831755593602, "flores_n_languages": 214, "mbpp_pass1": 0.218, "humaneval_pass1": 0.1524390243902439, "generation_spec": { "version": "v2-2026-07-30", "bos_context": true, "eos_stop": true, "token_healing": true, "truncation": "left_to_max_length_minus_max_gen", "stop_truncation": "earliest_match_all_terms", "stop_match_decode": "specials_kept" }, "token_healing": true, "gsm8k_exact_match_strict": 0.17892342683851403, "gsm8k_exact_match_flexible": 0.2100075815011372 }, "metric_gaps": { "blimp_acc": "no full-128k-superbpe-clean-fineweb2full-capped-hybridwindow-t110k-v130k-mathcode-scratch_blimp_code_bpb.json", "code_bpb_mean": "no full-128k-superbpe-clean-fineweb2full-capped-hybridwindow-t110k-v130k-mathcode-scratch_blimp_code_bpb.json; no full-128k-superbpe-clean-fineweb2full-capped-hybridwindow-t110k-v130k-mathcode-scratch_code_bpb.json" }, "registry_note": "#10, clean pretok, slug `superbpe-clean-fineweb2full-capped-hybridwindow-t110k-v130k`. SuperBPE ablation (t110k/v130k transition point).", "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a139/cmeister/tokenizer-lm/checkpoints/full-128k-superbpe-clean-fineweb2full-capped-hybridwindow-t110k-v130k-mathcode-scratch", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "full-128k-superbpe-clean-fineweb2full-plus2-cappedv2-hybridwindow-t110k-v130k-consv2": { "run": "full-128k-superbpe-clean-fineweb2full-plus2-cappedv2-hybridwindow-t110k-v130k-consv2", "works": [ "paper_a_apertus_track" ], "training_regime": "std-1B", "architecture": { "n_layer": 24, "n_embd": 1536, "n_head": 12, "n_kv_head": 12, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 130004, "total_parameters": 1281099530 }, "training": { "step": 8827, "total_batch_size": 1048576, "token_budget": 9255780352, "total_bytes_consumed": 61543706497, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "full-128k" }, "tokenizer_slug": "superbpe-clean-fineweb2full-plus2-cappedv2-hybridwindow-t110k-v130k-consv2", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 130000, "metrics": { "val_bpb": 0.7322648630229084, "blimp_acc": 0.8177313432835819, "code_bpb_mean": 0.5352055661202627, "blimp_scoring": "bos_prefixed", "flores_mean_bpb": 3.0648179432560387, "flores_n_languages": 214, "mbpp_pass1": 0.084, "humaneval_pass1": 0.07317073170731707, "generation_spec": { "version": "v2-2026-07-30", "bos_context": true, "eos_stop": true, "token_healing": true, "truncation": "left_to_max_length_minus_max_gen", "stop_truncation": "earliest_match_all_terms", "stop_match_decode": "specials_kept" }, "token_healing": true, "gsm8k_exact_match_strict": 0.03866565579984837, "gsm8k_exact_match_flexible": 0.043214556482183475 }, "metric_gaps": {}, "registry_note": "SuperBPE on the clean-multi plus2 (v2) capped-v2 hybrid-window base, stage-2 transition t110k; fineweb2full, consv2 data; norm=NFC; SuperBPE; vocab ~130,000; Apertus-2 development ablation 'SuperBPE-plus2v2-cv2-t110k'; USED - Apertus-2 track ablation; excluded from main paper panel", "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a139/cmeister/tokenizer-lm/checkpoints/full-128k-superbpe-clean-fineweb2full-plus2-cappedv2-hybridwindow-t110k-v130k-consv2", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "full-128k-superbpe-clean-fineweb2full-plus2-cappedv2-hybridwindow-t110k-v130k-consv2-mathcode-scratch": { "run": "full-128k-superbpe-clean-fineweb2full-plus2-cappedv2-hybridwindow-t110k-v130k-consv2-mathcode-scratch", "works": [ "blog" ], "training_regime": "mathcode-20B-from-scratch", "architecture": { "n_layer": 24, "n_embd": 1536, "n_head": 12, "n_kv_head": 12, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 130004, "total_parameters": 1281099530 }, "training": { "step": 19073, "total_batch_size": 1048576, "token_budget": 19999490048, "total_bytes_consumed": 132980980403, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "full-128k-mathcode-scratch" }, "tokenizer_slug": "superbpe-clean-fineweb2full-plus2-cappedv2-hybridwindow-t110k-v130k-consv2", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 130000, "metrics": { "val_bpb": 0.34330320732271896, "flores_mean_bpb": 3.559493631071275, "flores_n_languages": 214, "mbpp_pass1": 0.246, "humaneval_pass1": 0.14634146341463414, "generation_spec": { "version": "v2-2026-07-30", "bos_context": true, "eos_stop": true, "token_healing": true, "truncation": "left_to_max_length_minus_max_gen", "stop_truncation": "earliest_match_all_terms", "stop_match_decode": "specials_kept" }, "token_healing": true, "gsm8k_exact_match_strict": 0.16603487490523122, "gsm8k_exact_match_flexible": 0.20166793025018953 }, "metric_gaps": { "blimp_acc": "no full-128k-superbpe-clean-fineweb2full-plus2-cappedv2-hybridwindow-t110k-v130k-consv2-mathcode-scratch_blimp_code_bpb.json", "code_bpb_mean": "no full-128k-superbpe-clean-fineweb2full-plus2-cappedv2-hybridwindow-t110k-v130k-consv2-mathcode-scratch_blimp_code_bpb.json; no full-128k-superbpe-clean-fineweb2full-plus2-cappedv2-hybridwindow-t110k-v130k-consv2-mathcode-scratch_code_bpb.json" }, "registry_note": "#16, clean pretok, slug `superbpe-clean-fineweb2full-plus2-cappedv2-hybridwindow-t110k-v130k-consv2`. SuperBPE on clean-multi + `plus2` + `cap_v2` + hw + `consv2` data, stage-2 transition 110k \u2192 130k. Specials added post-hoc (vocab 130000 \u2192 130004, bos_id=130000).", "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a139/cmeister/tokenizer-lm/checkpoints/full-128k-superbpe-clean-fineweb2full-plus2-cappedv2-hybridwindow-t110k-v130k-consv2-mathcode-scratch", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "full-128k-superbpe-gpt4-fineweb2full-hybridwindow-mathcode-scratch": { "run": "full-128k-superbpe-gpt4-fineweb2full-hybridwindow-mathcode-scratch", "works": [ "blog" ], "training_regime": "mathcode-20B-from-scratch", "architecture": { "n_layer": 24, "n_embd": 1536, "n_head": 12, "n_kv_head": 12, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 128004, "total_parameters": 1271957258 }, "training": { "step": 19073, "total_batch_size": 1048576, "token_budget": 19999490048, "total_bytes_consumed": 126294692304, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "full-128k-mathcode-scratch" }, "tokenizer_slug": "superbpe-gpt4-fineweb2full-hybridwindow", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 128000, "metrics": { "val_bpb": 0.34104167417539477, "flores_mean_bpb": 3.6204409397662, "flores_n_languages": 214, "mbpp_pass1": 0.162, "humaneval_pass1": 0.16463414634146342, "generation_spec": { "version": "v2-2026-07-30", "bos_context": true, "eos_stop": true, "token_healing": true, "truncation": "left_to_max_length_minus_max_gen", "stop_truncation": "earliest_match_all_terms", "stop_match_decode": "specials_kept" }, "token_healing": true, "gsm8k_exact_match_strict": 0.17134192570128887, "gsm8k_exact_match_flexible": 0.20773313115996966 }, "metric_gaps": { "blimp_acc": "no full-128k-superbpe-gpt4-fineweb2full-hybridwindow-mathcode-scratch_blimp_code_bpb.json", "code_bpb_mean": "no full-128k-superbpe-gpt4-fineweb2full-hybridwindow-mathcode-scratch_blimp_code_bpb.json; no full-128k-superbpe-gpt4-fineweb2full-hybridwindow-mathcode-scratch_code_bpb.json" }, "registry_note": "#12, gpt4 pretok, slug `superbpe-gpt4-fineweb2full-hybridwindow`. SuperBPE on older GPT-4 regex (uncapped, {1,3}-digit).", "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a139/cmeister/tokenizer-lm/checkpoints/full-128k-superbpe-gpt4-fineweb2full-hybridwindow-mathcode-scratch", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "full-128k-superbpe-gpt4o-balanced": { "run": "full-128k-superbpe-gpt4o-balanced", "works": [ "paper_a", "tokeval", "tokeval_cross_scale" ], "training_regime": "std-1B", "architecture": { "n_layer": 24, "n_embd": 1536, "n_head": 12, "n_kv_head": 12, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 128004, "total_parameters": 1271957258 }, "training": { "step": 8797, "total_batch_size": 1048576, "token_budget": 9224323072, "total_bytes_consumed": 70221489492, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "full-128k" }, "tokenizer_slug": "superbpe-gpt4o-balanced", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 128000, "metrics": { "val_bpb": 0.7236170040912355, "blimp_acc": 0.8035373134328355, "code_bpb_mean": 0.5249302535411448, "blimp_scoring": "bos_prefixed", "flores_mean_bpb": 2.6393678806347665, "flores_n_languages": 215, "mbpp_pass1": 0.052, "humaneval_pass1": 0.07317073170731707, "generation_spec": { "version": "v2-2026-07-30", "bos_context": true, "eos_stop": true, "token_healing": true, "truncation": "left_to_max_length_minus_max_gen", "stop_truncation": "earliest_match_all_terms", "stop_match_decode": "specials_kept" }, "token_healing": true, "gsm8k_exact_match_strict": 0.03335860500379075, "gsm8k_exact_match_flexible": 0.03411675511751327 }, "metric_gaps": {}, "registry_note": "SuperBPE stage-1 GPT-4o regex (90k inherited merges) / stage-2 reduced GPT-4o regex (38k superword merges); base = gpt4o-balanced-bpe; balanced (D2, halved for stage-2: balanced_half.json) data; norm=none (inherited from gpt4o-balanced-bpe base); SuperBPE; vocab 128,004; Phase-5 fixed retrain (2026-05-18) of the combining-mark-bug Run 1; USED - panels.json main21 (camera-ready main panel)", "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a139/cmeister/tokenizer-lm/checkpoints/full-128k-superbpe-gpt4o-balanced", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "full-128k-superbpe-gpt4o-balanced-mathcode-scratch": { "run": "full-128k-superbpe-gpt4o-balanced-mathcode-scratch", "works": [ "blog", "tokeval" ], "training_regime": "mathcode-20B-from-scratch", "architecture": { "n_layer": 24, "n_embd": 1536, "n_head": 12, "n_kv_head": 12, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 128004, "total_parameters": 1271957258 }, "training": { "step": 19073, "total_batch_size": 1048576, "token_budget": 19999490048, "total_bytes_consumed": 152249001828, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "full-128k-mathcode-scratch" }, "tokenizer_slug": "superbpe-gpt4o-balanced", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 128000, "metrics": { "val_bpb": 0.33859987485722765, "code_bpb_mean": 0.38337076287095095, "flores_mean_bpb": 3.332138865321979, "flores_n_languages": 214, "mbpp_pass1": 0.068, "humaneval_pass1": 0.15853658536585366, "generation_spec": { "version": "v2-2026-07-30", "bos_context": true, "eos_stop": true, "token_healing": true, "truncation": "left_to_max_length_minus_max_gen", "stop_truncation": "earliest_match_all_terms", "stop_match_decode": "specials_kept" }, "token_healing": true, "gsm8k_exact_match_strict": 0.18043972706595907, "gsm8k_exact_match_flexible": 0.2266868840030326 }, "metric_gaps": { "blimp_acc": "no full-128k-superbpe-gpt4o-balanced-mathcode-scratch_blimp_code_bpb.json" }, "registry_note": "#35, gpt4o pretok, slug `superbpe-gpt4o-balanced`. SuperBPE, gpt4o base (no norm, t90k), `balanced`, vocab 128,004. FINISHED (step 19073). MBPP 0.106 [0.080, 0.134].", "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a139/cmeister/tokenizer-lm/checkpoints/full-128k-superbpe-gpt4o-balanced-mathcode-scratch", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "full-128k-superbpe-pabpe-nfc-gpt4-balanced": { "run": "full-128k-superbpe-pabpe-nfc-gpt4-balanced", "works": [ "paper_a", "tokeval" ], "training_regime": "std-1B", "architecture": { "n_layer": 24, "n_embd": 1536, "n_head": 12, "n_kv_head": 12, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 128004, "total_parameters": 1271957258 }, "training": { "step": 8797, "total_batch_size": 1048576, "token_budget": 9224323072, "total_bytes_consumed": 71113918751, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "full-128k" }, "tokenizer_slug": "superbpe-pabpe-nfc-gpt4-balanced", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 128000, "metrics": { "val_bpb": 0.729206724155245, "blimp_acc": 0.8013283582089552, "code_bpb_mean": 0.5276386372741176, "blimp_scoring": "bos_prefixed", "flores_mean_bpb": 2.6525497236572044, "flores_n_languages": 215, "mbpp_pass1": 0.064, "humaneval_pass1": 0.04878048780487805, "generation_spec": { "version": "v2-2026-07-30", "bos_context": true, "eos_stop": true, "token_healing": true, "truncation": "left_to_max_length_minus_max_gen", "stop_truncation": "earliest_match_all_terms", "stop_match_decode": "specials_kept" }, "token_healing": true, "gsm8k_exact_match_strict": 0.030326004548900682, "gsm8k_exact_match_flexible": 0.03790750568612585 }, "metric_gaps": {}, "registry_note": "SuperBPE stage-1 GPT-4o regex / stage-2 reduced GPT-4o regex; base = PA-BPE nfc_gpt4_balanced; balanced (D2, halved) data; norm=NFC (inherited from PA-BPE base); SuperBPE; vocab 128,004; Phase-5 fixed retrain of Run 2; USED - balanced-1B bootstrap panel (S6.12) + paper panel; not in panels.json main21", "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a139/cmeister/tokenizer-lm/checkpoints/full-128k-superbpe-pabpe-nfc-gpt4-balanced", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "full-128k-superbpe-pabpe-nfc-gpt4-t64k": { "run": "full-128k-superbpe-pabpe-nfc-gpt4-t64k", "works": [ "paper_a", "tokeval" ], "training_regime": "std-1B", "architecture": { "n_layer": 24, "n_embd": 1536, "n_head": 12, "n_kv_head": 12, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 128004, "total_parameters": 1271957258 }, "training": { "step": 8797, "total_batch_size": 1048576, "token_budget": 9224323072, "total_bytes_consumed": 69736009453, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "full-128k" }, "tokenizer_slug": "superbpe-pabpe-nfc-gpt4-t64k", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 128000, "metrics": { "val_bpb": 0.7293487935520814, "blimp_acc": 0.7918208955223883, "code_bpb_mean": 0.529837856895328, "blimp_scoring": "bos_prefixed", "flores_mean_bpb": 2.6521159809686967, "flores_n_languages": 215, "mbpp_pass1": 0.046, "humaneval_pass1": 0.04878048780487805, "generation_spec": { "version": "v2-2026-07-30", "bos_context": true, "eos_stop": true, "token_healing": true, "truncation": "left_to_max_length_minus_max_gen", "stop_truncation": "earliest_match_all_terms", "stop_match_decode": "specials_kept" }, "token_healing": true, "gsm8k_exact_match_strict": 0.026535253980288095, "gsm8k_exact_match_flexible": 0.028051554207733132 }, "metric_gaps": {}, "registry_note": "SuperBPE stage-1 GPT-4o regex (64k transition, more aggressive superword fusing) / stage-2 reduced GPT-4o regex; base = PA-BPE nfc_gpt4_balanced; balanced (D2, halved) data; norm=NFC; SuperBPE; vocab 128,004; Phase-5 fixed retrain of Run 4 (t64k transition point vs t90k); USED - balanced-1B bootstrap panel (S6.12) + paper panel; not in panels.json main21", "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a139/cmeister/tokenizer-lm/checkpoints/full-128k-superbpe-pabpe-nfc-gpt4-t64k", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "full-128k-whitespace-balanced-bpe": { "run": "full-128k-whitespace-balanced-bpe", "works": [ "paper_a", "tokeval" ], "training_regime": "std-1B", "architecture": { "n_layer": 24, "n_embd": 1536, "n_head": 12, "n_kv_head": 12, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 128256, "total_parameters": 1272841994 }, "training": { "step": 8800, "total_batch_size": 1048576, "token_budget": 9227468800, "total_bytes_consumed": 66097600800, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "full-128k" }, "tokenizer_slug": "whitespace-balanced-bpe", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 0, "metrics": { "val_bpb": 0.720734254582808, "blimp_acc": 0.8136268656716417, "code_bpb_mean": 0.5269486944981118, "blimp_scoring": "bos_prefixed", "flores_mean_bpb": 2.6271712384711567, "flores_n_languages": 214, "mbpp_pass1": 0.056, "humaneval_pass1": 0.08536585365853659, "generation_spec": { "version": "v2-2026-07-30", "bos_context": true, "eos_stop": true, "token_healing": true, "truncation": "left_to_max_length_minus_max_gen", "stop_truncation": "earliest_match_all_terms", "stop_match_decode": "specials_kept" }, "token_healing": true, "gsm8k_exact_match_strict": 0.03639120545868082, "gsm8k_exact_match_flexible": 0.04094010614101592 }, "metric_gaps": {}, "registry_note": "WhitespaceSplit(preserve)+ByteLevel (minimal-structure anchor); balanced (D2) data; norm=none; BPE; vocab 128,260; lower-end pretok anchor; forward-fusing (whitespace+following word one pretoken); USED - panels.json pretok-controls (camera-ready)", "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a139/cmeister/tokenizer-lm/checkpoints/full-128k-whitespace-balanced-bpe", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "full-128k-whitespace-balanced-bpe-mathcode-scratch": { "run": "full-128k-whitespace-balanced-bpe-mathcode-scratch", "works": [ "blog" ], "training_regime": "mathcode-20B-from-scratch", "architecture": { "n_layer": 24, "n_embd": 1536, "n_head": 12, "n_kv_head": 12, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 128256, "total_parameters": 1272841994 }, "training": { "step": 19073, "total_batch_size": 1048576, "token_budget": 19999490048, "total_bytes_consumed": 143259038643, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "full-128k-mathcode-scratch" }, "tokenizer_slug": "whitespace-balanced-bpe", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 0, "metrics": { "val_bpb": 0.3395392358308889, "code_bpb_mean": 0.3742340349376568, "flores_mean_bpb": 3.321987419368069, "flores_n_languages": 214, "mbpp_pass1": 0.102, "humaneval_pass1": 0.17682926829268292, "generation_spec": { "version": "v2-2026-07-30", "bos_context": true, "eos_stop": true, "token_healing": true, "truncation": "left_to_max_length_minus_max_gen", "stop_truncation": "earliest_match_all_terms", "stop_match_decode": "specials_kept" }, "token_healing": true, "gsm8k_exact_match_strict": 0.177407126611069, "gsm8k_exact_match_flexible": 0.20621683093252463 }, "metric_gaps": { "blimp_acc": "no full-128k-whitespace-balanced-bpe-mathcode-scratch_blimp_code_bpb.json" }, "registry_note": "#36, whitespace pretok, slug `whitespace-balanced-bpe`. BPE with minimal WhitespaceSplit + ByteLevel pretok (lower-end pretok anchor), `balanced`. FINISHED (step 19073). MBPP 0.006 [0.000, 0.014].", "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a139/cmeister/tokenizer-lm/checkpoints/full-128k-whitespace-balanced-bpe-mathcode-scratch", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "full-128k-whitespace-balanced-unigram": { "run": "full-128k-whitespace-balanced-unigram", "works": [ "paper_a" ], "training_regime": "std-1B", "architecture": { "n_layer": 24, "n_embd": 1536, "n_head": 12, "n_kv_head": 12, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 128256, "total_parameters": 1272841994 }, "training": { "step": 8800, "total_batch_size": 1048576, "token_budget": 9227468800, "total_bytes_consumed": 73127700800, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "full-128k" }, "tokenizer_slug": "whitespace-balanced-unigram", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 0, "metrics": { "val_bpb": 0.8400830729672958, "blimp_acc": 0.8051940298507458, "code_bpb_mean": 0.6091013373748696, "blimp_scoring": "bos_prefixed", "flores_mean_bpb": 2.5814421169817967, "flores_n_languages": 214, "mbpp_pass1": 0.04, "humaneval_pass1": 0.0, "generation_spec": { "version": "v2-2026-07-30", "bos_context": true, "eos_stop": true, "token_healing": false, "truncation": "left_to_max_length_minus_max_gen", "stop_truncation": "earliest_match_all_terms", "stop_match_decode": "specials_kept" }, "token_healing": false, "gsm8k_exact_match_strict": 0.030326004548900682, "gsm8k_exact_match_flexible": 0.032600454890068235 }, "metric_gaps": {}, "registry_note": "Metaspace(replacement=U+2581)+ByteFallback (SPM native mode); balanced (D2) data; norm=none; UnigramLM (Google sentencepiece, mode=native; tuned kwargs matched to UNIGRAM_TUNED_KWARGS); vocab 128,260; algorithm-axis Unigram counterpart to whitespace-balanced-bpe; trained via SPM not HF (HF UnigramTrainer NaN-panics on WhitespaceSplit+CJK); USED - S5e AXES-2 whitespace comparison + balanced-1B bootstrap panel (S6.12); not in panels.json groups", "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a139/cmeister/tokenizer-lm/checkpoints/full-128k-whitespace-balanced-unigram", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "full-128k-whitespace-multiheavy-bpe": { "run": "full-128k-whitespace-multiheavy-bpe", "works": [ "paper_a", "tokeval" ], "training_regime": "std-1B", "architecture": { "n_layer": 24, "n_embd": 1536, "n_head": 12, "n_kv_head": 12, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 128256, "total_parameters": 1272841994 }, "training": { "step": 8800, "total_batch_size": 1048576, "token_budget": 9227468800, "total_bytes_consumed": 70848465600, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "full-128k" }, "tokenizer_slug": "whitespace-multiheavy-bpe", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 0, "metrics": { "val_bpb": 0.7200585315424582, "blimp_acc": 0.8089552238805972, "code_bpb_mean": 0.5272055135520145, "blimp_scoring": "bos_prefixed", "flores_mean_bpb": 2.6288794528747474, "flores_n_languages": 215, "mbpp_pass1": 0.072, "humaneval_pass1": 0.07317073170731707, "generation_spec": { "version": "v2-2026-07-30", "bos_context": true, "eos_stop": true, "token_healing": true, "truncation": "left_to_max_length_minus_max_gen", "stop_truncation": "earliest_match_all_terms", "stop_match_decode": "specials_kept" }, "token_healing": true, "gsm8k_exact_match_strict": 0.02350265352539803, "gsm8k_exact_match_flexible": 0.030326004548900682 }, "metric_gaps": {}, "registry_note": "WhitespaceSplit(preserve)+ByteLevel; multiheavy (30 FineWeb-2 lang weights doubled 0.30->0.60; english 0.175, math/code 0.0875 each) data; norm=none; BPE; vocab 128,256; data-compensation test: does doubling the multilingual budget offset whitespace-pretok's multilingual damage? (S5e: no, MBPP stays 0.002); USED - panels.json pretok-controls (camera-ready)", "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a139/cmeister/tokenizer-lm/checkpoints/full-128k-whitespace-multiheavy-bpe", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "pilot-128k-apertus-wdfix": { "run": "pilot-128k-apertus-wdfix", "works": [ "tokeval_cross_scale" ], "training_regime": "cross-scale-d16-300M", "architecture": { "n_layer": 16, "n_embd": 1024, "n_head": 8, "n_kv_head": 8, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 131072, "total_parameters": 605029178 }, "training": { "step": 3370, "total_batch_size": 1048576, "token_budget": 3533701120, "total_bytes_consumed": 23542361680, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "pilot-128k" }, "tokenizer_slug": "apertus", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 1, "metrics": { "val_bpb": 0.8210034941193963, "blimp_acc": 0.7977014925373136, "code_bpb_mean": 0.6115311977869082, "blimp_scoring": "bos_prefixed", "flores_mean_bpb": 2.826757759606348, "flores_n_languages": 215 }, "metric_gaps": { "mbpp_pass1": "no pilot-128k-apertus-wdfix_code_v2.json", "humaneval_pass1": "no pilot-128k-apertus-wdfix_code_v2.json", "gsm8k_exact_match_strict": "no pilot-128k-apertus-wdfix_gsm8k_v2.json" }, "registry_note": null, "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a139/cmeister/tokenizer-lm/checkpoints/pilot-128k-apertus-wdfix", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "pilot-128k-claude-balanced-bpe": { "run": "pilot-128k-claude-balanced-bpe", "works": [ "tokeval_cross_scale" ], "training_regime": "cross-scale-d16-300M", "architecture": { "n_layer": 16, "n_embd": 1024, "n_head": 8, "n_kv_head": 8, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "default 15.0, run predates the configurable field", "vocab_size": 128256, "total_parameters": 596378426 }, "training": { "step": 3341, "total_batch_size": 1048576, "token_budget": 3503292416, "total_bytes_consumed": 26503180769, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "pilot-128k" }, "tokenizer_slug": "claude-balanced-bpe", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 0, "metrics": { "val_bpb": 0.8130946827374702, "blimp_acc": 0.8036716417910448, "code_bpb_mean": 0.5993885155419614, "blimp_scoring": "bos_prefixed", "flores_mean_bpb": 2.74010354533894, "flores_n_languages": 215 }, "metric_gaps": { "mbpp_pass1": "no pilot-128k-claude-balanced-bpe_code_v2.json", "humaneval_pass1": "no pilot-128k-claude-balanced-bpe_code_v2.json", "gsm8k_exact_match_strict": "no pilot-128k-claude-balanced-bpe_gsm8k_v2.json" }, "registry_note": null, "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a139/cmeister/tokenizer-lm/checkpoints/pilot-128k-claude-balanced-bpe", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "pilot-128k-claude-balanced-nfc-bpe": { "run": "pilot-128k-claude-balanced-nfc-bpe", "works": [ "tokeval_cross_scale" ], "training_regime": "cross-scale-d16-300M", "architecture": { "n_layer": 16, "n_embd": 1024, "n_head": 8, "n_kv_head": 8, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "default 15.0, run predates the configurable field", "vocab_size": 128256, "total_parameters": 596378426 }, "training": { "step": 3341, "total_batch_size": 1048576, "token_budget": 3503292416, "total_bytes_consumed": 26503127313, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "pilot-128k" }, "tokenizer_slug": "claude-balanced-nfc-bpe", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 0, "metrics": { "val_bpb": 0.8137996131939872, "blimp_acc": 0.6210746268656718, "code_bpb_mean": 0.6015235015799788, "blimp_scoring": null, "flores_mean_bpb": 2.7355970218623393, "flores_n_languages": 215 }, "metric_gaps": { "mbpp_pass1": "no pilot-128k-claude-balanced-nfc-bpe_code_v2.json", "humaneval_pass1": "no pilot-128k-claude-balanced-nfc-bpe_code_v2.json", "gsm8k_exact_match_strict": "no pilot-128k-claude-balanced-nfc-bpe_gsm8k_v2.json" }, "registry_note": null, "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a139/cmeister/tokenizer-lm/checkpoints/pilot-128k-claude-balanced-nfc-bpe", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "pilot-128k-claude-balanced-unigram": { "run": "pilot-128k-claude-balanced-unigram", "works": [ "tokeval_cross_scale" ], "training_regime": "cross-scale-d16-300M", "architecture": { "n_layer": 16, "n_embd": 1024, "n_head": 8, "n_kv_head": 8, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "default 15.0, run predates the configurable field", "vocab_size": 128256, "total_parameters": 596378426 }, "training": { "step": 3341, "total_batch_size": 1048576, "token_budget": 3503292416, "total_bytes_consumed": 22735755575, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "pilot-128k" }, "tokenizer_slug": "claude-balanced-unigram", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 0, "metrics": { "val_bpb": 0.8470803384113235, "blimp_acc": 0.5728656716417911, "code_bpb_mean": 0.6425699405251949, "blimp_scoring": null, "flores_mean_bpb": 2.7912330346922847, "flores_n_languages": 215 }, "metric_gaps": { "mbpp_pass1": "no pilot-128k-claude-balanced-unigram_code_v2.json", "humaneval_pass1": "no pilot-128k-claude-balanced-unigram_code_v2.json", "gsm8k_exact_match_strict": "no pilot-128k-claude-balanced-unigram_gsm8k_v2.json" }, "registry_note": null, "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a139/cmeister/tokenizer-lm/checkpoints/pilot-128k-claude-balanced-unigram", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "pilot-128k-claude-english-bpe": { "run": "pilot-128k-claude-english-bpe", "works": [ "tokeval_cross_scale" ], "training_regime": "cross-scale-d16-300M", "architecture": { "n_layer": 16, "n_embd": 1024, "n_head": 8, "n_kv_head": 8, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "default 15.0, run predates the configurable field", "vocab_size": 128256, "total_parameters": 596378426 }, "training": { "step": 3341, "total_batch_size": 1048576, "token_budget": 3503292416, "total_bytes_consumed": 25009823930, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "pilot-128k" }, "tokenizer_slug": "claude-english-bpe", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 0, "metrics": { "val_bpb": 0.8326818042159945, "blimp_acc": 0.6018955223880598, "code_bpb_mean": 0.6386502722656455, "blimp_scoring": null, "flores_mean_bpb": 2.7481626126932053, "flores_n_languages": 215 }, "metric_gaps": { "mbpp_pass1": "no pilot-128k-claude-english-bpe_code_v2.json", "humaneval_pass1": "no pilot-128k-claude-english-bpe_code_v2.json", "gsm8k_exact_match_strict": "no pilot-128k-claude-english-bpe_gsm8k_v2.json" }, "registry_note": null, "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a139/cmeister/tokenizer-lm/checkpoints/pilot-128k-claude-english-bpe", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "pilot-128k-gpt4o-balanced-bpe-wdfix": { "run": "pilot-128k-gpt4o-balanced-bpe-wdfix", "works": [ "tokeval_cross_scale" ], "training_regime": "cross-scale-d16-300M", "architecture": { "n_layer": 16, "n_embd": 1024, "n_head": 8, "n_kv_head": 8, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 128256, "total_parameters": 596378426 }, "training": { "step": 3341, "total_batch_size": 1048576, "token_budget": 3503292416, "total_bytes_consumed": 25861307849, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "pilot-128k" }, "tokenizer_slug": "gpt4o-balanced-bpe", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 0, "metrics": { "val_bpb": 0.8066623035629523, "blimp_acc": 0.8082985074626864, "code_bpb_mean": 0.5994759350843143, "blimp_scoring": "bos_prefixed", "flores_mean_bpb": 2.7343951601156684, "flores_n_languages": 215 }, "metric_gaps": { "mbpp_pass1": "no pilot-128k-gpt4o-balanced-bpe-wdfix_code_v2.json", "humaneval_pass1": "no pilot-128k-gpt4o-balanced-bpe-wdfix_code_v2.json", "gsm8k_exact_match_strict": "no pilot-128k-gpt4o-balanced-bpe-wdfix_gsm8k_v2.json" }, "registry_note": null, "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a139/cmeister/tokenizer-lm/checkpoints/pilot-128k-gpt4o-balanced-bpe-wdfix", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "pilot-128k-gpt4o-balanced-nfc-bpe-wdfix": { "run": "pilot-128k-gpt4o-balanced-nfc-bpe-wdfix", "works": [ "tokeval_cross_scale" ], "training_regime": "cross-scale-d16-300M", "architecture": { "n_layer": 16, "n_embd": 1024, "n_head": 8, "n_kv_head": 8, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 128256, "total_parameters": 596378426 }, "training": { "step": 3341, "total_batch_size": 1048576, "token_budget": 3503292416, "total_bytes_consumed": 25861802317, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "pilot-128k" }, "tokenizer_slug": "gpt4o-balanced-nfc-bpe", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 0, "metrics": { "val_bpb": 0.8064211739874031, "blimp_acc": 0.8022089552238807, "code_bpb_mean": 0.5986240643292752, "blimp_scoring": "bos_prefixed", "flores_mean_bpb": 2.7400664324472417, "flores_n_languages": 215 }, "metric_gaps": { "mbpp_pass1": "no pilot-128k-gpt4o-balanced-nfc-bpe-wdfix_code_v2.json", "humaneval_pass1": "no pilot-128k-gpt4o-balanced-nfc-bpe-wdfix_code_v2.json", "gsm8k_exact_match_strict": "no pilot-128k-gpt4o-balanced-nfc-bpe-wdfix_gsm8k_v2.json" }, "registry_note": null, "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a139/cmeister/tokenizer-lm/checkpoints/pilot-128k-gpt4o-balanced-nfc-bpe-wdfix", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "pilot-128k-gpt4o-balanced-unigram-wdfix": { "run": "pilot-128k-gpt4o-balanced-unigram-wdfix", "works": [ "tokeval_cross_scale" ], "training_regime": "cross-scale-d16-300M", "architecture": { "n_layer": 16, "n_embd": 1024, "n_head": 8, "n_kv_head": 8, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 128256, "total_parameters": 596378426 }, "training": { "step": 3341, "total_batch_size": 1048576, "token_budget": 3503292416, "total_bytes_consumed": 22752717832, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "pilot-128k" }, "tokenizer_slug": "gpt4o-balanced-unigram", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 0, "metrics": { "val_bpb": 0.8407208082454437, "blimp_acc": 0.8078955223880595, "code_bpb_mean": 0.6417422373886053, "blimp_scoring": "bos_prefixed", "flores_mean_bpb": 2.78582454208114, "flores_n_languages": 215 }, "metric_gaps": { "mbpp_pass1": "no pilot-128k-gpt4o-balanced-unigram-wdfix_code_v2.json", "humaneval_pass1": "no pilot-128k-gpt4o-balanced-unigram-wdfix_code_v2.json", "gsm8k_exact_match_strict": "no pilot-128k-gpt4o-balanced-unigram-wdfix_gsm8k_v2.json" }, "registry_note": null, "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a139/cmeister/tokenizer-lm/checkpoints/pilot-128k-gpt4o-balanced-unigram-wdfix", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "pilot-128k-gpt4o-code-bpe-wdfix": { "run": "pilot-128k-gpt4o-code-bpe-wdfix", "works": [ "tokeval_cross_scale" ], "training_regime": "cross-scale-d16-300M", "architecture": { "n_layer": 16, "n_embd": 1024, "n_head": 8, "n_kv_head": 8, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 128256, "total_parameters": 596378426 }, "training": { "step": 3341, "total_batch_size": 1048576, "token_budget": 3503292416, "total_bytes_consumed": 22652143709, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "pilot-128k" }, "tokenizer_slug": "gpt4o-code-bpe", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 0, "metrics": { "val_bpb": 0.8281893407285823, "blimp_acc": 0.8007014925373133, "code_bpb_mean": 0.6246381299311335, "blimp_scoring": "bos_prefixed", "flores_mean_bpb": 2.7373735478233345, "flores_n_languages": 215 }, "metric_gaps": { "mbpp_pass1": "no pilot-128k-gpt4o-code-bpe-wdfix_code_v2.json", "humaneval_pass1": "no pilot-128k-gpt4o-code-bpe-wdfix_code_v2.json", "gsm8k_exact_match_strict": "no pilot-128k-gpt4o-code-bpe-wdfix_gsm8k_v2.json" }, "registry_note": null, "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a139/cmeister/tokenizer-lm/checkpoints/pilot-128k-gpt4o-code-bpe-wdfix", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "pilot-128k-gpt4o-english-bpe-wdfix": { "run": "pilot-128k-gpt4o-english-bpe-wdfix", "works": [ "tokeval_cross_scale" ], "training_regime": "cross-scale-d16-300M", "architecture": { "n_layer": 16, "n_embd": 1024, "n_head": 8, "n_kv_head": 8, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 128256, "total_parameters": 596378426 }, "training": { "step": 3341, "total_batch_size": 1048576, "token_budget": 3503292416, "total_bytes_consumed": 24726677521, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "pilot-128k" }, "tokenizer_slug": "gpt4o-english-bpe", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 0, "metrics": { "val_bpb": 0.8326820796903395, "blimp_acc": 0.8165970149253732, "code_bpb_mean": 0.6484054540133121, "blimp_scoring": "bos_prefixed", "flores_mean_bpb": 2.749240218556993, "flores_n_languages": 215 }, "metric_gaps": { "mbpp_pass1": "no pilot-128k-gpt4o-english-bpe-wdfix_code_v2.json", "humaneval_pass1": "no pilot-128k-gpt4o-english-bpe-wdfix_code_v2.json", "gsm8k_exact_match_strict": "no pilot-128k-gpt4o-english-bpe-wdfix_gsm8k_v2.json" }, "registry_note": null, "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a139/cmeister/tokenizer-lm/checkpoints/pilot-128k-gpt4o-english-bpe-wdfix", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "pilot-128k-llama3-wdfix": { "run": "pilot-128k-llama3-wdfix", "works": [ "tokeval_cross_scale" ], "training_regime": "cross-scale-d16-300M", "architecture": { "n_layer": 16, "n_embd": 1024, "n_head": 8, "n_kv_head": 8, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 128256, "total_parameters": 596378426 }, "training": { "step": 3341, "total_batch_size": 1048576, "token_budget": 3503292416, "total_bytes_consumed": 22614256769, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "pilot-128k" }, "tokenizer_slug": "llama3", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 128000, "metrics": { "val_bpb": 0.8192633301410291, "blimp_acc": 0.8085820895522388, "code_bpb_mean": 0.6220985225846258, "blimp_scoring": "bos_prefixed", "flores_mean_bpb": 2.7514998067269696, "flores_n_languages": 215 }, "metric_gaps": { "mbpp_pass1": "no pilot-128k-llama3-wdfix_code_v2.json", "humaneval_pass1": "no pilot-128k-llama3-wdfix_code_v2.json", "gsm8k_exact_match_strict": "no pilot-128k-llama3-wdfix_gsm8k_v2.json" }, "registry_note": null, "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a139/cmeister/tokenizer-lm/checkpoints/pilot-128k-llama3-wdfix", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "pilot-128k-pabpe-nfc-gpt4-balanced": { "run": "pilot-128k-pabpe-nfc-gpt4-balanced", "works": [ "tokeval_cross_scale" ], "training_regime": "cross-scale-d16-300M", "architecture": { "n_layer": 16, "n_embd": 1024, "n_head": 8, "n_kv_head": 8, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 127826, "total_parameters": 595198778 }, "training": { "step": 3337, "total_batch_size": 1048576, "token_budget": 3499098112, "total_bytes_consumed": 28947283691, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "pilot-128k" }, "tokenizer_slug": "pabpe-nfc-gpt4-balanced", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 127822, "metrics": { "val_bpb": 0.8333108785145202, "blimp_acc": 0.5588805970149254, "code_bpb_mean": 0.6240723541441774, "blimp_scoring": null, "flores_mean_bpb": 2.760243938956747, "flores_n_languages": 215, "mbpp_pass1": 0.016, "humaneval_pass1": 0.024390243902439025, "generation_spec": { "version": "v2-2026-07-30", "bos_context": true, "eos_stop": true, "token_healing": true, "truncation": "left_to_max_length_minus_max_gen", "stop_truncation": "earliest_match_all_terms", "stop_match_decode": "specials_kept" }, "token_healing": true, "gsm8k_exact_match_strict": 0.011372251705837756, "gsm8k_exact_match_flexible": 0.013646702047005308 }, "metric_gaps": {}, "registry_note": null, "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a139/cmeister/tokenizer-lm/checkpoints/pilot-128k-pabpe-nfc-gpt4-balanced", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "pilot-128k-punct-balanced-bpe-wdfix": { "run": "pilot-128k-punct-balanced-bpe-wdfix", "works": [ "tokeval_cross_scale" ], "training_regime": "cross-scale-d16-300M", "architecture": { "n_layer": 16, "n_embd": 1024, "n_head": 8, "n_kv_head": 8, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 128256, "total_parameters": 596378426 }, "training": { "step": 3341, "total_batch_size": 1048576, "token_budget": 3503292416, "total_bytes_consumed": 26678596633, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "pilot-128k" }, "tokenizer_slug": "punct-balanced-bpe", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 0, "metrics": { "val_bpb": 0.8178602531852399, "blimp_acc": 0.8050149253731345, "code_bpb_mean": 0.6131823552459643, "blimp_scoring": "bos_prefixed", "flores_mean_bpb": 2.7370705187894253, "flores_n_languages": 215 }, "metric_gaps": { "mbpp_pass1": "no pilot-128k-punct-balanced-bpe-wdfix_code_v2.json", "humaneval_pass1": "no pilot-128k-punct-balanced-bpe-wdfix_code_v2.json", "gsm8k_exact_match_strict": "no pilot-128k-punct-balanced-bpe-wdfix_gsm8k_v2.json" }, "registry_note": null, "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a139/cmeister/tokenizer-lm/checkpoints/pilot-128k-punct-balanced-bpe-wdfix", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "pilot-128k-punct-english-bpe-wdfix": { "run": "pilot-128k-punct-english-bpe-wdfix", "works": [ "tokeval_cross_scale" ], "training_regime": "cross-scale-d16-300M", "architecture": { "n_layer": 16, "n_embd": 1024, "n_head": 8, "n_kv_head": 8, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 128256, "total_parameters": 596378426 }, "training": { "step": 3341, "total_batch_size": 1048576, "token_budget": 3503292416, "total_bytes_consumed": 24786224164, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "pilot-128k" }, "tokenizer_slug": "punct-english-bpe", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 0, "metrics": { "val_bpb": 0.8333547427451476, "blimp_acc": 0.801268656716418, "code_bpb_mean": 0.6463363568675289, "blimp_scoring": "bos_prefixed", "flores_mean_bpb": 2.7478260022807928, "flores_n_languages": 215 }, "metric_gaps": { "mbpp_pass1": "no pilot-128k-punct-english-bpe-wdfix_code_v2.json", "humaneval_pass1": "no pilot-128k-punct-english-bpe-wdfix_code_v2.json", "gsm8k_exact_match_strict": "no pilot-128k-punct-english-bpe-wdfix_gsm8k_v2.json" }, "registry_note": null, "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a139/cmeister/tokenizer-lm/checkpoints/pilot-128k-punct-english-bpe-wdfix", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "pilot-128k-rightalign-balanced-bpe-wdfix": { "run": "pilot-128k-rightalign-balanced-bpe-wdfix", "works": [ "tokeval_cross_scale" ], "training_regime": "cross-scale-d16-300M", "architecture": { "n_layer": 16, "n_embd": 1024, "n_head": 8, "n_kv_head": 8, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 128256, "total_parameters": 596378426 }, "training": { "step": 3341, "total_batch_size": 1048576, "token_budget": 3503292416, "total_bytes_consumed": 25861528355, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "pilot-128k" }, "tokenizer_slug": "rightalign-balanced-bpe", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 0, "metrics": { "val_bpb": 0.806576174927823, "blimp_acc": 0.8065522388059699, "code_bpb_mean": 0.5993113828341335, "blimp_scoring": "bos_prefixed", "flores_mean_bpb": 2.735799278750796, "flores_n_languages": 215 }, "metric_gaps": { "mbpp_pass1": "no pilot-128k-rightalign-balanced-bpe-wdfix_code_v2.json", "humaneval_pass1": "no pilot-128k-rightalign-balanced-bpe-wdfix_code_v2.json", "gsm8k_exact_match_strict": "no pilot-128k-rightalign-balanced-bpe-wdfix_gsm8k_v2.json" }, "registry_note": null, "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a139/cmeister/tokenizer-lm/checkpoints/pilot-128k-rightalign-balanced-bpe-wdfix", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "pilot-128k-rightalign-balanced-nfc-bpe-wdfix": { "run": "pilot-128k-rightalign-balanced-nfc-bpe-wdfix", "works": [ "tokeval_cross_scale" ], "training_regime": "cross-scale-d16-300M", "architecture": { "n_layer": 16, "n_embd": 1024, "n_head": 8, "n_kv_head": 8, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 128256, "total_parameters": 596378426 }, "training": { "step": 3341, "total_batch_size": 1048576, "token_budget": 3503292416, "total_bytes_consumed": 25862019482, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "pilot-128k" }, "tokenizer_slug": "rightalign-balanced-nfc-bpe", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 0, "metrics": { "val_bpb": 0.8064537576731613, "blimp_acc": 0.8079850746268656, "code_bpb_mean": 0.5993740119523097, "blimp_scoring": "bos_prefixed", "flores_mean_bpb": 2.7336603165642184, "flores_n_languages": 215 }, "metric_gaps": { "mbpp_pass1": "no pilot-128k-rightalign-balanced-nfc-bpe-wdfix_code_v2.json", "humaneval_pass1": "no pilot-128k-rightalign-balanced-nfc-bpe-wdfix_code_v2.json", "gsm8k_exact_match_strict": "no pilot-128k-rightalign-balanced-nfc-bpe-wdfix_gsm8k_v2.json" }, "registry_note": null, "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a139/cmeister/tokenizer-lm/checkpoints/pilot-128k-rightalign-balanced-nfc-bpe-wdfix", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "pilot-128k-rightalign-balanced-unigram-wdfix": { "run": "pilot-128k-rightalign-balanced-unigram-wdfix", "works": [ "tokeval_cross_scale" ], "training_regime": "cross-scale-d16-300M", "architecture": { "n_layer": 16, "n_embd": 1024, "n_head": 8, "n_kv_head": 8, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 128256, "total_parameters": 596378426 }, "training": { "step": 3341, "total_batch_size": 1048576, "token_budget": 3503292416, "total_bytes_consumed": 22752280161, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "pilot-128k" }, "tokenizer_slug": "rightalign-balanced-unigram", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 0, "metrics": { "val_bpb": 0.840835607389188, "blimp_acc": 0.8000149253731341, "code_bpb_mean": 0.6415251666607594, "blimp_scoring": "bos_prefixed", "flores_mean_bpb": 2.78969682999519, "flores_n_languages": 215 }, "metric_gaps": { "mbpp_pass1": "no pilot-128k-rightalign-balanced-unigram-wdfix_code_v2.json", "humaneval_pass1": "no pilot-128k-rightalign-balanced-unigram-wdfix_code_v2.json", "gsm8k_exact_match_strict": "no pilot-128k-rightalign-balanced-unigram-wdfix_gsm8k_v2.json" }, "registry_note": null, "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a139/cmeister/tokenizer-lm/checkpoints/pilot-128k-rightalign-balanced-unigram-wdfix", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "pilot-128k-superbpe-gpt4o-balanced": { "run": "pilot-128k-superbpe-gpt4o-balanced", "works": [ "tokeval_cross_scale" ], "training_regime": "cross-scale-d16-300M", "architecture": { "n_layer": 16, "n_embd": 1024, "n_head": 8, "n_kv_head": 8, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 128004, "total_parameters": 595788602 }, "training": { "step": 3339, "total_batch_size": 1048576, "token_budget": 3501195264, "total_bytes_consumed": 26653353804, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "pilot-128k" }, "tokenizer_slug": "superbpe-gpt4o-balanced", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 128000, "metrics": { "val_bpb": 0.8097538729950767, "blimp_acc": 0.5948955223880596, "code_bpb_mean": 0.5919398272220795, "blimp_scoring": null, "flores_mean_bpb": 2.728707690406157, "flores_n_languages": 215, "mbpp_pass1": 0.014, "humaneval_pass1": 0.024390243902439025, "generation_spec": { "version": "v2-2026-07-30", "bos_context": true, "eos_stop": true, "token_healing": true, "truncation": "left_to_max_length_minus_max_gen", "stop_truncation": "earliest_match_all_terms", "stop_match_decode": "specials_kept" }, "token_healing": true, "gsm8k_exact_match_strict": 0.01288855193328279, "gsm8k_exact_match_flexible": 0.01592115238817286 }, "metric_gaps": {}, "registry_note": null, "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a139/cmeister/tokenizer-lm/checkpoints/pilot-128k-superbpe-gpt4o-balanced", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "small-128k-apertus": { "run": "small-128k-apertus", "works": [ "tokeval_cross_scale" ], "training_regime": "cross-scale-d12", "architecture": { "n_layer": 12, "n_embd": 768, "n_head": 6, "n_kv_head": 6, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 131072, "total_parameters": 387514850 }, "training": { "step": 3337, "total_batch_size": 1048576, "token_budget": 3499098112, "total_bytes_consumed": 23311828168, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "small-128k" }, "tokenizer_slug": "swiss-ai/Apertus-70B-2509", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 1, "metrics": { "val_bpb": 0.865119424983589 }, "metric_gaps": { "blimp_acc": "no small-128k-apertus_blimp_code_bpb.json", "code_bpb_mean": "no small-128k-apertus_blimp_code_bpb.json; no small-128k-apertus_code_bpb.json", "flores_mean_bpb": "no small-128k-apertus.json", "mbpp_pass1": "no small-128k-apertus_code_v2.json", "humaneval_pass1": "no small-128k-apertus_code_v2.json", "gsm8k_exact_match_strict": "no small-128k-apertus_gsm8k_v2.json" }, "registry_note": null, "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a139/cmeister/tokenizer-lm/checkpoints/small-128k-apertus", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "small-128k-claude-balanced-bpe": { "run": "small-128k-claude-balanced-bpe", "works": [ "tokeval_cross_scale" ], "training_regime": "cross-scale-d12", "architecture": { "n_layer": 12, "n_embd": 768, "n_head": 6, "n_kv_head": 6, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 128256, "total_parameters": 381026786 }, "training": { "step": 3337, "total_batch_size": 1048576, "token_budget": 3499098112, "total_bytes_consumed": 26471449933, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "small-128k" }, "tokenizer_slug": "claude-balanced-bpe", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 0, "metrics": { "val_bpb": 0.8528457701568503 }, "metric_gaps": { "blimp_acc": "no small-128k-claude-balanced-bpe_blimp_code_bpb.json", "code_bpb_mean": "no small-128k-claude-balanced-bpe_blimp_code_bpb.json; no small-128k-claude-balanced-bpe_code_bpb.json", "flores_mean_bpb": "no small-128k-claude-balanced-bpe.json", "mbpp_pass1": "no small-128k-claude-balanced-bpe_code_v2.json", "humaneval_pass1": "no small-128k-claude-balanced-bpe_code_v2.json", "gsm8k_exact_match_strict": "no small-128k-claude-balanced-bpe_gsm8k_v2.json" }, "registry_note": null, "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a139/cmeister/tokenizer-lm/checkpoints/small-128k-claude-balanced-bpe", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "small-128k-claude-balanced-nfc-bpe": { "run": "small-128k-claude-balanced-nfc-bpe", "works": [ "tokeval_cross_scale" ], "training_regime": "cross-scale-d12", "architecture": { "n_layer": 12, "n_embd": 768, "n_head": 6, "n_kv_head": 6, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 128256, "total_parameters": 381026786 }, "training": { "step": 3337, "total_batch_size": 1048576, "token_budget": 3499098112, "total_bytes_consumed": 26471396541, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "small-128k" }, "tokenizer_slug": "claude-balanced-nfc-bpe", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 0, "metrics": { "val_bpb": 0.8523228950373269 }, "metric_gaps": { "blimp_acc": "no small-128k-claude-balanced-nfc-bpe_blimp_code_bpb.json", "code_bpb_mean": "no small-128k-claude-balanced-nfc-bpe_blimp_code_bpb.json; no small-128k-claude-balanced-nfc-bpe_code_bpb.json", "flores_mean_bpb": "no small-128k-claude-balanced-nfc-bpe.json", "mbpp_pass1": "no small-128k-claude-balanced-nfc-bpe_code_v2.json", "humaneval_pass1": "no small-128k-claude-balanced-nfc-bpe_code_v2.json", "gsm8k_exact_match_strict": "no small-128k-claude-balanced-nfc-bpe_gsm8k_v2.json" }, "registry_note": null, "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a139/cmeister/tokenizer-lm/checkpoints/small-128k-claude-balanced-nfc-bpe", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "small-128k-claude-balanced-unigram": { "run": "small-128k-claude-balanced-unigram", "works": [ "tokeval_cross_scale" ], "training_regime": "cross-scale-d12", "architecture": { "n_layer": 12, "n_embd": 768, "n_head": 6, "n_kv_head": 6, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 128256, "total_parameters": 381026786 }, "training": { "step": 3337, "total_batch_size": 1048576, "token_budget": 3499098112, "total_bytes_consumed": 22708535275, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "small-128k" }, "tokenizer_slug": "claude-balanced-unigram", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 0, "metrics": { "val_bpb": 0.888320485067197 }, "metric_gaps": { "blimp_acc": "no small-128k-claude-balanced-unigram_blimp_code_bpb.json", "code_bpb_mean": "no small-128k-claude-balanced-unigram_blimp_code_bpb.json; no small-128k-claude-balanced-unigram_code_bpb.json", "flores_mean_bpb": "no small-128k-claude-balanced-unigram.json", "mbpp_pass1": "no small-128k-claude-balanced-unigram_code_v2.json", "humaneval_pass1": "no small-128k-claude-balanced-unigram_code_v2.json", "gsm8k_exact_match_strict": "no small-128k-claude-balanced-unigram_gsm8k_v2.json" }, "registry_note": null, "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a139/cmeister/tokenizer-lm/checkpoints/small-128k-claude-balanced-unigram", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "small-128k-claude-english-bpe": { "run": "small-128k-claude-english-bpe", "works": [ "tokeval_cross_scale" ], "training_regime": "cross-scale-d12", "architecture": { "n_layer": 12, "n_embd": 768, "n_head": 6, "n_kv_head": 6, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 128256, "total_parameters": 381026786 }, "training": { "step": 3337, "total_batch_size": 1048576, "token_budget": 3499098112, "total_bytes_consumed": 24979881010, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "small-128k" }, "tokenizer_slug": "claude-english-bpe", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 0, "metrics": { "val_bpb": 0.8739654201888126 }, "metric_gaps": { "blimp_acc": "no small-128k-claude-english-bpe_blimp_code_bpb.json", "code_bpb_mean": "no small-128k-claude-english-bpe_blimp_code_bpb.json; no small-128k-claude-english-bpe_code_bpb.json", "flores_mean_bpb": "no small-128k-claude-english-bpe.json", "mbpp_pass1": "no small-128k-claude-english-bpe_code_v2.json", "humaneval_pass1": "no small-128k-claude-english-bpe_code_v2.json", "gsm8k_exact_match_strict": "no small-128k-claude-english-bpe_gsm8k_v2.json" }, "registry_note": null, "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a139/cmeister/tokenizer-lm/checkpoints/small-128k-claude-english-bpe", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "small-128k-gpt4o-balanced-bpe": { "run": "small-128k-gpt4o-balanced-bpe", "works": [ "tokeval_cross_scale" ], "training_regime": "cross-scale-d12", "architecture": { "n_layer": 12, "n_embd": 768, "n_head": 6, "n_kv_head": 6, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 128256, "total_parameters": 381026786 }, "training": { "step": 3337, "total_batch_size": 1048576, "token_budget": 3499098112, "total_bytes_consumed": 25830345493, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "small-128k" }, "tokenizer_slug": "gpt4o-balanced-bpe", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 0, "metrics": { "val_bpb": 0.8488644192005518 }, "metric_gaps": { "blimp_acc": "no small-128k-gpt4o-balanced-bpe_blimp_code_bpb.json", "code_bpb_mean": "no small-128k-gpt4o-balanced-bpe_blimp_code_bpb.json; no small-128k-gpt4o-balanced-bpe_code_bpb.json", "flores_mean_bpb": "no small-128k-gpt4o-balanced-bpe.json", "mbpp_pass1": "no small-128k-gpt4o-balanced-bpe_code_v2.json", "humaneval_pass1": "no small-128k-gpt4o-balanced-bpe_code_v2.json", "gsm8k_exact_match_strict": "no small-128k-gpt4o-balanced-bpe_gsm8k_v2.json" }, "registry_note": null, "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a139/cmeister/tokenizer-lm/checkpoints/small-128k-gpt4o-balanced-bpe", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "small-128k-gpt4o-balanced-nfc-bpe": { "run": "small-128k-gpt4o-balanced-nfc-bpe", "works": [ "tokeval_cross_scale" ], "training_regime": "cross-scale-d12", "architecture": { "n_layer": 12, "n_embd": 768, "n_head": 6, "n_kv_head": 6, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 128256, "total_parameters": 381026786 }, "training": { "step": 3337, "total_batch_size": 1048576, "token_budget": 3499098112, "total_bytes_consumed": 25830839369, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "small-128k" }, "tokenizer_slug": "gpt4o-balanced-nfc-bpe", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 0, "metrics": { "val_bpb": 0.8486585324292858 }, "metric_gaps": { "blimp_acc": "no small-128k-gpt4o-balanced-nfc-bpe_blimp_code_bpb.json", "code_bpb_mean": "no small-128k-gpt4o-balanced-nfc-bpe_blimp_code_bpb.json; no small-128k-gpt4o-balanced-nfc-bpe_code_bpb.json", "flores_mean_bpb": "no small-128k-gpt4o-balanced-nfc-bpe.json", "mbpp_pass1": "no small-128k-gpt4o-balanced-nfc-bpe_code_v2.json", "humaneval_pass1": "no small-128k-gpt4o-balanced-nfc-bpe_code_v2.json", "gsm8k_exact_match_strict": "no small-128k-gpt4o-balanced-nfc-bpe_gsm8k_v2.json" }, "registry_note": null, "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a139/cmeister/tokenizer-lm/checkpoints/small-128k-gpt4o-balanced-nfc-bpe", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "small-128k-gpt4o-balanced-unigram": { "run": "small-128k-gpt4o-balanced-unigram", "works": [ "tokeval_cross_scale" ], "training_regime": "cross-scale-d12", "architecture": { "n_layer": 12, "n_embd": 768, "n_head": 6, "n_kv_head": 6, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 128256, "total_parameters": 381026786 }, "training": { "step": 3337, "total_batch_size": 1048576, "token_budget": 3499098112, "total_bytes_consumed": 22725477224, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "small-128k" }, "tokenizer_slug": "gpt4o-balanced-unigram", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 0, "metrics": { "val_bpb": 0.8836673905213913 }, "metric_gaps": { "blimp_acc": "no small-128k-gpt4o-balanced-unigram_blimp_code_bpb.json", "code_bpb_mean": "no small-128k-gpt4o-balanced-unigram_blimp_code_bpb.json; no small-128k-gpt4o-balanced-unigram_code_bpb.json", "flores_mean_bpb": "no small-128k-gpt4o-balanced-unigram.json", "mbpp_pass1": "no small-128k-gpt4o-balanced-unigram_code_v2.json", "humaneval_pass1": "no small-128k-gpt4o-balanced-unigram_code_v2.json", "gsm8k_exact_match_strict": "no small-128k-gpt4o-balanced-unigram_gsm8k_v2.json" }, "registry_note": null, "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a139/cmeister/tokenizer-lm/checkpoints/small-128k-gpt4o-balanced-unigram", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "small-128k-gpt4o-code-bpe": { "run": "small-128k-gpt4o-code-bpe", "works": [ "tokeval_cross_scale" ], "training_regime": "cross-scale-d12", "architecture": { "n_layer": 12, "n_embd": 768, "n_head": 6, "n_kv_head": 6, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 128256, "total_parameters": 381026786 }, "training": { "step": 3337, "total_batch_size": 1048576, "token_budget": 3499098112, "total_bytes_consumed": 22625023513, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "small-128k" }, "tokenizer_slug": "gpt4o-code-bpe", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 0, "metrics": { "val_bpb": 0.8719229906379045 }, "metric_gaps": { "blimp_acc": "no small-128k-gpt4o-code-bpe_blimp_code_bpb.json", "code_bpb_mean": "no small-128k-gpt4o-code-bpe_blimp_code_bpb.json; no small-128k-gpt4o-code-bpe_code_bpb.json", "flores_mean_bpb": "no small-128k-gpt4o-code-bpe.json", "mbpp_pass1": "no small-128k-gpt4o-code-bpe_code_v2.json", "humaneval_pass1": "no small-128k-gpt4o-code-bpe_code_v2.json", "gsm8k_exact_match_strict": "no small-128k-gpt4o-code-bpe_gsm8k_v2.json" }, "registry_note": null, "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a139/cmeister/tokenizer-lm/checkpoints/small-128k-gpt4o-code-bpe", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "small-128k-gpt4o-english-bpe": { "run": "small-128k-gpt4o-english-bpe", "works": [ "tokeval_cross_scale" ], "training_regime": "cross-scale-d12", "architecture": { "n_layer": 12, "n_embd": 768, "n_head": 6, "n_kv_head": 6, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 128256, "total_parameters": 381026786 }, "training": { "step": 3337, "total_batch_size": 1048576, "token_budget": 3499098112, "total_bytes_consumed": 24697073597, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "small-128k" }, "tokenizer_slug": "gpt4o-english-bpe", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 0, "metrics": { "val_bpb": 0.8781429007208184 }, "metric_gaps": { "blimp_acc": "no small-128k-gpt4o-english-bpe_blimp_code_bpb.json", "code_bpb_mean": "no small-128k-gpt4o-english-bpe_blimp_code_bpb.json; no small-128k-gpt4o-english-bpe_code_bpb.json", "flores_mean_bpb": "no small-128k-gpt4o-english-bpe.json", "mbpp_pass1": "no small-128k-gpt4o-english-bpe_code_v2.json", "humaneval_pass1": "no small-128k-gpt4o-english-bpe_code_v2.json", "gsm8k_exact_match_strict": "no small-128k-gpt4o-english-bpe_gsm8k_v2.json" }, "registry_note": null, "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a139/cmeister/tokenizer-lm/checkpoints/small-128k-gpt4o-english-bpe", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "small-128k-llama3": { "run": "small-128k-llama3", "works": [ "tokeval_cross_scale" ], "training_regime": "cross-scale-d12", "architecture": { "n_layer": 12, "n_embd": 768, "n_head": 6, "n_kv_head": 6, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 128256, "total_parameters": 381026786 }, "training": { "step": 3337, "total_batch_size": 1048576, "token_budget": 3499098112, "total_bytes_consumed": 22587181933, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "small-128k" }, "tokenizer_slug": "NousResearch/Meta-Llama-3-8B", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 128000, "metrics": { "val_bpb": 0.8639087471134697 }, "metric_gaps": { "blimp_acc": "no small-128k-llama3_blimp_code_bpb.json", "code_bpb_mean": "no small-128k-llama3_blimp_code_bpb.json; no small-128k-llama3_code_bpb.json", "flores_mean_bpb": "no small-128k-llama3.json", "mbpp_pass1": "no small-128k-llama3_code_v2.json", "humaneval_pass1": "no small-128k-llama3_code_v2.json", "gsm8k_exact_match_strict": "no small-128k-llama3_gsm8k_v2.json" }, "registry_note": null, "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a139/cmeister/tokenizer-lm/checkpoints/small-128k-llama3", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "small-128k-pabpe-nfc-gpt4-balanced": { "run": "small-128k-pabpe-nfc-gpt4-balanced", "works": [ "tokeval_cross_scale" ], "training_regime": "cross-scale-d12", "architecture": { "n_layer": 12, "n_embd": 768, "n_head": 6, "n_kv_head": 6, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 127826, "total_parameters": 380142050 }, "training": { "step": 3337, "total_batch_size": 1048576, "token_budget": 3499098112, "total_bytes_consumed": 28947283691, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "small-128k" }, "tokenizer_slug": "pabpe-nfc-gpt4-balanced", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 127822, "metrics": { "val_bpb": 0.8801238225129392, "blimp_acc": 0.5636865671641792, "code_bpb_mean": 0.666750653407198, "blimp_scoring": null, "flores_mean_bpb": 2.818358169768302, "flores_n_languages": 214, "mbpp_pass1": 0.002, "humaneval_pass1": 0.03048780487804878, "generation_spec": { "version": "v2-2026-07-30", "bos_context": true, "eos_stop": true, "token_healing": true, "truncation": "left_to_max_length_minus_max_gen", "stop_truncation": "earliest_match_all_terms", "stop_match_decode": "specials_kept" }, "token_healing": true, "gsm8k_exact_match_strict": 0.02122820318423048, "gsm8k_exact_match_flexible": 0.025018953752843062 }, "metric_gaps": {}, "registry_note": null, "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a139/cmeister/tokenizer-lm/checkpoints/small-128k-pabpe-nfc-gpt4-balanced", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "small-128k-punct-balanced-bpe": { "run": "small-128k-punct-balanced-bpe", "works": [ "tokeval_cross_scale" ], "training_regime": "cross-scale-d12", "architecture": { "n_layer": 12, "n_embd": 768, "n_head": 6, "n_kv_head": 6, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 128256, "total_parameters": 381026786 }, "training": { "step": 3337, "total_batch_size": 1048576, "token_budget": 3499098112, "total_bytes_consumed": 26646655781, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "small-128k" }, "tokenizer_slug": "punct-balanced-bpe", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 0, "metrics": { "val_bpb": 0.8583778693432756 }, "metric_gaps": { "blimp_acc": "no small-128k-punct-balanced-bpe_blimp_code_bpb.json", "code_bpb_mean": "no small-128k-punct-balanced-bpe_blimp_code_bpb.json; no small-128k-punct-balanced-bpe_code_bpb.json", "flores_mean_bpb": "no small-128k-punct-balanced-bpe.json", "mbpp_pass1": "no small-128k-punct-balanced-bpe_code_v2.json", "humaneval_pass1": "no small-128k-punct-balanced-bpe_code_v2.json", "gsm8k_exact_match_strict": "no small-128k-punct-balanced-bpe_gsm8k_v2.json" }, "registry_note": null, "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a139/cmeister/tokenizer-lm/checkpoints/small-128k-punct-balanced-bpe", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "small-128k-punct-english-bpe": { "run": "small-128k-punct-english-bpe", "works": [ "tokeval_cross_scale" ], "training_regime": "cross-scale-d12", "architecture": { "n_layer": 12, "n_embd": 768, "n_head": 6, "n_kv_head": 6, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 128256, "total_parameters": 381026786 }, "training": { "step": 3337, "total_batch_size": 1048576, "token_budget": 3499098112, "total_bytes_consumed": 24756548948, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "small-128k" }, "tokenizer_slug": "punct-english-bpe", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 0, "metrics": { "val_bpb": 0.8779310065917464 }, "metric_gaps": { "blimp_acc": "no small-128k-punct-english-bpe_blimp_code_bpb.json", "code_bpb_mean": "no small-128k-punct-english-bpe_blimp_code_bpb.json; no small-128k-punct-english-bpe_code_bpb.json", "flores_mean_bpb": "no small-128k-punct-english-bpe.json", "mbpp_pass1": "no small-128k-punct-english-bpe_code_v2.json", "humaneval_pass1": "no small-128k-punct-english-bpe_code_v2.json", "gsm8k_exact_match_strict": "no small-128k-punct-english-bpe_gsm8k_v2.json" }, "registry_note": null, "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a139/cmeister/tokenizer-lm/checkpoints/small-128k-punct-english-bpe", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "small-128k-rightalign-balanced-bpe": { "run": "small-128k-rightalign-balanced-bpe", "works": [ "tokeval_cross_scale" ], "training_regime": "cross-scale-d12", "architecture": { "n_layer": 12, "n_embd": 768, "n_head": 6, "n_kv_head": 6, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 128256, "total_parameters": 381026786 }, "training": { "step": 3337, "total_batch_size": 1048576, "token_budget": 3499098112, "total_bytes_consumed": 25830565735, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "small-128k" }, "tokenizer_slug": "rightalign-balanced-bpe", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 0, "metrics": { "val_bpb": 0.8476906713771896 }, "metric_gaps": { "blimp_acc": "no small-128k-rightalign-balanced-bpe_blimp_code_bpb.json", "code_bpb_mean": "no small-128k-rightalign-balanced-bpe_blimp_code_bpb.json; no small-128k-rightalign-balanced-bpe_code_bpb.json", "flores_mean_bpb": "no small-128k-rightalign-balanced-bpe.json", "mbpp_pass1": "no small-128k-rightalign-balanced-bpe_code_v2.json", "humaneval_pass1": "no small-128k-rightalign-balanced-bpe_code_v2.json", "gsm8k_exact_match_strict": "no small-128k-rightalign-balanced-bpe_gsm8k_v2.json" }, "registry_note": null, "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a139/cmeister/tokenizer-lm/checkpoints/small-128k-rightalign-balanced-bpe", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "small-128k-rightalign-balanced-nfc-bpe": { "run": "small-128k-rightalign-balanced-nfc-bpe", "works": [ "tokeval_cross_scale" ], "training_regime": "cross-scale-d12", "architecture": { "n_layer": 12, "n_embd": 768, "n_head": 6, "n_kv_head": 6, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 128256, "total_parameters": 381026786 }, "training": { "step": 3337, "total_batch_size": 1048576, "token_budget": 3499098112, "total_bytes_consumed": 25831056274, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "small-128k" }, "tokenizer_slug": "rightalign-balanced-nfc-bpe", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 0, "metrics": { "val_bpb": 0.8475073186072606 }, "metric_gaps": { "blimp_acc": "no small-128k-rightalign-balanced-nfc-bpe_blimp_code_bpb.json", "code_bpb_mean": "no small-128k-rightalign-balanced-nfc-bpe_blimp_code_bpb.json; no small-128k-rightalign-balanced-nfc-bpe_code_bpb.json", "flores_mean_bpb": "no small-128k-rightalign-balanced-nfc-bpe.json", "mbpp_pass1": "no small-128k-rightalign-balanced-nfc-bpe_code_v2.json", "humaneval_pass1": "no small-128k-rightalign-balanced-nfc-bpe_code_v2.json", "gsm8k_exact_match_strict": "no small-128k-rightalign-balanced-nfc-bpe_gsm8k_v2.json" }, "registry_note": null, "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a139/cmeister/tokenizer-lm/checkpoints/small-128k-rightalign-balanced-nfc-bpe", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "small-128k-rightalign-balanced-unigram": { "run": "small-128k-rightalign-balanced-unigram", "works": [ "tokeval_cross_scale" ], "training_regime": "cross-scale-d12", "architecture": { "n_layer": 12, "n_embd": 768, "n_head": 6, "n_kv_head": 6, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 128256, "total_parameters": 381026786 }, "training": { "step": 3337, "total_batch_size": 1048576, "token_budget": 3499098112, "total_bytes_consumed": 22725040077, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "small-128k" }, "tokenizer_slug": "rightalign-balanced-unigram", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 0, "metrics": { "val_bpb": 0.8831570596770636 }, "metric_gaps": { "blimp_acc": "no small-128k-rightalign-balanced-unigram_blimp_code_bpb.json", "code_bpb_mean": "no small-128k-rightalign-balanced-unigram_blimp_code_bpb.json; no small-128k-rightalign-balanced-unigram_code_bpb.json", "flores_mean_bpb": "no small-128k-rightalign-balanced-unigram.json", "mbpp_pass1": "no small-128k-rightalign-balanced-unigram_code_v2.json", "humaneval_pass1": "no small-128k-rightalign-balanced-unigram_code_v2.json", "gsm8k_exact_match_strict": "no small-128k-rightalign-balanced-unigram_gsm8k_v2.json" }, "registry_note": null, "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a139/cmeister/tokenizer-lm/checkpoints/small-128k-rightalign-balanced-unigram", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "small-128k-superbpe-gpt4o-balanced": { "run": "small-128k-superbpe-gpt4o-balanced", "works": [ "tokeval_cross_scale" ], "training_regime": "cross-scale-d12", "architecture": { "n_layer": 12, "n_embd": 768, "n_head": 6, "n_kv_head": 6, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 128004, "total_parameters": 380584418 }, "training": { "step": 3337, "total_batch_size": 1048576, "token_budget": 3499098112, "total_bytes_consumed": 26637388932, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "small-128k" }, "tokenizer_slug": "superbpe-gpt4o-balanced", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 128000, "metrics": { "val_bpb": 0.8516819488448236, "blimp_acc": 0.5957014925373133, "code_bpb_mean": 0.6307925376771104, "blimp_scoring": null, "flores_mean_bpb": 2.789638578278612, "flores_n_languages": 214, "mbpp_pass1": 0.006, "humaneval_pass1": 0.012195121951219513, "generation_spec": { "version": "v2-2026-07-30", "bos_context": true, "eos_stop": true, "token_healing": true, "truncation": "left_to_max_length_minus_max_gen", "stop_truncation": "earliest_match_all_terms", "stop_match_decode": "specials_kept" }, "token_healing": true, "gsm8k_exact_match_strict": 0.009097801364670205, "gsm8k_exact_match_flexible": 0.013646702047005308 }, "metric_gaps": {}, "registry_note": null, "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a139/cmeister/tokenizer-lm/checkpoints/small-128k-superbpe-gpt4o-balanced", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "tiny-128k-apertus": { "run": "tiny-128k-apertus", "works": [ "tokeval_cross_scale" ], "training_regime": "cross-scale-d8", "architecture": { "n_layer": 8, "n_embd": 512, "n_head": 4, "n_kv_head": 4, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 131072, "total_parameters": 226754794 }, "training": { "step": 3337, "total_batch_size": 1048576, "token_budget": 3499098112, "total_bytes_consumed": 23311828168, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "tiny-128k" }, "tokenizer_slug": "swiss-ai/Apertus-70B-2509", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 1, "metrics": { "val_bpb": 0.9355125749278125 }, "metric_gaps": { "blimp_acc": "no tiny-128k-apertus_blimp_code_bpb.json", "code_bpb_mean": "no tiny-128k-apertus_blimp_code_bpb.json; no tiny-128k-apertus_code_bpb.json", "flores_mean_bpb": "no tiny-128k-apertus.json", "mbpp_pass1": "no tiny-128k-apertus_code_v2.json", "humaneval_pass1": "no tiny-128k-apertus_code_v2.json", "gsm8k_exact_match_strict": "no tiny-128k-apertus_gsm8k_v2.json" }, "registry_note": null, "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a139/cmeister/tokenizer-lm/checkpoints/tiny-128k-apertus", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "tiny-128k-claude-balanced-bpe": { "run": "tiny-128k-claude-balanced-bpe", "works": [ "tokeval_cross_scale" ], "training_regime": "cross-scale-d8", "architecture": { "n_layer": 8, "n_embd": 512, "n_head": 4, "n_kv_head": 4, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 128256, "total_parameters": 222429418 }, "training": { "step": 3337, "total_batch_size": 1048576, "token_budget": 3499098112, "total_bytes_consumed": 26471449933, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "tiny-128k" }, "tokenizer_slug": "claude-balanced-bpe", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 0, "metrics": { "val_bpb": 0.9203617131329531 }, "metric_gaps": { "blimp_acc": "no tiny-128k-claude-balanced-bpe_blimp_code_bpb.json", "code_bpb_mean": "no tiny-128k-claude-balanced-bpe_blimp_code_bpb.json; no tiny-128k-claude-balanced-bpe_code_bpb.json", "flores_mean_bpb": "no tiny-128k-claude-balanced-bpe.json", "mbpp_pass1": "no tiny-128k-claude-balanced-bpe_code_v2.json", "humaneval_pass1": "no tiny-128k-claude-balanced-bpe_code_v2.json", "gsm8k_exact_match_strict": "no tiny-128k-claude-balanced-bpe_gsm8k_v2.json" }, "registry_note": null, "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a139/cmeister/tokenizer-lm/checkpoints/tiny-128k-claude-balanced-bpe", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "tiny-128k-claude-balanced-nfc-bpe": { "run": "tiny-128k-claude-balanced-nfc-bpe", "works": [ "tokeval_cross_scale" ], "training_regime": "cross-scale-d8", "architecture": { "n_layer": 8, "n_embd": 512, "n_head": 4, "n_kv_head": 4, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 128256, "total_parameters": 222429418 }, "training": { "step": 3337, "total_batch_size": 1048576, "token_budget": 3499098112, "total_bytes_consumed": 26471396541, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "tiny-128k" }, "tokenizer_slug": "claude-balanced-nfc-bpe", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 0, "metrics": { "val_bpb": 0.92068400302786 }, "metric_gaps": { "blimp_acc": "no tiny-128k-claude-balanced-nfc-bpe_blimp_code_bpb.json", "code_bpb_mean": "no tiny-128k-claude-balanced-nfc-bpe_blimp_code_bpb.json; no tiny-128k-claude-balanced-nfc-bpe_code_bpb.json", "flores_mean_bpb": "no tiny-128k-claude-balanced-nfc-bpe.json", "mbpp_pass1": "no tiny-128k-claude-balanced-nfc-bpe_code_v2.json", "humaneval_pass1": "no tiny-128k-claude-balanced-nfc-bpe_code_v2.json", "gsm8k_exact_match_strict": "no tiny-128k-claude-balanced-nfc-bpe_gsm8k_v2.json" }, "registry_note": null, "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a139/cmeister/tokenizer-lm/checkpoints/tiny-128k-claude-balanced-nfc-bpe", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "tiny-128k-claude-balanced-unigram": { "run": "tiny-128k-claude-balanced-unigram", "works": [ "tokeval_cross_scale" ], "training_regime": "cross-scale-d8", "architecture": { "n_layer": 8, "n_embd": 512, "n_head": 4, "n_kv_head": 4, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 128256, "total_parameters": 222429418 }, "training": { "step": 3337, "total_batch_size": 1048576, "token_budget": 3499098112, "total_bytes_consumed": 22708535275, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "tiny-128k" }, "tokenizer_slug": "claude-balanced-unigram", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 0, "metrics": { "val_bpb": 0.9638156719834017 }, "metric_gaps": { "blimp_acc": "no tiny-128k-claude-balanced-unigram_blimp_code_bpb.json", "code_bpb_mean": "no tiny-128k-claude-balanced-unigram_blimp_code_bpb.json; no tiny-128k-claude-balanced-unigram_code_bpb.json", "flores_mean_bpb": "no tiny-128k-claude-balanced-unigram.json", "mbpp_pass1": "no tiny-128k-claude-balanced-unigram_code_v2.json", "humaneval_pass1": "no tiny-128k-claude-balanced-unigram_code_v2.json", "gsm8k_exact_match_strict": "no tiny-128k-claude-balanced-unigram_gsm8k_v2.json" }, "registry_note": null, "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a139/cmeister/tokenizer-lm/checkpoints/tiny-128k-claude-balanced-unigram", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "tiny-128k-claude-english-bpe": { "run": "tiny-128k-claude-english-bpe", "works": [ "tokeval_cross_scale" ], "training_regime": "cross-scale-d8", "architecture": { "n_layer": 8, "n_embd": 512, "n_head": 4, "n_kv_head": 4, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 128256, "total_parameters": 222429418 }, "training": { "step": 3337, "total_batch_size": 1048576, "token_budget": 3499098112, "total_bytes_consumed": 24979881010, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "tiny-128k" }, "tokenizer_slug": "claude-english-bpe", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 0, "metrics": { "val_bpb": 0.9462504048449769 }, "metric_gaps": { "blimp_acc": "no tiny-128k-claude-english-bpe_blimp_code_bpb.json", "code_bpb_mean": "no tiny-128k-claude-english-bpe_blimp_code_bpb.json; no tiny-128k-claude-english-bpe_code_bpb.json", "flores_mean_bpb": "no tiny-128k-claude-english-bpe.json", "mbpp_pass1": "no tiny-128k-claude-english-bpe_code_v2.json", "humaneval_pass1": "no tiny-128k-claude-english-bpe_code_v2.json", "gsm8k_exact_match_strict": "no tiny-128k-claude-english-bpe_gsm8k_v2.json" }, "registry_note": null, "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a139/cmeister/tokenizer-lm/checkpoints/tiny-128k-claude-english-bpe", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "tiny-128k-gpt4o-balanced-bpe": { "run": "tiny-128k-gpt4o-balanced-bpe", "works": [ "tokeval_cross_scale" ], "training_regime": "cross-scale-d8", "architecture": { "n_layer": 8, "n_embd": 512, "n_head": 4, "n_kv_head": 4, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 128256, "total_parameters": 222429418 }, "training": { "step": 3337, "total_batch_size": 1048576, "token_budget": 3499098112, "total_bytes_consumed": 25830345493, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "tiny-128k" }, "tokenizer_slug": "gpt4o-balanced-bpe", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 0, "metrics": { "val_bpb": 0.9171902254705996 }, "metric_gaps": { "blimp_acc": "no tiny-128k-gpt4o-balanced-bpe_blimp_code_bpb.json", "code_bpb_mean": "no tiny-128k-gpt4o-balanced-bpe_blimp_code_bpb.json; no tiny-128k-gpt4o-balanced-bpe_code_bpb.json", "flores_mean_bpb": "no tiny-128k-gpt4o-balanced-bpe.json", "mbpp_pass1": "no tiny-128k-gpt4o-balanced-bpe_code_v2.json", "humaneval_pass1": "no tiny-128k-gpt4o-balanced-bpe_code_v2.json", "gsm8k_exact_match_strict": "no tiny-128k-gpt4o-balanced-bpe_gsm8k_v2.json" }, "registry_note": null, "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a139/cmeister/tokenizer-lm/checkpoints/tiny-128k-gpt4o-balanced-bpe", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "tiny-128k-gpt4o-balanced-nfc-bpe": { "run": "tiny-128k-gpt4o-balanced-nfc-bpe", "works": [ "tokeval_cross_scale" ], "training_regime": "cross-scale-d8", "architecture": { "n_layer": 8, "n_embd": 512, "n_head": 4, "n_kv_head": 4, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 128256, "total_parameters": 222429418 }, "training": { "step": 3337, "total_batch_size": 1048576, "token_budget": 3499098112, "total_bytes_consumed": 25830839369, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "tiny-128k" }, "tokenizer_slug": "gpt4o-balanced-nfc-bpe", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 0, "metrics": { "val_bpb": 0.9169944413597259 }, "metric_gaps": { "blimp_acc": "no tiny-128k-gpt4o-balanced-nfc-bpe_blimp_code_bpb.json", "code_bpb_mean": "no tiny-128k-gpt4o-balanced-nfc-bpe_blimp_code_bpb.json; no tiny-128k-gpt4o-balanced-nfc-bpe_code_bpb.json", "flores_mean_bpb": "no tiny-128k-gpt4o-balanced-nfc-bpe.json", "mbpp_pass1": "no tiny-128k-gpt4o-balanced-nfc-bpe_code_v2.json", "humaneval_pass1": "no tiny-128k-gpt4o-balanced-nfc-bpe_code_v2.json", "gsm8k_exact_match_strict": "no tiny-128k-gpt4o-balanced-nfc-bpe_gsm8k_v2.json" }, "registry_note": null, "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a139/cmeister/tokenizer-lm/checkpoints/tiny-128k-gpt4o-balanced-nfc-bpe", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "tiny-128k-gpt4o-balanced-unigram": { "run": "tiny-128k-gpt4o-balanced-unigram", "works": [ "tokeval_cross_scale" ], "training_regime": "cross-scale-d8", "architecture": { "n_layer": 8, "n_embd": 512, "n_head": 4, "n_kv_head": 4, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 128256, "total_parameters": 222429418 }, "training": { "step": 3337, "total_batch_size": 1048576, "token_budget": 3499098112, "total_bytes_consumed": 22725477224, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "tiny-128k" }, "tokenizer_slug": "gpt4o-balanced-unigram", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 0, "metrics": { "val_bpb": 0.9614550260734541 }, "metric_gaps": { "blimp_acc": "no tiny-128k-gpt4o-balanced-unigram_blimp_code_bpb.json", "code_bpb_mean": "no tiny-128k-gpt4o-balanced-unigram_blimp_code_bpb.json; no tiny-128k-gpt4o-balanced-unigram_code_bpb.json", "flores_mean_bpb": "no tiny-128k-gpt4o-balanced-unigram.json", "mbpp_pass1": "no tiny-128k-gpt4o-balanced-unigram_code_v2.json", "humaneval_pass1": "no tiny-128k-gpt4o-balanced-unigram_code_v2.json", "gsm8k_exact_match_strict": "no tiny-128k-gpt4o-balanced-unigram_gsm8k_v2.json" }, "registry_note": null, "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a139/cmeister/tokenizer-lm/checkpoints/tiny-128k-gpt4o-balanced-unigram", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "tiny-128k-gpt4o-code-bpe": { "run": "tiny-128k-gpt4o-code-bpe", "works": [ "tokeval_cross_scale" ], "training_regime": "cross-scale-d8", "architecture": { "n_layer": 8, "n_embd": 512, "n_head": 4, "n_kv_head": 4, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 128256, "total_parameters": 222429418 }, "training": { "step": 3337, "total_batch_size": 1048576, "token_budget": 3499098112, "total_bytes_consumed": 22625023513, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "tiny-128k" }, "tokenizer_slug": "gpt4o-code-bpe", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 0, "metrics": { "val_bpb": 0.9463340638979787 }, "metric_gaps": { "blimp_acc": "no tiny-128k-gpt4o-code-bpe_blimp_code_bpb.json", "code_bpb_mean": "no tiny-128k-gpt4o-code-bpe_blimp_code_bpb.json; no tiny-128k-gpt4o-code-bpe_code_bpb.json", "flores_mean_bpb": "no tiny-128k-gpt4o-code-bpe.json", "mbpp_pass1": "no tiny-128k-gpt4o-code-bpe_code_v2.json", "humaneval_pass1": "no tiny-128k-gpt4o-code-bpe_code_v2.json", "gsm8k_exact_match_strict": "no tiny-128k-gpt4o-code-bpe_gsm8k_v2.json" }, "registry_note": null, "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a139/cmeister/tokenizer-lm/checkpoints/tiny-128k-gpt4o-code-bpe", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "tiny-128k-gpt4o-english-bpe": { "run": "tiny-128k-gpt4o-english-bpe", "works": [ "tokeval_cross_scale" ], "training_regime": "cross-scale-d8", "architecture": { "n_layer": 8, "n_embd": 512, "n_head": 4, "n_kv_head": 4, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 128256, "total_parameters": 222429418 }, "training": { "step": 3337, "total_batch_size": 1048576, "token_budget": 3499098112, "total_bytes_consumed": 24697073597, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "tiny-128k" }, "tokenizer_slug": "gpt4o-english-bpe", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 0, "metrics": { "val_bpb": 0.9515298429772641 }, "metric_gaps": { "blimp_acc": "no tiny-128k-gpt4o-english-bpe_blimp_code_bpb.json", "code_bpb_mean": "no tiny-128k-gpt4o-english-bpe_blimp_code_bpb.json; no tiny-128k-gpt4o-english-bpe_code_bpb.json", "flores_mean_bpb": "no tiny-128k-gpt4o-english-bpe.json", "mbpp_pass1": "no tiny-128k-gpt4o-english-bpe_code_v2.json", "humaneval_pass1": "no tiny-128k-gpt4o-english-bpe_code_v2.json", "gsm8k_exact_match_strict": "no tiny-128k-gpt4o-english-bpe_gsm8k_v2.json" }, "registry_note": null, "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a139/cmeister/tokenizer-lm/checkpoints/tiny-128k-gpt4o-english-bpe", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "tiny-128k-llama3": { "run": "tiny-128k-llama3", "works": [ "tokeval_cross_scale" ], "training_regime": "cross-scale-d8", "architecture": { "n_layer": 8, "n_embd": 512, "n_head": 4, "n_kv_head": 4, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 128256, "total_parameters": 222429418 }, "training": { "step": 3337, "total_batch_size": 1048576, "token_budget": 3499098112, "total_bytes_consumed": 22587181933, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "tiny-128k" }, "tokenizer_slug": "NousResearch/Meta-Llama-3-8B", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 128000, "metrics": { "val_bpb": 0.9359259465717317 }, "metric_gaps": { "blimp_acc": "no tiny-128k-llama3_blimp_code_bpb.json", "code_bpb_mean": "no tiny-128k-llama3_blimp_code_bpb.json; no tiny-128k-llama3_code_bpb.json", "flores_mean_bpb": "no tiny-128k-llama3.json", "mbpp_pass1": "no tiny-128k-llama3_code_v2.json", "humaneval_pass1": "no tiny-128k-llama3_code_v2.json", "gsm8k_exact_match_strict": "no tiny-128k-llama3_gsm8k_v2.json" }, "registry_note": null, "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a139/cmeister/tokenizer-lm/checkpoints/tiny-128k-llama3", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "tiny-128k-pabpe-nfc-gpt4-balanced": { "run": "tiny-128k-pabpe-nfc-gpt4-balanced", "works": [ "tokeval_cross_scale" ], "training_regime": "cross-scale-d8", "architecture": { "n_layer": 8, "n_embd": 512, "n_head": 4, "n_kv_head": 4, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 127826, "total_parameters": 221839594 }, "training": { "step": 3337, "total_batch_size": 1048576, "token_budget": 3499098112, "total_bytes_consumed": 28947283691, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "tiny-128k" }, "tokenizer_slug": "pabpe-nfc-gpt4-balanced", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 127822, "metrics": { "val_bpb": 0.9584538663771468, "blimp_acc": 0.5613283582089551, "code_bpb_mean": 0.7416731447732372, "blimp_scoring": null, "flores_mean_bpb": 2.908093316785798, "flores_n_languages": 214, "mbpp_pass1": 0.0, "humaneval_pass1": 0.0, "generation_spec": { "version": "v2-2026-07-30", "bos_context": true, "eos_stop": true, "token_healing": true, "truncation": "left_to_max_length_minus_max_gen", "stop_truncation": "earliest_match_all_terms", "stop_match_decode": "specials_kept" }, "token_healing": true, "gsm8k_exact_match_strict": 0.01288855193328279, "gsm8k_exact_match_flexible": 0.02577710386656558 }, "metric_gaps": {}, "registry_note": null, "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a139/cmeister/tokenizer-lm/checkpoints/tiny-128k-pabpe-nfc-gpt4-balanced", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "tiny-128k-punct-balanced-bpe": { "run": "tiny-128k-punct-balanced-bpe", "works": [ "tokeval_cross_scale" ], "training_regime": "cross-scale-d8", "architecture": { "n_layer": 8, "n_embd": 512, "n_head": 4, "n_kv_head": 4, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 128256, "total_parameters": 222429418 }, "training": { "step": 3337, "total_batch_size": 1048576, "token_budget": 3499098112, "total_bytes_consumed": 26646655781, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "tiny-128k" }, "tokenizer_slug": "punct-balanced-bpe", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 0, "metrics": { "val_bpb": 0.9268041882882746 }, "metric_gaps": { "blimp_acc": "no tiny-128k-punct-balanced-bpe_blimp_code_bpb.json", "code_bpb_mean": "no tiny-128k-punct-balanced-bpe_blimp_code_bpb.json; no tiny-128k-punct-balanced-bpe_code_bpb.json", "flores_mean_bpb": "no tiny-128k-punct-balanced-bpe.json", "mbpp_pass1": "no tiny-128k-punct-balanced-bpe_code_v2.json", "humaneval_pass1": "no tiny-128k-punct-balanced-bpe_code_v2.json", "gsm8k_exact_match_strict": "no tiny-128k-punct-balanced-bpe_gsm8k_v2.json" }, "registry_note": null, "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a139/cmeister/tokenizer-lm/checkpoints/tiny-128k-punct-balanced-bpe", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "tiny-128k-punct-english-bpe": { "run": "tiny-128k-punct-english-bpe", "works": [ "tokeval_cross_scale" ], "training_regime": "cross-scale-d8", "architecture": { "n_layer": 8, "n_embd": 512, "n_head": 4, "n_kv_head": 4, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 128256, "total_parameters": 222429418 }, "training": { "step": 3337, "total_batch_size": 1048576, "token_budget": 3499098112, "total_bytes_consumed": 24756548948, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "tiny-128k" }, "tokenizer_slug": "punct-english-bpe", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 0, "metrics": { "val_bpb": 0.9492255185989006 }, "metric_gaps": { "blimp_acc": "no tiny-128k-punct-english-bpe_blimp_code_bpb.json", "code_bpb_mean": "no tiny-128k-punct-english-bpe_blimp_code_bpb.json; no tiny-128k-punct-english-bpe_code_bpb.json", "flores_mean_bpb": "no tiny-128k-punct-english-bpe.json", "mbpp_pass1": "no tiny-128k-punct-english-bpe_code_v2.json", "humaneval_pass1": "no tiny-128k-punct-english-bpe_code_v2.json", "gsm8k_exact_match_strict": "no tiny-128k-punct-english-bpe_gsm8k_v2.json" }, "registry_note": null, "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a139/cmeister/tokenizer-lm/checkpoints/tiny-128k-punct-english-bpe", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "tiny-128k-rightalign-balanced-bpe": { "run": "tiny-128k-rightalign-balanced-bpe", "works": [ "tokeval_cross_scale" ], "training_regime": "cross-scale-d8", "architecture": { "n_layer": 8, "n_embd": 512, "n_head": 4, "n_kv_head": 4, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 128256, "total_parameters": 222429418 }, "training": { "step": 3337, "total_batch_size": 1048576, "token_budget": 3499098112, "total_bytes_consumed": 25830565735, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "tiny-128k" }, "tokenizer_slug": "rightalign-balanced-bpe", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 0, "metrics": { "val_bpb": 0.9179302641444103 }, "metric_gaps": { "blimp_acc": "no tiny-128k-rightalign-balanced-bpe_blimp_code_bpb.json", "code_bpb_mean": "no tiny-128k-rightalign-balanced-bpe_blimp_code_bpb.json; no tiny-128k-rightalign-balanced-bpe_code_bpb.json", "flores_mean_bpb": "no tiny-128k-rightalign-balanced-bpe.json", "mbpp_pass1": "no tiny-128k-rightalign-balanced-bpe_code_v2.json", "humaneval_pass1": "no tiny-128k-rightalign-balanced-bpe_code_v2.json", "gsm8k_exact_match_strict": "no tiny-128k-rightalign-balanced-bpe_gsm8k_v2.json" }, "registry_note": null, "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a139/cmeister/tokenizer-lm/checkpoints/tiny-128k-rightalign-balanced-bpe", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "tiny-128k-rightalign-balanced-nfc-bpe": { "run": "tiny-128k-rightalign-balanced-nfc-bpe", "works": [ "tokeval_cross_scale" ], "training_regime": "cross-scale-d8", "architecture": { "n_layer": 8, "n_embd": 512, "n_head": 4, "n_kv_head": 4, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 128256, "total_parameters": 222429418 }, "training": { "step": 3337, "total_batch_size": 1048576, "token_budget": 3499098112, "total_bytes_consumed": 25831056274, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "tiny-128k" }, "tokenizer_slug": "rightalign-balanced-nfc-bpe", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 0, "metrics": { "val_bpb": 0.9177500705611673 }, "metric_gaps": { "blimp_acc": "no tiny-128k-rightalign-balanced-nfc-bpe_blimp_code_bpb.json", "code_bpb_mean": "no tiny-128k-rightalign-balanced-nfc-bpe_blimp_code_bpb.json; no tiny-128k-rightalign-balanced-nfc-bpe_code_bpb.json", "flores_mean_bpb": "no tiny-128k-rightalign-balanced-nfc-bpe.json", "mbpp_pass1": "no tiny-128k-rightalign-balanced-nfc-bpe_code_v2.json", "humaneval_pass1": "no tiny-128k-rightalign-balanced-nfc-bpe_code_v2.json", "gsm8k_exact_match_strict": "no tiny-128k-rightalign-balanced-nfc-bpe_gsm8k_v2.json" }, "registry_note": null, "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a139/cmeister/tokenizer-lm/checkpoints/tiny-128k-rightalign-balanced-nfc-bpe", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "tiny-128k-rightalign-balanced-unigram": { "run": "tiny-128k-rightalign-balanced-unigram", "works": [ "tokeval_cross_scale" ], "training_regime": "cross-scale-d8", "architecture": { "n_layer": 8, "n_embd": 512, "n_head": 4, "n_kv_head": 4, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 128256, "total_parameters": 222429418 }, "training": { "step": 3337, "total_batch_size": 1048576, "token_budget": 3499098112, "total_bytes_consumed": 22725040077, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "tiny-128k" }, "tokenizer_slug": "rightalign-balanced-unigram", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 0, "metrics": { "val_bpb": 0.9614013673266325 }, "metric_gaps": { "blimp_acc": "no tiny-128k-rightalign-balanced-unigram_blimp_code_bpb.json", "code_bpb_mean": "no tiny-128k-rightalign-balanced-unigram_blimp_code_bpb.json; no tiny-128k-rightalign-balanced-unigram_code_bpb.json", "flores_mean_bpb": "no tiny-128k-rightalign-balanced-unigram.json", "mbpp_pass1": "no tiny-128k-rightalign-balanced-unigram_code_v2.json", "humaneval_pass1": "no tiny-128k-rightalign-balanced-unigram_code_v2.json", "gsm8k_exact_match_strict": "no tiny-128k-rightalign-balanced-unigram_gsm8k_v2.json" }, "registry_note": null, "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a139/cmeister/tokenizer-lm/checkpoints/tiny-128k-rightalign-balanced-unigram", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } }, "tiny-128k-superbpe-gpt4o-balanced": { "run": "tiny-128k-superbpe-gpt4o-balanced", "works": [ "tokeval_cross_scale" ], "training_regime": "cross-scale-d8", "architecture": { "n_layer": 8, "n_embd": 512, "n_head": 4, "n_kv_head": 4, "sequence_len": 2048, "window_pattern": "SSSL", "ve_dim": 128, "logit_softcap": 15.0, "logit_softcap_source": "meta", "vocab_size": 128004, "total_parameters": 222134506 }, "training": { "step": 3337, "total_batch_size": 1048576, "token_budget": 3499098112, "total_bytes_consumed": 26637388932, "seed": 42, "matrix_lr": 0.02, "embedding_lr": 0.3, "unembedding_lr": 0.008, "weight_decay": 0.28, "width_lr_exponent": -1.0, "config_name": "tiny-128k" }, "tokenizer_slug": "superbpe-gpt4o-balanced", "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", "bos_token_id": 128000, "metrics": { "val_bpb": 0.9208942139791034, "blimp_acc": 0.6045223880597016, "code_bpb_mean": 0.6974649709299585, "blimp_scoring": null, "flores_mean_bpb": 2.8663971214415374, "flores_n_languages": 214, "mbpp_pass1": 0.004, "humaneval_pass1": 0.018292682926829267, "generation_spec": { "version": "v2-2026-07-30", "bos_context": true, "eos_stop": true, "token_healing": true, "truncation": "left_to_max_length_minus_max_gen", "stop_truncation": "earliest_match_all_terms", "stop_match_decode": "specials_kept" }, "token_healing": true, "gsm8k_exact_match_strict": 0.008339651250947688, "gsm8k_exact_match_flexible": 0.022744503411675512 }, "metric_gaps": {}, "registry_note": null, "provenance": { "checkpoint_dir": "/capstor/store/cscs/swissai/a139/cmeister/tokenizer-lm/checkpoints/tiny-128k-superbpe-gpt4o-balanced", "repo_commit": "1a9a718", "nanochat_commit": "f919449", "nanochat_branch": "tokenizer-lm-patches" } } } }