{ "_generated_by": "scripts/hf_release/package_tokenizers.py", "_manifest_commit": "1a9a718", "_provenance_fields": { "empirical": "read from the shipped tokenizer file", "declared_experiments": "scripts/train_custom_tokenizers.py EXPERIMENTS row", "declared_run_manifest": "run_manifest.json written by the external trainer", "registry_note": "configs/model_registry_notes.json, verbatim, for one run" }, "counts": { "slugs": 94, "redistributable": 90, "pointer_only": 4, "without_declared_source": 19 }, "tokenizers": { "NousResearch/Meta-Llama-3-8B": { "slug": "NousResearch/Meta-Llama-3-8B", "runs": [ "full-128k-llama3", "full-128k-llama3-seed42", "small-128k-llama3", "tiny-128k-llama3" ], "n_runs": 4, "redistributable": false, "loadable_with": "transformers", "hub_id": "NousResearch/Meta-Llama-3-8B", "policy_note": "", "empirical": null, "declared_experiments": null, "declared_run_manifest": null, "registry_note": "LLaMA-3 tiktoken (off-the-shelf); n/a (off-the-shelf, Meta-Llama-3-8B pretraining mix) data; norm=n/a; off-the-shelf BPE; vocab 128,256; USED - panels.json main21 (camera-ready main panel)" }, "allmulti-equal-bpe": { "slug": "allmulti-equal-bpe", "runs": [ "full-128k-allmulti-equal-bpe" ], "n_runs": 1, "redistributable": true, "loadable_with": "transformers", "hub_id": null, "policy_note": "trained in this project", "empirical": { "model_type": "BPE", "n_base_tokens": 128256, "n_added_tokens": 4, "vocab_total": 128260, "added_tokens": [ "", "", "", "" ], "normalizer": null, "pre_tokenizer": { "type": "Sequence", "pretokenizers": [ { "type": "Split", "pattern": { "Regex": "[^\\r\\n\\p{L}\\p{N}]?[\\p{Lu}\\p{Lt}\\p{Lm}\\p{Lo}\\p{M}]*[\\p{Ll}\\p{Lm}\\p{Lo}\\p{M}]+(?i:'s|'t|'re|'ve|'m|'ll|'d)?|[^\\r\\n\\p{L}\\p{N}]?[\\p{Lu}\\p{Lt}\\p{Lm}\\p{Lo}\\p{M}]+[\\p{Ll}\\p{Lm}\\p{Lo}\\p{M}]*(?i:'s|'t|'re|'ve|'m|'ll|'d)?|\\p{N}{1,3}| ?[^\\s\\p{L}\\p{N}]+[\\r\\n/]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+" }, "behavior": "Isolated", "invert": false }, { "type": "ByteLevel", "add_prefix_space": false, "trim_offsets": true, "use_regex": false } ] }, "has_post_processor": false, "byte_fallback": false, "bos_token": "", "bos_token_id": 0 }, "declared_experiments": { "algorithm": "bpe", "pretokenizer_constant": "REGEX_GPT4O", "normalizer": null, "data_composition": "allmulti_equal", "description": "GPT-4o regex, equal-per-language multilingual data (30 langs, repeat-sampled), BPE", "trainer_kwargs": null }, "declared_run_manifest": null, "registry_note": "P2 GPT-4o regex; allmulti_equal (30 FineWeb-2 langs equal weight, max_files=50, repeat-sampled) data; norm=none; BPE; vocab 128,256; baseline of the tail-harm result; USED - S6d.4 equal-weighting ESTABLISHED result (rho=-0.595 vs log training weight, n=31 langs)" }, "apertus": { "slug": "apertus", "runs": [ "full-128k-apertus-mathcode-scratch", "pilot-128k-apertus-wdfix" ], "n_runs": 2, "redistributable": false, "loadable_with": "transformers", "hub_id": "swiss-ai/Apertus-70B-2509", "policy_note": "The papers call this tokenizer Mistral-Nemo: the Apertus and Mistral-Nemo tokenizers are the same artifact. These runs record the local mirror directory rather than the hub id.", "empirical": null, "declared_experiments": null, "declared_run_manifest": null, "registry_note": "#4, apertus pretok, slug `apertus`. Mistral-Nemo regex; see EXPERIMENTS_RESULTS.md \u00a73a for the RCA of the apertus-pretok MBPP code-gen breakage (real, reproducible effect, not an artifact)." }, "bpe-nfc-clean-balanced": { "slug": "bpe-nfc-clean-balanced", "runs": [ "full-128k-bpe-nfc-clean-balanced", "full-128k-bpe-nfc-clean-balanced-mathcode-scratch" ], "n_runs": 2, "redistributable": true, "loadable_with": "transformers", "hub_id": null, "policy_note": "trained in this project", "empirical": { "model_type": "BPE", "n_base_tokens": 128000, "n_added_tokens": 4, "vocab_total": 128004, "added_tokens": [ "", "", "", "" ], "normalizer": { "type": "NFC" }, "pre_tokenizer": { "type": "Sequence", "pretokenizers": [ { "type": "Split", "pattern": { "Regex": "[ ]?[\\p{Lu}\\p{Lt}\\p{Lm}\\p{Lo}\\p{M}]*[\\p{Ll}\\p{Lm}\\p{Lo}\\p{M}]+|[ ]?[\\p{Lu}\\p{Lt}\\p{Lm}\\p{Lo}\\p{M}]+[\\p{Ll}\\p{Lm}\\p{Lo}\\p{M}]*|\\p{N}| ?[^\\s\\p{L}\\p{N}]+|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+" }, "behavior": "Isolated", "invert": false }, { "type": "ByteLevel", "add_prefix_space": false, "trim_offsets": true, "use_regex": false } ] }, "has_post_processor": false, "byte_fallback": false, "bos_token": "", "bos_token_id": 128000 }, "declared_experiments": null, "declared_run_manifest": { "variant": "bpe_nfc_clean_multi_balanced", "variant_config": { "trainer": "bpe", "pretok": "clean_multi_nfc", "grouped_config": "configs/parity_aware_config_balanced.json", "special_tokens_override": [], "decoder_add_prefix_space": true, "output_group": "tokenizer-lm-toks", "output_dir_name": "bpe_nfc_clean_multi_balanced", "post_training_special_tokens": [ "", "", "", "" ] }, "grouped_config_path": "/users/cmeister747/pa_tokenizers_branch/configs/parity_aware_config_balanced.json", "vocab_size": 128000, "min_frequency": 2, "special_tokens": [], "start_time": "2026-05-17T18:59:07.507364", "elapsed_seconds": 1901.555356, "slurm_job_id": "2282607", "cli_args": [ "/users/cmeister747/pa_tokenizers_branch/train_tokenizer.py", "--variant", "bpe_nfc_clean_multi_balanced" ], "git": { "commit": "5d8381af9ec1417b078f7dce82bac68873da915e", "branch": "parity-aware-bpe", "dirty": false }, "per_group_summary": { "n_groups": 34, "full_block_in": "tokenizers/bpe-nfc-clean-balanced/run_manifest.json" } }, "registry_note": "clean-multi regex (REGEX_CLEAN); balanced (D2) data; norm=NFC; BPE; vocab 128,004; non-parity BPE baseline for the pabpe-nfc-clean-* family; also the vocab-ladder v128k anchor; USED - balanced-1B bootstrap panel (S6.12) + paper panel + S6h code analysis (n=21 main21 general-metric rows use bpe-nfc-clean-balanced's twin? no - not in main21 itself); not in panels.json main21" }, "bpe-nfc-clean-balanced-v128k": { "slug": "bpe-nfc-clean-balanced-v128k", "runs": [ "full-128k-bpe-nfc-clean-balanced-v128k" ], "n_runs": 1, "redistributable": true, "loadable_with": "transformers", "hub_id": null, "policy_note": "trained in this project", "empirical": { "model_type": "BPE", "n_base_tokens": 128000, "n_added_tokens": 4, "vocab_total": 128004, "added_tokens": [ "", "", "", "" ], "normalizer": { "type": "NFC" }, "pre_tokenizer": { "type": "Sequence", "pretokenizers": [ { "type": "Split", "pattern": { "Regex": "[ ]?[\\p{Lu}\\p{Lt}\\p{Lm}\\p{Lo}\\p{M}]*[\\p{Ll}\\p{Lm}\\p{Lo}\\p{M}]+|[ ]?[\\p{Lu}\\p{Lt}\\p{Lm}\\p{Lo}\\p{M}]+[\\p{Ll}\\p{Lm}\\p{Lo}\\p{M}]*|\\p{N}| ?[^\\s\\p{L}\\p{N}]+|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+" }, "behavior": "Isolated", "invert": false }, { "type": "ByteLevel", "add_prefix_space": false, "trim_offsets": true, "use_regex": false } ] }, "has_post_processor": false, "byte_fallback": false, "bos_token": "", "bos_token_id": 0 }, "declared_experiments": { "algorithm": "bpe", "pretokenizer_constant": "REGEX_CLEAN", "normalizer": { "type": "NFC" }, "data_composition": "balanced", "description": "Base clean-multi pretok, balanced data, NFC, vocab_size=128000 (vocab ladder anchor; also the baseline for bpe-nfc-clean-nomark-balanced)", "trainer_kwargs": null }, "declared_run_manifest": null, "registry_note": "clean-multi regex (REGEX_CLEAN); balanced (D2) data; norm=NFC; BPE; vocab ~128k target (byte-matched: 8797 steps to match the v128k anchor's training bytes); byte-matched vocab-size ladder (equal training BYTES across the 5 points, unequal steps); the 'clean single-recipe ladder' S6f said had never been run; USED - panels.json vocab-ladder (camera-ready); write-up pending EXPERIMENTS_RESULTS.md (EXPERIMENTS_PLAN.md 'V1/vocab ladder', evals landed 2026-07-24/29)" }, "bpe-nfc-clean-balanced-v160k": { "slug": "bpe-nfc-clean-balanced-v160k", "runs": [ "full-128k-bpe-nfc-clean-balanced-v160k" ], "n_runs": 1, "redistributable": true, "loadable_with": "transformers", "hub_id": null, "policy_note": "trained in this project", "empirical": { "model_type": "BPE", "n_base_tokens": 160000, "n_added_tokens": 4, "vocab_total": 160004, "added_tokens": [ "", "", "", "" ], "normalizer": { "type": "NFC" }, "pre_tokenizer": { "type": "Sequence", "pretokenizers": [ { "type": "Split", "pattern": { "Regex": "[ ]?[\\p{Lu}\\p{Lt}\\p{Lm}\\p{Lo}\\p{M}]*[\\p{Ll}\\p{Lm}\\p{Lo}\\p{M}]+|[ ]?[\\p{Lu}\\p{Lt}\\p{Lm}\\p{Lo}\\p{M}]+[\\p{Ll}\\p{Lm}\\p{Lo}\\p{M}]*|\\p{N}| ?[^\\s\\p{L}\\p{N}]+|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+" }, "behavior": "Isolated", "invert": false }, { "type": "ByteLevel", "add_prefix_space": false, "trim_offsets": true, "use_regex": false } ] }, "has_post_processor": false, "byte_fallback": false, "bos_token": "", "bos_token_id": 0 }, "declared_experiments": { "algorithm": "bpe", "pretokenizer_constant": "REGEX_CLEAN", "normalizer": { "type": "NFC" }, "data_composition": "balanced", "description": "Base clean-multi pretok, balanced data, NFC, vocab_size=160000 (vocab ladder)", "trainer_kwargs": null }, "declared_run_manifest": null, "registry_note": "clean-multi regex (REGEX_CLEAN); balanced (D2) data; norm=NFC; BPE; vocab ~160k target (byte-matched: 8667 steps to match the v128k anchor's training bytes); byte-matched vocab-size ladder (equal training BYTES across the 5 points, unequal steps); the 'clean single-recipe ladder' S6f said had never been run; USED - panels.json vocab-ladder (camera-ready); write-up pending EXPERIMENTS_RESULTS.md (EXPERIMENTS_PLAN.md 'V1/vocab ladder', evals landed 2026-07-24/29)" }, "bpe-nfc-clean-balanced-v200k": { "slug": "bpe-nfc-clean-balanced-v200k", "runs": [ "full-128k-bpe-nfc-clean-balanced-v200k" ], "n_runs": 1, "redistributable": true, "loadable_with": "transformers", "hub_id": null, "policy_note": "trained in this project", "empirical": { "model_type": "BPE", "n_base_tokens": 200000, "n_added_tokens": 4, "vocab_total": 200004, "added_tokens": [ "", "", "", "" ], "normalizer": { "type": "NFC" }, "pre_tokenizer": { "type": "Sequence", "pretokenizers": [ { "type": "Split", "pattern": { "Regex": "[ ]?[\\p{Lu}\\p{Lt}\\p{Lm}\\p{Lo}\\p{M}]*[\\p{Ll}\\p{Lm}\\p{Lo}\\p{M}]+|[ ]?[\\p{Lu}\\p{Lt}\\p{Lm}\\p{Lo}\\p{M}]+[\\p{Ll}\\p{Lm}\\p{Lo}\\p{M}]*|\\p{N}| ?[^\\s\\p{L}\\p{N}]+|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+" }, "behavior": "Isolated", "invert": false }, { "type": "ByteLevel", "add_prefix_space": false, "trim_offsets": true, "use_regex": false } ] }, "has_post_processor": false, "byte_fallback": false, "bos_token": "", "bos_token_id": 0 }, "declared_experiments": { "algorithm": "bpe", "pretokenizer_constant": "REGEX_CLEAN", "normalizer": { "type": "NFC" }, "data_composition": "balanced", "description": "Base clean-multi pretok, balanced data, NFC, vocab_size=200000 (vocab ladder)", "trainer_kwargs": null }, "declared_run_manifest": null, "registry_note": "clean-multi regex (REGEX_CLEAN); balanced (D2) data; norm=NFC; BPE; vocab ~200k target (byte-matched: 8552 steps to match the v128k anchor's training bytes); byte-matched vocab-size ladder (equal training BYTES across the 5 points, unequal steps); the 'clean single-recipe ladder' S6f said had never been run; USED - panels.json vocab-ladder (camera-ready); write-up pending EXPERIMENTS_RESULTS.md (EXPERIMENTS_PLAN.md 'V1/vocab ladder', evals landed 2026-07-24/29)" }, "bpe-nfc-clean-balanced-v64k": { "slug": "bpe-nfc-clean-balanced-v64k", "runs": [ "full-128k-bpe-nfc-clean-balanced-v64k" ], "n_runs": 1, "redistributable": true, "loadable_with": "transformers", "hub_id": null, "policy_note": "trained in this project", "empirical": { "model_type": "BPE", "n_base_tokens": 64000, "n_added_tokens": 4, "vocab_total": 64004, "added_tokens": [ "", "", "", "" ], "normalizer": { "type": "NFC" }, "pre_tokenizer": { "type": "Sequence", "pretokenizers": [ { "type": "Split", "pattern": { "Regex": "[ ]?[\\p{Lu}\\p{Lt}\\p{Lm}\\p{Lo}\\p{M}]*[\\p{Ll}\\p{Lm}\\p{Lo}\\p{M}]+|[ ]?[\\p{Lu}\\p{Lt}\\p{Lm}\\p{Lo}\\p{M}]+[\\p{Ll}\\p{Lm}\\p{Lo}\\p{M}]*|\\p{N}| ?[^\\s\\p{L}\\p{N}]+|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+" }, "behavior": "Isolated", "invert": false }, { "type": "ByteLevel", "add_prefix_space": false, "trim_offsets": true, "use_regex": false } ] }, "has_post_processor": false, "byte_fallback": false, "bos_token": "", "bos_token_id": 0 }, "declared_experiments": { "algorithm": "bpe", "pretokenizer_constant": "REGEX_CLEAN", "normalizer": { "type": "NFC" }, "data_composition": "balanced", "description": "Base clean-multi pretok, balanced data, NFC, vocab_size=64000 (vocab ladder)", "trainer_kwargs": null }, "declared_run_manifest": null, "registry_note": "clean-multi regex (REGEX_CLEAN); balanced (D2) data; norm=NFC; BPE; vocab ~64k target (byte-matched: 9299 steps to match the v128k anchor's training bytes); byte-matched vocab-size ladder (equal training BYTES across the 5 points, unequal steps); the 'clean single-recipe ladder' S6f said had never been run; USED - panels.json vocab-ladder (camera-ready); write-up pending EXPERIMENTS_RESULTS.md (EXPERIMENTS_PLAN.md 'V1/vocab ladder', evals landed 2026-07-24/29)" }, "bpe-nfc-clean-balanced-v96k": { "slug": "bpe-nfc-clean-balanced-v96k", "runs": [ "full-128k-bpe-nfc-clean-balanced-v96k" ], "n_runs": 1, "redistributable": true, "loadable_with": "transformers", "hub_id": null, "policy_note": "trained in this project", "empirical": { "model_type": "BPE", "n_base_tokens": 96000, "n_added_tokens": 4, "vocab_total": 96004, "added_tokens": [ "", "", "", "" ], "normalizer": { "type": "NFC" }, "pre_tokenizer": { "type": "Sequence", "pretokenizers": [ { "type": "Split", "pattern": { "Regex": "[ ]?[\\p{Lu}\\p{Lt}\\p{Lm}\\p{Lo}\\p{M}]*[\\p{Ll}\\p{Lm}\\p{Lo}\\p{M}]+|[ ]?[\\p{Lu}\\p{Lt}\\p{Lm}\\p{Lo}\\p{M}]+[\\p{Ll}\\p{Lm}\\p{Lo}\\p{M}]*|\\p{N}| ?[^\\s\\p{L}\\p{N}]+|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+" }, "behavior": "Isolated", "invert": false }, { "type": "ByteLevel", "add_prefix_space": false, "trim_offsets": true, "use_regex": false } ] }, "has_post_processor": false, "byte_fallback": false, "bos_token": "", "bos_token_id": 0 }, "declared_experiments": { "algorithm": "bpe", "pretokenizer_constant": "REGEX_CLEAN", "normalizer": { "type": "NFC" }, "data_composition": "balanced", "description": "Base clean-multi pretok, balanced data, NFC, vocab_size=96000 (vocab ladder)", "trainer_kwargs": null }, "declared_run_manifest": null, "registry_note": "clean-multi regex (REGEX_CLEAN); balanced (D2) data; norm=NFC; BPE; vocab ~96k target (byte-matched: 8986 steps to match the v128k anchor's training bytes); byte-matched vocab-size ladder (equal training BYTES across the 5 points, unequal steps); the 'clean single-recipe ladder' S6f said had never been run; USED - panels.json vocab-ladder (camera-ready); write-up pending EXPERIMENTS_RESULTS.md (EXPERIMENTS_PLAN.md 'V1/vocab ladder', evals landed 2026-07-24/29)" }, "bpe-nfc-clean-fineweb2full": { "slug": "bpe-nfc-clean-fineweb2full", "runs": [ "full-128k-bpe-nfc-clean-fineweb2full" ], "n_runs": 1, "redistributable": true, "loadable_with": "transformers", "hub_id": null, "policy_note": "trained in this project", "empirical": { "model_type": "BPE", "n_base_tokens": 127996, "n_added_tokens": 4, "vocab_total": 128000, "added_tokens": [ "", "", "", "" ], "normalizer": { "type": "NFC" }, "pre_tokenizer": { "type": "Sequence", "pretokenizers": [ { "type": "Split", "pattern": { "Regex": "[ ]?[\\p{Lu}\\p{Lt}\\p{Lm}\\p{Lo}\\p{M}]*[\\p{Ll}\\p{Lm}\\p{Lo}\\p{M}]+|[ ]?[\\p{Lu}\\p{Lt}\\p{Lm}\\p{Lo}\\p{M}]+[\\p{Ll}\\p{Lm}\\p{Lo}\\p{M}]*|\\p{N}| ?[^\\s\\p{L}\\p{N}]+|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+" }, "behavior": "Isolated", "invert": false }, { "type": "ByteLevel", "add_prefix_space": false, "trim_offsets": true, "use_regex": false } ] }, "has_post_processor": false, "byte_fallback": false, "bos_token": "", "bos_token_id": 127996 }, "declared_experiments": null, "declared_run_manifest": { "variant": "bpe_nfc_clean_multi_fineweb2full", "variant_config": { "trainer": "bpe", "pretok": "clean_multi_nfc", "grouped_config": "configs/parity_aware_config_grouped_fineweb2full_quota.json", "special_tokens_override": [], "decoder_add_prefix_space": true, "output_group": "tokenizer-lm-toks", "output_dir_name": "bpe_nfc_clean_multi_fineweb2full", "post_training_special_tokens": [ "", "", "", "" ] }, "grouped_config_path": "/users/cmeister747/pa_tokenizers_branch/configs/parity_aware_config_grouped_fineweb2full_quota.json", "vocab_size": 128000, "min_frequency": 2, "special_tokens": [], "start_time": "2026-05-22T07:43:23.365585", "elapsed_seconds": 6204.931363, "slurm_job_id": "2328522", "cli_args": [ "/users/cmeister747/pa_tokenizers_branch/train_tokenizer.py", "--variant", "bpe_nfc_clean_multi_fineweb2full" ], "git": { "commit": "5d8381af9ec1417b078f7dce82bac68873da915e", "branch": "parity-aware-bpe", "dirty": false }, "per_group_summary": { "n_groups": 25, "full_block_in": "tokenizers/bpe-nfc-clean-fineweb2full/run_manifest.json" } }, "registry_note": "clean-multi regex; fineweb2full (full FineWeb-2 corpus) data; norm=NFC; BPE; vocab 128,000; standard-BPE non-parity baseline for the pabpe-nfc-clean-fineweb2full family; USED - S5b fineweb2full PA-BPE/BPE analysis + S6e.1 cohort update; fineweb2full data variant, excluded from the main paper panel per PAPER_PANEL.md rule" }, "bpe-nfc-clean-nomark-balanced": { "slug": "bpe-nfc-clean-nomark-balanced", "runs": [ "full-128k-bpe-nfc-clean-nomark-balanced" ], "n_runs": 1, "redistributable": true, "loadable_with": "transformers", "hub_id": null, "policy_note": "trained in this project", "empirical": { "model_type": "BPE", "n_base_tokens": 128000, "n_added_tokens": 4, "vocab_total": 128004, "added_tokens": [ "", "", "", "" ], "normalizer": { "type": "NFC" }, "pre_tokenizer": { "type": "Sequence", "pretokenizers": [ { "type": "Split", "pattern": { "Regex": "[ ]?[\\p{Lu}\\p{Lt}\\p{Lm}\\p{Lo}]*[\\p{Ll}\\p{Lm}\\p{Lo}]+|[ ]?[\\p{Lu}\\p{Lt}\\p{Lm}\\p{Lo}]+[\\p{Ll}\\p{Lm}\\p{Lo}]*|\\p{N}| ?[^\\s\\p{L}\\p{N}]+|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+" }, "behavior": "Isolated", "invert": false }, { "type": "ByteLevel", "add_prefix_space": false, "trim_offsets": true, "use_regex": false } ] }, "has_post_processor": false, "byte_fallback": false, "bos_token": "", "bos_token_id": 0 }, "declared_experiments": { "algorithm": "bpe", "pretokenizer_constant": "REGEX_CLEAN_NOMARK", "normalizer": { "type": "NFC" }, "data_composition": "balanced", "description": "Combining-mark-split ablation of the clean pretok (\\p{M} removed), balanced data, NFC, vocab_size=128000 (crosslingual multilingual-hostile control)", "trainer_kwargs": null }, "declared_run_manifest": null, "registry_note": "clean-multi regex, combining-mark variant (REGEX_CLEAN_NOMARK: \\\\p{M} removed so combining marks split from base letters; documented in EXPERIMENTS_PLAN.md XL-pretok, located 2026-08-16); balanced (D2) data; norm=NFC; BPE; vocab 128,000 (verified from tokenizer.json 2026-08-16); combining-mark control run alongside the vocab-ladder cohort (CHRONOLOGICAL 2026-07-29: 'the eleventh' run, evals complete except MGSM, later fixed); USED - companion control for the vocab-ladder cohort (11th run in that batch, CHRONOLOGICAL 2026-07-29); not itself a panels.json entry; USED - crosslingual panel v1 rev5 Tier-1 member (user-approved inclusion 2026-08-16, configs/panels/crosslingual_panel_v1.json)" }, "bpe-nfc-gpt4-fineweb2full": { "slug": "bpe-nfc-gpt4-fineweb2full", "runs": [ "full-128k-bpe-nfc-gpt4-fineweb2full" ], "n_runs": 1, "redistributable": true, "loadable_with": "transformers", "hub_id": null, "policy_note": "trained in this project", "empirical": { "model_type": "BPE", "n_base_tokens": 127996, "n_added_tokens": 4, "vocab_total": 128000, "added_tokens": [ "", "", "", "" ], "normalizer": { "type": "NFC" }, "pre_tokenizer": { "type": "Sequence", "pretokenizers": [ { "type": "Split", "pattern": { "Regex": "[^\\r\\n\\p{L}\\p{N}]?[\\p{Lu}\\p{Lt}\\p{Lm}\\p{Lo}\\p{M}]*[\\p{Ll}\\p{Lm}\\p{Lo}\\p{M}]+(?i:'s|'t|'re|'ve|'m|'ll|'d)?|[^\\r\\n\\p{L}\\p{N}]?[\\p{Lu}\\p{Lt}\\p{Lm}\\p{Lo}\\p{M}]+[\\p{Ll}\\p{Lm}\\p{Lo}\\p{M}]*(?i:'s|'t|'re|'ve|'m|'ll|'d)?|\\p{N}{1,3}| ?[^\\s\\p{L}\\p{N}]+[\\r\\n/]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+" }, "behavior": "Isolated", "invert": false }, { "type": "ByteLevel", "add_prefix_space": false, "trim_offsets": true, "use_regex": false } ] }, "has_post_processor": false, "byte_fallback": false, "bos_token": "", "bos_token_id": 127996 }, "declared_experiments": null, "declared_run_manifest": { "variant": "bpe_nfc_gpt4_fineweb2full", "variant_config": { "trainer": "bpe", "pretok": "gpt4o_nfc", "grouped_config": "configs/parity_aware_config_grouped_fineweb2full_quota.json", "special_tokens_override": [], "decoder_add_prefix_space": true, "output_group": "tokenizer-lm-toks", "output_dir_name": "bpe_nfc_gpt4_fineweb2full", "post_training_special_tokens": [ "", "", "", "" ] }, "grouped_config_path": "/users/cmeister747/pa_tokenizers_branch/configs/parity_aware_config_grouped_fineweb2full_quota.json", "vocab_size": 128000, "min_frequency": 2, "special_tokens": [], "start_time": "2026-05-22T07:43:23.576450", "elapsed_seconds": 6515.053178, "slurm_job_id": "2328523", "cli_args": [ "/users/cmeister747/pa_tokenizers_branch/train_tokenizer.py", "--variant", "bpe_nfc_gpt4_fineweb2full" ], "git": { "commit": "5d8381af9ec1417b078f7dce82bac68873da915e", "branch": "parity-aware-bpe", "dirty": false }, "per_group_summary": { "n_groups": 25, "full_block_in": "tokenizers/bpe-nfc-gpt4-fineweb2full/run_manifest.json" } }, "registry_note": "GPT-4o regex; fineweb2full data; norm=NFC; BPE; vocab 128,000; standard-BPE non-parity baseline for the pabpe-nfc-gpt4-fineweb2full family; USED - S5b fineweb2full analysis + S6e.1; fineweb2full data variant, excluded from main paper panel" }, "bpe-nfc-plus2-balanced": { "slug": "bpe-nfc-plus2-balanced", "runs": [ "full-128k-bpe-nfc-plus2-balanced", "full-128k-bpe-nfc-plus2-balanced-mathcode-scratch", "full-128k-bpe-nfc-plus2-balanced-seed43-mathcode-scratch", "full-128k-bpe-nfc-plus2-balanced-seed44-mathcode-scratch" ], "n_runs": 4, "redistributable": true, "loadable_with": "transformers", "hub_id": null, "policy_note": "trained in this project", "empirical": { "model_type": "BPE", "n_base_tokens": 128256, "n_added_tokens": 4, "vocab_total": 128260, "added_tokens": [ "", "", "", "" ], "normalizer": { "type": "NFC" }, "pre_tokenizer": { "type": "Sequence", "pretokenizers": [ { "type": "Split", "pattern": { "Regex": "[ \\x{0F0B}\\x{0027}\\x{2019}]?[\\p{Lu}\\p{Lt}\\p{Lm}\\p{Lo}\\p{M}]*[\\p{Ll}\\p{Lm}\\p{Lo}\\p{M}]+|[ \\x{0F0B}\\x{0027}\\x{2019}]?[\\p{Lu}\\p{Lt}\\p{Lm}\\p{Lo}\\p{M}]+[\\p{Ll}\\p{Lm}\\p{Lo}\\p{M}]*|\\p{N}| ?[^\\s\\p{L}\\p{N}]{1,16}|\\s{0,16}[\\r\\n]{1,16}|\\s{1,16}(?!\\S)|\\s{1,16}" }, "behavior": "Isolated", "invert": false }, { "type": "ByteLevel", "add_prefix_space": false, "trim_offsets": true, "use_regex": false } ] }, "has_post_processor": false, "byte_fallback": false, "bos_token": "", "bos_token_id": 0 }, "declared_experiments": { "algorithm": "bpe", "pretokenizer_constant": "REGEX_CLEAN_PLUS2", "normalizer": { "type": "NFC" }, "data_composition": "balanced", "description": "Clean-multi pretok with plus2 prefix-only apostrophe/tsheg attachment, balanced data, NFC normalization", "trainer_kwargs": null }, "declared_run_manifest": null, "registry_note": "clean-multi plus2 (REGEX_CLEAN_PLUS2: attaches both apostrophe forms, contraction arm removed); balanced (D2) data; norm=NFC; BPE; vocab 128,256; balanced-data control for the plus2 apostrophe-attachment pretok (ported from the Apertus-v2 fw2full family); USED - PAPER_PANEL.md paper-eligible addition (#5, landed after the S6.12 balanced-1B panel was built; paper_bootstrap_balanced_panel n=34); not in panels.json main21" }, "bpe-nfc-plus3-balanced": { "slug": "bpe-nfc-plus3-balanced", "runs": [ "full-128k-bpe-nfc-plus3-balanced", "full-128k-bpe-nfc-plus3-balanced-mathcode-scratch" ], "n_runs": 2, "redistributable": true, "loadable_with": "transformers", "hub_id": null, "policy_note": "trained in this project", "empirical": { "model_type": "BPE", "n_base_tokens": 128256, "n_added_tokens": 4, "vocab_total": 128260, "added_tokens": [ "", "", "", "" ], "normalizer": { "type": "NFC" }, "pre_tokenizer": { "type": "Sequence", "pretokenizers": [ { "type": "Split", "pattern": { "Regex": "[ \\x{0F0B}\\x{0027}\\x{2019}]?[\\p{Lu}\\p{Lt}\\p{Lm}\\p{Lo}\\p{M}]*[\\p{Ll}\\p{Lm}\\p{Lo}\\p{M}]+(?:[\\x{0027}\\x{2019}](?!\\p{L}))?|[ \\x{0F0B}\\x{0027}\\x{2019}]?[\\p{Lu}\\p{Lt}\\p{Lm}\\p{Lo}\\p{M}]+[\\p{Ll}\\p{Lm}\\p{Lo}\\p{M}]*(?:[\\x{0027}\\x{2019}](?!\\p{L}))?|\\p{N}| ?[^\\s\\p{L}\\p{N}]{1,16}|\\s{0,16}[\\r\\n]{1,16}|\\s{1,16}(?!\\S)|\\s{1,16}" }, "behavior": "Isolated", "invert": false }, { "type": "ByteLevel", "add_prefix_space": false, "trim_offsets": true, "use_regex": false } ] }, "has_post_processor": false, "byte_fallback": false, "bos_token": "", "bos_token_id": 0 }, "declared_experiments": { "algorithm": "bpe", "pretokenizer_constant": "REGEX_CLEAN_PLUS3", "normalizer": { "type": "NFC" }, "data_composition": "balanced", "description": "Clean-multi pretok with plus3 apostrophe/tsheg attachment, balanced data, NFC normalization", "trainer_kwargs": null }, "declared_run_manifest": null, "registry_note": "clean-multi plus3 (REGEX_CLEAN_PLUS3: plus2 + guarded trailing-apostrophe attachment); balanced (D2) data; norm=NFC; BPE; vocab 128,256; balanced-data control for plus3; USED - PAPER_PANEL.md paper-eligible addition (#6); paper_bootstrap_balanced_panel n=34; not in panels.json main21" }, "claude-allmulti-equal-bpe": { "slug": "claude-allmulti-equal-bpe", "runs": [ "full-128k-claude-allmulti-equal-bpe" ], "n_runs": 1, "redistributable": true, "loadable_with": "transformers", "hub_id": null, "policy_note": "trained in this project", "empirical": { "model_type": "BPE", "n_base_tokens": 128256, "n_added_tokens": 4, "vocab_total": 128260, "added_tokens": [ "", "", "", "" ], "normalizer": null, "pre_tokenizer": { "type": "Sequence", "pretokenizers": [ { "type": "Split", "pattern": { "Regex": "(?i:[''\\u2019]s|[''\\u2019]t|[''\\u2019]re|[''\\u2019]ve|[''\\u2019]m|[''\\u2019]ll|[''\\u2019]d)|[ ]?[\\p{Lu}\\p{Lt}\\p{Lm}\\p{Lo}\\p{M}]*[\\p{Ll}\\p{Lm}\\p{Lo}\\p{M}]+(?i:[''\\u2019]s|[''\\u2019]t|[''\\u2019]re|[''\\u2019]ve|[''\\u2019]m|[''\\u2019]ll|[''\\u2019]d)?|[ ]?[\\p{Lu}\\p{Lt}\\p{Lm}\\p{Lo}\\p{M}]+[\\p{Ll}\\p{Lm}\\p{Lo}\\p{M}]*(?i:[''\\u2019]s|[''\\u2019]t|[''\\u2019]re|[''\\u2019]ve|[''\\u2019]m|[''\\u2019]ll|[''\\u2019]d)?|\\p{N}{1,3}|[ ]?[^\\s\\p{L}\\p{N}]+|[ ]+|[\\t]+|[\\n]+|[\\r]+|\\s+|\\S" }, "behavior": "Isolated", "invert": false }, { "type": "ByteLevel", "add_prefix_space": false, "trim_offsets": true, "use_regex": false } ] }, "has_post_processor": false, "byte_fallback": false, "bos_token": "", "bos_token_id": 0 }, "declared_experiments": { "algorithm": "bpe", "pretokenizer_constant": "REGEX_CLAUDE", "normalizer": null, "data_composition": "allmulti_equal", "description": "Claude pretok, equal-per-language data (repeat-sampled), BPE. Pairs with claude-balanced-bpe.", "trainer_kwargs": null }, "declared_run_manifest": null, "registry_note": "P3 Claude V1 regex; allmulti_equal data; norm=none; BPE; vocab 128,256; pretokenizer-generality arm, pairs with claude-balanced-bpe; USED - S6d.4 (rho=-0.558)" }, "claude-balanced-bpe": { "slug": "claude-balanced-bpe", "runs": [ "full-128k-claude-balanced-bpe", "full-128k-claude-balanced-bpe-mathcode-scratch", "full-128k-claude-balanced-bpe-seed42", "full-128k-claude-balanced-bpe-seed43-mathcode-scratch", "full-128k-claude-balanced-bpe-seed44-mathcode-scratch", "pilot-128k-claude-balanced-bpe", "small-128k-claude-balanced-bpe", "tiny-128k-claude-balanced-bpe" ], "n_runs": 8, "redistributable": true, "loadable_with": "transformers", "hub_id": null, "policy_note": "trained in this project", "empirical": { "model_type": "BPE", "n_base_tokens": 128256, "n_added_tokens": 4, "vocab_total": 128260, "added_tokens": [ "", "", "", "" ], "normalizer": null, "pre_tokenizer": { "type": "Sequence", "pretokenizers": [ { "type": "Split", "pattern": { "Regex": "(?i:[''\\u2019]s|[''\\u2019]t|[''\\u2019]re|[''\\u2019]ve|[''\\u2019]m|[''\\u2019]ll|[''\\u2019]d)|[ ]?[\\p{Lu}\\p{Lt}\\p{Lm}\\p{Lo}\\p{M}]*[\\p{Ll}\\p{Lm}\\p{Lo}\\p{M}]+(?i:[''\\u2019]s|[''\\u2019]t|[''\\u2019]re|[''\\u2019]ve|[''\\u2019]m|[''\\u2019]ll|[''\\u2019]d)?|[ ]?[\\p{Lu}\\p{Lt}\\p{Lm}\\p{Lo}\\p{M}]+[\\p{Ll}\\p{Lm}\\p{Lo}\\p{M}]*(?i:[''\\u2019]s|[''\\u2019]t|[''\\u2019]re|[''\\u2019]ve|[''\\u2019]m|[''\\u2019]ll|[''\\u2019]d)?|\\p{N}{1,3}|[ ]?[^\\s\\p{L}\\p{N}]+|[ ]+|[\\t]+|[\\n]+|[\\r]+|\\s+|\\S" }, "behavior": "Isolated", "invert": false }, { "type": "ByteLevel", "add_prefix_space": false, "trim_offsets": true, "use_regex": false } ] }, "has_post_processor": false, "byte_fallback": false, "bos_token": "", "bos_token_id": 0 }, "declared_experiments": { "algorithm": "bpe", "pretokenizer_constant": "REGEX_CLAUDE", "normalizer": null, "data_composition": "balanced", "description": "Claude pretok (case-level splits), balanced data", "trainer_kwargs": null }, "declared_run_manifest": null, "registry_note": "P3 Claude V1 regex (REGEX_CLAUDE); balanced (D2) data; norm=none; BPE; vocab 128,260; USED - panels.json main21 (camera-ready main panel)" }, "claude-balanced-nfc-bpe": { "slug": "claude-balanced-nfc-bpe", "runs": [ "full-128k-claude-balanced-nfc-bpe", "full-128k-claude-balanced-nfc-bpe-mathcode-scratch", "full-128k-claude-balanced-nfc-bpe-seed42", "pilot-128k-claude-balanced-nfc-bpe", "small-128k-claude-balanced-nfc-bpe", "tiny-128k-claude-balanced-nfc-bpe" ], "n_runs": 6, "redistributable": true, "loadable_with": "transformers", "hub_id": null, "policy_note": "trained in this project", "empirical": { "model_type": "BPE", "n_base_tokens": 128256, "n_added_tokens": 4, "vocab_total": 128260, "added_tokens": [ "", "", "", "" ], "normalizer": { "type": "NFC" }, "pre_tokenizer": { "type": "Sequence", "pretokenizers": [ { "type": "Split", "pattern": { "Regex": "(?i:[''\\u2019]s|[''\\u2019]t|[''\\u2019]re|[''\\u2019]ve|[''\\u2019]m|[''\\u2019]ll|[''\\u2019]d)|[ ]?[\\p{Lu}\\p{Lt}\\p{Lm}\\p{Lo}\\p{M}]*[\\p{Ll}\\p{Lm}\\p{Lo}\\p{M}]+(?i:[''\\u2019]s|[''\\u2019]t|[''\\u2019]re|[''\\u2019]ve|[''\\u2019]m|[''\\u2019]ll|[''\\u2019]d)?|[ ]?[\\p{Lu}\\p{Lt}\\p{Lm}\\p{Lo}\\p{M}]+[\\p{Ll}\\p{Lm}\\p{Lo}\\p{M}]*(?i:[''\\u2019]s|[''\\u2019]t|[''\\u2019]re|[''\\u2019]ve|[''\\u2019]m|[''\\u2019]ll|[''\\u2019]d)?|\\p{N}{1,3}|[ ]?[^\\s\\p{L}\\p{N}]+|[ ]+|[\\t]+|[\\n]+|[\\r]+|\\s+|\\S" }, "behavior": "Isolated", "invert": false }, { "type": "ByteLevel", "add_prefix_space": false, "trim_offsets": true, "use_regex": false } ] }, "has_post_processor": false, "byte_fallback": false, "bos_token": "", "bos_token_id": 0 }, "declared_experiments": { "algorithm": "bpe", "pretokenizer_constant": "REGEX_CLAUDE", "normalizer": { "type": "NFC" }, "data_composition": "balanced", "description": "Claude pretok, balanced data, NFC normalization", "trainer_kwargs": null }, "declared_run_manifest": null, "registry_note": "P3 Claude V1 regex; balanced (D2) data; norm=NFC; BPE; vocab 128,260; USED - panels.json main21 (camera-ready main panel)" }, "claude-balanced-unigram": { "slug": "claude-balanced-unigram", "runs": [ "full-128k-claude-balanced-unigram", "full-128k-claude-balanced-unigram-mathcode-scratch", "full-128k-claude-balanced-unigram-seed42", "pilot-128k-claude-balanced-unigram", "small-128k-claude-balanced-unigram", "tiny-128k-claude-balanced-unigram" ], "n_runs": 6, "redistributable": true, "loadable_with": "transformers", "hub_id": null, "policy_note": "trained in this project", "empirical": { "model_type": "Unigram", "n_base_tokens": 128256, "n_added_tokens": 4, "vocab_total": 128260, "added_tokens": [ "", "", "", "" ], "normalizer": null, "pre_tokenizer": { "type": "Sequence", "pretokenizers": [ { "type": "Split", "pattern": { "Regex": "(?i:[''\\u2019]s|[''\\u2019]t|[''\\u2019]re|[''\\u2019]ve|[''\\u2019]m|[''\\u2019]ll|[''\\u2019]d)|[ ]?[\\p{Lu}\\p{Lt}\\p{Lm}\\p{Lo}\\p{M}]*[\\p{Ll}\\p{Lm}\\p{Lo}\\p{M}]+(?i:[''\\u2019]s|[''\\u2019]t|[''\\u2019]re|[''\\u2019]ve|[''\\u2019]m|[''\\u2019]ll|[''\\u2019]d)?|[ ]?[\\p{Lu}\\p{Lt}\\p{Lm}\\p{Lo}\\p{M}]+[\\p{Ll}\\p{Lm}\\p{Lo}\\p{M}]*(?i:[''\\u2019]s|[''\\u2019]t|[''\\u2019]re|[''\\u2019]ve|[''\\u2019]m|[''\\u2019]ll|[''\\u2019]d)?|\\p{N}{1,3}|[ ]?[^\\s\\p{L}\\p{N}]+|[ ]+|[\\t]+|[\\n]+|[\\r]+|\\s+|\\S" }, "behavior": "Isolated", "invert": false }, { "type": "ByteLevel", "add_prefix_space": false, "trim_offsets": true, "use_regex": false } ] }, "has_post_processor": false, "byte_fallback": false, "bos_token": "", "bos_token_id": 0 }, "declared_experiments": { "algorithm": "unigram", "pretokenizer_constant": "REGEX_CLAUDE", "normalizer": null, "data_composition": "balanced", "description": "Claude pretok, balanced data, UnigramLM algorithm", "trainer_kwargs": null }, "declared_run_manifest": null, "registry_note": "P3 Claude V1 regex; balanced (D2) data; norm=none; UnigramLM (HF, default kwargs); vocab 128,260; USED - panels.json main21 (camera-ready main panel)" }, "claude-english-bpe": { "slug": "claude-english-bpe", "runs": [ "full-128k-claude-english-bpe", "full-128k-claude-english-bpe-mathcode-scratch", "full-128k-claude-english-bpe-seed42", "pilot-128k-claude-english-bpe", "small-128k-claude-english-bpe", "tiny-128k-claude-english-bpe" ], "n_runs": 6, "redistributable": true, "loadable_with": "transformers", "hub_id": null, "policy_note": "trained in this project", "empirical": { "model_type": "BPE", "n_base_tokens": 128256, "n_added_tokens": 4, "vocab_total": 128260, "added_tokens": [ "", "", "", "" ], "normalizer": null, "pre_tokenizer": { "type": "Sequence", "pretokenizers": [ { "type": "Split", "pattern": { "Regex": "(?i:[''\\u2019]s|[''\\u2019]t|[''\\u2019]re|[''\\u2019]ve|[''\\u2019]m|[''\\u2019]ll|[''\\u2019]d)|[ ]?[\\p{Lu}\\p{Lt}\\p{Lm}\\p{Lo}\\p{M}]*[\\p{Ll}\\p{Lm}\\p{Lo}\\p{M}]+(?i:[''\\u2019]s|[''\\u2019]t|[''\\u2019]re|[''\\u2019]ve|[''\\u2019]m|[''\\u2019]ll|[''\\u2019]d)?|[ ]?[\\p{Lu}\\p{Lt}\\p{Lm}\\p{Lo}\\p{M}]+[\\p{Ll}\\p{Lm}\\p{Lo}\\p{M}]*(?i:[''\\u2019]s|[''\\u2019]t|[''\\u2019]re|[''\\u2019]ve|[''\\u2019]m|[''\\u2019]ll|[''\\u2019]d)?|\\p{N}{1,3}|[ ]?[^\\s\\p{L}\\p{N}]+|[ ]+|[\\t]+|[\\n]+|[\\r]+|\\s+|\\S" }, "behavior": "Isolated", "invert": false }, { "type": "ByteLevel", "add_prefix_space": false, "trim_offsets": true, "use_regex": false } ] }, "has_post_processor": false, "byte_fallback": false, "bos_token": "", "bos_token_id": 0 }, "declared_experiments": { "algorithm": "bpe", "pretokenizer_constant": "REGEX_CLAUDE", "normalizer": null, "data_composition": "english", "description": "Claude pretok, English-only data", "trainer_kwargs": null }, "declared_run_manifest": null, "registry_note": "P3 Claude V1 regex; english (D1) data; norm=none; BPE; vocab 128,260; affected by the initial_alphabet byte-drop defect (same class as gpt4o-english-bpe); USED - panels.json main21 (camera-ready main panel)" }, "clauderev-balanced-bpe": { "slug": "clauderev-balanced-bpe", "runs": [ "full-128k-clauderev-balanced-bpe", "full-128k-clauderev-balanced-bpe-mathcode-scratch" ], "n_runs": 2, "redistributable": true, "loadable_with": "transformers", "hub_id": null, "policy_note": "trained in this project", "empirical": { "model_type": "BPE", "n_base_tokens": 128256, "n_added_tokens": 4, "vocab_total": 128260, "added_tokens": [ "", "", "", "" ], "normalizer": { "type": "Sequence", "normalizers": [ { "type": "NFD" }, { "type": "Replace", "pattern": { "Regex": "[\\x{000D}\\x{000B}\\x{000C}\\x{0085}]" }, "content": "" }, { "type": "Replace", "pattern": { "Regex": "[\\x{00A0}\\x{1680}\\x{2000}-\\x{200A}\\x{202F}\\x{205F}\\x{2028}\\x{2029}]" }, "content": " " } ] }, "pre_tokenizer": { "type": "Sequence", "pretokenizers": [ { "type": "Split", "pattern": { "Regex": "(?i:['\\u2019](?:s|t|re|ve|m|ll|d))|(?", "bos_token_id": 0 }, "declared_experiments": { "algorithm": "bpe", "pretokenizer_constant": "REGEX_CLAUDE_REV", "normalizer": { "type": "Sequence", "normalizers": [ { "type": "NFD" }, { "type": "Replace", "pattern": { "Regex": "[\\x{000D}\\x{000B}\\x{000C}\\x{0085}]" }, "content": "" }, { "type": "Replace", "pattern": { "Regex": "[\\x{00A0}\\x{1680}\\x{2000}-\\x{200A}\\x{202F}\\x{205F}\\x{2028}\\x{2029}]" }, "content": " " } ] }, "data_composition": "balanced", "description": "Revised-Claude pretok (2026-07: no camel-case/script split, lookbehind space-attach, contraction-suffix clause) + revised normalizer (NFD + CR/VT/FF/NEL delete + whitespace-fold), balanced data, BPE", "trainer_kwargs": null }, "declared_run_manifest": null, "registry_note": "REGEX_CLAUDE_REV (Claude V2: no camel-case split, no cross-script split, dedicated contraction clause, lookbehind-gated space-attach); balanced (D2) data; norm=NFD + delete CR/VT/FF/NEL + fold-whitespace-to-U+0020 (Sequence normalizer); BPE; vocab 128,256; Claude V2 spec (2026-07-05 build); BPB metrics were biased +6.40% by the NFD byte-denominator bug, FIXED 2026-07-13 (accuracy metrics MBPP/HumanEval/GSM8K/BLiMP/code_bpb unaffected); never enters a val_bpb/FLORES correlation per panels.json note; USED - panels.json pretok-controls (camera-ready)" }, "gpt2-balanced-bpe": { "slug": "gpt2-balanced-bpe", "runs": [ "full-128k-gpt2-balanced-bpe" ], "n_runs": 1, "redistributable": true, "loadable_with": "transformers", "hub_id": null, "policy_note": "trained in this project", "empirical": { "model_type": "BPE", "n_base_tokens": 128256, "n_added_tokens": 4, "vocab_total": 128260, "added_tokens": [ "", "", "", "" ], "normalizer": null, "pre_tokenizer": { "type": "ByteLevel", "add_prefix_space": false, "trim_offsets": true, "use_regex": true }, "has_post_processor": false, "byte_fallback": false, "bos_token": "", "bos_token_id": 0 }, "declared_experiments": { "algorithm": "bpe", "pretokenizer_constant": "ByteLevel", "normalizer": null, "data_composition": "balanced", "description": "GPT-2 default regex (ByteLevel use_regex=True), balanced data, BPE", "trainer_kwargs": null }, "declared_run_manifest": null, "registry_note": "plain ByteLevel(use_regex=True) i.e. GPT-2 tiktoken regex; balanced (D2) data; norm=none; BPE; vocab 128,260; reference/default regex missing from the original panel; USED - panels.json pretok-controls (camera-ready)" }, "gpt4o-allmulti-equal-nfc-bpe": { "slug": "gpt4o-allmulti-equal-nfc-bpe", "runs": [ "full-128k-gpt4o-allmulti-equal-nfc-bpe" ], "n_runs": 1, "redistributable": true, "loadable_with": "transformers", "hub_id": null, "policy_note": "trained in this project", "empirical": { "model_type": "BPE", "n_base_tokens": 128256, "n_added_tokens": 4, "vocab_total": 128260, "added_tokens": [ "", "", "", "" ], "normalizer": { "type": "NFC" }, "pre_tokenizer": { "type": "Sequence", "pretokenizers": [ { "type": "Split", "pattern": { "Regex": "[^\\r\\n\\p{L}\\p{N}]?[\\p{Lu}\\p{Lt}\\p{Lm}\\p{Lo}\\p{M}]*[\\p{Ll}\\p{Lm}\\p{Lo}\\p{M}]+(?i:'s|'t|'re|'ve|'m|'ll|'d)?|[^\\r\\n\\p{L}\\p{N}]?[\\p{Lu}\\p{Lt}\\p{Lm}\\p{Lo}\\p{M}]+[\\p{Ll}\\p{Lm}\\p{Lo}\\p{M}]*(?i:'s|'t|'re|'ve|'m|'ll|'d)?|\\p{N}{1,3}| ?[^\\s\\p{L}\\p{N}]+[\\r\\n/]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+" }, "behavior": "Isolated", "invert": false }, { "type": "ByteLevel", "add_prefix_space": false, "trim_offsets": true, "use_regex": false } ] }, "has_post_processor": false, "byte_fallback": false, "bos_token": "", "bos_token_id": 0 }, "declared_experiments": { "algorithm": "bpe", "pretokenizer_constant": "REGEX_GPT4O", "normalizer": { "type": "NFC" }, "data_composition": "allmulti_equal", "description": "GPT-4o regex + NFC, equal-per-language data (repeat-sampled), BPE. Pure normalizer contrast against allmulti-equal-bpe; pairs with gpt4o-balanced-nfc-bpe.", "trainer_kwargs": null }, "declared_run_manifest": null, "registry_note": "P2 GPT-4o regex; allmulti_equal data; norm=NFC; BPE; vocab 128,256; pure normalizer contrast against allmulti-equal-bpe; pairs with gpt4o-balanced-nfc-bpe; USED - S6d.4 (rho=-0.475)" }, "gpt4o-allmulti-equal-norepeat-bpe": { "slug": "gpt4o-allmulti-equal-norepeat-bpe", "runs": [ "full-128k-gpt4o-allmulti-equal-norepeat-bpe" ], "n_runs": 1, "redistributable": true, "loadable_with": "transformers", "hub_id": null, "policy_note": "trained in this project", "empirical": { "model_type": "BPE", "n_base_tokens": 128256, "n_added_tokens": 4, "vocab_total": 128260, "added_tokens": [ "", "", "", "" ], "normalizer": null, "pre_tokenizer": { "type": "Sequence", "pretokenizers": [ { "type": "Split", "pattern": { "Regex": "[^\\r\\n\\p{L}\\p{N}]?[\\p{Lu}\\p{Lt}\\p{Lm}\\p{Lo}\\p{M}]*[\\p{Ll}\\p{Lm}\\p{Lo}\\p{M}]+(?i:'s|'t|'re|'ve|'m|'ll|'d)?|[^\\r\\n\\p{L}\\p{N}]?[\\p{Lu}\\p{Lt}\\p{Lm}\\p{Lo}\\p{M}]+[\\p{Ll}\\p{Lm}\\p{Lo}\\p{M}]*(?i:'s|'t|'re|'ve|'m|'ll|'d)?|\\p{N}{1,3}| ?[^\\s\\p{L}\\p{N}]+[\\r\\n/]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+" }, "behavior": "Isolated", "invert": false }, { "type": "ByteLevel", "add_prefix_space": false, "trim_offsets": true, "use_regex": false } ] }, "has_post_processor": false, "byte_fallback": false, "bos_token": "", "bos_token_id": 0 }, "declared_experiments": { "algorithm": "bpe", "pretokenizer_constant": "REGEX_GPT4O", "normalizer": null, "data_composition": "allmulti_equal_norepeat", "description": "GPT-4o regex, equal per-language weights + max_files=50 but NO repeat-sampling (tail exhausts). Differs from allmulti-equal-bpe ONLY in repeat-sampling, so it isolates that factor.", "trainer_kwargs": null }, "declared_run_manifest": null, "registry_note": "P2 GPT-4o regex; allmulti_equal_norepeat (equal weights, max_files=50, NO repeat-sampling) data; norm=none; BPE; vocab 128,256; isolates repeat-sampling factor against allmulti-equal-bpe; USED - S6d.4 (rho=-0.581; repeat-sampling contributes at most ~1/4 of the tail penalty)" }, "gpt4o-balanced-bpe": { "slug": "gpt4o-balanced-bpe", "runs": [ "full-128k-gpt4o-balanced-bpe", "full-128k-gpt4o-balanced-bpe-seed42", "full-128k-gpt4o-balanced-bpe-seed43-mathcode-scratch", "full-128k-gpt4o-balanced-bpe-seed44-mathcode-scratch", "full-128k-mathcode-scratch", "pilot-128k-gpt4o-balanced-bpe-wdfix", "small-128k-gpt4o-balanced-bpe", "tiny-128k-gpt4o-balanced-bpe" ], "n_runs": 8, "redistributable": true, "loadable_with": "transformers", "hub_id": null, "policy_note": "trained in this project", "empirical": { "model_type": "BPE", "n_base_tokens": 128256, "n_added_tokens": 4, "vocab_total": 128260, "added_tokens": [ "", "", "", "" ], "normalizer": null, "pre_tokenizer": { "type": "Sequence", "pretokenizers": [ { "type": "Split", "pattern": { "Regex": "[^\\r\\n\\p{L}\\p{N}]?[\\p{Lu}\\p{Lt}\\p{Lm}\\p{Lo}\\p{M}]*[\\p{Ll}\\p{Lm}\\p{Lo}\\p{M}]+(?i:'s|'t|'re|'ve|'m|'ll|'d)?|[^\\r\\n\\p{L}\\p{N}]?[\\p{Lu}\\p{Lt}\\p{Lm}\\p{Lo}\\p{M}]+[\\p{Ll}\\p{Lm}\\p{Lo}\\p{M}]*(?i:'s|'t|'re|'ve|'m|'ll|'d)?|\\p{N}{1,3}| ?[^\\s\\p{L}\\p{N}]+[\\r\\n/]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+" }, "behavior": "Isolated", "invert": false }, { "type": "ByteLevel", "add_prefix_space": false, "trim_offsets": true, "use_regex": false } ] }, "has_post_processor": false, "byte_fallback": false, "bos_token": "", "bos_token_id": 0 }, "declared_experiments": { "algorithm": "bpe", "pretokenizer_constant": "REGEX_GPT4O", "normalizer": null, "data_composition": "balanced", "description": "GPT-4o regex, balanced multilingual data", "trainer_kwargs": null }, "declared_run_manifest": null, "registry_note": "P2 GPT-4o regex (REGEX_GPT4O); balanced (D2) data; norm=none; BPE; vocab 128,260; project baseline tokenizer (used as the fixed tokenizer for the softcap ablation and as the mathcode-scratch #1 baseline); USED - panels.json main21 (camera-ready main panel)" }, "gpt4o-balanced-maxfiles50-bpe": { "slug": "gpt4o-balanced-maxfiles50-bpe", "runs": [ "full-128k-gpt4o-balanced-maxfiles50-bpe" ], "n_runs": 1, "redistributable": true, "loadable_with": "transformers", "hub_id": null, "policy_note": "trained in this project", "empirical": { "model_type": "BPE", "n_base_tokens": 128256, "n_added_tokens": 4, "vocab_total": 128260, "added_tokens": [ "", "", "", "" ], "normalizer": null, "pre_tokenizer": { "type": "Sequence", "pretokenizers": [ { "type": "Split", "pattern": { "Regex": "[^\\r\\n\\p{L}\\p{N}]?[\\p{Lu}\\p{Lt}\\p{Lm}\\p{Lo}\\p{M}]*[\\p{Ll}\\p{Lm}\\p{Lo}\\p{M}]+(?i:'s|'t|'re|'ve|'m|'ll|'d)?|[^\\r\\n\\p{L}\\p{N}]?[\\p{Lu}\\p{Lt}\\p{Lm}\\p{Lo}\\p{M}]+[\\p{Ll}\\p{Lm}\\p{Lo}\\p{M}]*(?i:'s|'t|'re|'ve|'m|'ll|'d)?|\\p{N}{1,3}| ?[^\\s\\p{L}\\p{N}]+[\\r\\n/]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+" }, "behavior": "Isolated", "invert": false }, { "type": "ByteLevel", "add_prefix_space": false, "trim_offsets": true, "use_regex": false } ] }, "has_post_processor": false, "byte_fallback": false, "bos_token": "", "bos_token_id": 0 }, "declared_experiments": { "algorithm": "bpe", "pretokenizer_constant": "REGEX_GPT4O", "normalizer": null, "data_composition": "balanced_maxfiles50", "description": "GPT-4o regex, proportional (balanced) weights with max_files=50. Differs from gpt4o-balanced-bpe ONLY in max_files, so it isolates that factor. Expected NULL control.", "trainer_kwargs": null }, "declared_run_manifest": null, "registry_note": "P2 GPT-4o regex; balanced (proportional) weights, max_files raised to 50, no repeat data; norm=none; BPE; vocab 128,256; NULL CONTROL for the max_files cap; USED - S6d.4 null control (rho=+0.004, p=0.98 - exonerates max_files as the driver)" }, "gpt4o-balanced-nfc-bpe": { "slug": "gpt4o-balanced-nfc-bpe", "runs": [ "full-128k-gpt4o-balanced-nfc-bpe", "full-128k-gpt4o-balanced-nfc-bpe-mathcode-scratch", "full-128k-gpt4o-balanced-nfc-bpe-seed42", "pilot-128k-gpt4o-balanced-nfc-bpe-wdfix", "small-128k-gpt4o-balanced-nfc-bpe", "tiny-128k-gpt4o-balanced-nfc-bpe" ], "n_runs": 6, "redistributable": true, "loadable_with": "transformers", "hub_id": null, "policy_note": "trained in this project", "empirical": { "model_type": "BPE", "n_base_tokens": 128256, "n_added_tokens": 4, "vocab_total": 128260, "added_tokens": [ "", "", "", "" ], "normalizer": { "type": "NFC" }, "pre_tokenizer": { "type": "Sequence", "pretokenizers": [ { "type": "Split", "pattern": { "Regex": "[^\\r\\n\\p{L}\\p{N}]?[\\p{Lu}\\p{Lt}\\p{Lm}\\p{Lo}\\p{M}]*[\\p{Ll}\\p{Lm}\\p{Lo}\\p{M}]+(?i:'s|'t|'re|'ve|'m|'ll|'d)?|[^\\r\\n\\p{L}\\p{N}]?[\\p{Lu}\\p{Lt}\\p{Lm}\\p{Lo}\\p{M}]+[\\p{Ll}\\p{Lm}\\p{Lo}\\p{M}]*(?i:'s|'t|'re|'ve|'m|'ll|'d)?|\\p{N}{1,3}| ?[^\\s\\p{L}\\p{N}]+[\\r\\n/]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+" }, "behavior": "Isolated", "invert": false }, { "type": "ByteLevel", "add_prefix_space": false, "trim_offsets": true, "use_regex": false } ] }, "has_post_processor": false, "byte_fallback": false, "bos_token": "", "bos_token_id": 0 }, "declared_experiments": { "algorithm": "bpe", "pretokenizer_constant": "REGEX_GPT4O", "normalizer": { "type": "NFC" }, "data_composition": "balanced", "description": "GPT-4o regex, balanced data, NFC normalization", "trainer_kwargs": null }, "declared_run_manifest": null, "registry_note": "P2 GPT-4o regex; balanced (D2) data; norm=NFC; BPE; vocab 128,260; USED - panels.json main21 (camera-ready main panel)" }, "gpt4o-balanced-unigram": { "slug": "gpt4o-balanced-unigram", "runs": [ "full-128k-gpt4o-balanced-unigram", "full-128k-gpt4o-balanced-unigram-mathcode-scratch", "full-128k-gpt4o-balanced-unigram-seed42", "pilot-128k-gpt4o-balanced-unigram-wdfix", "small-128k-gpt4o-balanced-unigram", "tiny-128k-gpt4o-balanced-unigram" ], "n_runs": 6, "redistributable": true, "loadable_with": "transformers", "hub_id": null, "policy_note": "trained in this project", "empirical": { "model_type": "Unigram", "n_base_tokens": 128256, "n_added_tokens": 4, "vocab_total": 128260, "added_tokens": [ "", "", "", "" ], "normalizer": null, "pre_tokenizer": { "type": "Sequence", "pretokenizers": [ { "type": "Split", "pattern": { "Regex": "[^\\r\\n\\p{L}\\p{N}]?[\\p{Lu}\\p{Lt}\\p{Lm}\\p{Lo}\\p{M}]*[\\p{Ll}\\p{Lm}\\p{Lo}\\p{M}]+(?i:'s|'t|'re|'ve|'m|'ll|'d)?|[^\\r\\n\\p{L}\\p{N}]?[\\p{Lu}\\p{Lt}\\p{Lm}\\p{Lo}\\p{M}]+[\\p{Ll}\\p{Lm}\\p{Lo}\\p{M}]*(?i:'s|'t|'re|'ve|'m|'ll|'d)?|\\p{N}{1,3}| ?[^\\s\\p{L}\\p{N}]+[\\r\\n/]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+" }, "behavior": "Isolated", "invert": false }, { "type": "ByteLevel", "add_prefix_space": false, "trim_offsets": true, "use_regex": false } ] }, "has_post_processor": false, "byte_fallback": false, "bos_token": "", "bos_token_id": 0 }, "declared_experiments": { "algorithm": "unigram", "pretokenizer_constant": "REGEX_GPT4O", "normalizer": null, "data_composition": "balanced", "description": "GPT-4o regex, balanced data, UnigramLM algorithm", "trainer_kwargs": null }, "declared_run_manifest": null, "registry_note": "P2 GPT-4o regex; balanced (D2) data; norm=none; UnigramLM (HF UnigramTrainer, default kwargs); vocab 128,260; USED - panels.json main21 (camera-ready main panel)" }, "gpt4o-balanced-unigram-tuned": { "slug": "gpt4o-balanced-unigram-tuned", "runs": [ "full-128k-gpt4o-balanced-unigram-tuned" ], "n_runs": 1, "redistributable": true, "loadable_with": "transformers", "hub_id": null, "policy_note": "trained in this project", "empirical": { "model_type": "Unigram", "n_base_tokens": 128256, "n_added_tokens": 4, "vocab_total": 128260, "added_tokens": [ "", "", "", "" ], "normalizer": null, "pre_tokenizer": { "type": "Sequence", "pretokenizers": [ { "type": "Split", "pattern": { "Regex": "[^\\r\\n\\p{L}\\p{N}]?[\\p{Lu}\\p{Lt}\\p{Lm}\\p{Lo}\\p{M}]*[\\p{Ll}\\p{Lm}\\p{Lo}\\p{M}]+(?i:'s|'t|'re|'ve|'m|'ll|'d)?|[^\\r\\n\\p{L}\\p{N}]?[\\p{Lu}\\p{Lt}\\p{Lm}\\p{Lo}\\p{M}]+[\\p{Ll}\\p{Lm}\\p{Lo}\\p{M}]*(?i:'s|'t|'re|'ve|'m|'ll|'d)?|\\p{N}{1,3}| ?[^\\s\\p{L}\\p{N}]+[\\r\\n/]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+" }, "behavior": "Isolated", "invert": false }, { "type": "ByteLevel", "add_prefix_space": false, "trim_offsets": true, "use_regex": false } ] }, "has_post_processor": false, "byte_fallback": false, "bos_token": "", "bos_token_id": 0 }, "declared_experiments": { "algorithm": "unigram", "pretokenizer_constant": "REGEX_GPT4O", "normalizer": null, "data_composition": "balanced", "description": "GPT-4o regex, balanced data, UnigramLM with tuned hyperparameters", "trainer_kwargs": { "shrinking_factor": 0.7, "n_sub_iterations": 3, "max_piece_length": 64, "initial_alphabet": [ "\u0131", "\u00c5", "\u00b1", "}", "\u00cb", "_", "t", "\u00fa", "\u00be", "\u00d2", "\u00ea", "8", "\u012e", "\u00d4", "i", "\u00c2", "w", "r", "\u00b7", "\u013d", "\u00d0", "U", "p", "\u0121", "\u00c7", "R", "\u00ae", "\u0116", "\u00ac", "b", "\u00d7", "d", "?", "D", "\u00d1", "\u0143", "\u00e9", "\u00e4", "\u0129", "\u00b8", "W", "k", "\u00e8", "\u011b", "\u013b", "\u00c8", "\u0109", "\u00f8", "\u013e", "\u0103", "\u0112", "\u0111", "z", "L", "\u00e1", "\u00fb", "\u00a3", "\u00f6", "\u011c", "\u012d", "\u012c", "B", "C", "\u00df", "\u0122", "\u0120", "\u00b2", "\u00a7", "]", "5", "\u0139", ">", "`", "\u0108", "\u00ba", "\u0128", "(", "\u0115", "\u013a", "\u00f1", "f", "o", "\u00cd", "\u00da", "\u00a5", "\u012a", "\u010c", "\u00de", "x", "\u00e3", "\u00f7", "\u010a", "\u010e", "\u010f", "\u00c1", "\u00b0", "\"", "\u00ce", ".", "l", "J", "\u00ee", "F", "\u00a2", "\u00e0", "\u011d", "\u00bf", "Q", "\u00cc", "\u0117", "\u00cf", "\u0124", "Y", "$", "K", "\u00a9", "-", "\u00ef", "v", "^", "\u0105", "\u00fc", "\u0126", "\u00af", "\u0114", "\u0142", "I", "\u012b", "u", "<", "2", "\u00bb", "~", "\u0106", "n", "m", "\u0140", "\\", "1", "\u0127", "\u00e7", "6", "\u00db", "\u00d9", "\u00f5", "X", "\u00ab", "\u00eb", "'", "\u013c", "E", "\u00c0", "\u0135", "\u00dc", "\u00fd", "\u00c4", "\u0113", "\u0138", "\u00bd", "%", "@", "\u00ca", "3", "\u00ec", "\u00f2", "h", "y", "\u00b4", "\u00a4", "\u00b6", "&", "\u00ff", "\u011e", "g", "c", "\u00e6", "N", "\u0134", "4", "\u00f4", "\u00ed", "\u0118", "\u012f", "\u00b3", "V", "Z", "\u0141", "\u00b9", "\u0125", "\u0110", "O", "\u0100", "\u00f0", ")", "\u0136", "*", "P", "/", "\u00f9", "a", "\u0132", "\u011f", "\u0137", "!", "\u00d5", "\u00e5", "9", "A", "\u00fe", "q", "\u00d6", "e", "=", "\u0123", "\u00a8", "{", "\u00f3", "\u0133", "\u011a", "#", "\u00d3", "S", "G", "[", "\u010b", "\u0104", "s", "\u00c6", "\u0119", "\u0102", "\u010d", ":", "\u00a1", "\u00aa", "\u00d8", "\u0130", "j", "M", "7", "\u00a6", "\u0107", "\u013f", "\u0101", "0", "\u00c3", "\u00dd", ";", "H", "\u00b5", "+", "|", ",", "\u00c9", "T", "\u00bc", "\u00e2" ] } }, "declared_run_manifest": null, "registry_note": "P2 GPT-4o regex; balanced (D2) data; norm=none; UnigramLM (HF, UNIGRAM_TUNED_KWARGS: shrinking_factor=0.7, n_sub_iterations=3, max_piece_length=64, full byte initial_alphabet); vocab 128,260; tests whether 'Unigram loses' survives tuned HF UnigramTrainer hyperparameters; USED - balanced-1B bootstrap panel (S6.12, n=32/34) and paper_bootstrap_balanced_panel; not in panels.json main21 (superseded by default-kwargs gpt4o-balanced-unigram in the settled panel)" }, "gpt4o-code-bpe": { "slug": "gpt4o-code-bpe", "runs": [ "full-128k-gpt4o-code-bpe", "full-128k-gpt4o-code-bpe-mathcode-scratch", "full-128k-gpt4o-code-bpe-seed42", "pilot-128k-gpt4o-code-bpe-wdfix", "small-128k-gpt4o-code-bpe", "tiny-128k-gpt4o-code-bpe" ], "n_runs": 6, "redistributable": true, "loadable_with": "transformers", "hub_id": null, "policy_note": "trained in this project", "empirical": { "model_type": "BPE", "n_base_tokens": 128256, "n_added_tokens": 4, "vocab_total": 128260, "added_tokens": [ "", "", "", "" ], "normalizer": null, "pre_tokenizer": { "type": "Sequence", "pretokenizers": [ { "type": "Split", "pattern": { "Regex": "[^\\r\\n\\p{L}\\p{N}]?[\\p{Lu}\\p{Lt}\\p{Lm}\\p{Lo}\\p{M}]*[\\p{Ll}\\p{Lm}\\p{Lo}\\p{M}]+(?i:'s|'t|'re|'ve|'m|'ll|'d)?|[^\\r\\n\\p{L}\\p{N}]?[\\p{Lu}\\p{Lt}\\p{Lm}\\p{Lo}\\p{M}]+[\\p{Ll}\\p{Lm}\\p{Lo}\\p{M}]*(?i:'s|'t|'re|'ve|'m|'ll|'d)?|\\p{N}{1,3}| ?[^\\s\\p{L}\\p{N}]+[\\r\\n/]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+" }, "behavior": "Isolated", "invert": false }, { "type": "ByteLevel", "add_prefix_space": false, "trim_offsets": true, "use_regex": false } ] }, "has_post_processor": false, "byte_fallback": false, "bos_token": "", "bos_token_id": 0 }, "declared_experiments": { "algorithm": "bpe", "pretokenizer_constant": "REGEX_GPT4O", "normalizer": null, "data_composition": "code", "description": "GPT-4o regex, code-heavy data", "trainer_kwargs": null }, "declared_run_manifest": null, "registry_note": "P2 GPT-4o regex; code-heavy (D3) data; norm=none; BPE; vocab 128,260; USED - panels.json main21 (camera-ready main panel)" }, "gpt4o-codegrad00-bpe": { "slug": "gpt4o-codegrad00-bpe", "runs": [ "full-128k-gpt4o-codegrad00-bpe-mathcode-scratch" ], "n_runs": 1, "redistributable": true, "loadable_with": "transformers", "hub_id": null, "policy_note": "trained in this project", "empirical": { "model_type": "BPE", "n_base_tokens": 128256, "n_added_tokens": 4, "vocab_total": 128260, "added_tokens": [ "", "", "", "" ], "normalizer": null, "pre_tokenizer": { "type": "Sequence", "pretokenizers": [ { "type": "Split", "pattern": { "Regex": "[^\\r\\n\\p{L}\\p{N}]?[\\p{Lu}\\p{Lt}\\p{Lm}\\p{Lo}\\p{M}]*[\\p{Ll}\\p{Lm}\\p{Lo}\\p{M}]+(?i:'s|'t|'re|'ve|'m|'ll|'d)?|[^\\r\\n\\p{L}\\p{N}]?[\\p{Lu}\\p{Lt}\\p{Lm}\\p{Lo}\\p{M}]+[\\p{Ll}\\p{Lm}\\p{Lo}\\p{M}]*(?i:'s|'t|'re|'ve|'m|'ll|'d)?|\\p{N}{1,3}| ?[^\\s\\p{L}\\p{N}]+[\\r\\n/]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+" }, "behavior": "Isolated", "invert": false }, { "type": "ByteLevel", "add_prefix_space": false, "trim_offsets": true, "use_regex": false } ] }, "has_post_processor": false, "byte_fallback": false, "bos_token": "", "bos_token_id": 0 }, "declared_experiments": { "algorithm": "bpe", "pretokenizer_constant": "REGEX_GPT4O", "normalizer": null, "data_composition": "codegrad00", "description": "GPT-4o regex, code-fraction gradient c_w=0.00 (code omitted), BPE", "trainer_kwargs": null }, "declared_run_manifest": null, "registry_note": "#39, gpt4o pretok, slug `gpt4o-codegrad00-bpe`. AXES-2 code-fraction gradient, fusing regex (`REGEX_GPT4O`), code fraction 0%. Fused punct+newline vocab tokens: 482. FINISHED. MBPP 0.166 (generation_spec v1); v2 re-derivation (2026-08-04) 0.232, see \u00a73a.3. Not in panel." }, "gpt4o-codegrad05-bpe": { "slug": "gpt4o-codegrad05-bpe", "runs": [ "full-128k-gpt4o-codegrad05-bpe-mathcode-scratch" ], "n_runs": 1, "redistributable": true, "loadable_with": "transformers", "hub_id": null, "policy_note": "trained in this project", "empirical": { "model_type": "BPE", "n_base_tokens": 128256, "n_added_tokens": 4, "vocab_total": 128260, "added_tokens": [ "", "", "", "" ], "normalizer": null, "pre_tokenizer": { "type": "Sequence", "pretokenizers": [ { "type": "Split", "pattern": { "Regex": "[^\\r\\n\\p{L}\\p{N}]?[\\p{Lu}\\p{Lt}\\p{Lm}\\p{Lo}\\p{M}]*[\\p{Ll}\\p{Lm}\\p{Lo}\\p{M}]+(?i:'s|'t|'re|'ve|'m|'ll|'d)?|[^\\r\\n\\p{L}\\p{N}]?[\\p{Lu}\\p{Lt}\\p{Lm}\\p{Lo}\\p{M}]+[\\p{Ll}\\p{Lm}\\p{Lo}\\p{M}]*(?i:'s|'t|'re|'ve|'m|'ll|'d)?|\\p{N}{1,3}| ?[^\\s\\p{L}\\p{N}]+[\\r\\n/]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+" }, "behavior": "Isolated", "invert": false }, { "type": "ByteLevel", "add_prefix_space": false, "trim_offsets": true, "use_regex": false } ] }, "has_post_processor": false, "byte_fallback": false, "bos_token": "", "bos_token_id": 0 }, "declared_experiments": { "algorithm": "bpe", "pretokenizer_constant": "REGEX_GPT4O", "normalizer": null, "data_composition": "codegrad05", "description": "GPT-4o regex, code-fraction gradient c_w=0.05, BPE", "trainer_kwargs": null }, "declared_run_manifest": null, "registry_note": "#40, gpt4o pretok, slug `gpt4o-codegrad05-bpe`. AXES-2 code-fraction gradient, fusing regex (`REGEX_GPT4O`), code fraction 5%. Fused 749. FINISHED. MBPP 0.048 (v1); v2 0.050. Not in panel." }, "gpt4o-codegrad30-bpe": { "slug": "gpt4o-codegrad30-bpe", "runs": [ "full-128k-gpt4o-codegrad30-bpe-mathcode-scratch" ], "n_runs": 1, "redistributable": true, "loadable_with": "transformers", "hub_id": null, "policy_note": "trained in this project", "empirical": { "model_type": "BPE", "n_base_tokens": 128256, "n_added_tokens": 4, "vocab_total": 128260, "added_tokens": [ "", "", "", "" ], "normalizer": null, "pre_tokenizer": { "type": "Sequence", "pretokenizers": [ { "type": "Split", "pattern": { "Regex": "[^\\r\\n\\p{L}\\p{N}]?[\\p{Lu}\\p{Lt}\\p{Lm}\\p{Lo}\\p{M}]*[\\p{Ll}\\p{Lm}\\p{Lo}\\p{M}]+(?i:'s|'t|'re|'ve|'m|'ll|'d)?|[^\\r\\n\\p{L}\\p{N}]?[\\p{Lu}\\p{Lt}\\p{Lm}\\p{Lo}\\p{M}]+[\\p{Ll}\\p{Lm}\\p{Lo}\\p{M}]*(?i:'s|'t|'re|'ve|'m|'ll|'d)?|\\p{N}{1,3}| ?[^\\s\\p{L}\\p{N}]+[\\r\\n/]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+" }, "behavior": "Isolated", "invert": false }, { "type": "ByteLevel", "add_prefix_space": false, "trim_offsets": true, "use_regex": false } ] }, "has_post_processor": false, "byte_fallback": false, "bos_token": "", "bos_token_id": 0 }, "declared_experiments": { "algorithm": "bpe", "pretokenizer_constant": "REGEX_GPT4O", "normalizer": null, "data_composition": "codegrad30", "description": "GPT-4o regex, code-fraction gradient c_w=0.30, BPE", "trainer_kwargs": null }, "declared_run_manifest": null, "registry_note": "#41, gpt4o pretok, slug `gpt4o-codegrad30-bpe`. AXES-2 code-fraction gradient, fusing regex (`REGEX_GPT4O`), code fraction 30%. Fused 1393. FINISHED. MBPP 0.048 (v1); v2 0.178. Not in panel." }, "gpt4o-codegrad50-bpe": { "slug": "gpt4o-codegrad50-bpe", "runs": [ "full-128k-gpt4o-codegrad50-bpe-mathcode-scratch" ], "n_runs": 1, "redistributable": true, "loadable_with": "transformers", "hub_id": null, "policy_note": "trained in this project", "empirical": { "model_type": "BPE", "n_base_tokens": 128256, "n_added_tokens": 4, "vocab_total": 128260, "added_tokens": [ "", "", "", "" ], "normalizer": null, "pre_tokenizer": { "type": "Sequence", "pretokenizers": [ { "type": "Split", "pattern": { "Regex": "[^\\r\\n\\p{L}\\p{N}]?[\\p{Lu}\\p{Lt}\\p{Lm}\\p{Lo}\\p{M}]*[\\p{Ll}\\p{Lm}\\p{Lo}\\p{M}]+(?i:'s|'t|'re|'ve|'m|'ll|'d)?|[^\\r\\n\\p{L}\\p{N}]?[\\p{Lu}\\p{Lt}\\p{Lm}\\p{Lo}\\p{M}]+[\\p{Ll}\\p{Lm}\\p{Lo}\\p{M}]*(?i:'s|'t|'re|'ve|'m|'ll|'d)?|\\p{N}{1,3}| ?[^\\s\\p{L}\\p{N}]+[\\r\\n/]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+" }, "behavior": "Isolated", "invert": false }, { "type": "ByteLevel", "add_prefix_space": false, "trim_offsets": true, "use_regex": false } ] }, "has_post_processor": false, "byte_fallback": false, "bos_token": "", "bos_token_id": 0 }, "declared_experiments": { "algorithm": "bpe", "pretokenizer_constant": "REGEX_GPT4O", "normalizer": null, "data_composition": "codegrad50", "description": "GPT-4o regex, code-fraction gradient c_w=0.50, BPE", "trainer_kwargs": null }, "declared_run_manifest": null, "registry_note": "#42, gpt4o pretok, slug `gpt4o-codegrad50-bpe`. AXES-2 code-fraction gradient, fusing regex (`REGEX_GPT4O`), code fraction 50%. Fused 1831. FINISHED. MBPP 0.108 (v1); v2 0.088. Not in panel." }, "gpt4o-english-bpe": { "slug": "gpt4o-english-bpe", "runs": [ "full-128k-gpt4o-english-bpe", "full-128k-gpt4o-english-bpe-mathcode-scratch", "full-128k-gpt4o-english-bpe-seed42", "pilot-128k-gpt4o-english-bpe-wdfix", "small-128k-gpt4o-english-bpe", "tiny-128k-gpt4o-english-bpe" ], "n_runs": 6, "redistributable": true, "loadable_with": "transformers", "hub_id": null, "policy_note": "trained in this project", "empirical": { "model_type": "BPE", "n_base_tokens": 128256, "n_added_tokens": 4, "vocab_total": 128260, "added_tokens": [ "", "", "", "" ], "normalizer": null, "pre_tokenizer": { "type": "Sequence", "pretokenizers": [ { "type": "Split", "pattern": { "Regex": "[^\\r\\n\\p{L}\\p{N}]?[\\p{Lu}\\p{Lt}\\p{Lm}\\p{Lo}\\p{M}]*[\\p{Ll}\\p{Lm}\\p{Lo}\\p{M}]+(?i:'s|'t|'re|'ve|'m|'ll|'d)?|[^\\r\\n\\p{L}\\p{N}]?[\\p{Lu}\\p{Lt}\\p{Lm}\\p{Lo}\\p{M}]+[\\p{Ll}\\p{Lm}\\p{Lo}\\p{M}]*(?i:'s|'t|'re|'ve|'m|'ll|'d)?|\\p{N}{1,3}| ?[^\\s\\p{L}\\p{N}]+[\\r\\n/]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+" }, "behavior": "Isolated", "invert": false }, { "type": "ByteLevel", "add_prefix_space": false, "trim_offsets": true, "use_regex": false } ] }, "has_post_processor": false, "byte_fallback": false, "bos_token": "", "bos_token_id": 0 }, "declared_experiments": { "algorithm": "bpe", "pretokenizer_constant": "REGEX_GPT4O", "normalizer": null, "data_composition": "english", "description": "GPT-4o regex, English-only data", "trainer_kwargs": null }, "declared_run_manifest": null, "registry_note": "P2 GPT-4o regex; english (D1) data; norm=none; BPE; vocab 128,260; affected by the initial_alphabet byte-drop defect (drops 32 control bytes incl. '{' and DEL); see gpt4o-english-fullbyte-bpe control; USED - panels.json main21 (camera-ready main panel)" }, "gpt4o-english-fullbyte-bpe": { "slug": "gpt4o-english-fullbyte-bpe", "runs": [ "full-128k-gpt4o-english-fullbyte-bpe", "full-128k-gpt4o-english-fullbyte-bpe-mathcode-scratch" ], "n_runs": 2, "redistributable": true, "loadable_with": "transformers", "hub_id": null, "policy_note": "trained in this project", "empirical": { "model_type": "BPE", "n_base_tokens": 128256, "n_added_tokens": 4, "vocab_total": 128260, "added_tokens": [ "", "", "", "" ], "normalizer": null, "pre_tokenizer": { "type": "Sequence", "pretokenizers": [ { "type": "Split", "pattern": { "Regex": "[^\\r\\n\\p{L}\\p{N}]?[\\p{Lu}\\p{Lt}\\p{Lm}\\p{Lo}\\p{M}]*[\\p{Ll}\\p{Lm}\\p{Lo}\\p{M}]+(?i:'s|'t|'re|'ve|'m|'ll|'d)?|[^\\r\\n\\p{L}\\p{N}]?[\\p{Lu}\\p{Lt}\\p{Lm}\\p{Lo}\\p{M}]+[\\p{Ll}\\p{Lm}\\p{Lo}\\p{M}]*(?i:'s|'t|'re|'ve|'m|'ll|'d)?|\\p{N}{1,3}| ?[^\\s\\p{L}\\p{N}]+[\\r\\n/]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+" }, "behavior": "Isolated", "invert": false }, { "type": "ByteLevel", "add_prefix_space": false, "trim_offsets": true, "use_regex": false } ] }, "has_post_processor": false, "byte_fallback": false, "bos_token": "", "bos_token_id": 0 }, "declared_experiments": { "algorithm": "bpe", "pretokenizer_constant": "REGEX_GPT4O", "normalizer": null, "data_composition": "english", "description": "GPT-4o regex, English-only data, full byte alphabet seeded (control for the missing-initial_alphabet byte-drop defect; matched pair vs gpt4o-english-bpe)", "trainer_kwargs": { "full_byte_alphabet": true } }, "declared_run_manifest": null, "registry_note": "P2 GPT-4o regex; english (D1) data; norm=none; BPE (trainer_kwargs full_byte_alphabet=True: initial_alphabet forced to the complete 256-entry ByteLevel alphabet); vocab 128,256; isolates the initial_alphabet byte-drop defect vs gpt4o-english-bpe (209/256 -> 256/256 bytes present; 47 fewer merges); USED - panels.json pretok-controls (camera-ready)" }, "gpt4onl-balanced-bpe": { "slug": "gpt4onl-balanced-bpe", "runs": [ "full-128k-gpt4onl-balanced-bpe-mathcode-scratch", "full-128k-gpt4onl-balanced-bpe-seed43-mathcode-scratch", "full-128k-gpt4onl-balanced-bpe-seed44-mathcode-scratch" ], "n_runs": 3, "redistributable": true, "loadable_with": "transformers", "hub_id": null, "policy_note": "trained in this project", "empirical": { "model_type": "BPE", "n_base_tokens": 128256, "n_added_tokens": 4, "vocab_total": 128260, "added_tokens": [ "", "", "", "" ], "normalizer": null, "pre_tokenizer": { "type": "Sequence", "pretokenizers": [ { "type": "Split", "pattern": { "Regex": "[^\\r\\n\\p{L}\\p{N}]?[\\p{Lu}\\p{Lt}\\p{Lm}\\p{Lo}\\p{M}]*[\\p{Ll}\\p{Lm}\\p{Lo}\\p{M}]+(?i:'s|'t|'re|'ve|'m|'ll|'d)?|[^\\r\\n\\p{L}\\p{N}]?[\\p{Lu}\\p{Lt}\\p{Lm}\\p{Lo}\\p{M}]+[\\p{Ll}\\p{Lm}\\p{Lo}\\p{M}]*(?i:'s|'t|'re|'ve|'m|'ll|'d)?|\\p{N}{1,3}| ?[^\\s\\p{L}\\p{N}]+|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+" }, "behavior": "Isolated", "invert": false }, { "type": "ByteLevel", "add_prefix_space": false, "trim_offsets": true, "use_regex": false } ] }, "has_post_processor": false, "byte_fallback": false, "bos_token": "", "bos_token_id": 0 }, "declared_experiments": { "algorithm": "bpe", "pretokenizer_constant": "REGEX_GPT4O_NONL", "normalizer": null, "data_composition": "balanced", "description": "GPT-4o regex with the punct-arm newline-fusion tail removed (AXES-2 single-mechanism ablation vs gpt4o-balanced-bpe), balanced data, BPE", "trainer_kwargs": null }, "declared_run_manifest": null, "registry_note": "#47, gpt4onl pretok, slug `gpt4onl-balanced-bpe`. AXES-2 single-mechanism pair: non-fusing regex on the balanced composition, fused 0. FINISHED. v1: MBPP 0.158, HumanEval 0.0183, GSM8K 0.242. Fusing counterpart is the unslugged baseline `full-128k-mathcode-scratch` (gpt4o-balanced-bpe, same recipe/step). Pair v1: MBPP 0.024 vs 0.158, HumanEval 0.0854 vs 0.0183 - benchmarks disagree on sign; both have clean prompt boundaries (\u00a73a.6), not the HumanEval artifact. v2 (2026-08-04): this run MBPP 0.224/HumanEval 0.1890; baseline MBPP 0.174/HumanEval 0.1646. Not in panel." }, "gpt4onl-codegrad00-bpe": { "slug": "gpt4onl-codegrad00-bpe", "runs": [ "full-128k-gpt4onl-codegrad00-bpe-mathcode-scratch" ], "n_runs": 1, "redistributable": true, "loadable_with": "transformers", "hub_id": null, "policy_note": "trained in this project", "empirical": { "model_type": "BPE", "n_base_tokens": 128256, "n_added_tokens": 4, "vocab_total": 128260, "added_tokens": [ "", "", "", "" ], "normalizer": null, "pre_tokenizer": { "type": "Sequence", "pretokenizers": [ { "type": "Split", "pattern": { "Regex": "[^\\r\\n\\p{L}\\p{N}]?[\\p{Lu}\\p{Lt}\\p{Lm}\\p{Lo}\\p{M}]*[\\p{Ll}\\p{Lm}\\p{Lo}\\p{M}]+(?i:'s|'t|'re|'ve|'m|'ll|'d)?|[^\\r\\n\\p{L}\\p{N}]?[\\p{Lu}\\p{Lt}\\p{Lm}\\p{Lo}\\p{M}]+[\\p{Ll}\\p{Lm}\\p{Lo}\\p{M}]*(?i:'s|'t|'re|'ve|'m|'ll|'d)?|\\p{N}{1,3}| ?[^\\s\\p{L}\\p{N}]+|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+" }, "behavior": "Isolated", "invert": false }, { "type": "ByteLevel", "add_prefix_space": false, "trim_offsets": true, "use_regex": false } ] }, "has_post_processor": false, "byte_fallback": false, "bos_token": "", "bos_token_id": 0 }, "declared_experiments": { "algorithm": "bpe", "pretokenizer_constant": "REGEX_GPT4O_NONL", "normalizer": null, "data_composition": "codegrad00", "description": "GPT-4o-NoNL regex, code-fraction gradient c_w=0.00 (code omitted), BPE", "trainer_kwargs": null }, "declared_run_manifest": null, "registry_note": "#43, gpt4onl pretok, slug `gpt4onl-codegrad00-bpe`. AXES-2 code-fraction gradient, non-fusing regex (REGEX_GPT4O_NONL: punct arm loses trailing [\\r\\n/]*, else byte-identical to REGEX_GPT4O), tokenizer-corpus code fraction 0%. Fused 0 by construction. FINISHED. MBPP 0.232 (generation_spec v1), the highest of any run in the 20B mathcode family (v1-scored, like-for-like with the panel). v2 re-derivation (2026-08-04) 0.248, see \u00a73a.3. Not in panel." }, "gpt4onl-codegrad05-bpe": { "slug": "gpt4onl-codegrad05-bpe", "runs": [ "full-128k-gpt4onl-codegrad05-bpe-mathcode-scratch" ], "n_runs": 1, "redistributable": true, "loadable_with": "transformers", "hub_id": null, "policy_note": "trained in this project", "empirical": { "model_type": "BPE", "n_base_tokens": 128256, "n_added_tokens": 4, "vocab_total": 128260, "added_tokens": [ "", "", "", "" ], "normalizer": null, "pre_tokenizer": { "type": "Sequence", "pretokenizers": [ { "type": "Split", "pattern": { "Regex": "[^\\r\\n\\p{L}\\p{N}]?[\\p{Lu}\\p{Lt}\\p{Lm}\\p{Lo}\\p{M}]*[\\p{Ll}\\p{Lm}\\p{Lo}\\p{M}]+(?i:'s|'t|'re|'ve|'m|'ll|'d)?|[^\\r\\n\\p{L}\\p{N}]?[\\p{Lu}\\p{Lt}\\p{Lm}\\p{Lo}\\p{M}]+[\\p{Ll}\\p{Lm}\\p{Lo}\\p{M}]*(?i:'s|'t|'re|'ve|'m|'ll|'d)?|\\p{N}{1,3}| ?[^\\s\\p{L}\\p{N}]+|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+" }, "behavior": "Isolated", "invert": false }, { "type": "ByteLevel", "add_prefix_space": false, "trim_offsets": true, "use_regex": false } ] }, "has_post_processor": false, "byte_fallback": false, "bos_token": "", "bos_token_id": 0 }, "declared_experiments": { "algorithm": "bpe", "pretokenizer_constant": "REGEX_GPT4O_NONL", "normalizer": null, "data_composition": "codegrad05", "description": "GPT-4o-NoNL regex, code-fraction gradient c_w=0.05, BPE", "trainer_kwargs": null }, "declared_run_manifest": null, "registry_note": "#44, gpt4onl pretok, slug `gpt4onl-codegrad05-bpe`. AXES-2 code-fraction gradient, non-fusing regex (`REGEX_GPT4O_NONL`), code fraction 5%. Fused 0. FINISHED. MBPP 0.204 (v1); v2 0.206. Not in panel." }, "gpt4onl-codegrad30-bpe": { "slug": "gpt4onl-codegrad30-bpe", "runs": [ "full-128k-gpt4onl-codegrad30-bpe-mathcode-scratch" ], "n_runs": 1, "redistributable": true, "loadable_with": "transformers", "hub_id": null, "policy_note": "trained in this project", "empirical": { "model_type": "BPE", "n_base_tokens": 128256, "n_added_tokens": 4, "vocab_total": 128260, "added_tokens": [ "", "", "", "" ], "normalizer": null, "pre_tokenizer": { "type": "Sequence", "pretokenizers": [ { "type": "Split", "pattern": { "Regex": "[^\\r\\n\\p{L}\\p{N}]?[\\p{Lu}\\p{Lt}\\p{Lm}\\p{Lo}\\p{M}]*[\\p{Ll}\\p{Lm}\\p{Lo}\\p{M}]+(?i:'s|'t|'re|'ve|'m|'ll|'d)?|[^\\r\\n\\p{L}\\p{N}]?[\\p{Lu}\\p{Lt}\\p{Lm}\\p{Lo}\\p{M}]+[\\p{Ll}\\p{Lm}\\p{Lo}\\p{M}]*(?i:'s|'t|'re|'ve|'m|'ll|'d)?|\\p{N}{1,3}| ?[^\\s\\p{L}\\p{N}]+|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+" }, "behavior": "Isolated", "invert": false }, { "type": "ByteLevel", "add_prefix_space": false, "trim_offsets": true, "use_regex": false } ] }, "has_post_processor": false, "byte_fallback": false, "bos_token": "", "bos_token_id": 0 }, "declared_experiments": { "algorithm": "bpe", "pretokenizer_constant": "REGEX_GPT4O_NONL", "normalizer": null, "data_composition": "codegrad30", "description": "GPT-4o-NoNL regex, code-fraction gradient c_w=0.30, BPE", "trainer_kwargs": null }, "declared_run_manifest": null, "registry_note": "#45, gpt4onl pretok, slug `gpt4onl-codegrad30-bpe`. AXES-2 code-fraction gradient, non-fusing regex (`REGEX_GPT4O_NONL`), code fraction 30%. Fused 0. FINISHED. MBPP 0.178 (v1); v2 0.224. Not in panel." }, "gpt4onl-codegrad50-bpe": { "slug": "gpt4onl-codegrad50-bpe", "runs": [ "full-128k-gpt4onl-codegrad50-bpe-mathcode-scratch" ], "n_runs": 1, "redistributable": true, "loadable_with": "transformers", "hub_id": null, "policy_note": "trained in this project", "empirical": { "model_type": "BPE", "n_base_tokens": 128256, "n_added_tokens": 4, "vocab_total": 128260, "added_tokens": [ "", "", "", "" ], "normalizer": null, "pre_tokenizer": { "type": "Sequence", "pretokenizers": [ { "type": "Split", "pattern": { "Regex": "[^\\r\\n\\p{L}\\p{N}]?[\\p{Lu}\\p{Lt}\\p{Lm}\\p{Lo}\\p{M}]*[\\p{Ll}\\p{Lm}\\p{Lo}\\p{M}]+(?i:'s|'t|'re|'ve|'m|'ll|'d)?|[^\\r\\n\\p{L}\\p{N}]?[\\p{Lu}\\p{Lt}\\p{Lm}\\p{Lo}\\p{M}]+[\\p{Ll}\\p{Lm}\\p{Lo}\\p{M}]*(?i:'s|'t|'re|'ve|'m|'ll|'d)?|\\p{N}{1,3}| ?[^\\s\\p{L}\\p{N}]+|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+" }, "behavior": "Isolated", "invert": false }, { "type": "ByteLevel", "add_prefix_space": false, "trim_offsets": true, "use_regex": false } ] }, "has_post_processor": false, "byte_fallback": false, "bos_token": "", "bos_token_id": 0 }, "declared_experiments": { "algorithm": "bpe", "pretokenizer_constant": "REGEX_GPT4O_NONL", "normalizer": null, "data_composition": "codegrad50", "description": "GPT-4o-NoNL regex, code-fraction gradient c_w=0.50, BPE", "trainer_kwargs": null }, "declared_run_manifest": null, "registry_note": "#46, gpt4onl pretok, slug `gpt4onl-codegrad50-bpe`. AXES-2 code-fraction gradient, non-fusing regex (`REGEX_GPT4O_NONL`), code fraction 50%. Fused 0. FINISHED. MBPP 0.174 (v1); v2 0.212. Not in panel." }, "highmid-bpe": { "slug": "highmid-bpe", "runs": [ "full-128k-highmid-bpe" ], "n_runs": 1, "redistributable": true, "loadable_with": "transformers", "hub_id": null, "policy_note": "trained in this project", "empirical": { "model_type": "BPE", "n_base_tokens": 128256, "n_added_tokens": 4, "vocab_total": 128260, "added_tokens": [ "", "", "", "" ], "normalizer": null, "pre_tokenizer": { "type": "Sequence", "pretokenizers": [ { "type": "Split", "pattern": { "Regex": "[^\\r\\n\\p{L}\\p{N}]?[\\p{Lu}\\p{Lt}\\p{Lm}\\p{Lo}\\p{M}]*[\\p{Ll}\\p{Lm}\\p{Lo}\\p{M}]+(?i:'s|'t|'re|'ve|'m|'ll|'d)?|[^\\r\\n\\p{L}\\p{N}]?[\\p{Lu}\\p{Lt}\\p{Lm}\\p{Lo}\\p{M}]+[\\p{Ll}\\p{Lm}\\p{Lo}\\p{M}]*(?i:'s|'t|'re|'ve|'m|'ll|'d)?|\\p{N}{1,3}| ?[^\\s\\p{L}\\p{N}]+[\\r\\n/]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+" }, "behavior": "Isolated", "invert": false }, { "type": "ByteLevel", "add_prefix_space": false, "trim_offsets": true, "use_regex": false } ] }, "has_post_processor": false, "byte_fallback": false, "bos_token": "", "bos_token_id": 0 }, "declared_experiments": { "algorithm": "bpe", "pretokenizer_constant": "REGEX_GPT4O", "normalizer": null, "data_composition": "highmid", "description": "GPT-4o regex, high+mid-resource subset (21 langs), BPE", "trainer_kwargs": null }, "declared_run_manifest": null, "registry_note": "P2 GPT-4o regex; highmid (21 high+mid-resource langs + English + math/code) data; norm=none; BPE; vocab 128,260; intermediate-language-coverage probe (tokenizer sees 21 langs); USED - S6d.3 coverage ladder; superseded within panels.json main21 by highmid-unigram" }, "highmid-unigram": { "slug": "highmid-unigram", "runs": [ "full-128k-highmid-unigram" ], "n_runs": 1, "redistributable": true, "loadable_with": "transformers", "hub_id": null, "policy_note": "trained in this project", "empirical": { "model_type": "Unigram", "n_base_tokens": 128256, "n_added_tokens": 4, "vocab_total": 128260, "added_tokens": [ "", "", "", "" ], "normalizer": null, "pre_tokenizer": { "type": "Sequence", "pretokenizers": [ { "type": "Split", "pattern": { "Regex": "[^\\r\\n\\p{L}\\p{N}]?[\\p{Lu}\\p{Lt}\\p{Lm}\\p{Lo}\\p{M}]*[\\p{Ll}\\p{Lm}\\p{Lo}\\p{M}]+(?i:'s|'t|'re|'ve|'m|'ll|'d)?|[^\\r\\n\\p{L}\\p{N}]?[\\p{Lu}\\p{Lt}\\p{Lm}\\p{Lo}\\p{M}]+[\\p{Ll}\\p{Lm}\\p{Lo}\\p{M}]*(?i:'s|'t|'re|'ve|'m|'ll|'d)?|\\p{N}{1,3}| ?[^\\s\\p{L}\\p{N}]+[\\r\\n/]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+" }, "behavior": "Isolated", "invert": false }, { "type": "ByteLevel", "add_prefix_space": false, "trim_offsets": true, "use_regex": false } ] }, "has_post_processor": false, "byte_fallback": false, "bos_token": "", "bos_token_id": 0 }, "declared_experiments": { "algorithm": "unigram", "pretokenizer_constant": "REGEX_GPT4O", "normalizer": null, "data_composition": "highmid", "description": "GPT-4o regex, high+mid-resource subset (21 langs), UnigramLM", "trainer_kwargs": null }, "declared_run_manifest": null, "registry_note": "P2 GPT-4o regex; highmid (21 langs) data; norm=none; UnigramLM (HF, default kwargs); vocab 128,260; Unigram counterpart to highmid-bpe; USED - panels.json main21 (camera-ready main panel)" }, "highres-bpe": { "slug": "highres-bpe", "runs": [ "full-128k-highres-bpe" ], "n_runs": 1, "redistributable": true, "loadable_with": "transformers", "hub_id": null, "policy_note": "trained in this project", "empirical": { "model_type": "BPE", "n_base_tokens": 128256, "n_added_tokens": 4, "vocab_total": 128260, "added_tokens": [ "", "", "", "" ], "normalizer": null, "pre_tokenizer": { "type": "Sequence", "pretokenizers": [ { "type": "Split", "pattern": { "Regex": "[^\\r\\n\\p{L}\\p{N}]?[\\p{Lu}\\p{Lt}\\p{Lm}\\p{Lo}\\p{M}]*[\\p{Ll}\\p{Lm}\\p{Lo}\\p{M}]+(?i:'s|'t|'re|'ve|'m|'ll|'d)?|[^\\r\\n\\p{L}\\p{N}]?[\\p{Lu}\\p{Lt}\\p{Lm}\\p{Lo}\\p{M}]+[\\p{Ll}\\p{Lm}\\p{Lo}\\p{M}]*(?i:'s|'t|'re|'ve|'m|'ll|'d)?|\\p{N}{1,3}| ?[^\\s\\p{L}\\p{N}]+[\\r\\n/]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+" }, "behavior": "Isolated", "invert": false }, { "type": "ByteLevel", "add_prefix_space": false, "trim_offsets": true, "use_regex": false } ] }, "has_post_processor": false, "byte_fallback": false, "bos_token": "", "bos_token_id": 0 }, "declared_experiments": { "algorithm": "bpe", "pretokenizer_constant": "REGEX_GPT4O", "normalizer": null, "data_composition": "highres", "description": "GPT-4o regex, high-resource-only subset (6 langs), BPE", "trainer_kwargs": null }, "declared_run_manifest": null, "registry_note": "P2 GPT-4o regex; highres (6 high-resource langs + English + math/code) data; norm=none; BPE; vocab 128,260; intermediate-language-coverage probe (tokenizer sees 6 langs, LM trains on all 30); USED - S6d.3 coverage ladder (english-only/highres/highmid/balanced GPT-4o-BPE comparison); superseded within panels.json main21 by highres-unigram" }, "highres-unigram": { "slug": "highres-unigram", "runs": [ "full-128k-highres-unigram" ], "n_runs": 1, "redistributable": true, "loadable_with": "transformers", "hub_id": null, "policy_note": "trained in this project", "empirical": { "model_type": "Unigram", "n_base_tokens": 128256, "n_added_tokens": 4, "vocab_total": 128260, "added_tokens": [ "", "", "", "" ], "normalizer": null, "pre_tokenizer": { "type": "Sequence", "pretokenizers": [ { "type": "Split", "pattern": { "Regex": "[^\\r\\n\\p{L}\\p{N}]?[\\p{Lu}\\p{Lt}\\p{Lm}\\p{Lo}\\p{M}]*[\\p{Ll}\\p{Lm}\\p{Lo}\\p{M}]+(?i:'s|'t|'re|'ve|'m|'ll|'d)?|[^\\r\\n\\p{L}\\p{N}]?[\\p{Lu}\\p{Lt}\\p{Lm}\\p{Lo}\\p{M}]+[\\p{Ll}\\p{Lm}\\p{Lo}\\p{M}]*(?i:'s|'t|'re|'ve|'m|'ll|'d)?|\\p{N}{1,3}| ?[^\\s\\p{L}\\p{N}]+[\\r\\n/]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+" }, "behavior": "Isolated", "invert": false }, { "type": "ByteLevel", "add_prefix_space": false, "trim_offsets": true, "use_regex": false } ] }, "has_post_processor": false, "byte_fallback": false, "bos_token": "", "bos_token_id": 0 }, "declared_experiments": { "algorithm": "unigram", "pretokenizer_constant": "REGEX_GPT4O", "normalizer": null, "data_composition": "highres", "description": "GPT-4o regex, high-resource-only subset (6 langs), UnigramLM", "trainer_kwargs": null }, "declared_run_manifest": null, "registry_note": "P2 GPT-4o regex; highres (6 langs) data; norm=none; UnigramLM (HF, default kwargs); vocab 128,260; Unigram counterpart to highres-bpe, completes the {BPE,Unigram}x{highres,highmid,balanced} grid; USED - panels.json main21 (camera-ready main panel)" }, "llama3": { "slug": "llama3", "runs": [ "full-128k-llama3-mathcode-scratch", "pilot-128k-llama3-wdfix" ], "n_runs": 2, "redistributable": false, "loadable_with": "transformers", "hub_id": "NousResearch/Meta-Llama-3-8B", "policy_note": "These runs record the local mirror directory rather than the hub id; the file is the same off-the-shelf tokenizer.", "empirical": null, "declared_experiments": null, "declared_run_manifest": null, "registry_note": "#32, llama3 pretok, slug `llama3`. Off-the-shelf Meta-Llama-3 tokenizer, vocab 128,256. FINISHED (step 19073). MBPP 0.166 [0.134, 0.198]." }, "pabpe-nfc-apertus-fineweb2full-capped-tuned-hybridwindow": { "slug": "pabpe-nfc-apertus-fineweb2full-capped-tuned-hybridwindow", "runs": [ "full-128k-pabpe-nfc-apertus-fineweb2full-capped-tuned-hybridwindow", "full-128k-pabpe-nfc-apertus-fineweb2full-capped-tuned-hybridwindow-mathcode-scratch" ], "n_runs": 2, "redistributable": true, "loadable_with": "transformers", "hub_id": null, "policy_note": "trained in this project", "empirical": { "model_type": "BPE", "n_base_tokens": 127831, "n_added_tokens": 4, "vocab_total": 127835, "added_tokens": [ "", "", "", "" ], "normalizer": { "type": "NFC" }, "pre_tokenizer": { "type": "Sequence", "pretokenizers": [ { "type": "Split", "pattern": { "Regex": "[^\\r\\n\\p{L}\\p{N}]?[\\p{Lu}\\p{Lt}\\p{Lm}\\p{Lo}\\p{M}]*[\\p{Ll}\\p{Lm}\\p{Lo}\\p{M}]+|[^\\r\\n\\p{L}\\p{N}]?[\\p{Lu}\\p{Lt}\\p{Lm}\\p{Lo}\\p{M}]+[\\p{Ll}\\p{Lm}\\p{Lo}\\p{M}]*|\\p{N}| ?[^\\s\\p{L}\\p{N}]{1,16}[\\r\\n]{0,16}|\\s{0,16}[\\r\\n]{1,16}|\\s{1,16}(?!\\S)|\\s{1,16}" }, "behavior": "Isolated", "invert": false }, { "type": "ByteLevel", "add_prefix_space": false, "trim_offsets": true, "use_regex": false } ] }, "has_post_processor": false, "byte_fallback": false, "bos_token": "", "bos_token_id": 127831 }, "declared_experiments": null, "declared_run_manifest": { "variant": "pa_bpe_nfc_apertus_capped_hybrid_window_tuned", "variant_config": { "trainer": "parity-bpe", "pretok": "apertus_capped_nfc", "variant": "window", "global_merges": 64000, "window_size": 100, "alpha": 2.0, "signal": "ratios", "grouped_config": "configs/parity_aware_config_grouped_fineweb2full_quota_tuned.json", "special_tokens_override": [], "decoder_add_prefix_space": true, "output_group": "tokenizer-lm-toks", "output_dir_name": "nfc_apertus_capped_hybrid_window_tuned", "post_training_special_tokens": [ "", "", "", "" ] }, "grouped_config_path": "/users/cmeister747/pa_tokenizers_branch/configs/parity_aware_config_grouped_fineweb2full_quota_tuned.json", "vocab_size": 128000, "min_frequency": 2, "special_tokens": [], "start_time": "2026-05-23T10:29:11.168479", "elapsed_seconds": 7230.190345, "slurm_job_id": "2353211", "cli_args": [ "/users/cmeister747/pa_tokenizers_branch/train_tokenizer.py", "--variant", "pa_bpe_nfc_apertus_capped_hybrid_window_tuned" ], "git": { "commit": "5d8381af9ec1417b078f7dce82bac68873da915e", "branch": "parity-aware-bpe", "dirty": false }, "per_group_summary": { "n_groups": 25, "full_block_in": "tokenizers/pabpe-nfc-apertus-fineweb2full-capped-tuned-hybridwindow/run_manifest.json" } }, "registry_note": "PA-BPE hybrid global+window, apertus/Mistral-Nemo pretok regex; fineweb2full, capped + tuned data; norm=NFC; PA-BPE (hybrid global+window); vocab 127,835; 2x2 candidate; the apertus-pretok arm later diagnosed with the MBPP-collapse RCA (fuses code line-endings into superwords) at 20B scale; USED - Apertus-2 development track candidate; excluded from main paper panel" }, "pabpe-nfc-clean-balanced": { "slug": "pabpe-nfc-clean-balanced", "runs": [ "full-128k-pabpe-nfc-clean-balanced" ], "n_runs": 1, "redistributable": true, "loadable_with": "transformers", "hub_id": null, "policy_note": "trained in this project", "empirical": { "model_type": "BPE", "n_base_tokens": 127832, "n_added_tokens": 4, "vocab_total": 127836, "added_tokens": [ "", "", "", "" ], "normalizer": { "type": "NFC" }, "pre_tokenizer": { "type": "Sequence", "pretokenizers": [ { "type": "Split", "pattern": { "Regex": "[ ]?[\\p{Lu}\\p{Lt}\\p{Lm}\\p{Lo}\\p{M}]*[\\p{Ll}\\p{Lm}\\p{Lo}\\p{M}]+|[ ]?[\\p{Lu}\\p{Lt}\\p{Lm}\\p{Lo}\\p{M}]+[\\p{Ll}\\p{Lm}\\p{Lo}\\p{M}]*|\\p{N}| ?[^\\s\\p{L}\\p{N}]+|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+" }, "behavior": "Isolated", "invert": false }, { "type": "ByteLevel", "add_prefix_space": false, "trim_offsets": true, "use_regex": false } ] }, "has_post_processor": false, "byte_fallback": false, "bos_token": "", "bos_token_id": 127832 }, "declared_experiments": null, "declared_run_manifest": { "variant": "pa_bpe_nfc_clean_multi_balanced", "variant_config": { "trainer": "parity-bpe", "pretok": "clean_multi_nfc", "variant": "base", "global_merges": 0, "signal": "ratios", "grouped_config": "configs/parity_aware_config_balanced.json", "special_tokens_override": [], "decoder_add_prefix_space": true, "output_group": "tokenizer-lm-toks", "output_dir_name": "nfc_clean_multi_balanced", "post_training_special_tokens": [ "", "", "", "" ] }, "grouped_config_path": "/users/cmeister747/pa_tokenizers_branch/configs/parity_aware_config_balanced.json", "vocab_size": 128000, "min_frequency": 2, "special_tokens": [], "start_time": "2026-05-16T00:20:05.420579", "elapsed_seconds": 2555.150446, "slurm_job_id": "2257133", "cli_args": [ "/users/cmeister747/pa_tokenizers_branch/train_tokenizer.py", "--variant", "pa_bpe_nfc_clean_multi_balanced" ], "git": { "commit": "b2ceac78388c06deb786755b60d6c1d335cb6c25", "branch": "parity-aware-bpe", "dirty": false }, "per_group_summary": { "n_groups": 34, "full_block_in": "tokenizers/pabpe-nfc-clean-balanced/run_manifest.json" } }, "registry_note": "PA-BPE (parity, base) with clean-multi regex; balanced (D2) data; norm=NFC; PA-BPE (base); vocab 127,836; same data as pabpe-nfc-gpt4-balanced (2a), different pretok regex (controlled pretok ablation); bug-fixed 2026-05-16 replacement (2b) for retired pabpe-balanced; USED - balanced-1B bootstrap panel (S6.12) + paper panel; not in panels.json main21" }, "pabpe-nfc-clean-balanced-hybridwindow": { "slug": "pabpe-nfc-clean-balanced-hybridwindow", "runs": [ "full-128k-pabpe-nfc-clean-balanced-hybridwindow-mathcode-scratch" ], "n_runs": 1, "redistributable": true, "loadable_with": "transformers", "hub_id": null, "policy_note": "trained in this project", "empirical": { "model_type": "BPE", "n_base_tokens": 127832, "n_added_tokens": 4, "vocab_total": 127836, "added_tokens": [ "", "", "", "" ], "normalizer": { "type": "NFC" }, "pre_tokenizer": { "type": "Sequence", "pretokenizers": [ { "type": "Split", "pattern": { "Regex": "[ ]?[\\p{Lu}\\p{Lt}\\p{Lm}\\p{Lo}\\p{M}]*[\\p{Ll}\\p{Lm}\\p{Lo}\\p{M}]+|[ ]?[\\p{Lu}\\p{Lt}\\p{Lm}\\p{Lo}\\p{M}]+[\\p{Ll}\\p{Lm}\\p{Lo}\\p{M}]*|\\p{N}| ?[^\\s\\p{L}\\p{N}]+|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+" }, "behavior": "Isolated", "invert": false }, { "type": "ByteLevel", "add_prefix_space": false, "trim_offsets": true, "use_regex": false } ] }, "has_post_processor": false, "byte_fallback": false, "bos_token": "", "bos_token_id": 127832 }, "declared_experiments": null, "declared_run_manifest": { "variant": "pa_bpe_nfc_clean_multi_balanced_hybrid_window", "variant_config": { "trainer": "parity-bpe", "pretok": "clean_multi_nfc", "variant": "window", "global_merges": 64000, "window_size": 100, "alpha": 2.0, "signal": "ratios", "grouped_config": "configs/parity_aware_config_balanced.json", "special_tokens_override": [], "decoder_add_prefix_space": true, "output_group": "tokenizer-lm-toks", "output_dir_name": "nfc_clean_multi_balanced_hybrid_window", "post_training_special_tokens": [ "", "", "", "" ] }, "grouped_config_path": "/users/cmeister747/pa_tokenizers_branch/configs/parity_aware_config_balanced.json", "vocab_size": 128000, "min_frequency": 2, "special_tokens": [], "start_time": "2026-05-16T15:19:40.285398", "elapsed_seconds": 2324.797585, "slurm_job_id": "2260977", "cli_args": [ "/users/cmeister747/pa_tokenizers_branch/train_tokenizer.py", "--variant", "pa_bpe_nfc_clean_multi_balanced_hybrid_window" ], "git": { "commit": "7172cc669840d0dcce4170f52ef0430eb2a3d899", "branch": "parity-aware-bpe", "dirty": false }, "per_group_summary": { "n_groups": 34, "full_block_in": "tokenizers/pabpe-nfc-clean-balanced-hybridwindow/run_manifest.json" } }, "registry_note": "#6, clean pretok, slug `pabpe-nfc-clean-balanced-hybridwindow`. PA-BPE on clean pretok (balanced, HW)." }, "pabpe-nfc-clean-fineweb2full": { "slug": "pabpe-nfc-clean-fineweb2full", "runs": [ "full-128k-pabpe-nfc-clean-fineweb2full" ], "n_runs": 1, "redistributable": true, "loadable_with": "transformers", "hub_id": null, "policy_note": "trained in this project", "empirical": { "model_type": "BPE", "n_base_tokens": 127831, "n_added_tokens": 4, "vocab_total": 127835, "added_tokens": [ "", "", "", "" ], "normalizer": { "type": "NFC" }, "pre_tokenizer": { "type": "Sequence", "pretokenizers": [ { "type": "Split", "pattern": { "Regex": "[ ]?[\\p{Lu}\\p{Lt}\\p{Lm}\\p{Lo}\\p{M}]*[\\p{Ll}\\p{Lm}\\p{Lo}\\p{M}]+|[ ]?[\\p{Lu}\\p{Lt}\\p{Lm}\\p{Lo}\\p{M}]+[\\p{Ll}\\p{Lm}\\p{Lo}\\p{M}]*|\\p{N}| ?[^\\s\\p{L}\\p{N}]+|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+" }, "behavior": "Isolated", "invert": false }, { "type": "ByteLevel", "add_prefix_space": false, "trim_offsets": true, "use_regex": false } ] }, "has_post_processor": false, "byte_fallback": false, "bos_token": "", "bos_token_id": 127831 }, "declared_experiments": null, "declared_run_manifest": { "variant": "pa_bpe_nfc_clean_multi_fineweb2full", "variant_config": { "trainer": "parity-bpe", "pretok": "clean_multi_nfc", "variant": "base", "global_merges": 0, "signal": "ratios", "grouped_config": "configs/parity_aware_config_grouped_fineweb2full_quota.json", "special_tokens_override": [], "decoder_add_prefix_space": true, "output_group": "tokenizer-lm-toks", "output_dir_name": "nfc_clean_multi_fineweb2full", "post_training_special_tokens": [ "", "", "", "" ] }, "grouped_config_path": "/users/cmeister747/pa_tokenizers_branch/configs/parity_aware_config_grouped_fineweb2full_quota.json", "vocab_size": 128000, "min_frequency": 2, "special_tokens": [], "start_time": "2026-05-22T07:43:23.883861", "elapsed_seconds": 8150.928394, "slurm_job_id": "2328520", "cli_args": [ "/users/cmeister747/pa_tokenizers_branch/train_tokenizer.py", "--variant", "pa_bpe_nfc_clean_multi_fineweb2full" ], "git": { "commit": "5d8381af9ec1417b078f7dce82bac68873da915e", "branch": "parity-aware-bpe", "dirty": false }, "per_group_summary": { "n_groups": 25, "full_block_in": "tokenizers/pabpe-nfc-clean-fineweb2full/run_manifest.json" } }, "registry_note": "PA-BPE (parity, base) with clean-multi regex; fineweb2full data; norm=NFC; PA-BPE (base); vocab 127,835; USED - S5b fineweb2full analysis (best all-FLORES BPB 2.848 but worst val_bpb/BLiMP/code_bpb of the six); fineweb2full data variant, excluded from main paper panel" }, "pabpe-nfc-clean-fineweb2full-capped-tuned-hybridwindow": { "slug": "pabpe-nfc-clean-fineweb2full-capped-tuned-hybridwindow", "runs": [ "full-128k-pabpe-nfc-clean-fineweb2full-capped-tuned-hybridwindow", "full-128k-pabpe-nfc-clean-fineweb2full-capped-tuned-hybridwindow-mathcode-scratch" ], "n_runs": 2, "redistributable": true, "loadable_with": "transformers", "hub_id": null, "policy_note": "trained in this project", "empirical": { "model_type": "BPE", "n_base_tokens": 127831, "n_added_tokens": 4, "vocab_total": 127835, "added_tokens": [ "", "", "", "" ], "normalizer": { "type": "NFC" }, "pre_tokenizer": { "type": "Sequence", "pretokenizers": [ { "type": "Split", "pattern": { "Regex": "[ ]?[\\p{Lu}\\p{Lt}\\p{Lm}\\p{Lo}\\p{M}]*[\\p{Ll}\\p{Lm}\\p{Lo}\\p{M}]+|[ ]?[\\p{Lu}\\p{Lt}\\p{Lm}\\p{Lo}\\p{M}]+[\\p{Ll}\\p{Lm}\\p{Lo}\\p{M}]*|\\p{N}| ?[^\\s\\p{L}\\p{N}]{1,16}|\\s{0,16}[\\r\\n]{1,16}|\\s{1,16}(?!\\S)|\\s{1,16}" }, "behavior": "Isolated", "invert": false }, { "type": "ByteLevel", "add_prefix_space": false, "trim_offsets": true, "use_regex": false } ] }, "has_post_processor": false, "byte_fallback": false, "bos_token": "", "bos_token_id": 127831 }, "declared_experiments": null, "declared_run_manifest": { "variant": "pa_bpe_nfc_clean_multi_capped_hybrid_window_tuned", "variant_config": { "trainer": "parity-bpe", "pretok": "clean_multi_capped_nfc", "variant": "window", "global_merges": 64000, "window_size": 100, "alpha": 2.0, "signal": "ratios", "grouped_config": "configs/parity_aware_config_grouped_fineweb2full_quota_tuned.json", "special_tokens_override": [], "decoder_add_prefix_space": true, "output_group": "tokenizer-lm-toks", "output_dir_name": "nfc_clean_multi_capped_hybrid_window_tuned", "post_training_special_tokens": [ "", "", "", "" ] }, "grouped_config_path": "/users/cmeister747/pa_tokenizers_branch/configs/parity_aware_config_grouped_fineweb2full_quota_tuned.json", "vocab_size": 128000, "min_frequency": 2, "special_tokens": [], "start_time": "2026-05-23T18:26:08.559566", "elapsed_seconds": 7026.863542, "slurm_job_id": "2353210", "cli_args": [ "/users/cmeister747/pa_tokenizers_branch/train_tokenizer.py", "--variant", "pa_bpe_nfc_clean_multi_capped_hybrid_window_tuned" ], "git": { "commit": "5d8381af9ec1417b078f7dce82bac68873da915e", "branch": "parity-aware-bpe", "dirty": false }, "per_group_summary": { "n_groups": 25, "full_block_in": "tokenizers/pabpe-nfc-clean-fineweb2full-capped-tuned-hybridwindow/run_manifest.json" } }, "registry_note": "PA-BPE hybrid global+window, clean-multi regex; fineweb2full, capped ({1,16} repeat-run cap) + quota-tuned ('tuned'/v5 hand-tuned per-family weighting) data; norm=NFC; PA-BPE (hybrid global+window); vocab 127,835; one of 4 Apertus-2-track candidates in the original 2x2 (pretok x {apertus,clean}) design; USED - Apertus-2 development track candidate; excluded from main paper panel (fineweb2full data variant, PAPER_PANEL.md rule)" }, "pabpe-nfc-clean-fineweb2full-consv2-engfull-eu3-gm110k": { "slug": "pabpe-nfc-clean-fineweb2full-consv2-engfull-eu3-gm110k", "runs": [ "full-128k-pabpe-nfc-clean-fineweb2full-consv2-engfull-eu3-gm110k", "full-128k-pabpe-nfc-clean-fineweb2full-consv2-engfull-eu3-gm110k-mathcode-scratch" ], "n_runs": 2, "redistributable": true, "loadable_with": "transformers", "hub_id": null, "policy_note": "trained in this project", "empirical": { "model_type": "BPE", "n_base_tokens": 131072, "n_added_tokens": 124, "vocab_total": 131196, "added_tokens": [ "", "", "", "", "<|system_start|>", "<|system_end|>", "<|developer_start|>", "<|developer_end|>", "<|user_start|>", "<|user_end|>", "<|assistant_start|>", "<|assistant_end|>", "<|inner_prefix|>", "<|inner_suffix|>", "<|tools_prefix|>", "<|tools_suffix|>", "<|tool_output_start|>", "<|tool_output_end|>", "<|image|>", "<|audio|>", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "" ], "normalizer": { "type": "NFC" }, "pre_tokenizer": { "type": "Sequence", "pretokenizers": [ { "type": "Split", "pattern": { "Regex": "[ \\x{0F0B}\\x{0027}\\x{2019}]?(?[^\\s\\p{N}])\\k{7}|[ \\x{0F0B}\\x{0027}\\x{2019}]?[\\p{Lu}\\p{Lt}\\p{Lm}\\p{Lo}\\p{M}]*[\\p{Ll}\\p{Lm}\\p{Lo}\\p{M}]+|[ \\x{0F0B}\\x{0027}\\x{2019}]?[\\p{Lu}\\p{Lt}\\p{Lm}\\p{Lo}\\p{M}]+[\\p{Ll}\\p{Lm}\\p{Lo}\\p{M}]*|\\p{N}| ?[^\\s\\p{L}\\p{N}]{1,16}|\\s{0,16}[\\r\\n]{1,16}|\\s{1,16}(?!\\S)|\\s{1,16}" }, "behavior": "Isolated", "invert": false }, { "type": "ByteLevel", "add_prefix_space": false, "trim_offsets": true, "use_regex": false } ] }, "has_post_processor": true, "byte_fallback": false, "bos_token": "", "bos_token_id": 1 }, "declared_experiments": null, "declared_run_manifest": null, "registry_note": "PA-BPE hybrid global+window (gm110k), clean-multi regex; fineweb2full, consv2 engfull_eu3 (full English FineWeb-1 sample + European boost + Arabic fix) data; norm=NFC; PA-BPE (hybrid global+window); vocab 131,072 + 124 specials; shipped Apertus-v2 candidate 'preliminary_enh' (English-preserving); roster #22; USED - Apertus-2 track shipped candidate; S6f Contrast B (131k arm); excluded from main paper panel" }, "pabpe-nfc-clean-fineweb2full-consv2-eudata-gm110k": { "slug": "pabpe-nfc-clean-fineweb2full-consv2-eudata-gm110k", "runs": [ "full-128k-pabpe-nfc-clean-fineweb2full-consv2-eudata-gm110k" ], "n_runs": 1, "redistributable": true, "loadable_with": "transformers", "hub_id": null, "policy_note": "trained in this project", "empirical": { "model_type": "BPE", "n_base_tokens": 131072, "n_added_tokens": 124, "vocab_total": 131196, "added_tokens": [ "", "", "", "", "<|system_start|>", "<|system_end|>", "<|developer_start|>", "<|developer_end|>", "<|user_start|>", "<|user_end|>", "<|assistant_start|>", "<|assistant_end|>", "<|inner_prefix|>", "<|inner_suffix|>", "<|tools_prefix|>", "<|tools_suffix|>", "<|tool_output_start|>", "<|tool_output_end|>", "<|image|>", "<|audio|>", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "" ], "normalizer": { "type": "NFC" }, "pre_tokenizer": { "type": "Sequence", "pretokenizers": [ { "type": "Split", "pattern": { "Regex": "[ \\x{0F0B}\\x{0027}\\x{2019}]?(?[^\\s])\\k{7}|[ \\x{0F0B}\\x{0027}\\x{2019}]?[\\p{Lu}\\p{Lt}\\p{Lm}\\p{Lo}\\p{M}]*[\\p{Ll}\\p{Lm}\\p{Lo}\\p{M}]+|[ \\x{0F0B}\\x{0027}\\x{2019}]?[\\p{Lu}\\p{Lt}\\p{Lm}\\p{Lo}\\p{M}]+[\\p{Ll}\\p{Lm}\\p{Lo}\\p{M}]*|\\p{N}| ?[^\\s\\p{L}\\p{N}]{1,16}|\\s{0,16}[\\r\\n]{1,16}|\\s{1,16}(?!\\S)|\\s{1,16}" }, "behavior": "Isolated", "invert": false }, { "type": "ByteLevel", "add_prefix_space": false, "trim_offsets": true, "use_regex": false } ] }, "has_post_processor": false, "byte_fallback": false, "bos_token": "", "bos_token_id": 1 }, "declared_experiments": null, "declared_run_manifest": { "variant": "pa_bpe_nfc_clean_multi_plus2_repcap8_capped_hybrid_window_tuned_consv2_eudata_gm110k_v131k_sp124_eng5g", "variant_config": { "trainer": "parity-bpe", "pretok": "clean_multi_plus2_repcap8_capped_nfc", "variant": "window", "global_merges": 110000, "window_size": 100, "alpha": 2.0, "signal": "ratios", "vocab_size": 131072, "grouped_config": "configs/parity_aware_config_grouped_fineweb2full_quota_tuned_consv2_eudata_eng5g.json", "special_tokens_override": [ "", "", "", "", "<|system_start|>", "<|system_end|>", "<|developer_start|>", "<|developer_end|>", "<|user_start|>", "<|user_end|>", "<|assistant_start|>", "<|assistant_end|>", "<|inner_prefix|>", "<|inner_suffix|>", "<|tools_prefix|>", "<|tools_suffix|>", "<|tool_output_start|>", "<|tool_output_end|>", "<|image|>", "<|audio|>", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "" ], "decoder_add_prefix_space": true, "output_group": "tokenizer-lm-toks", "output_dir_name": "nfc_clean_multi_plus2_repcap8_capped_hybrid_window_tuned_consv2_eudata_gm110k_v131k_sp124_eng5g" }, "grouped_config_path": "/users/cmeister747/pa_tokenizers_branch/configs/parity_aware_config_grouped_fineweb2full_quota_tuned_consv2_eudata_eng5g.json", "vocab_size": 131072, "min_frequency": 2, "special_tokens": [ "", "", "", "", "<|system_start|>", "<|system_end|>", "<|developer_start|>", "<|developer_end|>", "<|user_start|>", "<|user_end|>", "<|assistant_start|>", "<|assistant_end|>", "<|inner_prefix|>", "<|inner_suffix|>", "<|tools_prefix|>", "<|tools_suffix|>", "<|tool_output_start|>", "<|tool_output_end|>", "<|image|>", "<|audio|>", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "" ], "start_time": "2026-06-17T21:19:35.605876", "elapsed_seconds": 5611.718146, "slurm_job_id": "2554545", "cli_args": [ "/users/cmeister747/pa_tokenizers_branch/train_tokenizer.py", "--variant", "pa_bpe_nfc_clean_multi_plus2_repcap8_capped_hybrid_window_tuned_consv2_eudata_gm110k_v131k_sp124_eng5g" ], "git": { "commit": "5d8381af9ec1417b078f7dce82bac68873da915e", "branch": "parity-aware-bpe", "dirty": true }, "per_group_summary": { "n_groups": 25, "full_block_in": "tokenizers/pabpe-nfc-clean-fineweb2full-consv2-eudata-gm110k/run_manifest.json" } }, "registry_note": "PA-BPE hybrid global+window (gm110k), clean-multi regex; fineweb2full, consv2 + eudata data; norm=NFC; PA-BPE (hybrid global+window); vocab ~131,196; same eudata config, higher global-merges warmup; USED - S6f Contrast B (131k arm) + Apertus-2 development track; excluded from main paper panel" }, "pabpe-nfc-clean-fineweb2full-consv2-eudata-gm90k": { "slug": "pabpe-nfc-clean-fineweb2full-consv2-eudata-gm90k", "runs": [ "full-128k-pabpe-nfc-clean-fineweb2full-consv2-eudata-gm90k" ], "n_runs": 1, "redistributable": true, "loadable_with": "transformers", "hub_id": null, "policy_note": "trained in this project", "empirical": { "model_type": "BPE", "n_base_tokens": 131072, "n_added_tokens": 124, "vocab_total": 131196, "added_tokens": [ "", "", "", "", "<|system_start|>", "<|system_end|>", "<|developer_start|>", "<|developer_end|>", "<|user_start|>", "<|user_end|>", "<|assistant_start|>", "<|assistant_end|>", "<|inner_prefix|>", "<|inner_suffix|>", "<|tools_prefix|>", "<|tools_suffix|>", "<|tool_output_start|>", "<|tool_output_end|>", "<|image|>", "<|audio|>", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "" ], "normalizer": { "type": "NFC" }, "pre_tokenizer": { "type": "Sequence", "pretokenizers": [ { "type": "Split", "pattern": { "Regex": "[ \\x{0F0B}\\x{0027}\\x{2019}]?(?[^\\s])\\k{7}|[ \\x{0F0B}\\x{0027}\\x{2019}]?[\\p{Lu}\\p{Lt}\\p{Lm}\\p{Lo}\\p{M}]*[\\p{Ll}\\p{Lm}\\p{Lo}\\p{M}]+|[ \\x{0F0B}\\x{0027}\\x{2019}]?[\\p{Lu}\\p{Lt}\\p{Lm}\\p{Lo}\\p{M}]+[\\p{Ll}\\p{Lm}\\p{Lo}\\p{M}]*|\\p{N}| ?[^\\s\\p{L}\\p{N}]{1,16}|\\s{0,16}[\\r\\n]{1,16}|\\s{1,16}(?!\\S)|\\s{1,16}" }, "behavior": "Isolated", "invert": false }, { "type": "ByteLevel", "add_prefix_space": false, "trim_offsets": true, "use_regex": false } ] }, "has_post_processor": false, "byte_fallback": false, "bos_token": "", "bos_token_id": 1 }, "declared_experiments": null, "declared_run_manifest": { "variant": "pa_bpe_nfc_clean_multi_plus2_repcap8_capped_hybrid_window_tuned_consv2_eudata_gm90k_v131k_sp124_eng5g", "variant_config": { "trainer": "parity-bpe", "pretok": "clean_multi_plus2_repcap8_capped_nfc", "variant": "window", "global_merges": 90000, "window_size": 100, "alpha": 2.0, "signal": "ratios", "vocab_size": 131072, "grouped_config": "configs/parity_aware_config_grouped_fineweb2full_quota_tuned_consv2_eudata_eng5g.json", "special_tokens_override": [ "", "", "", "", "<|system_start|>", "<|system_end|>", "<|developer_start|>", "<|developer_end|>", "<|user_start|>", "<|user_end|>", "<|assistant_start|>", "<|assistant_end|>", "<|inner_prefix|>", "<|inner_suffix|>", "<|tools_prefix|>", "<|tools_suffix|>", "<|tool_output_start|>", "<|tool_output_end|>", "<|image|>", "<|audio|>", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "" ], "decoder_add_prefix_space": true, "output_group": "tokenizer-lm-toks", "output_dir_name": "nfc_clean_multi_plus2_repcap8_capped_hybrid_window_tuned_consv2_eudata_gm90k_v131k_sp124_eng5g" }, "grouped_config_path": "/users/cmeister747/pa_tokenizers_branch/configs/parity_aware_config_grouped_fineweb2full_quota_tuned_consv2_eudata_eng5g.json", "vocab_size": 131072, "min_frequency": 2, "special_tokens": [ "", "", "", "", "<|system_start|>", "<|system_end|>", "<|developer_start|>", "<|developer_end|>", "<|user_start|>", "<|user_end|>", "<|assistant_start|>", "<|assistant_end|>", "<|inner_prefix|>", "<|inner_suffix|>", "<|tools_prefix|>", "<|tools_suffix|>", "<|tool_output_start|>", "<|tool_output_end|>", "<|image|>", "<|audio|>", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "" ], "start_time": "2026-06-17T21:19:32.179033", "elapsed_seconds": 5852.504629, "slurm_job_id": "2554539", "cli_args": [ "/users/cmeister747/pa_tokenizers_branch/train_tokenizer.py", "--variant", "pa_bpe_nfc_clean_multi_plus2_repcap8_capped_hybrid_window_tuned_consv2_eudata_gm90k_v131k_sp124_eng5g" ], "git": { "commit": "5d8381af9ec1417b078f7dce82bac68873da915e", "branch": "parity-aware-bpe", "dirty": true }, "per_group_summary": { "n_groups": 25, "full_block_in": "tokenizers/pabpe-nfc-clean-fineweb2full-consv2-eudata-gm90k/run_manifest.json" } }, "registry_note": "PA-BPE hybrid global+window (gm90k), clean-multi regex, plus2/repcap8/capped/tuned base; fineweb2full, consv2 quota formula + eudata (European-data-quota grouped config) data; norm=NFC; PA-BPE (hybrid global+window); vocab ~131,196 (131,072 target + 124 specials); earlier eudata sweep point, superseded by the shipped consv2 candidates; USED - S6f Contrast B (PA-BPE consv2 family, 131k arm, n=12) + Apertus-2 development track; excluded from main paper panel" }, "pabpe-nfc-clean-fineweb2full-consv2-eudata4-gm100k": { "slug": "pabpe-nfc-clean-fineweb2full-consv2-eudata4-gm100k", "runs": [ "full-128k-pabpe-nfc-clean-fineweb2full-consv2-eudata4-gm100k" ], "n_runs": 1, "redistributable": true, "loadable_with": "transformers", "hub_id": null, "policy_note": "trained in this project", "empirical": { "model_type": "BPE", "n_base_tokens": 131072, "n_added_tokens": 124, "vocab_total": 131196, "added_tokens": [ "", "", "", "", "<|system_start|>", "<|system_end|>", "<|developer_start|>", "<|developer_end|>", "<|user_start|>", "<|user_end|>", "<|assistant_start|>", "<|assistant_end|>", "<|inner_prefix|>", "<|inner_suffix|>", "<|tools_prefix|>", "<|tools_suffix|>", "<|tool_output_start|>", "<|tool_output_end|>", "<|image|>", "<|audio|>", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "" ], "normalizer": { "type": "NFC" }, "pre_tokenizer": { "type": "Sequence", "pretokenizers": [ { "type": "Split", "pattern": { "Regex": "[ \\x{0F0B}\\x{0027}\\x{2019}]?(?[^\\s])\\k{7}|[ \\x{0F0B}\\x{0027}\\x{2019}]?[\\p{Lu}\\p{Lt}\\p{Lm}\\p{Lo}\\p{M}]*[\\p{Ll}\\p{Lm}\\p{Lo}\\p{M}]+|[ \\x{0F0B}\\x{0027}\\x{2019}]?[\\p{Lu}\\p{Lt}\\p{Lm}\\p{Lo}\\p{M}]+[\\p{Ll}\\p{Lm}\\p{Lo}\\p{M}]*|\\p{N}| ?[^\\s\\p{L}\\p{N}]{1,16}|\\s{0,16}[\\r\\n]{1,16}|\\s{1,16}(?!\\S)|\\s{1,16}" }, "behavior": "Isolated", "invert": false }, { "type": "ByteLevel", "add_prefix_space": false, "trim_offsets": true, "use_regex": false } ] }, "has_post_processor": false, "byte_fallback": false, "bos_token": "", "bos_token_id": 1 }, "declared_experiments": null, "declared_run_manifest": { "variant": "pa_bpe_nfc_clean_multi_plus2_repcap8_capped_hybrid_window_tuned_consv2_eudata4_gm100k_v131k_sp124_eng5g", "variant_config": { "trainer": "parity-bpe", "pretok": "clean_multi_plus2_repcap8_capped_nfc", "variant": "window", "global_merges": 100000, "window_size": 100, "alpha": 2.0, "signal": "ratios", "vocab_size": 131072, "grouped_config": "configs/parity_aware_config_grouped_fineweb2full_quota_tuned_consv2_eudata4_eng5g.json", "special_tokens_override": [ "", "", "", "", "<|system_start|>", "<|system_end|>", "<|developer_start|>", "<|developer_end|>", "<|user_start|>", "<|user_end|>", "<|assistant_start|>", "<|assistant_end|>", "<|inner_prefix|>", "<|inner_suffix|>", "<|tools_prefix|>", "<|tools_suffix|>", "<|tool_output_start|>", "<|tool_output_end|>", "<|image|>", "<|audio|>", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "" ], "decoder_add_prefix_space": true, "output_group": "tokenizer-lm-toks", "output_dir_name": "nfc_clean_multi_plus2_repcap8_capped_hybrid_window_tuned_consv2_eudata4_gm100k_v131k_sp124_eng5g" }, "grouped_config_path": "/users/cmeister747/pa_tokenizers_branch/configs/parity_aware_config_grouped_fineweb2full_quota_tuned_consv2_eudata4_eng5g.json", "vocab_size": 131072, "min_frequency": 2, "special_tokens": [ "", "", "", "", "<|system_start|>", "<|system_end|>", "<|developer_start|>", "<|developer_end|>", "<|user_start|>", "<|user_end|>", "<|assistant_start|>", "<|assistant_end|>", "<|inner_prefix|>", "<|inner_suffix|>", "<|tools_prefix|>", "<|tools_suffix|>", "<|tool_output_start|>", "<|tool_output_end|>", "<|image|>", "<|audio|>", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "" ], "start_time": "2026-06-17T21:34:19.615240", "elapsed_seconds": 5219.069353, "slurm_job_id": "2554820", "cli_args": [ "/users/cmeister747/pa_tokenizers_branch/train_tokenizer.py", "--variant", "pa_bpe_nfc_clean_multi_plus2_repcap8_capped_hybrid_window_tuned_consv2_eudata4_gm100k_v131k_sp124_eng5g" ], "git": { "commit": "5d8381af9ec1417b078f7dce82bac68873da915e", "branch": "parity-aware-bpe", "dirty": true }, "per_group_summary": { "n_groups": 25, "full_block_in": "tokenizers/pabpe-nfc-clean-fineweb2full-consv2-eudata4-gm100k/run_manifest.json" } }, "registry_note": "PA-BPE hybrid global+window (gm100k), clean-multi regex; fineweb2full, consv2 + eudata4 variant data; norm=NFC; PA-BPE (hybrid global+window); vocab ~131,196; USED - S6f Contrast B (131k arm) + Apertus-2 development track; excluded from main paper panel" }, "pabpe-nfc-clean-fineweb2full-consv2-eusino-v2c-frde-kr120-gm130k-v200k": { "slug": "pabpe-nfc-clean-fineweb2full-consv2-eusino-v2c-frde-kr120-gm130k-v200k", "runs": [ "full-128k-pabpe-nfc-clean-fineweb2full-consv2-eusino-v2c-frde-kr120-gm130k-v200k", "full-128k-pabpe-nfc-clean-fineweb2full-consv2-eusino-v2c-frde-kr120-gm130k-v200k-mathcode-scratch" ], "n_runs": 2, "redistributable": true, "loadable_with": "transformers", "hub_id": null, "policy_note": "trained in this project", "empirical": { "model_type": "BPE", "n_base_tokens": 200000, "n_added_tokens": 124, "vocab_total": 200124, "added_tokens": [ "", "", "", "", "<|system_start|>", "<|system_end|>", "<|developer_start|>", "<|developer_end|>", "<|user_start|>", "<|user_end|>", "<|assistant_start|>", "<|assistant_end|>", "<|inner_prefix|>", "<|inner_suffix|>", "<|tools_prefix|>", "<|tools_suffix|>", "<|tool_output_start|>", "<|tool_output_end|>", "<|image|>", "<|audio|>", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "" ], "normalizer": { "type": "NFC" }, "pre_tokenizer": { "type": "Sequence", "pretokenizers": [ { "type": "Split", "pattern": { "Regex": "[ \\x{0F0B}\\x{0027}\\x{2019}]?(?[^\\s\\p{N}])\\k{7}|[ \\x{0F0B}\\x{0027}\\x{2019}]?[\\p{Lu}\\p{Lt}\\p{Lm}\\p{Lo}\\p{M}]*[\\p{Ll}\\p{Lm}\\p{Lo}\\p{M}]+|[ \\x{0F0B}\\x{0027}\\x{2019}]?[\\p{Lu}\\p{Lt}\\p{Lm}\\p{Lo}\\p{M}]+[\\p{Ll}\\p{Lm}\\p{Lo}\\p{M}]*|\\p{N}| ?[^\\s\\p{L}\\p{N}]{1,16}|\\s{0,16}[\\r\\n]{1,16}|\\s{1,16}(?!\\S)|\\s{1,16}" }, "behavior": "Isolated", "invert": false }, { "type": "ByteLevel", "add_prefix_space": false, "trim_offsets": true, "use_regex": false } ] }, "has_post_processor": false, "byte_fallback": false, "bos_token": "", "bos_token_id": 1 }, "declared_experiments": null, "declared_run_manifest": { "variant": "pa_bpe_nfc_clean_multi_plus2_repcap8_capped_hybrid_window_tuned_consv2_eusino_v2c_frde_kr120_gm130k_v200k_sp124_eng5g", "variant_config": { "trainer": "parity-bpe", "pretok": "clean_multi_plus2_repcap8_capped_nfc", "variant": "window", "global_merges": 130000, "window_size": 100, "alpha": 2.0, "signal": "ratios", "vocab_size": 200000, "grouped_config": "configs/parity_aware_config_grouped_fineweb2full_quota_tuned_consv2_eusino_v2c_frde_kr120_eng5g.json", "special_tokens_override": [ "", "", "", "", "<|system_start|>", "<|system_end|>", "<|developer_start|>", "<|developer_end|>", "<|user_start|>", "<|user_end|>", "<|assistant_start|>", "<|assistant_end|>", "<|inner_prefix|>", "<|inner_suffix|>", "<|tools_prefix|>", "<|tools_suffix|>", "<|tool_output_start|>", "<|tool_output_end|>", "<|image|>", "<|audio|>", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "" ], "decoder_add_prefix_space": true, "output_group": "tokenizer-lm-toks", "output_dir_name": "nfc_clean_multi_plus2_repcap8_capped_hybrid_window_tuned_consv2_eusino_v2c_frde_kr120_gm130k_v200k_sp124_eng5g" }, "grouped_config_path": "/users/cmeister747/pa_tokenizers_branch/configs/parity_aware_config_grouped_fineweb2full_quota_tuned_consv2_eusino_v2c_frde_kr120_eng5g.json", "vocab_size": 200000, "min_frequency": 2, "special_tokens": [ "", "", "", "", "<|system_start|>", "<|system_end|>", "<|developer_start|>", "<|developer_end|>", "<|user_start|>", "<|user_end|>", "<|assistant_start|>", "<|assistant_end|>", "<|inner_prefix|>", "<|inner_suffix|>", "<|tools_prefix|>", "<|tools_suffix|>", "<|tool_output_start|>", "<|tool_output_end|>", "<|image|>", "<|audio|>", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "" ], "start_time": "2026-06-22T10:31:53.137663", "elapsed_seconds": 6071.042492, "slurm_job_id": "2592411", "cli_args": [ "train_tokenizer.py", "--variant", "pa_bpe_nfc_clean_multi_plus2_repcap8_capped_hybrid_window_tuned_consv2_eusino_v2c_frde_kr120_gm130k_v200k_sp124_eng5g" ], "git": { "commit": "5d8381af9ec1417b078f7dce82bac68873da915e", "branch": "parity-aware-bpe", "dirty": true }, "per_group_summary": { "n_groups": 25, "full_block_in": "tokenizers/pabpe-nfc-clean-fineweb2full-consv2-eusino-v2c-frde-kr120-gm130k-v200k/run_manifest.json" } }, "registry_note": "PA-BPE hybrid global+window (gm130k), clean-multi regex; fineweb2full, consv2 eusino_v2c + French/German boost + kr120 (+120MB Korean) data; norm=NFC; PA-BPE (hybrid global+window); vocab 200,064; the shipped preliminary_mul_200k recipe; roster #38; USED - S6f Contrast B (200k arm, n=2) + Apertus-2 track shipped candidate; excluded from main paper panel" }, "pabpe-nfc-clean-fineweb2full-consv2-eusino-v2c-gm120k-v200k": { "slug": "pabpe-nfc-clean-fineweb2full-consv2-eusino-v2c-gm120k-v200k", "runs": [ "full-128k-pabpe-nfc-clean-fineweb2full-consv2-eusino-v2c-gm120k-v200k", "full-128k-pabpe-nfc-clean-fineweb2full-consv2-eusino-v2c-gm120k-v200k-mathcode-scratch" ], "n_runs": 2, "redistributable": true, "loadable_with": "transformers", "hub_id": null, "policy_note": "trained in this project", "empirical": { "model_type": "BPE", "n_base_tokens": 200000, "n_added_tokens": 124, "vocab_total": 200124, "added_tokens": [ "", "", "", "", "<|system_start|>", "<|system_end|>", "<|developer_start|>", "<|developer_end|>", "<|user_start|>", "<|user_end|>", "<|assistant_start|>", "<|assistant_end|>", "<|inner_prefix|>", "<|inner_suffix|>", "<|tools_prefix|>", "<|tools_suffix|>", "<|tool_output_start|>", "<|tool_output_end|>", "<|image|>", "<|audio|>", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "" ], "normalizer": { "type": "NFC" }, "pre_tokenizer": { "type": "Sequence", "pretokenizers": [ { "type": "Split", "pattern": { "Regex": "[ \\x{0F0B}\\x{0027}\\x{2019}]?(?[^\\s\\p{N}])\\k{7}|[ \\x{0F0B}\\x{0027}\\x{2019}]?[\\p{Lu}\\p{Lt}\\p{Lm}\\p{Lo}\\p{M}]*[\\p{Ll}\\p{Lm}\\p{Lo}\\p{M}]+|[ \\x{0F0B}\\x{0027}\\x{2019}]?[\\p{Lu}\\p{Lt}\\p{Lm}\\p{Lo}\\p{M}]+[\\p{Ll}\\p{Lm}\\p{Lo}\\p{M}]*|\\p{N}| ?[^\\s\\p{L}\\p{N}]{1,16}|\\s{0,16}[\\r\\n]{1,16}|\\s{1,16}(?!\\S)|\\s{1,16}" }, "behavior": "Isolated", "invert": false }, { "type": "ByteLevel", "add_prefix_space": false, "trim_offsets": true, "use_regex": false } ] }, "has_post_processor": true, "byte_fallback": false, "bos_token": "", "bos_token_id": 1 }, "declared_experiments": null, "declared_run_manifest": null, "registry_note": "PA-BPE hybrid global+window (gm120k), clean-multi regex; fineweb2full, consv2 eusino_v2c (European+Sinotibetan rebalance) data; norm=NFC; PA-BPE (hybrid global+window); vocab 200,000; the 200k predecessor LM evaluated for shipped candidate 'preliminary_mul_200k'; roster #25; USED - S6f Contrast B (200k arm, n=2) + Apertus-2 track shipped candidate; excluded from main paper panel" }, "pabpe-nfc-clean-fineweb2full-consv2-frde2-gm110k": { "slug": "pabpe-nfc-clean-fineweb2full-consv2-frde2-gm110k", "runs": [ "full-128k-pabpe-nfc-clean-fineweb2full-consv2-frde2-gm110k", "full-128k-pabpe-nfc-clean-fineweb2full-consv2-frde2-gm110k-mathcode-scratch" ], "n_runs": 2, "redistributable": true, "loadable_with": "transformers", "hub_id": null, "policy_note": "trained in this project", "empirical": { "model_type": "BPE", "n_base_tokens": 131072, "n_added_tokens": 124, "vocab_total": 131196, "added_tokens": [ "", "", "", "", "<|system_start|>", "<|system_end|>", "<|developer_start|>", "<|developer_end|>", "<|user_start|>", "<|user_end|>", "<|assistant_start|>", "<|assistant_end|>", "<|inner_prefix|>", "<|inner_suffix|>", "<|tools_prefix|>", "<|tools_suffix|>", "<|tool_output_start|>", "<|tool_output_end|>", "<|image|>", "<|audio|>", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "" ], "normalizer": { "type": "NFC" }, "pre_tokenizer": { "type": "Sequence", "pretokenizers": [ { "type": "Split", "pattern": { "Regex": "[ \\x{0F0B}\\x{0027}\\x{2019}]?(?[^\\s\\p{N}])\\k{7}|[ \\x{0F0B}\\x{0027}\\x{2019}]?[\\p{Lu}\\p{Lt}\\p{Lm}\\p{Lo}\\p{M}]*[\\p{Ll}\\p{Lm}\\p{Lo}\\p{M}]+|[ \\x{0F0B}\\x{0027}\\x{2019}]?[\\p{Lu}\\p{Lt}\\p{Lm}\\p{Lo}\\p{M}]+[\\p{Ll}\\p{Lm}\\p{Lo}\\p{M}]*|\\p{N}| ?[^\\s\\p{L}\\p{N}]{1,16}|\\s{0,16}[\\r\\n]{1,16}|\\s{1,16}(?!\\S)|\\s{1,16}" }, "behavior": "Isolated", "invert": false }, { "type": "ByteLevel", "add_prefix_space": false, "trim_offsets": true, "use_regex": false } ] }, "has_post_processor": true, "byte_fallback": false, "bos_token": "", "bos_token_id": 1 }, "declared_experiments": null, "declared_run_manifest": null, "registry_note": "PA-BPE hybrid global+window (gm110k), clean-multi regex; fineweb2full, consv2 frde2 (more French/German, less Chinese) data; norm=NFC; PA-BPE (hybrid global+window); vocab 131,072 + 124 specials; shipped Apertus-v2 candidate 'preliminary_euh' (EU-dense, least fair of the four); roster #23; USED - Apertus-2 track shipped candidate; S6f Contrast B (131k arm); excluded from main paper panel" }, "pabpe-nfc-clean-fineweb2full-consv2-plus3-repcap8": { "slug": "pabpe-nfc-clean-fineweb2full-consv2-plus3-repcap8", "runs": [ "full-128k-pabpe-nfc-clean-fineweb2full-consv2-plus3-repcap8", "full-128k-pabpe-nfc-clean-fineweb2full-consv2-plus3-repcap8-mathcode-scratch" ], "n_runs": 2, "redistributable": true, "loadable_with": "transformers", "hub_id": null, "policy_note": "trained in this project", "empirical": { "model_type": "BPE", "n_base_tokens": 131017, "n_added_tokens": 124, "vocab_total": 131141, "added_tokens": [ "", "", "", "", "<|system_start|>", "<|system_end|>", "<|developer_start|>", "<|developer_end|>", "<|user_start|>", "<|user_end|>", "<|assistant_start|>", "<|assistant_end|>", "<|inner_prefix|>", "<|inner_suffix|>", "<|tools_prefix|>", "<|tools_suffix|>", "<|tool_output_start|>", "<|tool_output_end|>", "<|image|>", "<|audio|>", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "" ], "normalizer": { "type": "NFC" }, "pre_tokenizer": { "type": "Sequence", "pretokenizers": [ { "type": "Split", "pattern": { "Regex": "[ \\x{0F0B}\\x{0027}\\x{2019}]?(?[^\\s])\\k{7}|[ \\x{0F0B}\\x{0027}\\x{2019}]?(?:(?!(?.)\\k{8})[\\p{Lu}\\p{Lt}\\p{Lm}\\p{Lo}\\p{M}])*(?:(?!(?.)\\k{8})[\\p{Ll}\\p{Lm}\\p{Lo}\\p{M}])+(?:[\\x{0027}\\x{2019}](?!\\p{L}))?|[ \\x{0F0B}\\x{0027}\\x{2019}]?(?:(?!(?.)\\k{8})[\\p{Lu}\\p{Lt}\\p{Lm}\\p{Lo}\\p{M}])+(?:(?!(?.)\\k{8})[\\p{Ll}\\p{Lm}\\p{Lo}\\p{M}])*(?:[\\x{0027}\\x{2019}](?!\\p{L}))?|\\p{N}| ?(?:(?!(?

.)\\k

{8})[^\\s\\p{L}\\p{N}]){1,16}|\\s{0,16}[\\r\\n]{1,16}|\\s{1,16}(?!\\S)|\\s{1,16}" }, "behavior": "Isolated", "invert": false }, { "type": "ByteLevel", "add_prefix_space": false, "trim_offsets": true, "use_regex": false } ] }, "has_post_processor": false, "byte_fallback": false, "bos_token": "", "bos_token_id": 1 }, "declared_experiments": null, "declared_run_manifest": null, "registry_note": "PA-BPE hybrid global+window, clean-multi plus3 regex + repcap8 (8+ char run cap); fineweb2full, consv2 baseline (reparam ratios) data; norm=NFC; PA-BPE (hybrid global+window); vocab 131,017 (55-token ParityBpe shortfall vs 131,072 target); shipped Apertus-v2 candidate 'preliminary_mul' (balanced baseline, fairest of the four); roster #24; USED - Apertus-2 track shipped candidate; S6f Contrast B (131k arm); excluded from main paper panel" }, "pabpe-nfc-clean-fineweb2full-consv2-prelim-mul-v131072": { "slug": "pabpe-nfc-clean-fineweb2full-consv2-prelim-mul-v131072", "runs": [ "full-128k-pabpe-nfc-clean-fineweb2full-consv2-prelim-mul-v131072", "full-128k-pabpe-nfc-clean-fineweb2full-consv2-prelim-mul-v131072-mathcode-scratch" ], "n_runs": 2, "redistributable": true, "loadable_with": "transformers", "hub_id": null, "policy_note": "trained in this project", "empirical": { "model_type": "BPE", "n_base_tokens": 131072, "n_added_tokens": 124, "vocab_total": 131196, "added_tokens": [ "", "", "", "", "<|system_start|>", "<|system_end|>", "<|developer_start|>", "<|developer_end|>", "<|user_start|>", "<|user_end|>", "<|assistant_start|>", "<|assistant_end|>", "<|inner_prefix|>", "<|inner_suffix|>", "<|tools_prefix|>", "<|tools_suffix|>", "<|tool_output_start|>", "<|tool_output_end|>", "<|image|>", "<|audio|>", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "" ], "normalizer": { "type": "NFC" }, "pre_tokenizer": { "type": "Sequence", "pretokenizers": [ { "type": "Split", "pattern": { "Regex": "[ \\x{0F0B}\\x{0027}\\x{2019}]?(?[^\\s])\\k{7}|[ \\x{0F0B}\\x{0027}\\x{2019}]?[\\p{Lu}\\p{Lt}\\p{Lm}\\p{Lo}\\p{M}]*[\\p{Ll}\\p{Lm}\\p{Lo}\\p{M}]+(?:[\\x{0027}\\x{2019}](?!\\p{L}))?|[ \\x{0F0B}\\x{0027}\\x{2019}]?[\\p{Lu}\\p{Lt}\\p{Lm}\\p{Lo}\\p{M}]+[\\p{Ll}\\p{Lm}\\p{Lo}\\p{M}]*(?:[\\x{0027}\\x{2019}](?!\\p{L}))?|\\p{N}| ?[^\\s\\p{L}\\p{N}]{1,16}|\\s{0,16}[\\r\\n]{1,16}|\\s{1,16}(?!\\S)|\\s{1,16}" }, "behavior": "Isolated", "invert": false }, { "type": "ByteLevel", "add_prefix_space": false, "trim_offsets": true, "use_regex": false } ] }, "has_post_processor": true, "byte_fallback": false, "bos_token": "", "bos_token_id": 1 }, "declared_experiments": null, "declared_run_manifest": null, "registry_note": "PA-BPE hybrid global+window, clean-multi plus3 + repcap8; fineweb2full, consv2 reparam (preliminary_mul) data; norm=NFC; PA-BPE (hybrid global+window); vocab 131,072; the preliminary_mul build registered under an explicit slug (distinct md5 from the roster #24 twin); USED - S6h code analysis is not applicable (20B-only); Apertus-2 track; excluded from main paper panel" }, "pabpe-nfc-clean-fineweb2full-hybridwindow": { "slug": "pabpe-nfc-clean-fineweb2full-hybridwindow", "runs": [ "full-128k-pabpe-nfc-clean-fineweb2full-hybridwindow" ], "n_runs": 1, "redistributable": true, "loadable_with": "transformers", "hub_id": null, "policy_note": "trained in this project", "empirical": { "model_type": "BPE", "n_base_tokens": 127831, "n_added_tokens": 4, "vocab_total": 127835, "added_tokens": [ "", "", "", "" ], "normalizer": { "type": "NFC" }, "pre_tokenizer": { "type": "Sequence", "pretokenizers": [ { "type": "Split", "pattern": { "Regex": "[ ]?[\\p{Lu}\\p{Lt}\\p{Lm}\\p{Lo}\\p{M}]*[\\p{Ll}\\p{Lm}\\p{Lo}\\p{M}]+|[ ]?[\\p{Lu}\\p{Lt}\\p{Lm}\\p{Lo}\\p{M}]+[\\p{Ll}\\p{Lm}\\p{Lo}\\p{M}]*|\\p{N}| ?[^\\s\\p{L}\\p{N}]+|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+" }, "behavior": "Isolated", "invert": false }, { "type": "ByteLevel", "add_prefix_space": false, "trim_offsets": true, "use_regex": false } ] }, "has_post_processor": false, "byte_fallback": false, "bos_token": "", "bos_token_id": 127831 }, "declared_experiments": null, "declared_run_manifest": { "variant": "pa_bpe_nfc_clean_multi_fineweb2full_hybrid_window", "variant_config": { "trainer": "parity-bpe", "pretok": "clean_multi_nfc", "variant": "window", "global_merges": 64000, "window_size": 100, "alpha": 2.0, "signal": "ratios", "grouped_config": "configs/parity_aware_config_grouped_fineweb2full_quota.json", "special_tokens_override": [], "decoder_add_prefix_space": true, "output_group": "tokenizer-lm-toks", "output_dir_name": "nfc_clean_multi_fineweb2full_hybrid_window", "post_training_special_tokens": [ "", "", "", "" ] }, "grouped_config_path": "/users/cmeister747/pa_tokenizers_branch/configs/parity_aware_config_grouped_fineweb2full_quota.json", "vocab_size": 128000, "min_frequency": 2, "special_tokens": [], "start_time": "2026-05-22T07:43:23.362987", "elapsed_seconds": 7228.839727, "slurm_job_id": "2328521", "cli_args": [ "/users/cmeister747/pa_tokenizers_branch/train_tokenizer.py", "--variant", "pa_bpe_nfc_clean_multi_fineweb2full_hybrid_window" ], "git": { "commit": "5d8381af9ec1417b078f7dce82bac68873da915e", "branch": "parity-aware-bpe", "dirty": false }, "per_group_summary": { "n_groups": 25, "full_block_in": "tokenizers/pabpe-nfc-clean-fineweb2full-hybridwindow/run_manifest.json" } }, "registry_note": "PA-BPE hybrid global+window (gm=64000,W=100,a=2.0) with clean-multi regex; fineweb2full data; norm=NFC; PA-BPE (hybrid global+window); vocab 127,835; hybrid-window pulls val_bpb/BLiMP/code_bpb back toward BPE-like at the cost of most of the all-FLORES gain; USED - S5b fineweb2full analysis; fineweb2full data variant, excluded from main paper panel" }, "pabpe-nfc-clean-fineweb2full-plus2-capped-tuned-hybridwindow": { "slug": "pabpe-nfc-clean-fineweb2full-plus2-capped-tuned-hybridwindow", "runs": [ "full-128k-pabpe-nfc-clean-fineweb2full-plus2-capped-tuned-hybridwindow", "full-128k-pabpe-nfc-clean-fineweb2full-plus2-capped-tuned-hybridwindow-mathcode-scratch" ], "n_runs": 2, "redistributable": true, "loadable_with": "transformers", "hub_id": null, "policy_note": "trained in this project", "empirical": { "model_type": "BPE", "n_base_tokens": 127831, "n_added_tokens": 4, "vocab_total": 127835, "added_tokens": [ "", "", "", "" ], "normalizer": { "type": "NFC" }, "pre_tokenizer": { "type": "Sequence", "pretokenizers": [ { "type": "Split", "pattern": { "Regex": "[ \\x{0F0B}\\x{0027}\\x{2019}]?[\\p{Lu}\\p{Lt}\\p{Lm}\\p{Lo}\\p{M}]*[\\p{Ll}\\p{Lm}\\p{Lo}\\p{M}]+|[ \\x{0F0B}\\x{0027}\\x{2019}]?[\\p{Lu}\\p{Lt}\\p{Lm}\\p{Lo}\\p{M}]+[\\p{Ll}\\p{Lm}\\p{Lo}\\p{M}]*|\\p{N}| ?[^\\s\\p{L}\\p{N}]{1,16}|\\s{0,16}[\\r\\n]{1,16}|\\s{1,16}(?!\\S)|\\s{1,16}" }, "behavior": "Isolated", "invert": false }, { "type": "ByteLevel", "add_prefix_space": false, "trim_offsets": true, "use_regex": false } ] }, "has_post_processor": false, "byte_fallback": false, "bos_token": "", "bos_token_id": 127831 }, "declared_experiments": null, "declared_run_manifest": { "variant": "pa_bpe_nfc_clean_multi_plus2_capped_hybrid_window_tuned", "variant_config": { "trainer": "parity-bpe", "pretok": "clean_multi_plus2_capped_nfc", "variant": "window", "global_merges": 64000, "window_size": 100, "alpha": 2.0, "signal": "ratios", "grouped_config": "configs/parity_aware_config_grouped_fineweb2full_quota_tuned.json", "special_tokens_override": [], "decoder_add_prefix_space": true, "output_group": "tokenizer-lm-toks", "output_dir_name": "nfc_clean_multi_plus2_capped_hybrid_window_tuned", "post_training_special_tokens": [ "", "", "", "" ] }, "grouped_config_path": "/users/cmeister747/pa_tokenizers_branch/configs/parity_aware_config_grouped_fineweb2full_quota_tuned.json", "vocab_size": 128000, "min_frequency": 2, "special_tokens": [], "start_time": "2026-06-02T17:49:32.228735", "elapsed_seconds": 7130.059512, "slurm_job_id": "2458438", "cli_args": [ "/users/cmeister747/pa_tokenizers_branch/train_tokenizer.py", "--variant", "pa_bpe_nfc_clean_multi_plus2_capped_hybrid_window_tuned" ], "git": { "commit": "5d8381af9ec1417b078f7dce82bac68873da915e", "branch": "parity-aware-bpe", "dirty": false }, "per_group_summary": { "n_groups": 25, "full_block_in": "tokenizers/pabpe-nfc-clean-fineweb2full-plus2-capped-tuned-hybridwindow/run_manifest.json" } }, "registry_note": "PA-BPE hybrid global+window, clean-multi plus2 regex; fineweb2full, capped + tuned (v5) data; norm=NFC; PA-BPE (hybrid global+window); vocab ~131,196; plus2 tuned-data baseline; USED - S6f Contrast B (131k arm) + Apertus-2 track; excluded from main paper panel" }, "pabpe-nfc-clean-fineweb2full-plus2-capped-tuned-hybridwindow-consv2-tailcuts-gm90k-v130-sp-eng5g": { "slug": "pabpe-nfc-clean-fineweb2full-plus2-capped-tuned-hybridwindow-consv2-tailcuts-gm90k-v130-sp-eng5g", "runs": [ "full-128k-pabpe-nfc-clean-fineweb2full-plus2-capped-tuned-hybridwindow-consv2-tailcuts-gm90k-v130-sp-eng5g", "full-128k-pabpe-nfc-clean-fineweb2full-plus2-capped-tuned-hybridwindow-consv2-tailcuts-gm90k-v130-sp-eng5g-mathcode-scratch" ], "n_runs": 2, "redistributable": true, "loadable_with": "transformers", "hub_id": null, "policy_note": "trained in this project", "empirical": { "model_type": "BPE", "n_base_tokens": 130929, "n_added_tokens": 99, "vocab_total": 131028, "added_tokens": [ "", "", "", "", "[/INST]", "[AVAILABLE_TOOLS]", "[/AVAILABLE_TOOLS]", "[TOOL_RESULTS]", "[/TOOL_RESULTS]", "[TOOL_CALLS]", "", "[PREFIX]", "[MIDDLE]", "[SUFFIX]", "\\begin{", "\\end{", "\\text{", "\\boxed{", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "<|fim_begin|>", "<|fim_hole|>", "<|fim_end|>", "<|system_start|>", "<|system_end|>", "<|developer_start|>", "<|developer_end|>", "<|user_start|>", "<|user_end|>", "<|assistant_start|>", "<|assistant_end|>", "<|inner_prefix|>", "<|inner_suffix|>", "<|tools_prefix|>", "<|tools_suffix|>", "<|tool_output_start|>", "<|tool_output_end|>", "<|image|>", "<|audio|>", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "" ], "normalizer": { "type": "NFC" }, "pre_tokenizer": { "type": "Sequence", "pretokenizers": [ { "type": "Split", "pattern": { "Regex": "[ \\x{0F0B}\\x{0027}\\x{2019}]?[\\p{Lu}\\p{Lt}\\p{Lm}\\p{Lo}\\p{M}]*[\\p{Ll}\\p{Lm}\\p{Lo}\\p{M}]+|[ \\x{0F0B}\\x{0027}\\x{2019}]?[\\p{Lu}\\p{Lt}\\p{Lm}\\p{Lo}\\p{M}]+[\\p{Ll}\\p{Lm}\\p{Lo}\\p{M}]*|\\p{N}| ?[^\\s\\p{L}\\p{N}]{1,16}|\\s{0,16}[\\r\\n]{1,16}|\\s{1,16}(?!\\S)|\\s{1,16}" }, "behavior": "Isolated", "invert": false }, { "type": "ByteLevel", "add_prefix_space": false, "trim_offsets": true, "use_regex": false } ] }, "has_post_processor": false, "byte_fallback": false, "bos_token": "", "bos_token_id": 1 }, "declared_experiments": null, "declared_run_manifest": { "variant": "pa_bpe_nfc_clean_multi_plus2_capped_hybrid_window_tuned_consv2_tailcuts_gm90k_v130_sp_eng5g", "variant_config": { "trainer": "parity-bpe", "pretok": "clean_multi_plus2_capped_nfc", "variant": "window", "global_merges": 90000, "window_size": 100, "alpha": 2.0, "signal": "ratios", "vocab_size": 130999, "grouped_config": "configs/parity_aware_config_grouped_fineweb2full_quota_tuned_consv2_tailcuts_eng5g.json", "special_tokens_override": [ "", "", "", "", "[/INST]", "[AVAILABLE_TOOLS]", "[/AVAILABLE_TOOLS]", "[TOOL_RESULTS]", "[/TOOL_RESULTS]", "[TOOL_CALLS]", "", "[PREFIX]", "[MIDDLE]", "[SUFFIX]", "\\begin{", "\\end{", "\\text{", "\\boxed{", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "<|fim_begin|>", "<|fim_hole|>", "<|fim_end|>", "<|system_start|>", "<|system_end|>", "<|developer_start|>", "<|developer_end|>", "<|user_start|>", "<|user_end|>", "<|assistant_start|>", "<|assistant_end|>", "<|inner_prefix|>", "<|inner_suffix|>", "<|tools_prefix|>", "<|tools_suffix|>", "<|tool_output_start|>", "<|tool_output_end|>", "<|image|>", "<|audio|>", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "" ], "decoder_add_prefix_space": true, "output_group": "tokenizer-lm-toks", "output_dir_name": "nfc_clean_multi_plus2_capped_hybrid_window_tuned_consv2_tailcuts_gm90k_v130_sp_eng5g" }, "grouped_config_path": "/users/cmeister747/pa_tokenizers_branch/configs/parity_aware_config_grouped_fineweb2full_quota_tuned_consv2_tailcuts_eng5g.json", "vocab_size": 130999, "min_frequency": 2, "special_tokens": [ "", "", "", "", "[/INST]", "[AVAILABLE_TOOLS]", "[/AVAILABLE_TOOLS]", "[TOOL_RESULTS]", "[/TOOL_RESULTS]", "[TOOL_CALLS]", "", "[PREFIX]", "[MIDDLE]", "[SUFFIX]", "\\begin{", "\\end{", "\\text{", "\\boxed{", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "<|fim_begin|>", "<|fim_hole|>", "<|fim_end|>", "<|system_start|>", "<|system_end|>", "<|developer_start|>", "<|developer_end|>", "<|user_start|>", "<|user_end|>", "<|assistant_start|>", "<|assistant_end|>", "<|inner_prefix|>", "<|inner_suffix|>", "<|tools_prefix|>", "<|tools_suffix|>", "<|tool_output_start|>", "<|tool_output_end|>", "<|image|>", "<|audio|>", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "" ], "start_time": "2026-06-05T18:34:53.428167", "elapsed_seconds": 6850.657703, "slurm_job_id": "2476236", "cli_args": [ "/users/cmeister747/pa_tokenizers_branch/train_tokenizer.py", "--variant", "pa_bpe_nfc_clean_multi_plus2_capped_hybrid_window_tuned_consv2_tailcuts_gm90k_v130_sp_eng5g" ], "git": { "commit": "5d8381af9ec1417b078f7dce82bac68873da915e", "branch": "parity-aware-bpe", "dirty": false }, "per_group_summary": { "n_groups": 25, "full_block_in": "tokenizers/pabpe-nfc-clean-fineweb2full-plus2-capped-tuned-hybridwindow-consv2-tailcuts-gm90k-v130-sp-eng5g/run_manifest.json" } }, "registry_note": "PA-BPE hybrid global+window (gm90k), clean-multi plus2 regex; fineweb2full, consv2 + tailcuts (6 families demoted to ratio 1.0) + eng5g (+5GB English) data; norm=NFC; PA-BPE (hybrid global+window); vocab ~131,196 (~131k + full Apertus chat-template specials, ~99 tokens); Apertus-2 development ablation 'PA-Clean-plus2-A8'; USED - S6f Contrast B (131k arm) + Apertus-2 track ablation; excluded from main paper panel" }, "pabpe-nfc-clean-fineweb2full-plus3-capped-tuned-hybridwindow-consv2": { "slug": "pabpe-nfc-clean-fineweb2full-plus3-capped-tuned-hybridwindow-consv2", "runs": [ "full-128k-pabpe-nfc-clean-fineweb2full-plus3-capped-tuned-hybridwindow-consv2", "full-128k-pabpe-nfc-clean-fineweb2full-plus3-capped-tuned-hybridwindow-consv2-mathcode-scratch" ], "n_runs": 2, "redistributable": true, "loadable_with": "transformers", "hub_id": null, "policy_note": "trained in this project", "empirical": { "model_type": "BPE", "n_base_tokens": 127831, "n_added_tokens": 4, "vocab_total": 127835, "added_tokens": [ "", "", "", "" ], "normalizer": { "type": "NFC" }, "pre_tokenizer": { "type": "Sequence", "pretokenizers": [ { "type": "Split", "pattern": { "Regex": "[ \\x{0F0B}\\x{0027}\\x{2019}]?[\\p{Lu}\\p{Lt}\\p{Lm}\\p{Lo}\\p{M}]*[\\p{Ll}\\p{Lm}\\p{Lo}\\p{M}]+(?:[\\x{0027}\\x{2019}](?!\\p{L}))?|[ \\x{0F0B}\\x{0027}\\x{2019}]?[\\p{Lu}\\p{Lt}\\p{Lm}\\p{Lo}\\p{M}]+[\\p{Ll}\\p{Lm}\\p{Lo}\\p{M}]*(?:[\\x{0027}\\x{2019}](?!\\p{L}))?|\\p{N}| ?[^\\s\\p{L}\\p{N}]{1,16}|\\s{0,16}[\\r\\n]{1,16}|\\s{1,16}(?!\\S)|\\s{1,16}" }, "behavior": "Isolated", "invert": false }, { "type": "ByteLevel", "add_prefix_space": false, "trim_offsets": true, "use_regex": false } ] }, "has_post_processor": false, "byte_fallback": false, "bos_token": "", "bos_token_id": 127831 }, "declared_experiments": null, "declared_run_manifest": { "variant": "pa_bpe_nfc_clean_multi_plus3_capped_hybrid_window_tuned_consv2", "variant_config": { "trainer": "parity-bpe", "pretok": "clean_multi_plus3_capped_nfc", "variant": "window", "global_merges": 64000, "window_size": 100, "alpha": 2.0, "signal": "ratios", "grouped_config": "configs/parity_aware_config_grouped_fineweb2full_quota_tuned_consv2.json", "special_tokens_override": [], "decoder_add_prefix_space": true, "output_group": "tokenizer-lm-toks", "output_dir_name": "nfc_clean_multi_plus3_capped_hybrid_window_tuned_consv2", "post_training_special_tokens": [ "", "", "", "" ] }, "grouped_config_path": "/users/cmeister747/pa_tokenizers_branch/configs/parity_aware_config_grouped_fineweb2full_quota_tuned_consv2.json", "vocab_size": 128000, "min_frequency": 2, "special_tokens": [], "start_time": "2026-06-04T00:07:42.220508", "elapsed_seconds": 7096.755123, "slurm_job_id": "2466634", "cli_args": [ "/users/cmeister747/pa_tokenizers_branch/train_tokenizer.py", "--variant", "pa_bpe_nfc_clean_multi_plus3_capped_hybrid_window_tuned_consv2" ], "git": { "commit": "5d8381af9ec1417b078f7dce82bac68873da915e", "branch": "parity-aware-bpe", "dirty": false }, "per_group_summary": { "n_groups": 25, "full_block_in": "tokenizers/pabpe-nfc-clean-fineweb2full-plus3-capped-tuned-hybridwindow-consv2/run_manifest.json" } }, "registry_note": "PA-BPE hybrid global+window, clean-multi plus3 regex; fineweb2full, consv2 (no tailcuts) data; norm=NFC; PA-BPE (hybrid global+window); vocab ~131,196; plus3 consv2 baseline; USED - S6f Contrast B (131k arm) + Apertus-2 track; excluded from main paper panel" }, "pabpe-nfc-clean-fineweb2full-plus3-capped-tuned-hybridwindow-consv2-tailcuts-gm70k-v130-sp-eng5g": { "slug": "pabpe-nfc-clean-fineweb2full-plus3-capped-tuned-hybridwindow-consv2-tailcuts-gm70k-v130-sp-eng5g", "runs": [ "full-128k-pabpe-nfc-clean-fineweb2full-plus3-capped-tuned-hybridwindow-consv2-tailcuts-gm70k-v130-sp-eng5g", "full-128k-pabpe-nfc-clean-fineweb2full-plus3-capped-tuned-hybridwindow-consv2-tailcuts-gm70k-v130-sp-eng5g-mathcode-scratch" ], "n_runs": 2, "redistributable": true, "loadable_with": "transformers", "hub_id": null, "policy_note": "trained in this project", "empirical": { "model_type": "BPE", "n_base_tokens": 130929, "n_added_tokens": 99, "vocab_total": 131028, "added_tokens": [ "", "", "", "", "[/INST]", "[AVAILABLE_TOOLS]", "[/AVAILABLE_TOOLS]", "[TOOL_RESULTS]", "[/TOOL_RESULTS]", "[TOOL_CALLS]", "", "[PREFIX]", "[MIDDLE]", "[SUFFIX]", "\\begin{", "\\end{", "\\text{", "\\boxed{", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "<|fim_begin|>", "<|fim_hole|>", "<|fim_end|>", "<|system_start|>", "<|system_end|>", "<|developer_start|>", "<|developer_end|>", "<|user_start|>", "<|user_end|>", "<|assistant_start|>", "<|assistant_end|>", "<|inner_prefix|>", "<|inner_suffix|>", "<|tools_prefix|>", "<|tools_suffix|>", "<|tool_output_start|>", "<|tool_output_end|>", "<|image|>", "<|audio|>", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "" ], "normalizer": { "type": "NFC" }, "pre_tokenizer": { "type": "Sequence", "pretokenizers": [ { "type": "Split", "pattern": { "Regex": "[ \\x{0F0B}\\x{0027}\\x{2019}]?[\\p{Lu}\\p{Lt}\\p{Lm}\\p{Lo}\\p{M}]*[\\p{Ll}\\p{Lm}\\p{Lo}\\p{M}]+(?:[\\x{0027}\\x{2019}](?!\\p{L}))?|[ \\x{0F0B}\\x{0027}\\x{2019}]?[\\p{Lu}\\p{Lt}\\p{Lm}\\p{Lo}\\p{M}]+[\\p{Ll}\\p{Lm}\\p{Lo}\\p{M}]*(?:[\\x{0027}\\x{2019}](?!\\p{L}))?|\\p{N}| ?[^\\s\\p{L}\\p{N}]{1,16}|\\s{0,16}[\\r\\n]{1,16}|\\s{1,16}(?!\\S)|\\s{1,16}" }, "behavior": "Isolated", "invert": false }, { "type": "ByteLevel", "add_prefix_space": false, "trim_offsets": true, "use_regex": false } ] }, "has_post_processor": false, "byte_fallback": false, "bos_token": "", "bos_token_id": 1 }, "declared_experiments": null, "declared_run_manifest": { "variant": "pa_bpe_nfc_clean_multi_plus3_capped_hybrid_window_tuned_consv2_tailcuts_gm70k_v130_sp_eng5g", "variant_config": { "trainer": "parity-bpe", "pretok": "clean_multi_plus3_capped_nfc", "variant": "window", "global_merges": 70000, "window_size": 100, "alpha": 2.0, "signal": "ratios", "vocab_size": 130999, "grouped_config": "configs/parity_aware_config_grouped_fineweb2full_quota_tuned_consv2_tailcuts_eng5g.json", "special_tokens_override": [ "", "", "", "", "[/INST]", "[AVAILABLE_TOOLS]", "[/AVAILABLE_TOOLS]", "[TOOL_RESULTS]", "[/TOOL_RESULTS]", "[TOOL_CALLS]", "", "[PREFIX]", "[MIDDLE]", "[SUFFIX]", "\\begin{", "\\end{", "\\text{", "\\boxed{", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "<|fim_begin|>", "<|fim_hole|>", "<|fim_end|>", "<|system_start|>", "<|system_end|>", "<|developer_start|>", "<|developer_end|>", "<|user_start|>", "<|user_end|>", "<|assistant_start|>", "<|assistant_end|>", "<|inner_prefix|>", "<|inner_suffix|>", "<|tools_prefix|>", "<|tools_suffix|>", "<|tool_output_start|>", "<|tool_output_end|>", "<|image|>", "<|audio|>", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "" ], "decoder_add_prefix_space": true, "output_group": "tokenizer-lm-toks", "output_dir_name": "nfc_clean_multi_plus3_capped_hybrid_window_tuned_consv2_tailcuts_gm70k_v130_sp_eng5g" }, "grouped_config_path": "/users/cmeister747/pa_tokenizers_branch/configs/parity_aware_config_grouped_fineweb2full_quota_tuned_consv2_tailcuts_eng5g.json", "vocab_size": 130999, "min_frequency": 2, "special_tokens": [ "", "", "", "", "[/INST]", "[AVAILABLE_TOOLS]", "[/AVAILABLE_TOOLS]", "[TOOL_RESULTS]", "[/TOOL_RESULTS]", "[TOOL_CALLS]", "", "[PREFIX]", "[MIDDLE]", "[SUFFIX]", "\\begin{", "\\end{", "\\text{", "\\boxed{", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "<|fim_begin|>", "<|fim_hole|>", "<|fim_end|>", "<|system_start|>", "<|system_end|>", "<|developer_start|>", "<|developer_end|>", "<|user_start|>", "<|user_end|>", "<|assistant_start|>", "<|assistant_end|>", "<|inner_prefix|>", "<|inner_suffix|>", "<|tools_prefix|>", "<|tools_suffix|>", "<|tool_output_start|>", "<|tool_output_end|>", "<|image|>", "<|audio|>", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "" ], "start_time": "2026-06-05T15:57:20.559966", "elapsed_seconds": 7154.459067, "slurm_job_id": "2475069", "cli_args": [ "/users/cmeister747/pa_tokenizers_branch/train_tokenizer.py", "--variant", "pa_bpe_nfc_clean_multi_plus3_capped_hybrid_window_tuned_consv2_tailcuts_gm70k_v130_sp_eng5g" ], "git": { "commit": "5d8381af9ec1417b078f7dce82bac68873da915e", "branch": "parity-aware-bpe", "dirty": false }, "per_group_summary": { "n_groups": 25, "full_block_in": "tokenizers/pabpe-nfc-clean-fineweb2full-plus3-capped-tuned-hybridwindow-consv2-tailcuts-gm70k-v130-sp-eng5g/run_manifest.json" } }, "registry_note": "PA-BPE hybrid global+window (gm70k), clean-multi plus3 regex; fineweb2full, consv2 + tailcuts + eng5g data; norm=NFC; PA-BPE (hybrid global+window); vocab ~131,196; Apertus-2 development ablation 'PA-Clean-plus3-A6'; USED - S6f Contrast B (131k arm) + Apertus-2 track ablation; excluded from main paper panel" }, "pabpe-nfc-clean-fineweb2full-plus3-capped-tuned-hybridwindow-consv2-tailcuts-gm90k-v130-sp-eng5g": { "slug": "pabpe-nfc-clean-fineweb2full-plus3-capped-tuned-hybridwindow-consv2-tailcuts-gm90k-v130-sp-eng5g", "runs": [ "full-128k-pabpe-nfc-clean-fineweb2full-plus3-capped-tuned-hybridwindow-consv2-tailcuts-gm90k-v130-sp-eng5g", "full-128k-pabpe-nfc-clean-fineweb2full-plus3-capped-tuned-hybridwindow-consv2-tailcuts-gm90k-v130-sp-eng5g-mathcode-scratch" ], "n_runs": 2, "redistributable": true, "loadable_with": "transformers", "hub_id": null, "policy_note": "trained in this project", "empirical": { "model_type": "BPE", "n_base_tokens": 130929, "n_added_tokens": 99, "vocab_total": 131028, "added_tokens": [ "", "", "", "", "[/INST]", "[AVAILABLE_TOOLS]", "[/AVAILABLE_TOOLS]", "[TOOL_RESULTS]", "[/TOOL_RESULTS]", "[TOOL_CALLS]", "", "[PREFIX]", "[MIDDLE]", "[SUFFIX]", "\\begin{", "\\end{", "\\text{", "\\boxed{", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "<|fim_begin|>", "<|fim_hole|>", "<|fim_end|>", "<|system_start|>", "<|system_end|>", "<|developer_start|>", "<|developer_end|>", "<|user_start|>", "<|user_end|>", "<|assistant_start|>", "<|assistant_end|>", "<|inner_prefix|>", "<|inner_suffix|>", "<|tools_prefix|>", "<|tools_suffix|>", "<|tool_output_start|>", "<|tool_output_end|>", "<|image|>", "<|audio|>", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "" ], "normalizer": { "type": "NFC" }, "pre_tokenizer": { "type": "Sequence", "pretokenizers": [ { "type": "Split", "pattern": { "Regex": "[ \\x{0F0B}\\x{0027}\\x{2019}]?[\\p{Lu}\\p{Lt}\\p{Lm}\\p{Lo}\\p{M}]*[\\p{Ll}\\p{Lm}\\p{Lo}\\p{M}]+(?:[\\x{0027}\\x{2019}](?!\\p{L}))?|[ \\x{0F0B}\\x{0027}\\x{2019}]?[\\p{Lu}\\p{Lt}\\p{Lm}\\p{Lo}\\p{M}]+[\\p{Ll}\\p{Lm}\\p{Lo}\\p{M}]*(?:[\\x{0027}\\x{2019}](?!\\p{L}))?|\\p{N}| ?[^\\s\\p{L}\\p{N}]{1,16}|\\s{0,16}[\\r\\n]{1,16}|\\s{1,16}(?!\\S)|\\s{1,16}" }, "behavior": "Isolated", "invert": false }, { "type": "ByteLevel", "add_prefix_space": false, "trim_offsets": true, "use_regex": false } ] }, "has_post_processor": false, "byte_fallback": false, "bos_token": "", "bos_token_id": 1 }, "declared_experiments": null, "declared_run_manifest": { "variant": "pa_bpe_nfc_clean_multi_plus3_capped_hybrid_window_tuned_consv2_tailcuts_gm90k_v130_sp_eng5g", "variant_config": { "trainer": "parity-bpe", "pretok": "clean_multi_plus3_capped_nfc", "variant": "window", "global_merges": 90000, "window_size": 100, "alpha": 2.0, "signal": "ratios", "vocab_size": 130999, "grouped_config": "configs/parity_aware_config_grouped_fineweb2full_quota_tuned_consv2_tailcuts_eng5g.json", "special_tokens_override": [ "", "", "", "", "[/INST]", "[AVAILABLE_TOOLS]", "[/AVAILABLE_TOOLS]", "[TOOL_RESULTS]", "[/TOOL_RESULTS]", "[TOOL_CALLS]", "", "[PREFIX]", "[MIDDLE]", "[SUFFIX]", "\\begin{", "\\end{", "\\text{", "\\boxed{", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "<|fim_begin|>", "<|fim_hole|>", "<|fim_end|>", "<|system_start|>", "<|system_end|>", "<|developer_start|>", "<|developer_end|>", "<|user_start|>", "<|user_end|>", "<|assistant_start|>", "<|assistant_end|>", "<|inner_prefix|>", "<|inner_suffix|>", "<|tools_prefix|>", "<|tools_suffix|>", "<|tool_output_start|>", "<|tool_output_end|>", "<|image|>", "<|audio|>", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "" ], "decoder_add_prefix_space": true, "output_group": "tokenizer-lm-toks", "output_dir_name": "nfc_clean_multi_plus3_capped_hybrid_window_tuned_consv2_tailcuts_gm90k_v130_sp_eng5g" }, "grouped_config_path": "/users/cmeister747/pa_tokenizers_branch/configs/parity_aware_config_grouped_fineweb2full_quota_tuned_consv2_tailcuts_eng5g.json", "vocab_size": 130999, "min_frequency": 2, "special_tokens": [ "", "", "", "", "[/INST]", "[AVAILABLE_TOOLS]", "[/AVAILABLE_TOOLS]", "[TOOL_RESULTS]", "[/TOOL_RESULTS]", "[TOOL_CALLS]", "", "[PREFIX]", "[MIDDLE]", "[SUFFIX]", "\\begin{", "\\end{", "\\text{", "\\boxed{", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "<|fim_begin|>", "<|fim_hole|>", "<|fim_end|>", "<|system_start|>", "<|system_end|>", "<|developer_start|>", "<|developer_end|>", "<|user_start|>", "<|user_end|>", "<|assistant_start|>", "<|assistant_end|>", "<|inner_prefix|>", "<|inner_suffix|>", "<|tools_prefix|>", "<|tools_suffix|>", "<|tool_output_start|>", "<|tool_output_end|>", "<|image|>", "<|audio|>", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "" ], "start_time": "2026-06-05T15:57:20.375628", "elapsed_seconds": 7320.610917, "slurm_job_id": "2475071", "cli_args": [ "/users/cmeister747/pa_tokenizers_branch/train_tokenizer.py", "--variant", "pa_bpe_nfc_clean_multi_plus3_capped_hybrid_window_tuned_consv2_tailcuts_gm90k_v130_sp_eng5g" ], "git": { "commit": "5d8381af9ec1417b078f7dce82bac68873da915e", "branch": "parity-aware-bpe", "dirty": false }, "per_group_summary": { "n_groups": 25, "full_block_in": "tokenizers/pabpe-nfc-clean-fineweb2full-plus3-capped-tuned-hybridwindow-consv2-tailcuts-gm90k-v130-sp-eng5g/run_manifest.json" } }, "registry_note": "PA-BPE hybrid global+window (gm90k), clean-multi plus3 regex; fineweb2full, consv2 + tailcuts + eng5g data; norm=NFC; PA-BPE (hybrid global+window); vocab ~131,196; Apertus-2 development ablation 'PA-Clean-plus3-A8'; USED - S6f Contrast B (131k arm) + Apertus-2 track ablation; excluded from main paper panel" }, "pabpe-nfc-clean-fw2full-plus3-cap-hw-consv2-gm90k-tailcuts-eng5g-repcap8": { "slug": "pabpe-nfc-clean-fw2full-plus3-cap-hw-consv2-gm90k-tailcuts-eng5g-repcap8", "runs": [ "full-128k-pabpe-nfc-clean-fw2full-plus3-cap-hw-consv2-gm90k-tailcuts-eng5g-repcap8", "full-128k-pabpe-nfc-clean-fw2full-plus3-cap-hw-consv2-gm90k-tailcuts-eng5g-repcap8-mathcode-scratch" ], "n_runs": 2, "redistributable": true, "loadable_with": "transformers", "hub_id": null, "policy_note": "trained in this project", "empirical": { "model_type": "BPE", "n_base_tokens": 131017, "n_added_tokens": 124, "vocab_total": 131141, "added_tokens": [ "", "", "", "", "<|system_start|>", "<|system_end|>", "<|developer_start|>", "<|developer_end|>", "<|user_start|>", "<|user_end|>", "<|assistant_start|>", "<|assistant_end|>", "<|inner_prefix|>", "<|inner_suffix|>", "<|tools_prefix|>", "<|tools_suffix|>", "<|tool_output_start|>", "<|tool_output_end|>", "<|image|>", "<|audio|>", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "" ], "normalizer": { "type": "NFC" }, "pre_tokenizer": { "type": "Sequence", "pretokenizers": [ { "type": "Split", "pattern": { "Regex": "[ \\x{0F0B}\\x{0027}\\x{2019}]?(?[^\\s])\\k{7}|[ \\x{0F0B}\\x{0027}\\x{2019}]?(?:(?!(?.)\\k{8})[\\p{Lu}\\p{Lt}\\p{Lm}\\p{Lo}\\p{M}])*(?:(?!(?.)\\k{8})[\\p{Ll}\\p{Lm}\\p{Lo}\\p{M}])+(?:[\\x{0027}\\x{2019}](?!\\p{L}))?|[ \\x{0F0B}\\x{0027}\\x{2019}]?(?:(?!(?.)\\k{8})[\\p{Lu}\\p{Lt}\\p{Lm}\\p{Lo}\\p{M}])+(?:(?!(?.)\\k{8})[\\p{Ll}\\p{Lm}\\p{Lo}\\p{M}])*(?:[\\x{0027}\\x{2019}](?!\\p{L}))?|\\p{N}| ?(?:(?!(?

.)\\k

{8})[^\\s\\p{L}\\p{N}]){1,16}|\\s{0,16}[\\r\\n]{1,16}|\\s{1,16}(?!\\S)|\\s{1,16}" }, "behavior": "Isolated", "invert": false }, { "type": "ByteLevel", "add_prefix_space": false, "trim_offsets": true, "use_regex": false } ] }, "has_post_processor": false, "byte_fallback": false, "bos_token": "", "bos_token_id": 1 }, "declared_experiments": null, "declared_run_manifest": { "variant": "pa_bpe_nfc_clean_multi_plus3_repcap8_capped_hybrid_window_tuned_consv2_tailcuts_gm90k_v131k_sp124_eng5g_fr", "variant_config": { "trainer": "parity-bpe", "pretok": "clean_multi_plus3_repcap8_capped_nfc", "variant": "window", "global_merges": 90000, "window_size": 100, "alpha": 2.0, "signal": "ratios", "vocab_size": 131072, "grouped_config": "configs/parity_aware_config_grouped_fineweb2full_quota_tuned_consv2_tailcuts_eng5g.json", "special_tokens_override": [ "", "", "", "", "<|system_start|>", "<|system_end|>", "<|developer_start|>", "<|developer_end|>", "<|user_start|>", "<|user_end|>", "<|assistant_start|>", "<|assistant_end|>", "<|inner_prefix|>", "<|inner_suffix|>", "<|tools_prefix|>", "<|tools_suffix|>", "<|tool_output_start|>", "<|tool_output_end|>", "<|image|>", "<|audio|>", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "" ], "decoder_add_prefix_space": true, "output_group": "tokenizer-lm-toks", "output_dir_name": "nfc_clean_multi_plus3_repcap8_capped_hybrid_window_tuned_consv2_tailcuts_gm90k_v131k_sp124_eng5g_fr" }, "grouped_config_path": "/users/cmeister747/pa_tokenizers_branch/configs/parity_aware_config_grouped_fineweb2full_quota_tuned_consv2_tailcuts_eng5g.json", "vocab_size": 131072, "min_frequency": 2, "special_tokens": [ "", "", "", "", "<|system_start|>", "<|system_end|>", "<|developer_start|>", "<|developer_end|>", "<|user_start|>", "<|user_end|>", "<|assistant_start|>", "<|assistant_end|>", "<|inner_prefix|>", "<|inner_suffix|>", "<|tools_prefix|>", "<|tools_suffix|>", "<|tool_output_start|>", "<|tool_output_end|>", "<|image|>", "<|audio|>", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "" ], "start_time": "2026-06-08T17:50:22.080123", "elapsed_seconds": 7204.537718, "slurm_job_id": "2497771", "cli_args": [ "/users/cmeister747/pa_tokenizers_branch/train_tokenizer.py", "--variant", "pa_bpe_nfc_clean_multi_plus3_repcap8_capped_hybrid_window_tuned_consv2_tailcuts_gm90k_v131k_sp124_eng5g_fr" ], "git": { "commit": "5d8381af9ec1417b078f7dce82bac68873da915e", "branch": "parity-aware-bpe", "dirty": false }, "per_group_summary": { "n_groups": 25, "full_block_in": "tokenizers/pabpe-nfc-clean-fw2full-plus3-cap-hw-consv2-gm90k-tailcuts-eng5g-repcap8/run_manifest.json" } }, "registry_note": "PA-BPE hybrid global+window (gm90k), clean-multi plus3 regex + repcap8; fineweb2full, consv2 + tailcuts + eng5g data; norm=NFC; PA-BPE (hybrid global+window); vocab ~131,141 (131,017 + 124 specials; Apertus precedent for >128k vocab); Apertus-2 development ablation 'PA-Clean-plus3-repcap8fr-A8'; USED - S6f Contrast B (131k arm) + Apertus-2 track ablation; excluded from main paper panel" }, "pabpe-nfc-gpt4-balanced": { "slug": "pabpe-nfc-gpt4-balanced", "runs": [ "full-128k-pabpe-nfc-gpt4-balanced", "full-128k-pabpe-nfc-gpt4-balanced-mathcode-scratch", "pilot-128k-pabpe-nfc-gpt4-balanced", "small-128k-pabpe-nfc-gpt4-balanced", "tiny-128k-pabpe-nfc-gpt4-balanced" ], "n_runs": 5, "redistributable": true, "loadable_with": "transformers", "hub_id": null, "policy_note": "trained in this project", "empirical": { "model_type": "BPE", "n_base_tokens": 127822, "n_added_tokens": 4, "vocab_total": 127826, "added_tokens": [ "", "", "", "" ], "normalizer": { "type": "NFC" }, "pre_tokenizer": { "type": "Sequence", "pretokenizers": [ { "type": "Split", "pattern": { "Regex": "[^\\r\\n\\p{L}\\p{N}]?[\\p{Lu}\\p{Lt}\\p{Lm}\\p{Lo}\\p{M}]*[\\p{Ll}\\p{Lm}\\p{Lo}\\p{M}]+(?i:'s|'t|'re|'ve|'m|'ll|'d)?|[^\\r\\n\\p{L}\\p{N}]?[\\p{Lu}\\p{Lt}\\p{Lm}\\p{Lo}\\p{M}]+[\\p{Ll}\\p{Lm}\\p{Lo}\\p{M}]*(?i:'s|'t|'re|'ve|'m|'ll|'d)?|\\p{N}{1,3}| ?[^\\s\\p{L}\\p{N}]+[\\r\\n/]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+" }, "behavior": "Isolated", "invert": false }, { "type": "ByteLevel", "add_prefix_space": false, "trim_offsets": true, "use_regex": false } ] }, "has_post_processor": false, "byte_fallback": false, "bos_token": "", "bos_token_id": 127822 }, "declared_experiments": null, "declared_run_manifest": { "variant": "pa_bpe_nfc_gpt4_balanced", "variant_config": { "trainer": "parity-bpe", "pretok": "gpt4o_nfc", "variant": "base", "global_merges": 0, "signal": "ratios", "grouped_config": "configs/parity_aware_config_balanced.json", "special_tokens_override": [], "decoder_add_prefix_space": true, "output_group": "tokenizer-lm-toks", "output_dir_name": "nfc_gpt4_balanced", "post_training_special_tokens": [ "", "", "", "" ] }, "grouped_config_path": "/users/cmeister747/pa_tokenizers_branch/configs/parity_aware_config_balanced.json", "vocab_size": 128000, "min_frequency": 2, "special_tokens": [], "start_time": "2026-05-15T21:37:46.656038", "elapsed_seconds": 2543.241044, "slurm_job_id": "2256968", "cli_args": [ "/users/cmeister747/pa_tokenizers_branch/train_tokenizer.py", "--variant", "pa_bpe_nfc_gpt4_balanced" ], "git": { "commit": "b2ceac78388c06deb786755b60d6c1d335cb6c25", "branch": "parity-aware-bpe", "dirty": false }, "per_group_summary": { "n_groups": 34, "full_block_in": "tokenizers/pabpe-nfc-gpt4-balanced/run_manifest.json" } }, "registry_note": "PA-BPE (parity, grouped=25 script-family units) with GPT-4o regex; balanced (D2) data; norm=NFC; PA-BPE (ParityBpeTrainer); vocab 127,826; bug-fixed 2026-05-16 re-export of the retired pabpe-balanced (2a); replaces the newline-stripping-bug original; USED - panels.json main21 (camera-ready main panel)" }, "pabpe-nfc-gpt4-balanced-hybridwindow": { "slug": "pabpe-nfc-gpt4-balanced-hybridwindow", "runs": [ "full-128k-pabpe-nfc-gpt4-balanced-hybridwindow" ], "n_runs": 1, "redistributable": true, "loadable_with": "transformers", "hub_id": null, "policy_note": "trained in this project", "empirical": { "model_type": "BPE", "n_base_tokens": 127822, "n_added_tokens": 4, "vocab_total": 127826, "added_tokens": [ "", "", "", "" ], "normalizer": { "type": "NFC" }, "pre_tokenizer": { "type": "Sequence", "pretokenizers": [ { "type": "Split", "pattern": { "Regex": "[^\\r\\n\\p{L}\\p{N}]?[\\p{Lu}\\p{Lt}\\p{Lm}\\p{Lo}\\p{M}]*[\\p{Ll}\\p{Lm}\\p{Lo}\\p{M}]+(?i:'s|'t|'re|'ve|'m|'ll|'d)?|[^\\r\\n\\p{L}\\p{N}]?[\\p{Lu}\\p{Lt}\\p{Lm}\\p{Lo}\\p{M}]+[\\p{Ll}\\p{Lm}\\p{Lo}\\p{M}]*(?i:'s|'t|'re|'ve|'m|'ll|'d)?|\\p{N}{1,3}| ?[^\\s\\p{L}\\p{N}]+[\\r\\n/]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+" }, "behavior": "Isolated", "invert": false }, { "type": "ByteLevel", "add_prefix_space": false, "trim_offsets": true, "use_regex": false } ] }, "has_post_processor": false, "byte_fallback": false, "bos_token": "", "bos_token_id": 127822 }, "declared_experiments": null, "declared_run_manifest": { "variant": "pa_bpe_nfc_gpt4_balanced_hybrid_window", "variant_config": { "trainer": "parity-bpe", "pretok": "gpt4o_nfc", "variant": "window", "global_merges": 64000, "window_size": 100, "alpha": 2.0, "signal": "ratios", "grouped_config": "configs/parity_aware_config_balanced.json", "special_tokens_override": [], "decoder_add_prefix_space": true, "output_group": "tokenizer-lm-toks", "output_dir_name": "nfc_gpt4_balanced_hybrid_window", "post_training_special_tokens": [ "", "", "", "" ] }, "grouped_config_path": "/users/cmeister747/pa_tokenizers_branch/configs/parity_aware_config_balanced.json", "vocab_size": 128000, "min_frequency": 2, "special_tokens": [], "start_time": "2026-05-16T15:19:37.996073", "elapsed_seconds": 2379.645128, "slurm_job_id": "2260974", "cli_args": [ "/users/cmeister747/pa_tokenizers_branch/train_tokenizer.py", "--variant", "pa_bpe_nfc_gpt4_balanced_hybrid_window" ], "git": { "commit": "7172cc669840d0dcce4170f52ef0430eb2a3d899", "branch": "parity-aware-bpe", "dirty": false }, "per_group_summary": { "n_groups": 34, "full_block_in": "tokenizers/pabpe-nfc-gpt4-balanced-hybridwindow/run_manifest.json" } }, "registry_note": "PA-BPE hybrid global+window (global_merges=64000, window=100, alpha=2.0, signal=ratios) with GPT-4o regex; balanced (D2) data; norm=NFC; PA-BPE (hybrid global+window); vocab 127,826; tests whether the hybrid-window scheme improves over base PA-BPE at fixed data/pretok; USED - balanced-1B bootstrap panel (S6.12) + paper panel; not in panels.json main21" }, "pabpe-nfc-gpt4-fineweb2full": { "slug": "pabpe-nfc-gpt4-fineweb2full", "runs": [ "full-128k-pabpe-nfc-gpt4-fineweb2full" ], "n_runs": 1, "redistributable": true, "loadable_with": "transformers", "hub_id": null, "policy_note": "trained in this project", "empirical": { "model_type": "BPE", "n_base_tokens": 127821, "n_added_tokens": 4, "vocab_total": 127825, "added_tokens": [ "", "", "", "" ], "normalizer": { "type": "NFC" }, "pre_tokenizer": { "type": "Sequence", "pretokenizers": [ { "type": "Split", "pattern": { "Regex": "[^\\r\\n\\p{L}\\p{N}]?[\\p{Lu}\\p{Lt}\\p{Lm}\\p{Lo}\\p{M}]*[\\p{Ll}\\p{Lm}\\p{Lo}\\p{M}]+(?i:'s|'t|'re|'ve|'m|'ll|'d)?|[^\\r\\n\\p{L}\\p{N}]?[\\p{Lu}\\p{Lt}\\p{Lm}\\p{Lo}\\p{M}]+[\\p{Ll}\\p{Lm}\\p{Lo}\\p{M}]*(?i:'s|'t|'re|'ve|'m|'ll|'d)?|\\p{N}{1,3}| ?[^\\s\\p{L}\\p{N}]+[\\r\\n/]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+" }, "behavior": "Isolated", "invert": false }, { "type": "ByteLevel", "add_prefix_space": false, "trim_offsets": true, "use_regex": false } ] }, "has_post_processor": false, "byte_fallback": false, "bos_token": "", "bos_token_id": 127821 }, "declared_experiments": null, "declared_run_manifest": { "variant": "pa_bpe_nfc_gpt4_fineweb2full", "variant_config": { "trainer": "parity-bpe", "pretok": "gpt4o_nfc", "variant": "base", "global_merges": 0, "signal": "ratios", "grouped_config": "configs/parity_aware_config_grouped_fineweb2full_quota.json", "special_tokens_override": [], "decoder_add_prefix_space": true, "output_group": "tokenizer-lm-toks", "output_dir_name": "nfc_gpt4_fineweb2full", "post_training_special_tokens": [ "", "", "", "" ] }, "grouped_config_path": "/users/cmeister747/pa_tokenizers_branch/configs/parity_aware_config_grouped_fineweb2full_quota.json", "vocab_size": 128000, "min_frequency": 2, "special_tokens": [], "start_time": "2026-05-22T07:43:23.652650", "elapsed_seconds": 8217.121987, "slurm_job_id": "2328518", "cli_args": [ "/users/cmeister747/pa_tokenizers_branch/train_tokenizer.py", "--variant", "pa_bpe_nfc_gpt4_fineweb2full" ], "git": { "commit": "5d8381af9ec1417b078f7dce82bac68873da915e", "branch": "parity-aware-bpe", "dirty": false }, "per_group_summary": { "n_groups": 25, "full_block_in": "tokenizers/pabpe-nfc-gpt4-fineweb2full/run_manifest.json" } }, "registry_note": "PA-BPE (parity, base) with GPT-4o regex; fineweb2full data; norm=NFC; PA-BPE (base); vocab 127,825; USED - S5b fineweb2full analysis; fineweb2full data variant, excluded from main paper panel" }, "pabpe-nfc-gpt4-fineweb2full-hybridwindow": { "slug": "pabpe-nfc-gpt4-fineweb2full-hybridwindow", "runs": [ "full-128k-pabpe-nfc-gpt4-fineweb2full-hybridwindow" ], "n_runs": 1, "redistributable": true, "loadable_with": "transformers", "hub_id": null, "policy_note": "trained in this project", "empirical": { "model_type": "BPE", "n_base_tokens": 127821, "n_added_tokens": 4, "vocab_total": 127825, "added_tokens": [ "", "", "", "" ], "normalizer": { "type": "NFC" }, "pre_tokenizer": { "type": "Sequence", "pretokenizers": [ { "type": "Split", "pattern": { "Regex": "[^\\r\\n\\p{L}\\p{N}]?[\\p{Lu}\\p{Lt}\\p{Lm}\\p{Lo}\\p{M}]*[\\p{Ll}\\p{Lm}\\p{Lo}\\p{M}]+(?i:'s|'t|'re|'ve|'m|'ll|'d)?|[^\\r\\n\\p{L}\\p{N}]?[\\p{Lu}\\p{Lt}\\p{Lm}\\p{Lo}\\p{M}]+[\\p{Ll}\\p{Lm}\\p{Lo}\\p{M}]*(?i:'s|'t|'re|'ve|'m|'ll|'d)?|\\p{N}{1,3}| ?[^\\s\\p{L}\\p{N}]+[\\r\\n/]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+" }, "behavior": "Isolated", "invert": false }, { "type": "ByteLevel", "add_prefix_space": false, "trim_offsets": true, "use_regex": false } ] }, "has_post_processor": false, "byte_fallback": false, "bos_token": "", "bos_token_id": 127821 }, "declared_experiments": null, "declared_run_manifest": { "variant": "pa_bpe_nfc_gpt4_fineweb2full_hybrid_window", "variant_config": { "trainer": "parity-bpe", "pretok": "gpt4o_nfc", "variant": "window", "global_merges": 64000, "window_size": 100, "alpha": 2.0, "signal": "ratios", "grouped_config": "configs/parity_aware_config_grouped_fineweb2full_quota.json", "special_tokens_override": [], "decoder_add_prefix_space": true, "output_group": "tokenizer-lm-toks", "output_dir_name": "nfc_gpt4_fineweb2full_hybrid_window", "post_training_special_tokens": [ "", "", "", "" ] }, "grouped_config_path": "/users/cmeister747/pa_tokenizers_branch/configs/parity_aware_config_grouped_fineweb2full_quota.json", "vocab_size": 128000, "min_frequency": 2, "special_tokens": [], "start_time": "2026-05-22T07:43:23.607801", "elapsed_seconds": 7587.888544, "slurm_job_id": "2328519", "cli_args": [ "/users/cmeister747/pa_tokenizers_branch/train_tokenizer.py", "--variant", "pa_bpe_nfc_gpt4_fineweb2full_hybrid_window" ], "git": { "commit": "5d8381af9ec1417b078f7dce82bac68873da915e", "branch": "parity-aware-bpe", "dirty": false }, "per_group_summary": { "n_groups": 25, "full_block_in": "tokenizers/pabpe-nfc-gpt4-fineweb2full-hybridwindow/run_manifest.json" } }, "registry_note": "PA-BPE hybrid global+window with GPT-4o regex; fineweb2full data; norm=NFC; PA-BPE (hybrid global+window); vocab 127,825; USED - S5b fineweb2full analysis; fineweb2full data variant, excluded from main paper panel" }, "punct-allmulti-equal-bpe": { "slug": "punct-allmulti-equal-bpe", "runs": [ "full-128k-punct-allmulti-equal-bpe" ], "n_runs": 1, "redistributable": true, "loadable_with": "transformers", "hub_id": null, "policy_note": "trained in this project", "empirical": { "model_type": "BPE", "n_base_tokens": 128256, "n_added_tokens": 4, "vocab_total": 128260, "added_tokens": [ "", "", "", "" ], "normalizer": null, "pre_tokenizer": { "type": "Sequence", "pretokenizers": [ { "type": "Punctuation", "behavior": "Isolated" }, { "type": "ByteLevel", "add_prefix_space": false, "trim_offsets": true, "use_regex": true } ] }, "has_post_processor": false, "byte_fallback": false, "bos_token": "", "bos_token_id": 0 }, "declared_experiments": { "algorithm": "bpe", "pretokenizer_constant": "Punctuation+ByteLevel", "normalizer": null, "data_composition": "allmulti_equal", "description": "Punctuation+ByteLevel pretok, equal-per-language data (repeat-sampled), BPE. Pairs with punct-balanced-bpe.", "trainer_kwargs": null }, "declared_run_manifest": null, "registry_note": "P1 Punctuation+ByteLevel; allmulti_equal data; norm=none; BPE; vocab 128,256; pairs with punct-balanced-bpe; USED - S6d.4 (rho=-0.375, weakest of the four - punct also compresses the tail least)" }, "punct-balanced-bpe": { "slug": "punct-balanced-bpe", "runs": [ "full-128k-punct-balanced-bpe", "full-128k-punct-balanced-bpe-mathcode-scratch", "full-128k-punct-balanced-bpe-seed42", "full-128k-punct-balanced-bpe-seed42-mathcode-scratch", "full-128k-punct-balanced-bpe-seed43-mathcode-scratch", "full-128k-punct-balanced-bpe-seed44-mathcode-scratch", "pilot-128k-punct-balanced-bpe-wdfix", "small-128k-punct-balanced-bpe", "tiny-128k-punct-balanced-bpe" ], "n_runs": 9, "redistributable": true, "loadable_with": "transformers", "hub_id": null, "policy_note": "trained in this project", "empirical": { "model_type": "BPE", "n_base_tokens": 128256, "n_added_tokens": 4, "vocab_total": 128260, "added_tokens": [ "", "", "", "" ], "normalizer": null, "pre_tokenizer": { "type": "Sequence", "pretokenizers": [ { "type": "Punctuation", "behavior": "Isolated" }, { "type": "ByteLevel", "add_prefix_space": false, "trim_offsets": true, "use_regex": true } ] }, "has_post_processor": false, "byte_fallback": false, "bos_token": "", "bos_token_id": 0 }, "declared_experiments": { "algorithm": "bpe", "pretokenizer_constant": "Punctuation+ByteLevel", "normalizer": null, "data_composition": "balanced", "description": "Punctuation + ByteLevel pretok, balanced data", "trainer_kwargs": null }, "declared_run_manifest": null, "registry_note": "P1 Punctuation+ByteLevel; balanced (D2) data; norm=none; BPE; vocab 128,260; USED - panels.json main21 (camera-ready main panel)" }, "punct-balancedhalf-superbpe-t64k-copy": { "slug": "punct-balancedhalf-superbpe-t64k-copy", "runs": [ "full-128k-punct-balancedhalf-superbpe-t64k-copy" ], "n_runs": 1, "redistributable": true, "loadable_with": "transformers", "hub_id": null, "policy_note": "trained in this project", "empirical": { "model_type": "BPE", "n_base_tokens": 128000, "n_added_tokens": 4, "vocab_total": 128004, "added_tokens": [ "", "", "", "" ], "normalizer": null, "pre_tokenizer": { "type": "Sequence", "pretokenizers": [ { "type": "Split", "pattern": { "Regex": "\\s+(?!\\S)|\\s*[\\r\\n]+" }, "behavior": "Isolated", "invert": false }, { "type": "ByteLevel", "add_prefix_space": false, "trim_offsets": true, "use_regex": false } ] }, "has_post_processor": false, "byte_fallback": false, "bos_token": "", "bos_token_id": 128000 }, "declared_experiments": null, "declared_run_manifest": null, "registry_note": "SuperBPE stage-1 Punct(Isolated)+ByteLevel base (64k transition) / stage-2 minimal regex (allows superwords to span punctuation); balanced-half (D2 halved) data; norm=none (inherited from punct base); SuperBPE; vocab 128,004; the only SuperBPE variant EXEMPT from the S9.5 combining-mark bug (its stage-2 regex has no [^\\s\\p{L}\\p{N}] alternative); USED - balanced-1B bootstrap panel (S6.12) + paper panel; not in panels.json main21" }, "punct-english-bpe": { "slug": "punct-english-bpe", "runs": [ "full-128k-punct-english-bpe", "full-128k-punct-english-bpe-mathcode-scratch", "full-128k-punct-english-bpe-seed42", "pilot-128k-punct-english-bpe-wdfix", "small-128k-punct-english-bpe", "tiny-128k-punct-english-bpe" ], "n_runs": 6, "redistributable": true, "loadable_with": "transformers", "hub_id": null, "policy_note": "trained in this project", "empirical": { "model_type": "BPE", "n_base_tokens": 128256, "n_added_tokens": 4, "vocab_total": 128260, "added_tokens": [ "", "", "", "" ], "normalizer": null, "pre_tokenizer": { "type": "Sequence", "pretokenizers": [ { "type": "Punctuation", "behavior": "Isolated" }, { "type": "ByteLevel", "add_prefix_space": false, "trim_offsets": true, "use_regex": true } ] }, "has_post_processor": false, "byte_fallback": false, "bos_token": "", "bos_token_id": 0 }, "declared_experiments": { "algorithm": "bpe", "pretokenizer_constant": "Punctuation+ByteLevel", "normalizer": null, "data_composition": "english", "description": "Punctuation + ByteLevel pretok, English-only data", "trainer_kwargs": null }, "declared_run_manifest": null, "registry_note": "P1 Punctuation+ByteLevel; english (D1) data; norm=none; BPE; vocab 128,260; USED - panels.json main21 (camera-ready main panel)" }, "rightalign-balanced-bpe": { "slug": "rightalign-balanced-bpe", "runs": [ "full-128k-rightalign-balanced-bpe", "full-128k-rightalign-balanced-bpe-mathcode-scratch", "full-128k-rightalign-balanced-bpe-seed42", "pilot-128k-rightalign-balanced-bpe-wdfix", "small-128k-rightalign-balanced-bpe", "tiny-128k-rightalign-balanced-bpe" ], "n_runs": 6, "redistributable": true, "loadable_with": "transformers", "hub_id": null, "policy_note": "trained in this project", "empirical": { "model_type": "BPE", "n_base_tokens": 128256, "n_added_tokens": 4, "vocab_total": 128260, "added_tokens": [ "", "", "", "" ], "normalizer": null, "pre_tokenizer": { "type": "Sequence", "pretokenizers": [ { "type": "Split", "pattern": { "Regex": "[^\\r\\n\\p{L}\\p{N}]?[\\p{Lu}\\p{Lt}\\p{Lm}\\p{Lo}\\p{M}]*[\\p{Ll}\\p{Lm}\\p{Lo}\\p{M}]+(?i:'s|'t|'re|'ve|'m|'ll|'d)?|[^\\r\\n\\p{L}\\p{N}]?[\\p{Lu}\\p{Lt}\\p{Lm}\\p{Lo}\\p{M}]+[\\p{Ll}\\p{Lm}\\p{Lo}\\p{M}]*(?i:'s|'t|'re|'ve|'m|'ll|'d)?|\\p{N}{1,3}(?=(?:\\p{N}{3})*(?:\\P{N}|$))| ?[^\\s\\p{L}\\p{N}]+[\\r\\n/]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+" }, "behavior": "Isolated", "invert": false }, { "type": "ByteLevel", "add_prefix_space": false, "trim_offsets": true, "use_regex": false } ] }, "has_post_processor": false, "byte_fallback": false, "bos_token": "", "bos_token_id": 0 }, "declared_experiments": { "algorithm": "bpe", "pretokenizer_constant": "REGEX_GPT4O_RIGHTALIGN", "normalizer": null, "data_composition": "balanced", "description": "GPT-4o regex with right-aligned 3-digit parsing, balanced data", "trainer_kwargs": null }, "declared_run_manifest": null, "registry_note": "P4 right-aligned digits (REGEX_GPT4O_RIGHTALIGN); balanced (D2) data; norm=none; BPE; vocab 128,260; USED - panels.json main21 (camera-ready main panel)" }, "rightalign-balanced-nfc-bpe": { "slug": "rightalign-balanced-nfc-bpe", "runs": [ "full-128k-rightalign-balanced-nfc-bpe", "full-128k-rightalign-balanced-nfc-bpe-mathcode-scratch", "full-128k-rightalign-balanced-nfc-bpe-seed42", "pilot-128k-rightalign-balanced-nfc-bpe-wdfix", "small-128k-rightalign-balanced-nfc-bpe", "tiny-128k-rightalign-balanced-nfc-bpe" ], "n_runs": 6, "redistributable": true, "loadable_with": "transformers", "hub_id": null, "policy_note": "trained in this project", "empirical": { "model_type": "BPE", "n_base_tokens": 128256, "n_added_tokens": 4, "vocab_total": 128260, "added_tokens": [ "", "", "", "" ], "normalizer": { "type": "NFC" }, "pre_tokenizer": { "type": "Sequence", "pretokenizers": [ { "type": "Split", "pattern": { "Regex": "[^\\r\\n\\p{L}\\p{N}]?[\\p{Lu}\\p{Lt}\\p{Lm}\\p{Lo}\\p{M}]*[\\p{Ll}\\p{Lm}\\p{Lo}\\p{M}]+(?i:'s|'t|'re|'ve|'m|'ll|'d)?|[^\\r\\n\\p{L}\\p{N}]?[\\p{Lu}\\p{Lt}\\p{Lm}\\p{Lo}\\p{M}]+[\\p{Ll}\\p{Lm}\\p{Lo}\\p{M}]*(?i:'s|'t|'re|'ve|'m|'ll|'d)?|\\p{N}{1,3}(?=(?:\\p{N}{3})*(?:\\P{N}|$))| ?[^\\s\\p{L}\\p{N}]+[\\r\\n/]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+" }, "behavior": "Isolated", "invert": false }, { "type": "ByteLevel", "add_prefix_space": false, "trim_offsets": true, "use_regex": false } ] }, "has_post_processor": false, "byte_fallback": false, "bos_token": "", "bos_token_id": 0 }, "declared_experiments": { "algorithm": "bpe", "pretokenizer_constant": "REGEX_GPT4O_RIGHTALIGN", "normalizer": { "type": "NFC" }, "data_composition": "balanced", "description": "Right-aligned digits, balanced data, NFC normalization", "trainer_kwargs": null }, "declared_run_manifest": null, "registry_note": "P4 right-aligned digits; balanced (D2) data; norm=NFC; BPE; vocab 128,260; USED - panels.json main21 (camera-ready main panel)" }, "rightalign-balanced-unigram": { "slug": "rightalign-balanced-unigram", "runs": [ "full-128k-rightalign-balanced-unigram", "full-128k-rightalign-balanced-unigram-mathcode-scratch", "full-128k-rightalign-balanced-unigram-seed42", "pilot-128k-rightalign-balanced-unigram-wdfix", "small-128k-rightalign-balanced-unigram", "tiny-128k-rightalign-balanced-unigram" ], "n_runs": 6, "redistributable": true, "loadable_with": "transformers", "hub_id": null, "policy_note": "trained in this project", "empirical": { "model_type": "Unigram", "n_base_tokens": 128256, "n_added_tokens": 4, "vocab_total": 128260, "added_tokens": [ "", "", "", "" ], "normalizer": null, "pre_tokenizer": { "type": "Sequence", "pretokenizers": [ { "type": "Split", "pattern": { "Regex": "[^\\r\\n\\p{L}\\p{N}]?[\\p{Lu}\\p{Lt}\\p{Lm}\\p{Lo}\\p{M}]*[\\p{Ll}\\p{Lm}\\p{Lo}\\p{M}]+(?i:'s|'t|'re|'ve|'m|'ll|'d)?|[^\\r\\n\\p{L}\\p{N}]?[\\p{Lu}\\p{Lt}\\p{Lm}\\p{Lo}\\p{M}]+[\\p{Ll}\\p{Lm}\\p{Lo}\\p{M}]*(?i:'s|'t|'re|'ve|'m|'ll|'d)?|\\p{N}{1,3}(?=(?:\\p{N}{3})*(?:\\P{N}|$))| ?[^\\s\\p{L}\\p{N}]+[\\r\\n/]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+" }, "behavior": "Isolated", "invert": false }, { "type": "ByteLevel", "add_prefix_space": false, "trim_offsets": true, "use_regex": false } ] }, "has_post_processor": false, "byte_fallback": false, "bos_token": "", "bos_token_id": 0 }, "declared_experiments": { "algorithm": "unigram", "pretokenizer_constant": "REGEX_GPT4O_RIGHTALIGN", "normalizer": null, "data_composition": "balanced", "description": "Right-aligned digits, balanced data, UnigramLM algorithm", "trainer_kwargs": null }, "declared_run_manifest": null, "registry_note": "P4 right-aligned digits; balanced (D2) data; norm=none; UnigramLM (HF, default kwargs); vocab 128,260; USED - panels.json main21 (camera-ready main panel)" }, "scripttok-bpe-scriptenc_gpt4o_cb": { "slug": "scripttok-bpe-scriptenc_gpt4o_cb", "runs": [ "full-128k-scripttok-bpe-scriptenc_gpt4o_cb", "full-128k-scripttok-bpe-scriptenc_gpt4o_cb-mathcode-scratch" ], "n_runs": 2, "redistributable": true, "loadable_with": "script_tok", "hub_id": null, "policy_note": "trained in this project; the SCRIPT-encoding format needs the script_tok loader, stock AutoTokenizer cannot read it", "empirical": { "format": "tokenizer.json.gz", "note": "not a tokenizers-library file; not introspected here" }, "declared_experiments": null, "declared_run_manifest": null, "registry_note": "script_tok BPE, scriptenc_gpt4o_cb SCRIPT-encoding pretok; balanced (D2, script_tok's own fit sampler) data; norm=SCRIPT-encoding normalization; BPE (script_tok); vocab 128,256; requires pre-tokenized .npz shards; USED - panels.json structure-pretok (camera-ready)" }, "scripttok-mingram-scriptenc_cb": { "slug": "scripttok-mingram-scriptenc_cb", "runs": [ "full-128k-scripttok-mingram-scriptenc_cb", "full-128k-scripttok-mingram-scriptenc_cb-mathcode-scratch" ], "n_runs": 2, "redistributable": true, "loadable_with": "script_tok", "hub_id": null, "policy_note": "trained in this project; the SCRIPT-encoding format needs the script_tok loader, stock AutoTokenizer cannot read it", "empirical": { "format": "tokenizer.json.gz", "note": "not a tokenizers-library file; not introspected here" }, "declared_experiments": null, "declared_run_manifest": null, "registry_note": "script_tok MinGram, scriptenc_cb SCRIPT-encoding pretok (-n 126339), PUA/unassigned code points stripped; balanced (D2, script_tok's own fit sampler/composition/seed) data; norm=SCRIPT-encoding normalization (designed, applied identically train/eval); MinGram; vocab 128,256 (1,916 atomic + 126,339 learned + synthesized BOS); requires pre-tokenized .npz shards; forward-fusing (newline+following indentation fuse into one atomic pretoken, later diagnosed as the MBPP/HumanEval-collapse mechanism at 20B, S3a.1/3a.6/S6h); USED - panels.json main21 (camera-ready main panel)" }, "scripttok-mingram-scriptenc_cb_nl": { "slug": "scripttok-mingram-scriptenc_cb_nl", "runs": [ "full-128k-scripttok-mingram-scriptenc_cb_nl", "full-128k-scripttok-mingram-scriptenc_cb_nl-mathcode-scratch" ], "n_runs": 2, "redistributable": true, "loadable_with": "script_tok", "hub_id": null, "policy_note": "trained in this project; the SCRIPT-encoding format needs the script_tok loader, stock AutoTokenizer cannot read it", "empirical": { "format": "tokenizer.json.gz", "note": "not a tokenizers-library file; not introspected here" }, "declared_experiments": null, "declared_run_manifest": null, "registry_note": "script_tok MinGram, scriptenc_cb_nl (split_line_breaks=True: newline isolated from indentation, additive opt-in); balanced (D2, balanced_tlm10g_cb_nl corpus); SCRIPT-encoding normalization; MinGram; vocab 128,256 (1,916 atomic+126,339 learned+BOS); line-break-splitting ablation vs scripttok-mingram-scriptenc_cb (newline-fusion mechanism). USED - S3a.6/S3a.7/S6h newline-fusion analysis (std-1B twin of 20B pair, finished 2026-08-08); design-variant ablation, not a panels.json member." }, "superbpe-apertus-fineweb2full-capped-hybridwindow": { "slug": "superbpe-apertus-fineweb2full-capped-hybridwindow", "runs": [ "full-128k-superbpe-apertus-fineweb2full-capped-hybridwindow", "full-128k-superbpe-apertus-fineweb2full-capped-hybridwindow-mathcode-scratch" ], "n_runs": 2, "redistributable": true, "loadable_with": "transformers", "hub_id": null, "policy_note": "trained in this project", "empirical": { "model_type": "BPE", "n_base_tokens": 128000, "n_added_tokens": 4, "vocab_total": 128004, "added_tokens": [ "", "", "", "" ], "normalizer": { "type": "NFC" }, "pre_tokenizer": { "type": "Sequence", "pretokenizers": [ { "type": "Split", "pattern": { "Regex": "\\p{N}| ?(?:(?:[^\\s\\p{L}\\p{N}\\p{M}]\\p{M}*){1,16}[\\r\\n]{1,16}|(?:[^\\s\\p{L}\\p{N}\\p{M}]\\p{M}*){2,16})|\\s{0,16}[\\r\\n]{1,16}|\\s{1,16}(?!\\S)" }, "behavior": "Isolated", "invert": false }, { "type": "ByteLevel", "add_prefix_space": false, "trim_offsets": true, "use_regex": false } ] }, "has_post_processor": false, "byte_fallback": false, "bos_token": "", "bos_token_id": 128000 }, "declared_experiments": null, "declared_run_manifest": null, "registry_note": "SuperBPE on the apertus-pretok capped hybrid-window base; fineweb2full, capped data; norm=NFC; SuperBPE; vocab 128,004; 2x2 candidate; USED - Apertus-2 development track candidate; excluded from main paper panel" }, "superbpe-clean-c2-balanced": { "slug": "superbpe-clean-c2-balanced", "runs": [ "full-128k-superbpe-clean-c2-balanced", "full-128k-superbpe-clean-c2-balanced-mathcode-scratch" ], "n_runs": 2, "redistributable": true, "loadable_with": "transformers", "hub_id": null, "policy_note": "trained in this project", "empirical": { "model_type": "BPE", "n_base_tokens": 128000, "n_added_tokens": 4, "vocab_total": 128004, "added_tokens": [ "", "", "", "" ], "normalizer": { "type": "NFC" }, "pre_tokenizer": { "type": "Sequence", "pretokenizers": [ { "type": "Split", "pattern": { "Regex": "\\p{N}| ?[^\\s\\p{L}\\p{N}\\p{M}]\\p{M}*(?:[^\\s\\p{L}\\p{N}\\p{M}]\\p{M}*)*|\\s*[\\r\\n]+|\\s+(?!\\S)" }, "behavior": "Isolated", "invert": false }, { "type": "ByteLevel", "add_prefix_space": false, "trim_offsets": true, "use_regex": false } ] }, "has_post_processor": false, "byte_fallback": false, "bos_token": "", "bos_token_id": 128000 }, "declared_experiments": null, "declared_run_manifest": null, "registry_note": "SuperBPE stage-1 clean-multi (PA-BPE clean base) / stage-2 clean_multi C2 reduced regex; balanced (D2) data; norm=NFC (inherited from PA-BPE clean base); SuperBPE; vocab 128,004; Phase-5 clean_multi_c2_t90k stage-2 preset; USED - balanced-1B bootstrap panel (S6.12) + paper panel; not in panels.json main21" }, "superbpe-clean-c3-balanced": { "slug": "superbpe-clean-c3-balanced", "runs": [ "full-128k-superbpe-clean-c3-balanced", "full-128k-superbpe-clean-c3-balanced-mathcode-scratch" ], "n_runs": 2, "redistributable": true, "loadable_with": "transformers", "hub_id": null, "policy_note": "trained in this project", "empirical": { "model_type": "BPE", "n_base_tokens": 128000, "n_added_tokens": 4, "vocab_total": 128004, "added_tokens": [ "", "", "", "" ], "normalizer": { "type": "NFC" }, "pre_tokenizer": { "type": "Sequence", "pretokenizers": [ { "type": "Split", "pattern": { "Regex": "\\p{N}{1,3}| ?[^\\s\\p{L}\\p{N}\\p{M}]\\p{M}*(?:[^\\s\\p{L}\\p{N}\\p{M}]\\p{M}*)+|\\s*[\\r\\n]+|\\s+(?!\\S)" }, "behavior": "Isolated", "invert": false }, { "type": "ByteLevel", "add_prefix_space": false, "trim_offsets": true, "use_regex": false } ] }, "has_post_processor": false, "byte_fallback": false, "bos_token": "", "bos_token_id": 128000 }, "declared_experiments": null, "declared_run_manifest": null, "registry_note": "SuperBPE stage-1 clean-multi / stage-2 clean_multi C3 reduced regex; balanced (D2) data; norm=NFC; SuperBPE; vocab 128,004; Phase-5 clean_multi_c3_t90k stage-2 preset; USED - balanced-1B bootstrap panel (S6.12) + paper panel; not in panels.json main21" }, "superbpe-clean-fineweb2full-capped-hybridwindow": { "slug": "superbpe-clean-fineweb2full-capped-hybridwindow", "runs": [ "full-128k-superbpe-clean-fineweb2full-capped-hybridwindow", "full-128k-superbpe-clean-fineweb2full-capped-hybridwindow-mathcode-scratch" ], "n_runs": 2, "redistributable": true, "loadable_with": "transformers", "hub_id": null, "policy_note": "trained in this project", "empirical": { "model_type": "BPE", "n_base_tokens": 128000, "n_added_tokens": 4, "vocab_total": 128004, "added_tokens": [ "", "", "", "" ], "normalizer": { "type": "NFC" }, "pre_tokenizer": { "type": "Sequence", "pretokenizers": [ { "type": "Split", "pattern": { "Regex": "\\p{N}| ?(?:[^\\s\\p{L}\\p{N}\\p{M}]\\p{M}*){1,16}|\\s{0,16}[\\r\\n]{1,16}|\\s{1,16}(?!\\S)" }, "behavior": "Isolated", "invert": false }, { "type": "ByteLevel", "add_prefix_space": false, "trim_offsets": true, "use_regex": false } ] }, "has_post_processor": false, "byte_fallback": false, "bos_token": "", "bos_token_id": 128000 }, "declared_experiments": null, "declared_run_manifest": null, "registry_note": "SuperBPE on the clean-multi-pretok capped hybrid-window base; fineweb2full, capped data; norm=NFC; SuperBPE; vocab 128,004; 2x2 candidate (t90k/v128k); USED - Apertus-2 development track candidate + S6h code analysis roster (20B twin only, see MC table); excluded from main paper panel" }, "superbpe-clean-fineweb2full-capped-hybridwindow-t110k-v130k": { "slug": "superbpe-clean-fineweb2full-capped-hybridwindow-t110k-v130k", "runs": [ "full-128k-superbpe-clean-fineweb2full-capped-hybridwindow-t110k-v130k", "full-128k-superbpe-clean-fineweb2full-capped-hybridwindow-t110k-v130k-mathcode-scratch" ], "n_runs": 2, "redistributable": true, "loadable_with": "transformers", "hub_id": null, "policy_note": "trained in this project", "empirical": { "model_type": "BPE", "n_base_tokens": 130000, "n_added_tokens": 4, "vocab_total": 130004, "added_tokens": [ "", "", "", "" ], "normalizer": { "type": "NFC" }, "pre_tokenizer": { "type": "Sequence", "pretokenizers": [ { "type": "Split", "pattern": { "Regex": "\\p{N}| ?(?:[^\\s\\p{L}\\p{N}\\p{M}]\\p{M}*){1,16}|\\s{0,16}[\\r\\n]{1,16}|\\s{1,16}(?!\\S)" }, "behavior": "Isolated", "invert": false }, { "type": "ByteLevel", "add_prefix_space": false, "trim_offsets": true, "use_regex": false } ] }, "has_post_processor": false, "byte_fallback": false, "bos_token": "", "bos_token_id": 130000 }, "declared_experiments": null, "declared_run_manifest": null, "registry_note": "SuperBPE, clean-multi pretok base; fineweb2full, capped data; norm=NFC; SuperBPE; vocab ~130,000; SuperBPE stage-2 transition-point ablation (t110k vs the 2x2 candidate's t90k); USED - Apertus-2 development track ablation; excluded from main paper panel" }, "superbpe-clean-fineweb2full-plus2-cappedv2-hybridwindow-t110k-v130k-consv2": { "slug": "superbpe-clean-fineweb2full-plus2-cappedv2-hybridwindow-t110k-v130k-consv2", "runs": [ "full-128k-superbpe-clean-fineweb2full-plus2-cappedv2-hybridwindow-t110k-v130k-consv2", "full-128k-superbpe-clean-fineweb2full-plus2-cappedv2-hybridwindow-t110k-v130k-consv2-mathcode-scratch" ], "n_runs": 2, "redistributable": true, "loadable_with": "transformers", "hub_id": null, "policy_note": "trained in this project", "empirical": { "model_type": "BPE", "n_base_tokens": 130000, "n_added_tokens": 4, "vocab_total": 130004, "added_tokens": [ "", "", "", "" ], "normalizer": { "type": "NFC" }, "pre_tokenizer": { "type": "Sequence", "pretokenizers": [ { "type": "Split", "pattern": { "Regex": "\\p{N}| ?(?:(?:[^\\s\\p{L}\\p{N}\\p{M}]\\p{M}*){2,16}|[^\\s\\p{L}\\p{N}\\p{M}\\x{0027}\\x{2019}\\x{0F0B}]\\p{M}*)|\\s{0,16}[\\r\\n]{1,16}|\\s{1,16}(?!\\S)" }, "behavior": "Isolated", "invert": false }, { "type": "ByteLevel", "add_prefix_space": false, "trim_offsets": true, "use_regex": false } ] }, "has_post_processor": false, "byte_fallback": false, "bos_token": "", "bos_token_id": 130000 }, "declared_experiments": null, "declared_run_manifest": null, "registry_note": "SuperBPE on the clean-multi plus2 (v2) capped-v2 hybrid-window base, stage-2 transition t110k; fineweb2full, consv2 data; norm=NFC; SuperBPE; vocab ~130,000; Apertus-2 development ablation 'SuperBPE-plus2v2-cv2-t110k'; USED - Apertus-2 track ablation; excluded from main paper panel" }, "superbpe-gpt4-fineweb2full-hybridwindow": { "slug": "superbpe-gpt4-fineweb2full-hybridwindow", "runs": [ "full-128k-superbpe-gpt4-fineweb2full-hybridwindow-mathcode-scratch" ], "n_runs": 1, "redistributable": true, "loadable_with": "transformers", "hub_id": null, "policy_note": "trained in this project", "empirical": { "model_type": "BPE", "n_base_tokens": 128000, "n_added_tokens": 4, "vocab_total": 128004, "added_tokens": [ "", "", "", "" ], "normalizer": { "type": "NFC" }, "pre_tokenizer": { "type": "Sequence", "pretokenizers": [ { "type": "Split", "pattern": { "Regex": "\\p{N}{1,3}| ?(?:[^\\s\\p{L}\\p{N}\\p{M}]\\p{M}*(?:[^\\s\\p{L}\\p{N}\\p{M}]\\p{M}*)*[\\r\\n/]+|[^\\s\\p{L}\\p{N}\\p{M}]\\p{M}*(?:[^\\s\\p{L}\\p{N}\\p{M}]\\p{M}*)+)|\\s*[\\r\\n]+|\\s+(?!\\S)" }, "behavior": "Isolated", "invert": false }, { "type": "ByteLevel", "add_prefix_space": false, "trim_offsets": true, "use_regex": false } ] }, "has_post_processor": false, "byte_fallback": false, "bos_token": "", "bos_token_id": 128000 }, "declared_experiments": null, "declared_run_manifest": null, "registry_note": "#12, gpt4 pretok, slug `superbpe-gpt4-fineweb2full-hybridwindow`. SuperBPE on older GPT-4 regex (uncapped, {1,3}-digit)." }, "superbpe-gpt4o-balanced": { "slug": "superbpe-gpt4o-balanced", "runs": [ "full-128k-superbpe-gpt4o-balanced", "full-128k-superbpe-gpt4o-balanced-mathcode-scratch", "pilot-128k-superbpe-gpt4o-balanced", "small-128k-superbpe-gpt4o-balanced", "tiny-128k-superbpe-gpt4o-balanced" ], "n_runs": 5, "redistributable": true, "loadable_with": "transformers", "hub_id": null, "policy_note": "trained in this project", "empirical": { "model_type": "BPE", "n_base_tokens": 128000, "n_added_tokens": 4, "vocab_total": 128004, "added_tokens": [ "", "", "", "" ], "normalizer": null, "pre_tokenizer": { "type": "Sequence", "pretokenizers": [ { "type": "Split", "pattern": { "Regex": "\\p{N}{1,3}| ?(?:[^\\s\\p{L}\\p{N}\\p{M}]\\p{M}*(?:[^\\s\\p{L}\\p{N}\\p{M}]\\p{M}*)*[\\r\\n/]+|[^\\s\\p{L}\\p{N}\\p{M}]\\p{M}*(?:[^\\s\\p{L}\\p{N}\\p{M}]\\p{M}*)+)|\\s*[\\r\\n]+|\\s+(?!\\S)" }, "behavior": "Isolated", "invert": false }, { "type": "ByteLevel", "add_prefix_space": false, "trim_offsets": true, "use_regex": false } ] }, "has_post_processor": false, "byte_fallback": false, "bos_token": "", "bos_token_id": 128000 }, "declared_experiments": null, "declared_run_manifest": null, "registry_note": "SuperBPE stage-1 GPT-4o regex (90k inherited merges) / stage-2 reduced GPT-4o regex (38k superword merges); base = gpt4o-balanced-bpe; balanced (D2, halved for stage-2: balanced_half.json) data; norm=none (inherited from gpt4o-balanced-bpe base); SuperBPE; vocab 128,004; Phase-5 fixed retrain (2026-05-18) of the combining-mark-bug Run 1; USED - panels.json main21 (camera-ready main panel)" }, "superbpe-pabpe-nfc-gpt4-balanced": { "slug": "superbpe-pabpe-nfc-gpt4-balanced", "runs": [ "full-128k-superbpe-pabpe-nfc-gpt4-balanced" ], "n_runs": 1, "redistributable": true, "loadable_with": "transformers", "hub_id": null, "policy_note": "trained in this project", "empirical": { "model_type": "BPE", "n_base_tokens": 128000, "n_added_tokens": 4, "vocab_total": 128004, "added_tokens": [ "", "", "", "" ], "normalizer": { "type": "NFC" }, "pre_tokenizer": { "type": "Sequence", "pretokenizers": [ { "type": "Split", "pattern": { "Regex": "\\p{N}{1,3}| ?(?:[^\\s\\p{L}\\p{N}\\p{M}]\\p{M}*(?:[^\\s\\p{L}\\p{N}\\p{M}]\\p{M}*)*[\\r\\n/]+|[^\\s\\p{L}\\p{N}\\p{M}]\\p{M}*(?:[^\\s\\p{L}\\p{N}\\p{M}]\\p{M}*)+)|\\s*[\\r\\n]+|\\s+(?!\\S)" }, "behavior": "Isolated", "invert": false }, { "type": "ByteLevel", "add_prefix_space": false, "trim_offsets": true, "use_regex": false } ] }, "has_post_processor": false, "byte_fallback": false, "bos_token": "", "bos_token_id": 128000 }, "declared_experiments": null, "declared_run_manifest": null, "registry_note": "SuperBPE stage-1 GPT-4o regex / stage-2 reduced GPT-4o regex; base = PA-BPE nfc_gpt4_balanced; balanced (D2, halved) data; norm=NFC (inherited from PA-BPE base); SuperBPE; vocab 128,004; Phase-5 fixed retrain of Run 2; USED - balanced-1B bootstrap panel (S6.12) + paper panel; not in panels.json main21" }, "superbpe-pabpe-nfc-gpt4-t64k": { "slug": "superbpe-pabpe-nfc-gpt4-t64k", "runs": [ "full-128k-superbpe-pabpe-nfc-gpt4-t64k" ], "n_runs": 1, "redistributable": true, "loadable_with": "transformers", "hub_id": null, "policy_note": "trained in this project", "empirical": { "model_type": "BPE", "n_base_tokens": 128000, "n_added_tokens": 4, "vocab_total": 128004, "added_tokens": [ "", "", "", "" ], "normalizer": { "type": "NFC" }, "pre_tokenizer": { "type": "Sequence", "pretokenizers": [ { "type": "Split", "pattern": { "Regex": "\\p{N}{1,3}| ?(?:[^\\s\\p{L}\\p{N}\\p{M}]\\p{M}*(?:[^\\s\\p{L}\\p{N}\\p{M}]\\p{M}*)*[\\r\\n/]+|[^\\s\\p{L}\\p{N}\\p{M}]\\p{M}*(?:[^\\s\\p{L}\\p{N}\\p{M}]\\p{M}*)+)|\\s*[\\r\\n]+|\\s+(?!\\S)" }, "behavior": "Isolated", "invert": false }, { "type": "ByteLevel", "add_prefix_space": false, "trim_offsets": true, "use_regex": false } ] }, "has_post_processor": false, "byte_fallback": false, "bos_token": "", "bos_token_id": 128000 }, "declared_experiments": null, "declared_run_manifest": null, "registry_note": "SuperBPE stage-1 GPT-4o regex (64k transition, more aggressive superword fusing) / stage-2 reduced GPT-4o regex; base = PA-BPE nfc_gpt4_balanced; balanced (D2, halved) data; norm=NFC; SuperBPE; vocab 128,004; Phase-5 fixed retrain of Run 4 (t64k transition point vs t90k); USED - balanced-1B bootstrap panel (S6.12) + paper panel; not in panels.json main21" }, "swiss-ai/Apertus-70B-2509": { "slug": "swiss-ai/Apertus-70B-2509", "runs": [ "full-128k-apertus", "full-128k-apertus-seed42", "small-128k-apertus", "tiny-128k-apertus" ], "n_runs": 4, "redistributable": false, "loadable_with": "transformers", "hub_id": "swiss-ai/Apertus-70B-2509", "policy_note": "The papers call this tokenizer Mistral-Nemo: the Apertus and Mistral-Nemo tokenizers are the same artifact.", "empirical": null, "declared_experiments": null, "declared_run_manifest": null, "registry_note": "Apertus/Mistral-Nemo (off-the-shelf); n/a (off-the-shelf, Apertus-70B-2509 pretraining mix) data; norm=n/a; off-the-shelf BPE; vocab 131,072; USED - panels.json main21 (camera-ready main panel)" }, "whitespace-balanced-bpe": { "slug": "whitespace-balanced-bpe", "runs": [ "full-128k-whitespace-balanced-bpe", "full-128k-whitespace-balanced-bpe-mathcode-scratch" ], "n_runs": 2, "redistributable": true, "loadable_with": "transformers", "hub_id": null, "policy_note": "trained in this project", "empirical": { "model_type": "BPE", "n_base_tokens": 128256, "n_added_tokens": 4, "vocab_total": 128260, "added_tokens": [ "", "", "", "" ], "normalizer": null, "pre_tokenizer": { "type": "Sequence", "pretokenizers": [ { "type": "Split", "pattern": { "Regex": "\\s+" }, "behavior": "MergedWithNext", "invert": false }, { "type": "ByteLevel", "add_prefix_space": false, "trim_offsets": true, "use_regex": false } ] }, "has_post_processor": false, "byte_fallback": false, "bos_token": "", "bos_token_id": 0 }, "declared_experiments": { "algorithm": "bpe", "pretokenizer_constant": "Split(\\s+)+ByteLevel", "normalizer": null, "data_composition": "balanced", "description": "Split(\\s+, MergedWithNext) + ByteLevel pretok, balanced data, BPE", "trainer_kwargs": null }, "declared_run_manifest": null, "registry_note": "WhitespaceSplit(preserve)+ByteLevel (minimal-structure anchor); balanced (D2) data; norm=none; BPE; vocab 128,260; lower-end pretok anchor; forward-fusing (whitespace+following word one pretoken); USED - panels.json pretok-controls (camera-ready)" }, "whitespace-balanced-unigram": { "slug": "whitespace-balanced-unigram", "runs": [ "full-128k-whitespace-balanced-unigram" ], "n_runs": 1, "redistributable": true, "loadable_with": "transformers", "hub_id": null, "policy_note": "trained in this project", "empirical": { "model_type": "Unigram", "n_base_tokens": 128256, "n_added_tokens": 0, "vocab_total": 128256, "added_tokens": [], "normalizer": null, "pre_tokenizer": { "type": "Metaspace", "replacement": "\u2581", "prepend_scheme": "always", "split": true }, "has_post_processor": false, "byte_fallback": true, "bos_token": "", "bos_token_id": 0 }, "declared_experiments": { "algorithm": "unigram", "pretokenizer_constant": "Metaspace", "normalizer": null, "data_composition": "balanced", "description": "Metaspace + ByteFallback, balanced data, UnigramLM via sentencepiece (SPM-native)", "trainer_kwargs": { "backend": "spm", "mode": "native", "shrinking_factor": 0.7, "num_sub_iterations": 3, "max_sentencepiece_length": 64 } }, "declared_run_manifest": null, "registry_note": "Metaspace(replacement=U+2581)+ByteFallback (SPM native mode); balanced (D2) data; norm=none; UnigramLM (Google sentencepiece, mode=native; tuned kwargs matched to UNIGRAM_TUNED_KWARGS); vocab 128,260; algorithm-axis Unigram counterpart to whitespace-balanced-bpe; trained via SPM not HF (HF UnigramTrainer NaN-panics on WhitespaceSplit+CJK); USED - S5e AXES-2 whitespace comparison + balanced-1B bootstrap panel (S6.12); not in panels.json groups" }, "whitespace-multiheavy-bpe": { "slug": "whitespace-multiheavy-bpe", "runs": [ "full-128k-whitespace-multiheavy-bpe" ], "n_runs": 1, "redistributable": true, "loadable_with": "transformers", "hub_id": null, "policy_note": "trained in this project", "empirical": { "model_type": "BPE", "n_base_tokens": 128256, "n_added_tokens": 4, "vocab_total": 128260, "added_tokens": [ "", "", "", "" ], "normalizer": null, "pre_tokenizer": { "type": "Sequence", "pretokenizers": [ { "type": "Split", "pattern": { "Regex": "\\s+" }, "behavior": "MergedWithNext", "invert": false }, { "type": "ByteLevel", "add_prefix_space": false, "trim_offsets": true, "use_regex": false } ] }, "has_post_processor": false, "byte_fallback": false, "bos_token": "", "bos_token_id": 0 }, "declared_experiments": { "algorithm": "bpe", "pretokenizer_constant": "Split(\\s+)+ByteLevel", "normalizer": null, "data_composition": "multiheavy", "description": "Split(\\s+, MergedWithNext) + ByteLevel pretok, multiheavy data (multilingual budget doubled to 0.60), BPE", "trainer_kwargs": null }, "declared_run_manifest": null, "registry_note": "WhitespaceSplit(preserve)+ByteLevel; multiheavy (30 FineWeb-2 lang weights doubled 0.30->0.60; english 0.175, math/code 0.0875 each) data; norm=none; BPE; vocab 128,256; data-compensation test: does doubling the multilingual budget offset whitespace-pretok's multilingual damage? (S5e: no, MBPP stays 0.002); USED - panels.json pretok-controls (camera-ready)" } } }