| { |
| "_generated_by": "scripts/hf_release/package_tokenizers.py", |
| "_manifest_commit": "1a9a718", |
| "_provenance_fields": { |
| "empirical": "read from the shipped tokenizer file", |
| "declared_experiments": "scripts/train_custom_tokenizers.py EXPERIMENTS row", |
| "declared_run_manifest": "run_manifest.json written by the external trainer", |
| "registry_note": "configs/model_registry_notes.json, verbatim, for one run" |
| }, |
| "counts": { |
| "slugs": 94, |
| "redistributable": 90, |
| "pointer_only": 4, |
| "without_declared_source": 19 |
| }, |
| "tokenizers": { |
| "NousResearch/Meta-Llama-3-8B": { |
| "slug": "NousResearch/Meta-Llama-3-8B", |
| "runs": [ |
| "full-128k-llama3", |
| "full-128k-llama3-seed42", |
| "small-128k-llama3", |
| "tiny-128k-llama3" |
| ], |
| "n_runs": 4, |
| "redistributable": false, |
| "loadable_with": "transformers", |
| "hub_id": "NousResearch/Meta-Llama-3-8B", |
| "policy_note": "", |
| "empirical": null, |
| "declared_experiments": null, |
| "declared_run_manifest": null, |
| "registry_note": "LLaMA-3 tiktoken (off-the-shelf); n/a (off-the-shelf, Meta-Llama-3-8B pretraining mix) data; norm=n/a; off-the-shelf BPE; vocab 128,256; USED - panels.json main21 (camera-ready main panel)" |
| }, |
| "allmulti-equal-bpe": { |
| "slug": "allmulti-equal-bpe", |
| "runs": [ |
| "full-128k-allmulti-equal-bpe" |
| ], |
| "n_runs": 1, |
| "redistributable": true, |
| "loadable_with": "transformers", |
| "hub_id": null, |
| "policy_note": "trained in this project", |
| "empirical": { |
| "model_type": "BPE", |
| "n_base_tokens": 128256, |
| "n_added_tokens": 4, |
| "vocab_total": 128260, |
| "added_tokens": [ |
| "<s>", |
| "</s>", |
| "<unk>", |
| "<pad>" |
| ], |
| "normalizer": null, |
| "pre_tokenizer": { |
| "type": "Sequence", |
| "pretokenizers": [ |
| { |
| "type": "Split", |
| "pattern": { |
| "Regex": "[^\\r\\n\\p{L}\\p{N}]?[\\p{Lu}\\p{Lt}\\p{Lm}\\p{Lo}\\p{M}]*[\\p{Ll}\\p{Lm}\\p{Lo}\\p{M}]+(?i:'s|'t|'re|'ve|'m|'ll|'d)?|[^\\r\\n\\p{L}\\p{N}]?[\\p{Lu}\\p{Lt}\\p{Lm}\\p{Lo}\\p{M}]+[\\p{Ll}\\p{Lm}\\p{Lo}\\p{M}]*(?i:'s|'t|'re|'ve|'m|'ll|'d)?|\\p{N}{1,3}| ?[^\\s\\p{L}\\p{N}]+[\\r\\n/]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+" |
| }, |
| "behavior": "Isolated", |
| "invert": false |
| }, |
| { |
| "type": "ByteLevel", |
| "add_prefix_space": false, |
| "trim_offsets": true, |
| "use_regex": false |
| } |
| ] |
| }, |
| "has_post_processor": false, |
| "byte_fallback": false, |
| "bos_token": "<s>", |
| "bos_token_id": 0 |
| }, |
| "declared_experiments": { |
| "algorithm": "bpe", |
| "pretokenizer_constant": "REGEX_GPT4O", |
| "normalizer": null, |
| "data_composition": "allmulti_equal", |
| "description": "GPT-4o regex, equal-per-language multilingual data (30 langs, repeat-sampled), BPE", |
| "trainer_kwargs": null |
| }, |
| "declared_run_manifest": null, |
| "registry_note": "P2 GPT-4o regex; allmulti_equal (30 FineWeb-2 langs equal weight, max_files=50, repeat-sampled) data; norm=none; BPE; vocab 128,256; baseline of the tail-harm result; USED - S6d.4 equal-weighting ESTABLISHED result (rho=-0.595 vs log training weight, n=31 langs)" |
| }, |
| "apertus": { |
| "slug": "apertus", |
| "runs": [ |
| "full-128k-apertus-mathcode-scratch", |
| "pilot-128k-apertus-wdfix" |
| ], |
| "n_runs": 2, |
| "redistributable": false, |
| "loadable_with": "transformers", |
| "hub_id": "swiss-ai/Apertus-70B-2509", |
| "policy_note": "The papers call this tokenizer Mistral-Nemo: the Apertus and Mistral-Nemo tokenizers are the same artifact. These runs record the local mirror directory rather than the hub id.", |
| "empirical": null, |
| "declared_experiments": null, |
| "declared_run_manifest": null, |
| "registry_note": "#4, apertus pretok, slug `apertus`. Mistral-Nemo regex; see EXPERIMENTS_RESULTS.md \u00a73a for the RCA of the apertus-pretok MBPP code-gen breakage (real, reproducible effect, not an artifact)." |
| }, |
| "bpe-nfc-clean-balanced": { |
| "slug": "bpe-nfc-clean-balanced", |
| "runs": [ |
| "full-128k-bpe-nfc-clean-balanced", |
| "full-128k-bpe-nfc-clean-balanced-mathcode-scratch" |
| ], |
| "n_runs": 2, |
| "redistributable": true, |
| "loadable_with": "transformers", |
| "hub_id": null, |
| "policy_note": "trained in this project", |
| "empirical": { |
| "model_type": "BPE", |
| "n_base_tokens": 128000, |
| "n_added_tokens": 4, |
| "vocab_total": 128004, |
| "added_tokens": [ |
| "<s>", |
| "</s>", |
| "<unk>", |
| "<pad>" |
| ], |
| "normalizer": { |
| "type": "NFC" |
| }, |
| "pre_tokenizer": { |
| "type": "Sequence", |
| "pretokenizers": [ |
| { |
| "type": "Split", |
| "pattern": { |
| "Regex": "[ ]?[\\p{Lu}\\p{Lt}\\p{Lm}\\p{Lo}\\p{M}]*[\\p{Ll}\\p{Lm}\\p{Lo}\\p{M}]+|[ ]?[\\p{Lu}\\p{Lt}\\p{Lm}\\p{Lo}\\p{M}]+[\\p{Ll}\\p{Lm}\\p{Lo}\\p{M}]*|\\p{N}| ?[^\\s\\p{L}\\p{N}]+|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+" |
| }, |
| "behavior": "Isolated", |
| "invert": false |
| }, |
| { |
| "type": "ByteLevel", |
| "add_prefix_space": false, |
| "trim_offsets": true, |
| "use_regex": false |
| } |
| ] |
| }, |
| "has_post_processor": false, |
| "byte_fallback": false, |
| "bos_token": "<s>", |
| "bos_token_id": 128000 |
| }, |
| "declared_experiments": null, |
| "declared_run_manifest": { |
| "variant": "bpe_nfc_clean_multi_balanced", |
| "variant_config": { |
| "trainer": "bpe", |
| "pretok": "clean_multi_nfc", |
| "grouped_config": "configs/parity_aware_config_balanced.json", |
| "special_tokens_override": [], |
| "decoder_add_prefix_space": true, |
| "output_group": "tokenizer-lm-toks", |
| "output_dir_name": "bpe_nfc_clean_multi_balanced", |
| "post_training_special_tokens": [ |
| "<s>", |
| "</s>", |
| "<unk>", |
| "<pad>" |
| ] |
| }, |
| "grouped_config_path": "/users/cmeister747/pa_tokenizers_branch/configs/parity_aware_config_balanced.json", |
| "vocab_size": 128000, |
| "min_frequency": 2, |
| "special_tokens": [], |
| "start_time": "2026-05-17T18:59:07.507364", |
| "elapsed_seconds": 1901.555356, |
| "slurm_job_id": "2282607", |
| "cli_args": [ |
| "/users/cmeister747/pa_tokenizers_branch/train_tokenizer.py", |
| "--variant", |
| "bpe_nfc_clean_multi_balanced" |
| ], |
| "git": { |
| "commit": "5d8381af9ec1417b078f7dce82bac68873da915e", |
| "branch": "parity-aware-bpe", |
| "dirty": false |
| }, |
| "per_group_summary": { |
| "n_groups": 34, |
| "full_block_in": "tokenizers/bpe-nfc-clean-balanced/run_manifest.json" |
| } |
| }, |
| "registry_note": "clean-multi regex (REGEX_CLEAN); balanced (D2) data; norm=NFC; BPE; vocab 128,004; non-parity BPE baseline for the pabpe-nfc-clean-* family; also the vocab-ladder v128k anchor; USED - balanced-1B bootstrap panel (S6.12) + paper panel + S6h code analysis (n=21 main21 general-metric rows use bpe-nfc-clean-balanced's twin? no - not in main21 itself); not in panels.json main21" |
| }, |
| "bpe-nfc-clean-balanced-v128k": { |
| "slug": "bpe-nfc-clean-balanced-v128k", |
| "runs": [ |
| "full-128k-bpe-nfc-clean-balanced-v128k" |
| ], |
| "n_runs": 1, |
| "redistributable": true, |
| "loadable_with": "transformers", |
| "hub_id": null, |
| "policy_note": "trained in this project", |
| "empirical": { |
| "model_type": "BPE", |
| "n_base_tokens": 128000, |
| "n_added_tokens": 4, |
| "vocab_total": 128004, |
| "added_tokens": [ |
| "<s>", |
| "</s>", |
| "<unk>", |
| "<pad>" |
| ], |
| "normalizer": { |
| "type": "NFC" |
| }, |
| "pre_tokenizer": { |
| "type": "Sequence", |
| "pretokenizers": [ |
| { |
| "type": "Split", |
| "pattern": { |
| "Regex": "[ ]?[\\p{Lu}\\p{Lt}\\p{Lm}\\p{Lo}\\p{M}]*[\\p{Ll}\\p{Lm}\\p{Lo}\\p{M}]+|[ ]?[\\p{Lu}\\p{Lt}\\p{Lm}\\p{Lo}\\p{M}]+[\\p{Ll}\\p{Lm}\\p{Lo}\\p{M}]*|\\p{N}| ?[^\\s\\p{L}\\p{N}]+|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+" |
| }, |
| "behavior": "Isolated", |
| "invert": false |
| }, |
| { |
| "type": "ByteLevel", |
| "add_prefix_space": false, |
| "trim_offsets": true, |
| "use_regex": false |
| } |
| ] |
| }, |
| "has_post_processor": false, |
| "byte_fallback": false, |
| "bos_token": "<s>", |
| "bos_token_id": 0 |
| }, |
| "declared_experiments": { |
| "algorithm": "bpe", |
| "pretokenizer_constant": "REGEX_CLEAN", |
| "normalizer": { |
| "type": "NFC" |
| }, |
| "data_composition": "balanced", |
| "description": "Base clean-multi pretok, balanced data, NFC, vocab_size=128000 (vocab ladder anchor; also the baseline for bpe-nfc-clean-nomark-balanced)", |
| "trainer_kwargs": null |
| }, |
| "declared_run_manifest": null, |
| "registry_note": "clean-multi regex (REGEX_CLEAN); balanced (D2) data; norm=NFC; BPE; vocab ~128k target (byte-matched: 8797 steps to match the v128k anchor's training bytes); byte-matched vocab-size ladder (equal training BYTES across the 5 points, unequal steps); the 'clean single-recipe ladder' S6f said had never been run; USED - panels.json vocab-ladder (camera-ready); write-up pending EXPERIMENTS_RESULTS.md (EXPERIMENTS_PLAN.md 'V1/vocab ladder', evals landed 2026-07-24/29)" |
| }, |
| "bpe-nfc-clean-balanced-v160k": { |
| "slug": "bpe-nfc-clean-balanced-v160k", |
| "runs": [ |
| "full-128k-bpe-nfc-clean-balanced-v160k" |
| ], |
| "n_runs": 1, |
| "redistributable": true, |
| "loadable_with": "transformers", |
| "hub_id": null, |
| "policy_note": "trained in this project", |
| "empirical": { |
| "model_type": "BPE", |
| "n_base_tokens": 160000, |
| "n_added_tokens": 4, |
| "vocab_total": 160004, |
| "added_tokens": [ |
| "<s>", |
| "</s>", |
| "<unk>", |
| "<pad>" |
| ], |
| "normalizer": { |
| "type": "NFC" |
| }, |
| "pre_tokenizer": { |
| "type": "Sequence", |
| "pretokenizers": [ |
| { |
| "type": "Split", |
| "pattern": { |
| "Regex": "[ ]?[\\p{Lu}\\p{Lt}\\p{Lm}\\p{Lo}\\p{M}]*[\\p{Ll}\\p{Lm}\\p{Lo}\\p{M}]+|[ ]?[\\p{Lu}\\p{Lt}\\p{Lm}\\p{Lo}\\p{M}]+[\\p{Ll}\\p{Lm}\\p{Lo}\\p{M}]*|\\p{N}| ?[^\\s\\p{L}\\p{N}]+|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+" |
| }, |
| "behavior": "Isolated", |
| "invert": false |
| }, |
| { |
| "type": "ByteLevel", |
| "add_prefix_space": false, |
| "trim_offsets": true, |
| "use_regex": false |
| } |
| ] |
| }, |
| "has_post_processor": false, |
| "byte_fallback": false, |
| "bos_token": "<s>", |
| "bos_token_id": 0 |
| }, |
| "declared_experiments": { |
| "algorithm": "bpe", |
| "pretokenizer_constant": "REGEX_CLEAN", |
| "normalizer": { |
| "type": "NFC" |
| }, |
| "data_composition": "balanced", |
| "description": "Base clean-multi pretok, balanced data, NFC, vocab_size=160000 (vocab ladder)", |
| "trainer_kwargs": null |
| }, |
| "declared_run_manifest": null, |
| "registry_note": "clean-multi regex (REGEX_CLEAN); balanced (D2) data; norm=NFC; BPE; vocab ~160k target (byte-matched: 8667 steps to match the v128k anchor's training bytes); byte-matched vocab-size ladder (equal training BYTES across the 5 points, unequal steps); the 'clean single-recipe ladder' S6f said had never been run; USED - panels.json vocab-ladder (camera-ready); write-up pending EXPERIMENTS_RESULTS.md (EXPERIMENTS_PLAN.md 'V1/vocab ladder', evals landed 2026-07-24/29)" |
| }, |
| "bpe-nfc-clean-balanced-v200k": { |
| "slug": "bpe-nfc-clean-balanced-v200k", |
| "runs": [ |
| "full-128k-bpe-nfc-clean-balanced-v200k" |
| ], |
| "n_runs": 1, |
| "redistributable": true, |
| "loadable_with": "transformers", |
| "hub_id": null, |
| "policy_note": "trained in this project", |
| "empirical": { |
| "model_type": "BPE", |
| "n_base_tokens": 200000, |
| "n_added_tokens": 4, |
| "vocab_total": 200004, |
| "added_tokens": [ |
| "<s>", |
| "</s>", |
| "<unk>", |
| "<pad>" |
| ], |
| "normalizer": { |
| "type": "NFC" |
| }, |
| "pre_tokenizer": { |
| "type": "Sequence", |
| "pretokenizers": [ |
| { |
| "type": "Split", |
| "pattern": { |
| "Regex": "[ ]?[\\p{Lu}\\p{Lt}\\p{Lm}\\p{Lo}\\p{M}]*[\\p{Ll}\\p{Lm}\\p{Lo}\\p{M}]+|[ ]?[\\p{Lu}\\p{Lt}\\p{Lm}\\p{Lo}\\p{M}]+[\\p{Ll}\\p{Lm}\\p{Lo}\\p{M}]*|\\p{N}| ?[^\\s\\p{L}\\p{N}]+|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+" |
| }, |
| "behavior": "Isolated", |
| "invert": false |
| }, |
| { |
| "type": "ByteLevel", |
| "add_prefix_space": false, |
| "trim_offsets": true, |
| "use_regex": false |
| } |
| ] |
| }, |
| "has_post_processor": false, |
| "byte_fallback": false, |
| "bos_token": "<s>", |
| "bos_token_id": 0 |
| }, |
| "declared_experiments": { |
| "algorithm": "bpe", |
| "pretokenizer_constant": "REGEX_CLEAN", |
| "normalizer": { |
| "type": "NFC" |
| }, |
| "data_composition": "balanced", |
| "description": "Base clean-multi pretok, balanced data, NFC, vocab_size=200000 (vocab ladder)", |
| "trainer_kwargs": null |
| }, |
| "declared_run_manifest": null, |
| "registry_note": "clean-multi regex (REGEX_CLEAN); balanced (D2) data; norm=NFC; BPE; vocab ~200k target (byte-matched: 8552 steps to match the v128k anchor's training bytes); byte-matched vocab-size ladder (equal training BYTES across the 5 points, unequal steps); the 'clean single-recipe ladder' S6f said had never been run; USED - panels.json vocab-ladder (camera-ready); write-up pending EXPERIMENTS_RESULTS.md (EXPERIMENTS_PLAN.md 'V1/vocab ladder', evals landed 2026-07-24/29)" |
| }, |
| "bpe-nfc-clean-balanced-v64k": { |
| "slug": "bpe-nfc-clean-balanced-v64k", |
| "runs": [ |
| "full-128k-bpe-nfc-clean-balanced-v64k" |
| ], |
| "n_runs": 1, |
| "redistributable": true, |
| "loadable_with": "transformers", |
| "hub_id": null, |
| "policy_note": "trained in this project", |
| "empirical": { |
| "model_type": "BPE", |
| "n_base_tokens": 64000, |
| "n_added_tokens": 4, |
| "vocab_total": 64004, |
| "added_tokens": [ |
| "<s>", |
| "</s>", |
| "<unk>", |
| "<pad>" |
| ], |
| "normalizer": { |
| "type": "NFC" |
| }, |
| "pre_tokenizer": { |
| "type": "Sequence", |
| "pretokenizers": [ |
| { |
| "type": "Split", |
| "pattern": { |
| "Regex": "[ ]?[\\p{Lu}\\p{Lt}\\p{Lm}\\p{Lo}\\p{M}]*[\\p{Ll}\\p{Lm}\\p{Lo}\\p{M}]+|[ ]?[\\p{Lu}\\p{Lt}\\p{Lm}\\p{Lo}\\p{M}]+[\\p{Ll}\\p{Lm}\\p{Lo}\\p{M}]*|\\p{N}| ?[^\\s\\p{L}\\p{N}]+|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+" |
| }, |
| "behavior": "Isolated", |
| "invert": false |
| }, |
| { |
| "type": "ByteLevel", |
| "add_prefix_space": false, |
| "trim_offsets": true, |
| "use_regex": false |
| } |
| ] |
| }, |
| "has_post_processor": false, |
| "byte_fallback": false, |
| "bos_token": "<s>", |
| "bos_token_id": 0 |
| }, |
| "declared_experiments": { |
| "algorithm": "bpe", |
| "pretokenizer_constant": "REGEX_CLEAN", |
| "normalizer": { |
| "type": "NFC" |
| }, |
| "data_composition": "balanced", |
| "description": "Base clean-multi pretok, balanced data, NFC, vocab_size=64000 (vocab ladder)", |
| "trainer_kwargs": null |
| }, |
| "declared_run_manifest": null, |
| "registry_note": "clean-multi regex (REGEX_CLEAN); balanced (D2) data; norm=NFC; BPE; vocab ~64k target (byte-matched: 9299 steps to match the v128k anchor's training bytes); byte-matched vocab-size ladder (equal training BYTES across the 5 points, unequal steps); the 'clean single-recipe ladder' S6f said had never been run; USED - panels.json vocab-ladder (camera-ready); write-up pending EXPERIMENTS_RESULTS.md (EXPERIMENTS_PLAN.md 'V1/vocab ladder', evals landed 2026-07-24/29)" |
| }, |
| "bpe-nfc-clean-balanced-v96k": { |
| "slug": "bpe-nfc-clean-balanced-v96k", |
| "runs": [ |
| "full-128k-bpe-nfc-clean-balanced-v96k" |
| ], |
| "n_runs": 1, |
| "redistributable": true, |
| "loadable_with": "transformers", |
| "hub_id": null, |
| "policy_note": "trained in this project", |
| "empirical": { |
| "model_type": "BPE", |
| "n_base_tokens": 96000, |
| "n_added_tokens": 4, |
| "vocab_total": 96004, |
| "added_tokens": [ |
| "<s>", |
| "</s>", |
| "<unk>", |
| "<pad>" |
| ], |
| "normalizer": { |
| "type": "NFC" |
| }, |
| "pre_tokenizer": { |
| "type": "Sequence", |
| "pretokenizers": [ |
| { |
| "type": "Split", |
| "pattern": { |
| "Regex": "[ ]?[\\p{Lu}\\p{Lt}\\p{Lm}\\p{Lo}\\p{M}]*[\\p{Ll}\\p{Lm}\\p{Lo}\\p{M}]+|[ ]?[\\p{Lu}\\p{Lt}\\p{Lm}\\p{Lo}\\p{M}]+[\\p{Ll}\\p{Lm}\\p{Lo}\\p{M}]*|\\p{N}| ?[^\\s\\p{L}\\p{N}]+|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+" |
| }, |
| "behavior": "Isolated", |
| "invert": false |
| }, |
| { |
| "type": "ByteLevel", |
| "add_prefix_space": false, |
| "trim_offsets": true, |
| "use_regex": false |
| } |
| ] |
| }, |
| "has_post_processor": false, |
| "byte_fallback": false, |
| "bos_token": "<s>", |
| "bos_token_id": 0 |
| }, |
| "declared_experiments": { |
| "algorithm": "bpe", |
| "pretokenizer_constant": "REGEX_CLEAN", |
| "normalizer": { |
| "type": "NFC" |
| }, |
| "data_composition": "balanced", |
| "description": "Base clean-multi pretok, balanced data, NFC, vocab_size=96000 (vocab ladder)", |
| "trainer_kwargs": null |
| }, |
| "declared_run_manifest": null, |
| "registry_note": "clean-multi regex (REGEX_CLEAN); balanced (D2) data; norm=NFC; BPE; vocab ~96k target (byte-matched: 8986 steps to match the v128k anchor's training bytes); byte-matched vocab-size ladder (equal training BYTES across the 5 points, unequal steps); the 'clean single-recipe ladder' S6f said had never been run; USED - panels.json vocab-ladder (camera-ready); write-up pending EXPERIMENTS_RESULTS.md (EXPERIMENTS_PLAN.md 'V1/vocab ladder', evals landed 2026-07-24/29)" |
| }, |
| "bpe-nfc-clean-fineweb2full": { |
| "slug": "bpe-nfc-clean-fineweb2full", |
| "runs": [ |
| "full-128k-bpe-nfc-clean-fineweb2full" |
| ], |
| "n_runs": 1, |
| "redistributable": true, |
| "loadable_with": "transformers", |
| "hub_id": null, |
| "policy_note": "trained in this project", |
| "empirical": { |
| "model_type": "BPE", |
| "n_base_tokens": 127996, |
| "n_added_tokens": 4, |
| "vocab_total": 128000, |
| "added_tokens": [ |
| "<s>", |
| "</s>", |
| "<unk>", |
| "<pad>" |
| ], |
| "normalizer": { |
| "type": "NFC" |
| }, |
| "pre_tokenizer": { |
| "type": "Sequence", |
| "pretokenizers": [ |
| { |
| "type": "Split", |
| "pattern": { |
| "Regex": "[ ]?[\\p{Lu}\\p{Lt}\\p{Lm}\\p{Lo}\\p{M}]*[\\p{Ll}\\p{Lm}\\p{Lo}\\p{M}]+|[ ]?[\\p{Lu}\\p{Lt}\\p{Lm}\\p{Lo}\\p{M}]+[\\p{Ll}\\p{Lm}\\p{Lo}\\p{M}]*|\\p{N}| ?[^\\s\\p{L}\\p{N}]+|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+" |
| }, |
| "behavior": "Isolated", |
| "invert": false |
| }, |
| { |
| "type": "ByteLevel", |
| "add_prefix_space": false, |
| "trim_offsets": true, |
| "use_regex": false |
| } |
| ] |
| }, |
| "has_post_processor": false, |
| "byte_fallback": false, |
| "bos_token": "<s>", |
| "bos_token_id": 127996 |
| }, |
| "declared_experiments": null, |
| "declared_run_manifest": { |
| "variant": "bpe_nfc_clean_multi_fineweb2full", |
| "variant_config": { |
| "trainer": "bpe", |
| "pretok": "clean_multi_nfc", |
| "grouped_config": "configs/parity_aware_config_grouped_fineweb2full_quota.json", |
| "special_tokens_override": [], |
| "decoder_add_prefix_space": true, |
| "output_group": "tokenizer-lm-toks", |
| "output_dir_name": "bpe_nfc_clean_multi_fineweb2full", |
| "post_training_special_tokens": [ |
| "<s>", |
| "</s>", |
| "<unk>", |
| "<pad>" |
| ] |
| }, |
| "grouped_config_path": "/users/cmeister747/pa_tokenizers_branch/configs/parity_aware_config_grouped_fineweb2full_quota.json", |
| "vocab_size": 128000, |
| "min_frequency": 2, |
| "special_tokens": [], |
| "start_time": "2026-05-22T07:43:23.365585", |
| "elapsed_seconds": 6204.931363, |
| "slurm_job_id": "2328522", |
| "cli_args": [ |
| "/users/cmeister747/pa_tokenizers_branch/train_tokenizer.py", |
| "--variant", |
| "bpe_nfc_clean_multi_fineweb2full" |
| ], |
| "git": { |
| "commit": "5d8381af9ec1417b078f7dce82bac68873da915e", |
| "branch": "parity-aware-bpe", |
| "dirty": false |
| }, |
| "per_group_summary": { |
| "n_groups": 25, |
| "full_block_in": "tokenizers/bpe-nfc-clean-fineweb2full/run_manifest.json" |
| } |
| }, |
| "registry_note": "clean-multi regex; fineweb2full (full FineWeb-2 corpus) data; norm=NFC; BPE; vocab 128,000; standard-BPE non-parity baseline for the pabpe-nfc-clean-fineweb2full family; USED - S5b fineweb2full PA-BPE/BPE analysis + S6e.1 cohort update; fineweb2full data variant, excluded from the main paper panel per PAPER_PANEL.md rule" |
| }, |
| "bpe-nfc-clean-nomark-balanced": { |
| "slug": "bpe-nfc-clean-nomark-balanced", |
| "runs": [ |
| "full-128k-bpe-nfc-clean-nomark-balanced" |
| ], |
| "n_runs": 1, |
| "redistributable": true, |
| "loadable_with": "transformers", |
| "hub_id": null, |
| "policy_note": "trained in this project", |
| "empirical": { |
| "model_type": "BPE", |
| "n_base_tokens": 128000, |
| "n_added_tokens": 4, |
| "vocab_total": 128004, |
| "added_tokens": [ |
| "<s>", |
| "</s>", |
| "<unk>", |
| "<pad>" |
| ], |
| "normalizer": { |
| "type": "NFC" |
| }, |
| "pre_tokenizer": { |
| "type": "Sequence", |
| "pretokenizers": [ |
| { |
| "type": "Split", |
| "pattern": { |
| "Regex": "[ ]?[\\p{Lu}\\p{Lt}\\p{Lm}\\p{Lo}]*[\\p{Ll}\\p{Lm}\\p{Lo}]+|[ ]?[\\p{Lu}\\p{Lt}\\p{Lm}\\p{Lo}]+[\\p{Ll}\\p{Lm}\\p{Lo}]*|\\p{N}| ?[^\\s\\p{L}\\p{N}]+|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+" |
| }, |
| "behavior": "Isolated", |
| "invert": false |
| }, |
| { |
| "type": "ByteLevel", |
| "add_prefix_space": false, |
| "trim_offsets": true, |
| "use_regex": false |
| } |
| ] |
| }, |
| "has_post_processor": false, |
| "byte_fallback": false, |
| "bos_token": "<s>", |
| "bos_token_id": 0 |
| }, |
| "declared_experiments": { |
| "algorithm": "bpe", |
| "pretokenizer_constant": "REGEX_CLEAN_NOMARK", |
| "normalizer": { |
| "type": "NFC" |
| }, |
| "data_composition": "balanced", |
| "description": "Combining-mark-split ablation of the clean pretok (\\p{M} removed), balanced data, NFC, vocab_size=128000 (crosslingual multilingual-hostile control)", |
| "trainer_kwargs": null |
| }, |
| "declared_run_manifest": null, |
| "registry_note": "clean-multi regex, combining-mark variant (REGEX_CLEAN_NOMARK: \\\\p{M} removed so combining marks split from base letters; documented in EXPERIMENTS_PLAN.md XL-pretok, located 2026-08-16); balanced (D2) data; norm=NFC; BPE; vocab 128,000 (verified from tokenizer.json 2026-08-16); combining-mark control run alongside the vocab-ladder cohort (CHRONOLOGICAL 2026-07-29: 'the eleventh' run, evals complete except MGSM, later fixed); USED - companion control for the vocab-ladder cohort (11th run in that batch, CHRONOLOGICAL 2026-07-29); not itself a panels.json entry; USED - crosslingual panel v1 rev5 Tier-1 member (user-approved inclusion 2026-08-16, configs/panels/crosslingual_panel_v1.json)" |
| }, |
| "bpe-nfc-gpt4-fineweb2full": { |
| "slug": "bpe-nfc-gpt4-fineweb2full", |
| "runs": [ |
| "full-128k-bpe-nfc-gpt4-fineweb2full" |
| ], |
| "n_runs": 1, |
| "redistributable": true, |
| "loadable_with": "transformers", |
| "hub_id": null, |
| "policy_note": "trained in this project", |
| "empirical": { |
| "model_type": "BPE", |
| "n_base_tokens": 127996, |
| "n_added_tokens": 4, |
| "vocab_total": 128000, |
| "added_tokens": [ |
| "<s>", |
| "</s>", |
| "<unk>", |
| "<pad>" |
| ], |
| "normalizer": { |
| "type": "NFC" |
| }, |
| "pre_tokenizer": { |
| "type": "Sequence", |
| "pretokenizers": [ |
| { |
| "type": "Split", |
| "pattern": { |
| "Regex": "[^\\r\\n\\p{L}\\p{N}]?[\\p{Lu}\\p{Lt}\\p{Lm}\\p{Lo}\\p{M}]*[\\p{Ll}\\p{Lm}\\p{Lo}\\p{M}]+(?i:'s|'t|'re|'ve|'m|'ll|'d)?|[^\\r\\n\\p{L}\\p{N}]?[\\p{Lu}\\p{Lt}\\p{Lm}\\p{Lo}\\p{M}]+[\\p{Ll}\\p{Lm}\\p{Lo}\\p{M}]*(?i:'s|'t|'re|'ve|'m|'ll|'d)?|\\p{N}{1,3}| ?[^\\s\\p{L}\\p{N}]+[\\r\\n/]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+" |
| }, |
| "behavior": "Isolated", |
| "invert": false |
| }, |
| { |
| "type": "ByteLevel", |
| "add_prefix_space": false, |
| "trim_offsets": true, |
| "use_regex": false |
| } |
| ] |
| }, |
| "has_post_processor": false, |
| "byte_fallback": false, |
| "bos_token": "<s>", |
| "bos_token_id": 127996 |
| }, |
| "declared_experiments": null, |
| "declared_run_manifest": { |
| "variant": "bpe_nfc_gpt4_fineweb2full", |
| "variant_config": { |
| "trainer": "bpe", |
| "pretok": "gpt4o_nfc", |
| "grouped_config": "configs/parity_aware_config_grouped_fineweb2full_quota.json", |
| "special_tokens_override": [], |
| "decoder_add_prefix_space": true, |
| "output_group": "tokenizer-lm-toks", |
| "output_dir_name": "bpe_nfc_gpt4_fineweb2full", |
| "post_training_special_tokens": [ |
| "<s>", |
| "</s>", |
| "<unk>", |
| "<pad>" |
| ] |
| }, |
| "grouped_config_path": "/users/cmeister747/pa_tokenizers_branch/configs/parity_aware_config_grouped_fineweb2full_quota.json", |
| "vocab_size": 128000, |
| "min_frequency": 2, |
| "special_tokens": [], |
| "start_time": "2026-05-22T07:43:23.576450", |
| "elapsed_seconds": 6515.053178, |
| "slurm_job_id": "2328523", |
| "cli_args": [ |
| "/users/cmeister747/pa_tokenizers_branch/train_tokenizer.py", |
| "--variant", |
| "bpe_nfc_gpt4_fineweb2full" |
| ], |
| "git": { |
| "commit": "5d8381af9ec1417b078f7dce82bac68873da915e", |
| "branch": "parity-aware-bpe", |
| "dirty": false |
| }, |
| "per_group_summary": { |
| "n_groups": 25, |
| "full_block_in": "tokenizers/bpe-nfc-gpt4-fineweb2full/run_manifest.json" |
| } |
| }, |
| "registry_note": "GPT-4o regex; fineweb2full data; norm=NFC; BPE; vocab 128,000; standard-BPE non-parity baseline for the pabpe-nfc-gpt4-fineweb2full family; USED - S5b fineweb2full analysis + S6e.1; fineweb2full data variant, excluded from main paper panel" |
| }, |
| "bpe-nfc-plus2-balanced": { |
| "slug": "bpe-nfc-plus2-balanced", |
| "runs": [ |
| "full-128k-bpe-nfc-plus2-balanced", |
| "full-128k-bpe-nfc-plus2-balanced-mathcode-scratch", |
| "full-128k-bpe-nfc-plus2-balanced-seed43-mathcode-scratch", |
| "full-128k-bpe-nfc-plus2-balanced-seed44-mathcode-scratch" |
| ], |
| "n_runs": 4, |
| "redistributable": true, |
| "loadable_with": "transformers", |
| "hub_id": null, |
| "policy_note": "trained in this project", |
| "empirical": { |
| "model_type": "BPE", |
| "n_base_tokens": 128256, |
| "n_added_tokens": 4, |
| "vocab_total": 128260, |
| "added_tokens": [ |
| "<s>", |
| "</s>", |
| "<unk>", |
| "<pad>" |
| ], |
| "normalizer": { |
| "type": "NFC" |
| }, |
| "pre_tokenizer": { |
| "type": "Sequence", |
| "pretokenizers": [ |
| { |
| "type": "Split", |
| "pattern": { |
| "Regex": "[ \\x{0F0B}\\x{0027}\\x{2019}]?[\\p{Lu}\\p{Lt}\\p{Lm}\\p{Lo}\\p{M}]*[\\p{Ll}\\p{Lm}\\p{Lo}\\p{M}]+|[ \\x{0F0B}\\x{0027}\\x{2019}]?[\\p{Lu}\\p{Lt}\\p{Lm}\\p{Lo}\\p{M}]+[\\p{Ll}\\p{Lm}\\p{Lo}\\p{M}]*|\\p{N}| ?[^\\s\\p{L}\\p{N}]{1,16}|\\s{0,16}[\\r\\n]{1,16}|\\s{1,16}(?!\\S)|\\s{1,16}" |
| }, |
| "behavior": "Isolated", |
| "invert": false |
| }, |
| { |
| "type": "ByteLevel", |
| "add_prefix_space": false, |
| "trim_offsets": true, |
| "use_regex": false |
| } |
| ] |
| }, |
| "has_post_processor": false, |
| "byte_fallback": false, |
| "bos_token": "<s>", |
| "bos_token_id": 0 |
| }, |
| "declared_experiments": { |
| "algorithm": "bpe", |
| "pretokenizer_constant": "REGEX_CLEAN_PLUS2", |
| "normalizer": { |
| "type": "NFC" |
| }, |
| "data_composition": "balanced", |
| "description": "Clean-multi pretok with plus2 prefix-only apostrophe/tsheg attachment, balanced data, NFC normalization", |
| "trainer_kwargs": null |
| }, |
| "declared_run_manifest": null, |
| "registry_note": "clean-multi plus2 (REGEX_CLEAN_PLUS2: attaches both apostrophe forms, contraction arm removed); balanced (D2) data; norm=NFC; BPE; vocab 128,256; balanced-data control for the plus2 apostrophe-attachment pretok (ported from the Apertus-v2 fw2full family); USED - PAPER_PANEL.md paper-eligible addition (#5, landed after the S6.12 balanced-1B panel was built; paper_bootstrap_balanced_panel n=34); not in panels.json main21" |
| }, |
| "bpe-nfc-plus3-balanced": { |
| "slug": "bpe-nfc-plus3-balanced", |
| "runs": [ |
| "full-128k-bpe-nfc-plus3-balanced", |
| "full-128k-bpe-nfc-plus3-balanced-mathcode-scratch" |
| ], |
| "n_runs": 2, |
| "redistributable": true, |
| "loadable_with": "transformers", |
| "hub_id": null, |
| "policy_note": "trained in this project", |
| "empirical": { |
| "model_type": "BPE", |
| "n_base_tokens": 128256, |
| "n_added_tokens": 4, |
| "vocab_total": 128260, |
| "added_tokens": [ |
| "<s>", |
| "</s>", |
| "<unk>", |
| "<pad>" |
| ], |
| "normalizer": { |
| "type": "NFC" |
| }, |
| "pre_tokenizer": { |
| "type": "Sequence", |
| "pretokenizers": [ |
| { |
| "type": "Split", |
| "pattern": { |
| "Regex": "[ \\x{0F0B}\\x{0027}\\x{2019}]?[\\p{Lu}\\p{Lt}\\p{Lm}\\p{Lo}\\p{M}]*[\\p{Ll}\\p{Lm}\\p{Lo}\\p{M}]+(?:[\\x{0027}\\x{2019}](?!\\p{L}))?|[ \\x{0F0B}\\x{0027}\\x{2019}]?[\\p{Lu}\\p{Lt}\\p{Lm}\\p{Lo}\\p{M}]+[\\p{Ll}\\p{Lm}\\p{Lo}\\p{M}]*(?:[\\x{0027}\\x{2019}](?!\\p{L}))?|\\p{N}| ?[^\\s\\p{L}\\p{N}]{1,16}|\\s{0,16}[\\r\\n]{1,16}|\\s{1,16}(?!\\S)|\\s{1,16}" |
| }, |
| "behavior": "Isolated", |
| "invert": false |
| }, |
| { |
| "type": "ByteLevel", |
| "add_prefix_space": false, |
| "trim_offsets": true, |
| "use_regex": false |
| } |
| ] |
| }, |
| "has_post_processor": false, |
| "byte_fallback": false, |
| "bos_token": "<s>", |
| "bos_token_id": 0 |
| }, |
| "declared_experiments": { |
| "algorithm": "bpe", |
| "pretokenizer_constant": "REGEX_CLEAN_PLUS3", |
| "normalizer": { |
| "type": "NFC" |
| }, |
| "data_composition": "balanced", |
| "description": "Clean-multi pretok with plus3 apostrophe/tsheg attachment, balanced data, NFC normalization", |
| "trainer_kwargs": null |
| }, |
| "declared_run_manifest": null, |
| "registry_note": "clean-multi plus3 (REGEX_CLEAN_PLUS3: plus2 + guarded trailing-apostrophe attachment); balanced (D2) data; norm=NFC; BPE; vocab 128,256; balanced-data control for plus3; USED - PAPER_PANEL.md paper-eligible addition (#6); paper_bootstrap_balanced_panel n=34; not in panels.json main21" |
| }, |
| "claude-allmulti-equal-bpe": { |
| "slug": "claude-allmulti-equal-bpe", |
| "runs": [ |
| "full-128k-claude-allmulti-equal-bpe" |
| ], |
| "n_runs": 1, |
| "redistributable": true, |
| "loadable_with": "transformers", |
| "hub_id": null, |
| "policy_note": "trained in this project", |
| "empirical": { |
| "model_type": "BPE", |
| "n_base_tokens": 128256, |
| "n_added_tokens": 4, |
| "vocab_total": 128260, |
| "added_tokens": [ |
| "<s>", |
| "</s>", |
| "<unk>", |
| "<pad>" |
| ], |
| "normalizer": null, |
| "pre_tokenizer": { |
| "type": "Sequence", |
| "pretokenizers": [ |
| { |
| "type": "Split", |
| "pattern": { |
| "Regex": "(?i:[''\\u2019]s|[''\\u2019]t|[''\\u2019]re|[''\\u2019]ve|[''\\u2019]m|[''\\u2019]ll|[''\\u2019]d)|[ ]?[\\p{Lu}\\p{Lt}\\p{Lm}\\p{Lo}\\p{M}]*[\\p{Ll}\\p{Lm}\\p{Lo}\\p{M}]+(?i:[''\\u2019]s|[''\\u2019]t|[''\\u2019]re|[''\\u2019]ve|[''\\u2019]m|[''\\u2019]ll|[''\\u2019]d)?|[ ]?[\\p{Lu}\\p{Lt}\\p{Lm}\\p{Lo}\\p{M}]+[\\p{Ll}\\p{Lm}\\p{Lo}\\p{M}]*(?i:[''\\u2019]s|[''\\u2019]t|[''\\u2019]re|[''\\u2019]ve|[''\\u2019]m|[''\\u2019]ll|[''\\u2019]d)?|\\p{N}{1,3}|[ ]?[^\\s\\p{L}\\p{N}]+|[ ]+|[\\t]+|[\\n]+|[\\r]+|\\s+|\\S" |
| }, |
| "behavior": "Isolated", |
| "invert": false |
| }, |
| { |
| "type": "ByteLevel", |
| "add_prefix_space": false, |
| "trim_offsets": true, |
| "use_regex": false |
| } |
| ] |
| }, |
| "has_post_processor": false, |
| "byte_fallback": false, |
| "bos_token": "<s>", |
| "bos_token_id": 0 |
| }, |
| "declared_experiments": { |
| "algorithm": "bpe", |
| "pretokenizer_constant": "REGEX_CLAUDE", |
| "normalizer": null, |
| "data_composition": "allmulti_equal", |
| "description": "Claude pretok, equal-per-language data (repeat-sampled), BPE. Pairs with claude-balanced-bpe.", |
| "trainer_kwargs": null |
| }, |
| "declared_run_manifest": null, |
| "registry_note": "P3 Claude V1 regex; allmulti_equal data; norm=none; BPE; vocab 128,256; pretokenizer-generality arm, pairs with claude-balanced-bpe; USED - S6d.4 (rho=-0.558)" |
| }, |
| "claude-balanced-bpe": { |
| "slug": "claude-balanced-bpe", |
| "runs": [ |
| "full-128k-claude-balanced-bpe", |
| "full-128k-claude-balanced-bpe-mathcode-scratch", |
| "full-128k-claude-balanced-bpe-seed42", |
| "full-128k-claude-balanced-bpe-seed43-mathcode-scratch", |
| "full-128k-claude-balanced-bpe-seed44-mathcode-scratch", |
| "pilot-128k-claude-balanced-bpe", |
| "small-128k-claude-balanced-bpe", |
| "tiny-128k-claude-balanced-bpe" |
| ], |
| "n_runs": 8, |
| "redistributable": true, |
| "loadable_with": "transformers", |
| "hub_id": null, |
| "policy_note": "trained in this project", |
| "empirical": { |
| "model_type": "BPE", |
| "n_base_tokens": 128256, |
| "n_added_tokens": 4, |
| "vocab_total": 128260, |
| "added_tokens": [ |
| "<s>", |
| "</s>", |
| "<unk>", |
| "<pad>" |
| ], |
| "normalizer": null, |
| "pre_tokenizer": { |
| "type": "Sequence", |
| "pretokenizers": [ |
| { |
| "type": "Split", |
| "pattern": { |
| "Regex": "(?i:[''\\u2019]s|[''\\u2019]t|[''\\u2019]re|[''\\u2019]ve|[''\\u2019]m|[''\\u2019]ll|[''\\u2019]d)|[ ]?[\\p{Lu}\\p{Lt}\\p{Lm}\\p{Lo}\\p{M}]*[\\p{Ll}\\p{Lm}\\p{Lo}\\p{M}]+(?i:[''\\u2019]s|[''\\u2019]t|[''\\u2019]re|[''\\u2019]ve|[''\\u2019]m|[''\\u2019]ll|[''\\u2019]d)?|[ ]?[\\p{Lu}\\p{Lt}\\p{Lm}\\p{Lo}\\p{M}]+[\\p{Ll}\\p{Lm}\\p{Lo}\\p{M}]*(?i:[''\\u2019]s|[''\\u2019]t|[''\\u2019]re|[''\\u2019]ve|[''\\u2019]m|[''\\u2019]ll|[''\\u2019]d)?|\\p{N}{1,3}|[ ]?[^\\s\\p{L}\\p{N}]+|[ ]+|[\\t]+|[\\n]+|[\\r]+|\\s+|\\S" |
| }, |
| "behavior": "Isolated", |
| "invert": false |
| }, |
| { |
| "type": "ByteLevel", |
| "add_prefix_space": false, |
| "trim_offsets": true, |
| "use_regex": false |
| } |
| ] |
| }, |
| "has_post_processor": false, |
| "byte_fallback": false, |
| "bos_token": "<s>", |
| "bos_token_id": 0 |
| }, |
| "declared_experiments": { |
| "algorithm": "bpe", |
| "pretokenizer_constant": "REGEX_CLAUDE", |
| "normalizer": null, |
| "data_composition": "balanced", |
| "description": "Claude pretok (case-level splits), balanced data", |
| "trainer_kwargs": null |
| }, |
| "declared_run_manifest": null, |
| "registry_note": "P3 Claude V1 regex (REGEX_CLAUDE); balanced (D2) data; norm=none; BPE; vocab 128,260; USED - panels.json main21 (camera-ready main panel)" |
| }, |
| "claude-balanced-nfc-bpe": { |
| "slug": "claude-balanced-nfc-bpe", |
| "runs": [ |
| "full-128k-claude-balanced-nfc-bpe", |
| "full-128k-claude-balanced-nfc-bpe-mathcode-scratch", |
| "full-128k-claude-balanced-nfc-bpe-seed42", |
| "pilot-128k-claude-balanced-nfc-bpe", |
| "small-128k-claude-balanced-nfc-bpe", |
| "tiny-128k-claude-balanced-nfc-bpe" |
| ], |
| "n_runs": 6, |
| "redistributable": true, |
| "loadable_with": "transformers", |
| "hub_id": null, |
| "policy_note": "trained in this project", |
| "empirical": { |
| "model_type": "BPE", |
| "n_base_tokens": 128256, |
| "n_added_tokens": 4, |
| "vocab_total": 128260, |
| "added_tokens": [ |
| "<s>", |
| "</s>", |
| "<unk>", |
| "<pad>" |
| ], |
| "normalizer": { |
| "type": "NFC" |
| }, |
| "pre_tokenizer": { |
| "type": "Sequence", |
| "pretokenizers": [ |
| { |
| "type": "Split", |
| "pattern": { |
| "Regex": "(?i:[''\\u2019]s|[''\\u2019]t|[''\\u2019]re|[''\\u2019]ve|[''\\u2019]m|[''\\u2019]ll|[''\\u2019]d)|[ ]?[\\p{Lu}\\p{Lt}\\p{Lm}\\p{Lo}\\p{M}]*[\\p{Ll}\\p{Lm}\\p{Lo}\\p{M}]+(?i:[''\\u2019]s|[''\\u2019]t|[''\\u2019]re|[''\\u2019]ve|[''\\u2019]m|[''\\u2019]ll|[''\\u2019]d)?|[ ]?[\\p{Lu}\\p{Lt}\\p{Lm}\\p{Lo}\\p{M}]+[\\p{Ll}\\p{Lm}\\p{Lo}\\p{M}]*(?i:[''\\u2019]s|[''\\u2019]t|[''\\u2019]re|[''\\u2019]ve|[''\\u2019]m|[''\\u2019]ll|[''\\u2019]d)?|\\p{N}{1,3}|[ ]?[^\\s\\p{L}\\p{N}]+|[ ]+|[\\t]+|[\\n]+|[\\r]+|\\s+|\\S" |
| }, |
| "behavior": "Isolated", |
| "invert": false |
| }, |
| { |
| "type": "ByteLevel", |
| "add_prefix_space": false, |
| "trim_offsets": true, |
| "use_regex": false |
| } |
| ] |
| }, |
| "has_post_processor": false, |
| "byte_fallback": false, |
| "bos_token": "<s>", |
| "bos_token_id": 0 |
| }, |
| "declared_experiments": { |
| "algorithm": "bpe", |
| "pretokenizer_constant": "REGEX_CLAUDE", |
| "normalizer": { |
| "type": "NFC" |
| }, |
| "data_composition": "balanced", |
| "description": "Claude pretok, balanced data, NFC normalization", |
| "trainer_kwargs": null |
| }, |
| "declared_run_manifest": null, |
| "registry_note": "P3 Claude V1 regex; balanced (D2) data; norm=NFC; BPE; vocab 128,260; USED - panels.json main21 (camera-ready main panel)" |
| }, |
| "claude-balanced-unigram": { |
| "slug": "claude-balanced-unigram", |
| "runs": [ |
| "full-128k-claude-balanced-unigram", |
| "full-128k-claude-balanced-unigram-mathcode-scratch", |
| "full-128k-claude-balanced-unigram-seed42", |
| "pilot-128k-claude-balanced-unigram", |
| "small-128k-claude-balanced-unigram", |
| "tiny-128k-claude-balanced-unigram" |
| ], |
| "n_runs": 6, |
| "redistributable": true, |
| "loadable_with": "transformers", |
| "hub_id": null, |
| "policy_note": "trained in this project", |
| "empirical": { |
| "model_type": "Unigram", |
| "n_base_tokens": 128256, |
| "n_added_tokens": 4, |
| "vocab_total": 128260, |
| "added_tokens": [ |
| "<s>", |
| "</s>", |
| "<unk>", |
| "<pad>" |
| ], |
| "normalizer": null, |
| "pre_tokenizer": { |
| "type": "Sequence", |
| "pretokenizers": [ |
| { |
| "type": "Split", |
| "pattern": { |
| "Regex": "(?i:[''\\u2019]s|[''\\u2019]t|[''\\u2019]re|[''\\u2019]ve|[''\\u2019]m|[''\\u2019]ll|[''\\u2019]d)|[ ]?[\\p{Lu}\\p{Lt}\\p{Lm}\\p{Lo}\\p{M}]*[\\p{Ll}\\p{Lm}\\p{Lo}\\p{M}]+(?i:[''\\u2019]s|[''\\u2019]t|[''\\u2019]re|[''\\u2019]ve|[''\\u2019]m|[''\\u2019]ll|[''\\u2019]d)?|[ ]?[\\p{Lu}\\p{Lt}\\p{Lm}\\p{Lo}\\p{M}]+[\\p{Ll}\\p{Lm}\\p{Lo}\\p{M}]*(?i:[''\\u2019]s|[''\\u2019]t|[''\\u2019]re|[''\\u2019]ve|[''\\u2019]m|[''\\u2019]ll|[''\\u2019]d)?|\\p{N}{1,3}|[ ]?[^\\s\\p{L}\\p{N}]+|[ ]+|[\\t]+|[\\n]+|[\\r]+|\\s+|\\S" |
| }, |
| "behavior": "Isolated", |
| "invert": false |
| }, |
| { |
| "type": "ByteLevel", |
| "add_prefix_space": false, |
| "trim_offsets": true, |
| "use_regex": false |
| } |
| ] |
| }, |
| "has_post_processor": false, |
| "byte_fallback": false, |
| "bos_token": "<s>", |
| "bos_token_id": 0 |
| }, |
| "declared_experiments": { |
| "algorithm": "unigram", |
| "pretokenizer_constant": "REGEX_CLAUDE", |
| "normalizer": null, |
| "data_composition": "balanced", |
| "description": "Claude pretok, balanced data, UnigramLM algorithm", |
| "trainer_kwargs": null |
| }, |
| "declared_run_manifest": null, |
| "registry_note": "P3 Claude V1 regex; balanced (D2) data; norm=none; UnigramLM (HF, default kwargs); vocab 128,260; USED - panels.json main21 (camera-ready main panel)" |
| }, |
| "claude-english-bpe": { |
| "slug": "claude-english-bpe", |
| "runs": [ |
| "full-128k-claude-english-bpe", |
| "full-128k-claude-english-bpe-mathcode-scratch", |
| "full-128k-claude-english-bpe-seed42", |
| "pilot-128k-claude-english-bpe", |
| "small-128k-claude-english-bpe", |
| "tiny-128k-claude-english-bpe" |
| ], |
| "n_runs": 6, |
| "redistributable": true, |
| "loadable_with": "transformers", |
| "hub_id": null, |
| "policy_note": "trained in this project", |
| "empirical": { |
| "model_type": "BPE", |
| "n_base_tokens": 128256, |
| "n_added_tokens": 4, |
| "vocab_total": 128260, |
| "added_tokens": [ |
| "<s>", |
| "</s>", |
| "<unk>", |
| "<pad>" |
| ], |
| "normalizer": null, |
| "pre_tokenizer": { |
| "type": "Sequence", |
| "pretokenizers": [ |
| { |
| "type": "Split", |
| "pattern": { |
| "Regex": "(?i:[''\\u2019]s|[''\\u2019]t|[''\\u2019]re|[''\\u2019]ve|[''\\u2019]m|[''\\u2019]ll|[''\\u2019]d)|[ ]?[\\p{Lu}\\p{Lt}\\p{Lm}\\p{Lo}\\p{M}]*[\\p{Ll}\\p{Lm}\\p{Lo}\\p{M}]+(?i:[''\\u2019]s|[''\\u2019]t|[''\\u2019]re|[''\\u2019]ve|[''\\u2019]m|[''\\u2019]ll|[''\\u2019]d)?|[ ]?[\\p{Lu}\\p{Lt}\\p{Lm}\\p{Lo}\\p{M}]+[\\p{Ll}\\p{Lm}\\p{Lo}\\p{M}]*(?i:[''\\u2019]s|[''\\u2019]t|[''\\u2019]re|[''\\u2019]ve|[''\\u2019]m|[''\\u2019]ll|[''\\u2019]d)?|\\p{N}{1,3}|[ ]?[^\\s\\p{L}\\p{N}]+|[ ]+|[\\t]+|[\\n]+|[\\r]+|\\s+|\\S" |
| }, |
| "behavior": "Isolated", |
| "invert": false |
| }, |
| { |
| "type": "ByteLevel", |
| "add_prefix_space": false, |
| "trim_offsets": true, |
| "use_regex": false |
| } |
| ] |
| }, |
| "has_post_processor": false, |
| "byte_fallback": false, |
| "bos_token": "<s>", |
| "bos_token_id": 0 |
| }, |
| "declared_experiments": { |
| "algorithm": "bpe", |
| "pretokenizer_constant": "REGEX_CLAUDE", |
| "normalizer": null, |
| "data_composition": "english", |
| "description": "Claude pretok, English-only data", |
| "trainer_kwargs": null |
| }, |
| "declared_run_manifest": null, |
| "registry_note": "P3 Claude V1 regex; english (D1) data; norm=none; BPE; vocab 128,260; affected by the initial_alphabet byte-drop defect (same class as gpt4o-english-bpe); USED - panels.json main21 (camera-ready main panel)" |
| }, |
| "clauderev-balanced-bpe": { |
| "slug": "clauderev-balanced-bpe", |
| "runs": [ |
| "full-128k-clauderev-balanced-bpe", |
| "full-128k-clauderev-balanced-bpe-mathcode-scratch" |
| ], |
| "n_runs": 2, |
| "redistributable": true, |
| "loadable_with": "transformers", |
| "hub_id": null, |
| "policy_note": "trained in this project", |
| "empirical": { |
| "model_type": "BPE", |
| "n_base_tokens": 128256, |
| "n_added_tokens": 4, |
| "vocab_total": 128260, |
| "added_tokens": [ |
| "<s>", |
| "</s>", |
| "<unk>", |
| "<pad>" |
| ], |
| "normalizer": { |
| "type": "Sequence", |
| "normalizers": [ |
| { |
| "type": "NFD" |
| }, |
| { |
| "type": "Replace", |
| "pattern": { |
| "Regex": "[\\x{000D}\\x{000B}\\x{000C}\\x{0085}]" |
| }, |
| "content": "" |
| }, |
| { |
| "type": "Replace", |
| "pattern": { |
| "Regex": "[\\x{00A0}\\x{1680}\\x{2000}-\\x{200A}\\x{202F}\\x{205F}\\x{2028}\\x{2029}]" |
| }, |
| "content": " " |
| } |
| ] |
| }, |
| "pre_tokenizer": { |
| "type": "Sequence", |
| "pretokenizers": [ |
| { |
| "type": "Split", |
| "pattern": { |
| "Regex": "(?i:['\\u2019](?:s|t|re|ve|m|ll|d))|(?<![\\p{N}\\s\\p{Han}\\p{Hangul}])[ ][\\p{L}\\p{M}]+|[\\p{L}\\p{M}]+|\\p{N}{1,3}|(?<![\\p{N}\\s\\p{Han}\\p{Hangul}])[ ][^\\s\\p{L}\\p{N}]+|[^\\s\\p{L}\\p{N}]+|[ ]+|[\\t]+|[\\n]+|\\s+" |
| }, |
| "behavior": "Isolated", |
| "invert": false |
| }, |
| { |
| "type": "ByteLevel", |
| "add_prefix_space": false, |
| "trim_offsets": true, |
| "use_regex": false |
| } |
| ] |
| }, |
| "has_post_processor": false, |
| "byte_fallback": false, |
| "bos_token": "<s>", |
| "bos_token_id": 0 |
| }, |
| "declared_experiments": { |
| "algorithm": "bpe", |
| "pretokenizer_constant": "REGEX_CLAUDE_REV", |
| "normalizer": { |
| "type": "Sequence", |
| "normalizers": [ |
| { |
| "type": "NFD" |
| }, |
| { |
| "type": "Replace", |
| "pattern": { |
| "Regex": "[\\x{000D}\\x{000B}\\x{000C}\\x{0085}]" |
| }, |
| "content": "" |
| }, |
| { |
| "type": "Replace", |
| "pattern": { |
| "Regex": "[\\x{00A0}\\x{1680}\\x{2000}-\\x{200A}\\x{202F}\\x{205F}\\x{2028}\\x{2029}]" |
| }, |
| "content": " " |
| } |
| ] |
| }, |
| "data_composition": "balanced", |
| "description": "Revised-Claude pretok (2026-07: no camel-case/script split, lookbehind space-attach, contraction-suffix clause) + revised normalizer (NFD + CR/VT/FF/NEL delete + whitespace-fold), balanced data, BPE", |
| "trainer_kwargs": null |
| }, |
| "declared_run_manifest": null, |
| "registry_note": "REGEX_CLAUDE_REV (Claude V2: no camel-case split, no cross-script split, dedicated contraction clause, lookbehind-gated space-attach); balanced (D2) data; norm=NFD + delete CR/VT/FF/NEL + fold-whitespace-to-U+0020 (Sequence normalizer); BPE; vocab 128,256; Claude V2 spec (2026-07-05 build); BPB metrics were biased +6.40% by the NFD byte-denominator bug, FIXED 2026-07-13 (accuracy metrics MBPP/HumanEval/GSM8K/BLiMP/code_bpb unaffected); never enters a val_bpb/FLORES correlation per panels.json note; USED - panels.json pretok-controls (camera-ready)" |
| }, |
| "gpt2-balanced-bpe": { |
| "slug": "gpt2-balanced-bpe", |
| "runs": [ |
| "full-128k-gpt2-balanced-bpe" |
| ], |
| "n_runs": 1, |
| "redistributable": true, |
| "loadable_with": "transformers", |
| "hub_id": null, |
| "policy_note": "trained in this project", |
| "empirical": { |
| "model_type": "BPE", |
| "n_base_tokens": 128256, |
| "n_added_tokens": 4, |
| "vocab_total": 128260, |
| "added_tokens": [ |
| "<s>", |
| "</s>", |
| "<unk>", |
| "<pad>" |
| ], |
| "normalizer": null, |
| "pre_tokenizer": { |
| "type": "ByteLevel", |
| "add_prefix_space": false, |
| "trim_offsets": true, |
| "use_regex": true |
| }, |
| "has_post_processor": false, |
| "byte_fallback": false, |
| "bos_token": "<s>", |
| "bos_token_id": 0 |
| }, |
| "declared_experiments": { |
| "algorithm": "bpe", |
| "pretokenizer_constant": "ByteLevel", |
| "normalizer": null, |
| "data_composition": "balanced", |
| "description": "GPT-2 default regex (ByteLevel use_regex=True), balanced data, BPE", |
| "trainer_kwargs": null |
| }, |
| "declared_run_manifest": null, |
| "registry_note": "plain ByteLevel(use_regex=True) i.e. GPT-2 tiktoken regex; balanced (D2) data; norm=none; BPE; vocab 128,260; reference/default regex missing from the original panel; USED - panels.json pretok-controls (camera-ready)" |
| }, |
| "gpt4o-allmulti-equal-nfc-bpe": { |
| "slug": "gpt4o-allmulti-equal-nfc-bpe", |
| "runs": [ |
| "full-128k-gpt4o-allmulti-equal-nfc-bpe" |
| ], |
| "n_runs": 1, |
| "redistributable": true, |
| "loadable_with": "transformers", |
| "hub_id": null, |
| "policy_note": "trained in this project", |
| "empirical": { |
| "model_type": "BPE", |
| "n_base_tokens": 128256, |
| "n_added_tokens": 4, |
| "vocab_total": 128260, |
| "added_tokens": [ |
| "<s>", |
| "</s>", |
| "<unk>", |
| "<pad>" |
| ], |
| "normalizer": { |
| "type": "NFC" |
| }, |
| "pre_tokenizer": { |
| "type": "Sequence", |
| "pretokenizers": [ |
| { |
| "type": "Split", |
| "pattern": { |
| "Regex": "[^\\r\\n\\p{L}\\p{N}]?[\\p{Lu}\\p{Lt}\\p{Lm}\\p{Lo}\\p{M}]*[\\p{Ll}\\p{Lm}\\p{Lo}\\p{M}]+(?i:'s|'t|'re|'ve|'m|'ll|'d)?|[^\\r\\n\\p{L}\\p{N}]?[\\p{Lu}\\p{Lt}\\p{Lm}\\p{Lo}\\p{M}]+[\\p{Ll}\\p{Lm}\\p{Lo}\\p{M}]*(?i:'s|'t|'re|'ve|'m|'ll|'d)?|\\p{N}{1,3}| ?[^\\s\\p{L}\\p{N}]+[\\r\\n/]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+" |
| }, |
| "behavior": "Isolated", |
| "invert": false |
| }, |
| { |
| "type": "ByteLevel", |
| "add_prefix_space": false, |
| "trim_offsets": true, |
| "use_regex": false |
| } |
| ] |
| }, |
| "has_post_processor": false, |
| "byte_fallback": false, |
| "bos_token": "<s>", |
| "bos_token_id": 0 |
| }, |
| "declared_experiments": { |
| "algorithm": "bpe", |
| "pretokenizer_constant": "REGEX_GPT4O", |
| "normalizer": { |
| "type": "NFC" |
| }, |
| "data_composition": "allmulti_equal", |
| "description": "GPT-4o regex + NFC, equal-per-language data (repeat-sampled), BPE. Pure normalizer contrast against allmulti-equal-bpe; pairs with gpt4o-balanced-nfc-bpe.", |
| "trainer_kwargs": null |
| }, |
| "declared_run_manifest": null, |
| "registry_note": "P2 GPT-4o regex; allmulti_equal data; norm=NFC; BPE; vocab 128,256; pure normalizer contrast against allmulti-equal-bpe; pairs with gpt4o-balanced-nfc-bpe; USED - S6d.4 (rho=-0.475)" |
| }, |
| "gpt4o-allmulti-equal-norepeat-bpe": { |
| "slug": "gpt4o-allmulti-equal-norepeat-bpe", |
| "runs": [ |
| "full-128k-gpt4o-allmulti-equal-norepeat-bpe" |
| ], |
| "n_runs": 1, |
| "redistributable": true, |
| "loadable_with": "transformers", |
| "hub_id": null, |
| "policy_note": "trained in this project", |
| "empirical": { |
| "model_type": "BPE", |
| "n_base_tokens": 128256, |
| "n_added_tokens": 4, |
| "vocab_total": 128260, |
| "added_tokens": [ |
| "<s>", |
| "</s>", |
| "<unk>", |
| "<pad>" |
| ], |
| "normalizer": null, |
| "pre_tokenizer": { |
| "type": "Sequence", |
| "pretokenizers": [ |
| { |
| "type": "Split", |
| "pattern": { |
| "Regex": "[^\\r\\n\\p{L}\\p{N}]?[\\p{Lu}\\p{Lt}\\p{Lm}\\p{Lo}\\p{M}]*[\\p{Ll}\\p{Lm}\\p{Lo}\\p{M}]+(?i:'s|'t|'re|'ve|'m|'ll|'d)?|[^\\r\\n\\p{L}\\p{N}]?[\\p{Lu}\\p{Lt}\\p{Lm}\\p{Lo}\\p{M}]+[\\p{Ll}\\p{Lm}\\p{Lo}\\p{M}]*(?i:'s|'t|'re|'ve|'m|'ll|'d)?|\\p{N}{1,3}| ?[^\\s\\p{L}\\p{N}]+[\\r\\n/]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+" |
| }, |
| "behavior": "Isolated", |
| "invert": false |
| }, |
| { |
| "type": "ByteLevel", |
| "add_prefix_space": false, |
| "trim_offsets": true, |
| "use_regex": false |
| } |
| ] |
| }, |
| "has_post_processor": false, |
| "byte_fallback": false, |
| "bos_token": "<s>", |
| "bos_token_id": 0 |
| }, |
| "declared_experiments": { |
| "algorithm": "bpe", |
| "pretokenizer_constant": "REGEX_GPT4O", |
| "normalizer": null, |
| "data_composition": "allmulti_equal_norepeat", |
| "description": "GPT-4o regex, equal per-language weights + max_files=50 but NO repeat-sampling (tail exhausts). Differs from allmulti-equal-bpe ONLY in repeat-sampling, so it isolates that factor.", |
| "trainer_kwargs": null |
| }, |
| "declared_run_manifest": null, |
| "registry_note": "P2 GPT-4o regex; allmulti_equal_norepeat (equal weights, max_files=50, NO repeat-sampling) data; norm=none; BPE; vocab 128,256; isolates repeat-sampling factor against allmulti-equal-bpe; USED - S6d.4 (rho=-0.581; repeat-sampling contributes at most ~1/4 of the tail penalty)" |
| }, |
| "gpt4o-balanced-bpe": { |
| "slug": "gpt4o-balanced-bpe", |
| "runs": [ |
| "full-128k-gpt4o-balanced-bpe", |
| "full-128k-gpt4o-balanced-bpe-seed42", |
| "full-128k-gpt4o-balanced-bpe-seed43-mathcode-scratch", |
| "full-128k-gpt4o-balanced-bpe-seed44-mathcode-scratch", |
| "full-128k-mathcode-scratch", |
| "pilot-128k-gpt4o-balanced-bpe-wdfix", |
| "small-128k-gpt4o-balanced-bpe", |
| "tiny-128k-gpt4o-balanced-bpe" |
| ], |
| "n_runs": 8, |
| "redistributable": true, |
| "loadable_with": "transformers", |
| "hub_id": null, |
| "policy_note": "trained in this project", |
| "empirical": { |
| "model_type": "BPE", |
| "n_base_tokens": 128256, |
| "n_added_tokens": 4, |
| "vocab_total": 128260, |
| "added_tokens": [ |
| "<s>", |
| "</s>", |
| "<unk>", |
| "<pad>" |
| ], |
| "normalizer": null, |
| "pre_tokenizer": { |
| "type": "Sequence", |
| "pretokenizers": [ |
| { |
| "type": "Split", |
| "pattern": { |
| "Regex": "[^\\r\\n\\p{L}\\p{N}]?[\\p{Lu}\\p{Lt}\\p{Lm}\\p{Lo}\\p{M}]*[\\p{Ll}\\p{Lm}\\p{Lo}\\p{M}]+(?i:'s|'t|'re|'ve|'m|'ll|'d)?|[^\\r\\n\\p{L}\\p{N}]?[\\p{Lu}\\p{Lt}\\p{Lm}\\p{Lo}\\p{M}]+[\\p{Ll}\\p{Lm}\\p{Lo}\\p{M}]*(?i:'s|'t|'re|'ve|'m|'ll|'d)?|\\p{N}{1,3}| ?[^\\s\\p{L}\\p{N}]+[\\r\\n/]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+" |
| }, |
| "behavior": "Isolated", |
| "invert": false |
| }, |
| { |
| "type": "ByteLevel", |
| "add_prefix_space": false, |
| "trim_offsets": true, |
| "use_regex": false |
| } |
| ] |
| }, |
| "has_post_processor": false, |
| "byte_fallback": false, |
| "bos_token": "<s>", |
| "bos_token_id": 0 |
| }, |
| "declared_experiments": { |
| "algorithm": "bpe", |
| "pretokenizer_constant": "REGEX_GPT4O", |
| "normalizer": null, |
| "data_composition": "balanced", |
| "description": "GPT-4o regex, balanced multilingual data", |
| "trainer_kwargs": null |
| }, |
| "declared_run_manifest": null, |
| "registry_note": "P2 GPT-4o regex (REGEX_GPT4O); balanced (D2) data; norm=none; BPE; vocab 128,260; project baseline tokenizer (used as the fixed tokenizer for the softcap ablation and as the mathcode-scratch #1 baseline); USED - panels.json main21 (camera-ready main panel)" |
| }, |
| "gpt4o-balanced-maxfiles50-bpe": { |
| "slug": "gpt4o-balanced-maxfiles50-bpe", |
| "runs": [ |
| "full-128k-gpt4o-balanced-maxfiles50-bpe" |
| ], |
| "n_runs": 1, |
| "redistributable": true, |
| "loadable_with": "transformers", |
| "hub_id": null, |
| "policy_note": "trained in this project", |
| "empirical": { |
| "model_type": "BPE", |
| "n_base_tokens": 128256, |
| "n_added_tokens": 4, |
| "vocab_total": 128260, |
| "added_tokens": [ |
| "<s>", |
| "</s>", |
| "<unk>", |
| "<pad>" |
| ], |
| "normalizer": null, |
| "pre_tokenizer": { |
| "type": "Sequence", |
| "pretokenizers": [ |
| { |
| "type": "Split", |
| "pattern": { |
| "Regex": "[^\\r\\n\\p{L}\\p{N}]?[\\p{Lu}\\p{Lt}\\p{Lm}\\p{Lo}\\p{M}]*[\\p{Ll}\\p{Lm}\\p{Lo}\\p{M}]+(?i:'s|'t|'re|'ve|'m|'ll|'d)?|[^\\r\\n\\p{L}\\p{N}]?[\\p{Lu}\\p{Lt}\\p{Lm}\\p{Lo}\\p{M}]+[\\p{Ll}\\p{Lm}\\p{Lo}\\p{M}]*(?i:'s|'t|'re|'ve|'m|'ll|'d)?|\\p{N}{1,3}| ?[^\\s\\p{L}\\p{N}]+[\\r\\n/]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+" |
| }, |
| "behavior": "Isolated", |
| "invert": false |
| }, |
| { |
| "type": "ByteLevel", |
| "add_prefix_space": false, |
| "trim_offsets": true, |
| "use_regex": false |
| } |
| ] |
| }, |
| "has_post_processor": false, |
| "byte_fallback": false, |
| "bos_token": "<s>", |
| "bos_token_id": 0 |
| }, |
| "declared_experiments": { |
| "algorithm": "bpe", |
| "pretokenizer_constant": "REGEX_GPT4O", |
| "normalizer": null, |
| "data_composition": "balanced_maxfiles50", |
| "description": "GPT-4o regex, proportional (balanced) weights with max_files=50. Differs from gpt4o-balanced-bpe ONLY in max_files, so it isolates that factor. Expected NULL control.", |
| "trainer_kwargs": null |
| }, |
| "declared_run_manifest": null, |
| "registry_note": "P2 GPT-4o regex; balanced (proportional) weights, max_files raised to 50, no repeat data; norm=none; BPE; vocab 128,256; NULL CONTROL for the max_files cap; USED - S6d.4 null control (rho=+0.004, p=0.98 - exonerates max_files as the driver)" |
| }, |
| "gpt4o-balanced-nfc-bpe": { |
| "slug": "gpt4o-balanced-nfc-bpe", |
| "runs": [ |
| "full-128k-gpt4o-balanced-nfc-bpe", |
| "full-128k-gpt4o-balanced-nfc-bpe-mathcode-scratch", |
| "full-128k-gpt4o-balanced-nfc-bpe-seed42", |
| "pilot-128k-gpt4o-balanced-nfc-bpe-wdfix", |
| "small-128k-gpt4o-balanced-nfc-bpe", |
| "tiny-128k-gpt4o-balanced-nfc-bpe" |
| ], |
| "n_runs": 6, |
| "redistributable": true, |
| "loadable_with": "transformers", |
| "hub_id": null, |
| "policy_note": "trained in this project", |
| "empirical": { |
| "model_type": "BPE", |
| "n_base_tokens": 128256, |
| "n_added_tokens": 4, |
| "vocab_total": 128260, |
| "added_tokens": [ |
| "<s>", |
| "</s>", |
| "<unk>", |
| "<pad>" |
| ], |
| "normalizer": { |
| "type": "NFC" |
| }, |
| "pre_tokenizer": { |
| "type": "Sequence", |
| "pretokenizers": [ |
| { |
| "type": "Split", |
| "pattern": { |
| "Regex": "[^\\r\\n\\p{L}\\p{N}]?[\\p{Lu}\\p{Lt}\\p{Lm}\\p{Lo}\\p{M}]*[\\p{Ll}\\p{Lm}\\p{Lo}\\p{M}]+(?i:'s|'t|'re|'ve|'m|'ll|'d)?|[^\\r\\n\\p{L}\\p{N}]?[\\p{Lu}\\p{Lt}\\p{Lm}\\p{Lo}\\p{M}]+[\\p{Ll}\\p{Lm}\\p{Lo}\\p{M}]*(?i:'s|'t|'re|'ve|'m|'ll|'d)?|\\p{N}{1,3}| ?[^\\s\\p{L}\\p{N}]+[\\r\\n/]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+" |
| }, |
| "behavior": "Isolated", |
| "invert": false |
| }, |
| { |
| "type": "ByteLevel", |
| "add_prefix_space": false, |
| "trim_offsets": true, |
| "use_regex": false |
| } |
| ] |
| }, |
| "has_post_processor": false, |
| "byte_fallback": false, |
| "bos_token": "<s>", |
| "bos_token_id": 0 |
| }, |
| "declared_experiments": { |
| "algorithm": "bpe", |
| "pretokenizer_constant": "REGEX_GPT4O", |
| "normalizer": { |
| "type": "NFC" |
| }, |
| "data_composition": "balanced", |
| "description": "GPT-4o regex, balanced data, NFC normalization", |
| "trainer_kwargs": null |
| }, |
| "declared_run_manifest": null, |
| "registry_note": "P2 GPT-4o regex; balanced (D2) data; norm=NFC; BPE; vocab 128,260; USED - panels.json main21 (camera-ready main panel)" |
| }, |
| "gpt4o-balanced-unigram": { |
| "slug": "gpt4o-balanced-unigram", |
| "runs": [ |
| "full-128k-gpt4o-balanced-unigram", |
| "full-128k-gpt4o-balanced-unigram-mathcode-scratch", |
| "full-128k-gpt4o-balanced-unigram-seed42", |
| "pilot-128k-gpt4o-balanced-unigram-wdfix", |
| "small-128k-gpt4o-balanced-unigram", |
| "tiny-128k-gpt4o-balanced-unigram" |
| ], |
| "n_runs": 6, |
| "redistributable": true, |
| "loadable_with": "transformers", |
| "hub_id": null, |
| "policy_note": "trained in this project", |
| "empirical": { |
| "model_type": "Unigram", |
| "n_base_tokens": 128256, |
| "n_added_tokens": 4, |
| "vocab_total": 128260, |
| "added_tokens": [ |
| "<s>", |
| "</s>", |
| "<unk>", |
| "<pad>" |
| ], |
| "normalizer": null, |
| "pre_tokenizer": { |
| "type": "Sequence", |
| "pretokenizers": [ |
| { |
| "type": "Split", |
| "pattern": { |
| "Regex": "[^\\r\\n\\p{L}\\p{N}]?[\\p{Lu}\\p{Lt}\\p{Lm}\\p{Lo}\\p{M}]*[\\p{Ll}\\p{Lm}\\p{Lo}\\p{M}]+(?i:'s|'t|'re|'ve|'m|'ll|'d)?|[^\\r\\n\\p{L}\\p{N}]?[\\p{Lu}\\p{Lt}\\p{Lm}\\p{Lo}\\p{M}]+[\\p{Ll}\\p{Lm}\\p{Lo}\\p{M}]*(?i:'s|'t|'re|'ve|'m|'ll|'d)?|\\p{N}{1,3}| ?[^\\s\\p{L}\\p{N}]+[\\r\\n/]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+" |
| }, |
| "behavior": "Isolated", |
| "invert": false |
| }, |
| { |
| "type": "ByteLevel", |
| "add_prefix_space": false, |
| "trim_offsets": true, |
| "use_regex": false |
| } |
| ] |
| }, |
| "has_post_processor": false, |
| "byte_fallback": false, |
| "bos_token": "<s>", |
| "bos_token_id": 0 |
| }, |
| "declared_experiments": { |
| "algorithm": "unigram", |
| "pretokenizer_constant": "REGEX_GPT4O", |
| "normalizer": null, |
| "data_composition": "balanced", |
| "description": "GPT-4o regex, balanced data, UnigramLM algorithm", |
| "trainer_kwargs": null |
| }, |
| "declared_run_manifest": null, |
| "registry_note": "P2 GPT-4o regex; balanced (D2) data; norm=none; UnigramLM (HF UnigramTrainer, default kwargs); vocab 128,260; USED - panels.json main21 (camera-ready main panel)" |
| }, |
| "gpt4o-balanced-unigram-tuned": { |
| "slug": "gpt4o-balanced-unigram-tuned", |
| "runs": [ |
| "full-128k-gpt4o-balanced-unigram-tuned" |
| ], |
| "n_runs": 1, |
| "redistributable": true, |
| "loadable_with": "transformers", |
| "hub_id": null, |
| "policy_note": "trained in this project", |
| "empirical": { |
| "model_type": "Unigram", |
| "n_base_tokens": 128256, |
| "n_added_tokens": 4, |
| "vocab_total": 128260, |
| "added_tokens": [ |
| "<s>", |
| "</s>", |
| "<unk>", |
| "<pad>" |
| ], |
| "normalizer": null, |
| "pre_tokenizer": { |
| "type": "Sequence", |
| "pretokenizers": [ |
| { |
| "type": "Split", |
| "pattern": { |
| "Regex": "[^\\r\\n\\p{L}\\p{N}]?[\\p{Lu}\\p{Lt}\\p{Lm}\\p{Lo}\\p{M}]*[\\p{Ll}\\p{Lm}\\p{Lo}\\p{M}]+(?i:'s|'t|'re|'ve|'m|'ll|'d)?|[^\\r\\n\\p{L}\\p{N}]?[\\p{Lu}\\p{Lt}\\p{Lm}\\p{Lo}\\p{M}]+[\\p{Ll}\\p{Lm}\\p{Lo}\\p{M}]*(?i:'s|'t|'re|'ve|'m|'ll|'d)?|\\p{N}{1,3}| ?[^\\s\\p{L}\\p{N}]+[\\r\\n/]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+" |
| }, |
| "behavior": "Isolated", |
| "invert": false |
| }, |
| { |
| "type": "ByteLevel", |
| "add_prefix_space": false, |
| "trim_offsets": true, |
| "use_regex": false |
| } |
| ] |
| }, |
| "has_post_processor": false, |
| "byte_fallback": false, |
| "bos_token": "<s>", |
| "bos_token_id": 0 |
| }, |
| "declared_experiments": { |
| "algorithm": "unigram", |
| "pretokenizer_constant": "REGEX_GPT4O", |
| "normalizer": null, |
| "data_composition": "balanced", |
| "description": "GPT-4o regex, balanced data, UnigramLM with tuned hyperparameters", |
| "trainer_kwargs": { |
| "shrinking_factor": 0.7, |
| "n_sub_iterations": 3, |
| "max_piece_length": 64, |
| "initial_alphabet": [ |
| "\u0131", |
| "\u00c5", |
| "\u00b1", |
| "}", |
| "\u00cb", |
| "_", |
| "t", |
| "\u00fa", |
| "\u00be", |
| "\u00d2", |
| "\u00ea", |
| "8", |
| "\u012e", |
| "\u00d4", |
| "i", |
| "\u00c2", |
| "w", |
| "r", |
| "\u00b7", |
| "\u013d", |
| "\u00d0", |
| "U", |
| "p", |
| "\u0121", |
| "\u00c7", |
| "R", |
| "\u00ae", |
| "\u0116", |
| "\u00ac", |
| "b", |
| "\u00d7", |
| "d", |
| "?", |
| "D", |
| "\u00d1", |
| "\u0143", |
| "\u00e9", |
| "\u00e4", |
| "\u0129", |
| "\u00b8", |
| "W", |
| "k", |
| "\u00e8", |
| "\u011b", |
| "\u013b", |
| "\u00c8", |
| "\u0109", |
| "\u00f8", |
| "\u013e", |
| "\u0103", |
| "\u0112", |
| "\u0111", |
| "z", |
| "L", |
| "\u00e1", |
| "\u00fb", |
| "\u00a3", |
| "\u00f6", |
| "\u011c", |
| "\u012d", |
| "\u012c", |
| "B", |
| "C", |
| "\u00df", |
| "\u0122", |
| "\u0120", |
| "\u00b2", |
| "\u00a7", |
| "]", |
| "5", |
| "\u0139", |
| ">", |
| "`", |
| "\u0108", |
| "\u00ba", |
| "\u0128", |
| "(", |
| "\u0115", |
| "\u013a", |
| "\u00f1", |
| "f", |
| "o", |
| "\u00cd", |
| "\u00da", |
| "\u00a5", |
| "\u012a", |
| "\u010c", |
| "\u00de", |
| "x", |
| "\u00e3", |
| "\u00f7", |
| "\u010a", |
| "\u010e", |
| "\u010f", |
| "\u00c1", |
| "\u00b0", |
| "\"", |
| "\u00ce", |
| ".", |
| "l", |
| "J", |
| "\u00ee", |
| "F", |
| "\u00a2", |
| "\u00e0", |
| "\u011d", |
| "\u00bf", |
| "Q", |
| "\u00cc", |
| "\u0117", |
| "\u00cf", |
| "\u0124", |
| "Y", |
| "$", |
| "K", |
| "\u00a9", |
| "-", |
| "\u00ef", |
| "v", |
| "^", |
| "\u0105", |
| "\u00fc", |
| "\u0126", |
| "\u00af", |
| "\u0114", |
| "\u0142", |
| "I", |
| "\u012b", |
| "u", |
| "<", |
| "2", |
| "\u00bb", |
| "~", |
| "\u0106", |
| "n", |
| "m", |
| "\u0140", |
| "\\", |
| "1", |
| "\u0127", |
| "\u00e7", |
| "6", |
| "\u00db", |
| "\u00d9", |
| "\u00f5", |
| "X", |
| "\u00ab", |
| "\u00eb", |
| "'", |
| "\u013c", |
| "E", |
| "\u00c0", |
| "\u0135", |
| "\u00dc", |
| "\u00fd", |
| "\u00c4", |
| "\u0113", |
| "\u0138", |
| "\u00bd", |
| "%", |
| "@", |
| "\u00ca", |
| "3", |
| "\u00ec", |
| "\u00f2", |
| "h", |
| "y", |
| "\u00b4", |
| "\u00a4", |
| "\u00b6", |
| "&", |
| "\u00ff", |
| "\u011e", |
| "g", |
| "c", |
| "\u00e6", |
| "N", |
| "\u0134", |
| "4", |
| "\u00f4", |
| "\u00ed", |
| "\u0118", |
| "\u012f", |
| "\u00b3", |
| "V", |
| "Z", |
| "\u0141", |
| "\u00b9", |
| "\u0125", |
| "\u0110", |
| "O", |
| "\u0100", |
| "\u00f0", |
| ")", |
| "\u0136", |
| "*", |
| "P", |
| "/", |
| "\u00f9", |
| "a", |
| "\u0132", |
| "\u011f", |
| "\u0137", |
| "!", |
| "\u00d5", |
| "\u00e5", |
| "9", |
| "A", |
| "\u00fe", |
| "q", |
| "\u00d6", |
| "e", |
| "=", |
| "\u0123", |
| "\u00a8", |
| "{", |
| "\u00f3", |
| "\u0133", |
| "\u011a", |
| "#", |
| "\u00d3", |
| "S", |
| "G", |
| "[", |
| "\u010b", |
| "\u0104", |
| "s", |
| "\u00c6", |
| "\u0119", |
| "\u0102", |
| "\u010d", |
| ":", |
| "\u00a1", |
| "\u00aa", |
| "\u00d8", |
| "\u0130", |
| "j", |
| "M", |
| "7", |
| "\u00a6", |
| "\u0107", |
| "\u013f", |
| "\u0101", |
| "0", |
| "\u00c3", |
| "\u00dd", |
| ";", |
| "H", |
| "\u00b5", |
| "+", |
| "|", |
| ",", |
| "\u00c9", |
| "T", |
| "\u00bc", |
| "\u00e2" |
| ] |
| } |
| }, |
| "declared_run_manifest": null, |
| "registry_note": "P2 GPT-4o regex; balanced (D2) data; norm=none; UnigramLM (HF, UNIGRAM_TUNED_KWARGS: shrinking_factor=0.7, n_sub_iterations=3, max_piece_length=64, full byte initial_alphabet); vocab 128,260; tests whether 'Unigram loses' survives tuned HF UnigramTrainer hyperparameters; USED - balanced-1B bootstrap panel (S6.12, n=32/34) and paper_bootstrap_balanced_panel; not in panels.json main21 (superseded by default-kwargs gpt4o-balanced-unigram in the settled panel)" |
| }, |
| "gpt4o-code-bpe": { |
| "slug": "gpt4o-code-bpe", |
| "runs": [ |
| "full-128k-gpt4o-code-bpe", |
| "full-128k-gpt4o-code-bpe-mathcode-scratch", |
| "full-128k-gpt4o-code-bpe-seed42", |
| "pilot-128k-gpt4o-code-bpe-wdfix", |
| "small-128k-gpt4o-code-bpe", |
| "tiny-128k-gpt4o-code-bpe" |
| ], |
| "n_runs": 6, |
| "redistributable": true, |
| "loadable_with": "transformers", |
| "hub_id": null, |
| "policy_note": "trained in this project", |
| "empirical": { |
| "model_type": "BPE", |
| "n_base_tokens": 128256, |
| "n_added_tokens": 4, |
| "vocab_total": 128260, |
| "added_tokens": [ |
| "<s>", |
| "</s>", |
| "<unk>", |
| "<pad>" |
| ], |
| "normalizer": null, |
| "pre_tokenizer": { |
| "type": "Sequence", |
| "pretokenizers": [ |
| { |
| "type": "Split", |
| "pattern": { |
| "Regex": "[^\\r\\n\\p{L}\\p{N}]?[\\p{Lu}\\p{Lt}\\p{Lm}\\p{Lo}\\p{M}]*[\\p{Ll}\\p{Lm}\\p{Lo}\\p{M}]+(?i:'s|'t|'re|'ve|'m|'ll|'d)?|[^\\r\\n\\p{L}\\p{N}]?[\\p{Lu}\\p{Lt}\\p{Lm}\\p{Lo}\\p{M}]+[\\p{Ll}\\p{Lm}\\p{Lo}\\p{M}]*(?i:'s|'t|'re|'ve|'m|'ll|'d)?|\\p{N}{1,3}| ?[^\\s\\p{L}\\p{N}]+[\\r\\n/]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+" |
| }, |
| "behavior": "Isolated", |
| "invert": false |
| }, |
| { |
| "type": "ByteLevel", |
| "add_prefix_space": false, |
| "trim_offsets": true, |
| "use_regex": false |
| } |
| ] |
| }, |
| "has_post_processor": false, |
| "byte_fallback": false, |
| "bos_token": "<s>", |
| "bos_token_id": 0 |
| }, |
| "declared_experiments": { |
| "algorithm": "bpe", |
| "pretokenizer_constant": "REGEX_GPT4O", |
| "normalizer": null, |
| "data_composition": "code", |
| "description": "GPT-4o regex, code-heavy data", |
| "trainer_kwargs": null |
| }, |
| "declared_run_manifest": null, |
| "registry_note": "P2 GPT-4o regex; code-heavy (D3) data; norm=none; BPE; vocab 128,260; USED - panels.json main21 (camera-ready main panel)" |
| }, |
| "gpt4o-codegrad00-bpe": { |
| "slug": "gpt4o-codegrad00-bpe", |
| "runs": [ |
| "full-128k-gpt4o-codegrad00-bpe-mathcode-scratch" |
| ], |
| "n_runs": 1, |
| "redistributable": true, |
| "loadable_with": "transformers", |
| "hub_id": null, |
| "policy_note": "trained in this project", |
| "empirical": { |
| "model_type": "BPE", |
| "n_base_tokens": 128256, |
| "n_added_tokens": 4, |
| "vocab_total": 128260, |
| "added_tokens": [ |
| "<s>", |
| "</s>", |
| "<unk>", |
| "<pad>" |
| ], |
| "normalizer": null, |
| "pre_tokenizer": { |
| "type": "Sequence", |
| "pretokenizers": [ |
| { |
| "type": "Split", |
| "pattern": { |
| "Regex": "[^\\r\\n\\p{L}\\p{N}]?[\\p{Lu}\\p{Lt}\\p{Lm}\\p{Lo}\\p{M}]*[\\p{Ll}\\p{Lm}\\p{Lo}\\p{M}]+(?i:'s|'t|'re|'ve|'m|'ll|'d)?|[^\\r\\n\\p{L}\\p{N}]?[\\p{Lu}\\p{Lt}\\p{Lm}\\p{Lo}\\p{M}]+[\\p{Ll}\\p{Lm}\\p{Lo}\\p{M}]*(?i:'s|'t|'re|'ve|'m|'ll|'d)?|\\p{N}{1,3}| ?[^\\s\\p{L}\\p{N}]+[\\r\\n/]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+" |
| }, |
| "behavior": "Isolated", |
| "invert": false |
| }, |
| { |
| "type": "ByteLevel", |
| "add_prefix_space": false, |
| "trim_offsets": true, |
| "use_regex": false |
| } |
| ] |
| }, |
| "has_post_processor": false, |
| "byte_fallback": false, |
| "bos_token": "<s>", |
| "bos_token_id": 0 |
| }, |
| "declared_experiments": { |
| "algorithm": "bpe", |
| "pretokenizer_constant": "REGEX_GPT4O", |
| "normalizer": null, |
| "data_composition": "codegrad00", |
| "description": "GPT-4o regex, code-fraction gradient c_w=0.00 (code omitted), BPE", |
| "trainer_kwargs": null |
| }, |
| "declared_run_manifest": null, |
| "registry_note": "#39, gpt4o pretok, slug `gpt4o-codegrad00-bpe`. AXES-2 code-fraction gradient, fusing regex (`REGEX_GPT4O`), code fraction 0%. Fused punct+newline vocab tokens: 482. FINISHED. MBPP 0.166 (generation_spec v1); v2 re-derivation (2026-08-04) 0.232, see \u00a73a.3. Not in panel." |
| }, |
| "gpt4o-codegrad05-bpe": { |
| "slug": "gpt4o-codegrad05-bpe", |
| "runs": [ |
| "full-128k-gpt4o-codegrad05-bpe-mathcode-scratch" |
| ], |
| "n_runs": 1, |
| "redistributable": true, |
| "loadable_with": "transformers", |
| "hub_id": null, |
| "policy_note": "trained in this project", |
| "empirical": { |
| "model_type": "BPE", |
| "n_base_tokens": 128256, |
| "n_added_tokens": 4, |
| "vocab_total": 128260, |
| "added_tokens": [ |
| "<s>", |
| "</s>", |
| "<unk>", |
| "<pad>" |
| ], |
| "normalizer": null, |
| "pre_tokenizer": { |
| "type": "Sequence", |
| "pretokenizers": [ |
| { |
| "type": "Split", |
| "pattern": { |
| "Regex": "[^\\r\\n\\p{L}\\p{N}]?[\\p{Lu}\\p{Lt}\\p{Lm}\\p{Lo}\\p{M}]*[\\p{Ll}\\p{Lm}\\p{Lo}\\p{M}]+(?i:'s|'t|'re|'ve|'m|'ll|'d)?|[^\\r\\n\\p{L}\\p{N}]?[\\p{Lu}\\p{Lt}\\p{Lm}\\p{Lo}\\p{M}]+[\\p{Ll}\\p{Lm}\\p{Lo}\\p{M}]*(?i:'s|'t|'re|'ve|'m|'ll|'d)?|\\p{N}{1,3}| ?[^\\s\\p{L}\\p{N}]+[\\r\\n/]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+" |
| }, |
| "behavior": "Isolated", |
| "invert": false |
| }, |
| { |
| "type": "ByteLevel", |
| "add_prefix_space": false, |
| "trim_offsets": true, |
| "use_regex": false |
| } |
| ] |
| }, |
| "has_post_processor": false, |
| "byte_fallback": false, |
| "bos_token": "<s>", |
| "bos_token_id": 0 |
| }, |
| "declared_experiments": { |
| "algorithm": "bpe", |
| "pretokenizer_constant": "REGEX_GPT4O", |
| "normalizer": null, |
| "data_composition": "codegrad05", |
| "description": "GPT-4o regex, code-fraction gradient c_w=0.05, BPE", |
| "trainer_kwargs": null |
| }, |
| "declared_run_manifest": null, |
| "registry_note": "#40, gpt4o pretok, slug `gpt4o-codegrad05-bpe`. AXES-2 code-fraction gradient, fusing regex (`REGEX_GPT4O`), code fraction 5%. Fused 749. FINISHED. MBPP 0.048 (v1); v2 0.050. Not in panel." |
| }, |
| "gpt4o-codegrad30-bpe": { |
| "slug": "gpt4o-codegrad30-bpe", |
| "runs": [ |
| "full-128k-gpt4o-codegrad30-bpe-mathcode-scratch" |
| ], |
| "n_runs": 1, |
| "redistributable": true, |
| "loadable_with": "transformers", |
| "hub_id": null, |
| "policy_note": "trained in this project", |
| "empirical": { |
| "model_type": "BPE", |
| "n_base_tokens": 128256, |
| "n_added_tokens": 4, |
| "vocab_total": 128260, |
| "added_tokens": [ |
| "<s>", |
| "</s>", |
| "<unk>", |
| "<pad>" |
| ], |
| "normalizer": null, |
| "pre_tokenizer": { |
| "type": "Sequence", |
| "pretokenizers": [ |
| { |
| "type": "Split", |
| "pattern": { |
| "Regex": "[^\\r\\n\\p{L}\\p{N}]?[\\p{Lu}\\p{Lt}\\p{Lm}\\p{Lo}\\p{M}]*[\\p{Ll}\\p{Lm}\\p{Lo}\\p{M}]+(?i:'s|'t|'re|'ve|'m|'ll|'d)?|[^\\r\\n\\p{L}\\p{N}]?[\\p{Lu}\\p{Lt}\\p{Lm}\\p{Lo}\\p{M}]+[\\p{Ll}\\p{Lm}\\p{Lo}\\p{M}]*(?i:'s|'t|'re|'ve|'m|'ll|'d)?|\\p{N}{1,3}| ?[^\\s\\p{L}\\p{N}]+[\\r\\n/]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+" |
| }, |
| "behavior": "Isolated", |
| "invert": false |
| }, |
| { |
| "type": "ByteLevel", |
| "add_prefix_space": false, |
| "trim_offsets": true, |
| "use_regex": false |
| } |
| ] |
| }, |
| "has_post_processor": false, |
| "byte_fallback": false, |
| "bos_token": "<s>", |
| "bos_token_id": 0 |
| }, |
| "declared_experiments": { |
| "algorithm": "bpe", |
| "pretokenizer_constant": "REGEX_GPT4O", |
| "normalizer": null, |
| "data_composition": "codegrad30", |
| "description": "GPT-4o regex, code-fraction gradient c_w=0.30, BPE", |
| "trainer_kwargs": null |
| }, |
| "declared_run_manifest": null, |
| "registry_note": "#41, gpt4o pretok, slug `gpt4o-codegrad30-bpe`. AXES-2 code-fraction gradient, fusing regex (`REGEX_GPT4O`), code fraction 30%. Fused 1393. FINISHED. MBPP 0.048 (v1); v2 0.178. Not in panel." |
| }, |
| "gpt4o-codegrad50-bpe": { |
| "slug": "gpt4o-codegrad50-bpe", |
| "runs": [ |
| "full-128k-gpt4o-codegrad50-bpe-mathcode-scratch" |
| ], |
| "n_runs": 1, |
| "redistributable": true, |
| "loadable_with": "transformers", |
| "hub_id": null, |
| "policy_note": "trained in this project", |
| "empirical": { |
| "model_type": "BPE", |
| "n_base_tokens": 128256, |
| "n_added_tokens": 4, |
| "vocab_total": 128260, |
| "added_tokens": [ |
| "<s>", |
| "</s>", |
| "<unk>", |
| "<pad>" |
| ], |
| "normalizer": null, |
| "pre_tokenizer": { |
| "type": "Sequence", |
| "pretokenizers": [ |
| { |
| "type": "Split", |
| "pattern": { |
| "Regex": "[^\\r\\n\\p{L}\\p{N}]?[\\p{Lu}\\p{Lt}\\p{Lm}\\p{Lo}\\p{M}]*[\\p{Ll}\\p{Lm}\\p{Lo}\\p{M}]+(?i:'s|'t|'re|'ve|'m|'ll|'d)?|[^\\r\\n\\p{L}\\p{N}]?[\\p{Lu}\\p{Lt}\\p{Lm}\\p{Lo}\\p{M}]+[\\p{Ll}\\p{Lm}\\p{Lo}\\p{M}]*(?i:'s|'t|'re|'ve|'m|'ll|'d)?|\\p{N}{1,3}| ?[^\\s\\p{L}\\p{N}]+[\\r\\n/]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+" |
| }, |
| "behavior": "Isolated", |
| "invert": false |
| }, |
| { |
| "type": "ByteLevel", |
| "add_prefix_space": false, |
| "trim_offsets": true, |
| "use_regex": false |
| } |
| ] |
| }, |
| "has_post_processor": false, |
| "byte_fallback": false, |
| "bos_token": "<s>", |
| "bos_token_id": 0 |
| }, |
| "declared_experiments": { |
| "algorithm": "bpe", |
| "pretokenizer_constant": "REGEX_GPT4O", |
| "normalizer": null, |
| "data_composition": "codegrad50", |
| "description": "GPT-4o regex, code-fraction gradient c_w=0.50, BPE", |
| "trainer_kwargs": null |
| }, |
| "declared_run_manifest": null, |
| "registry_note": "#42, gpt4o pretok, slug `gpt4o-codegrad50-bpe`. AXES-2 code-fraction gradient, fusing regex (`REGEX_GPT4O`), code fraction 50%. Fused 1831. FINISHED. MBPP 0.108 (v1); v2 0.088. Not in panel." |
| }, |
| "gpt4o-english-bpe": { |
| "slug": "gpt4o-english-bpe", |
| "runs": [ |
| "full-128k-gpt4o-english-bpe", |
| "full-128k-gpt4o-english-bpe-mathcode-scratch", |
| "full-128k-gpt4o-english-bpe-seed42", |
| "pilot-128k-gpt4o-english-bpe-wdfix", |
| "small-128k-gpt4o-english-bpe", |
| "tiny-128k-gpt4o-english-bpe" |
| ], |
| "n_runs": 6, |
| "redistributable": true, |
| "loadable_with": "transformers", |
| "hub_id": null, |
| "policy_note": "trained in this project", |
| "empirical": { |
| "model_type": "BPE", |
| "n_base_tokens": 128256, |
| "n_added_tokens": 4, |
| "vocab_total": 128260, |
| "added_tokens": [ |
| "<s>", |
| "</s>", |
| "<unk>", |
| "<pad>" |
| ], |
| "normalizer": null, |
| "pre_tokenizer": { |
| "type": "Sequence", |
| "pretokenizers": [ |
| { |
| "type": "Split", |
| "pattern": { |
| "Regex": "[^\\r\\n\\p{L}\\p{N}]?[\\p{Lu}\\p{Lt}\\p{Lm}\\p{Lo}\\p{M}]*[\\p{Ll}\\p{Lm}\\p{Lo}\\p{M}]+(?i:'s|'t|'re|'ve|'m|'ll|'d)?|[^\\r\\n\\p{L}\\p{N}]?[\\p{Lu}\\p{Lt}\\p{Lm}\\p{Lo}\\p{M}]+[\\p{Ll}\\p{Lm}\\p{Lo}\\p{M}]*(?i:'s|'t|'re|'ve|'m|'ll|'d)?|\\p{N}{1,3}| ?[^\\s\\p{L}\\p{N}]+[\\r\\n/]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+" |
| }, |
| "behavior": "Isolated", |
| "invert": false |
| }, |
| { |
| "type": "ByteLevel", |
| "add_prefix_space": false, |
| "trim_offsets": true, |
| "use_regex": false |
| } |
| ] |
| }, |
| "has_post_processor": false, |
| "byte_fallback": false, |
| "bos_token": "<s>", |
| "bos_token_id": 0 |
| }, |
| "declared_experiments": { |
| "algorithm": "bpe", |
| "pretokenizer_constant": "REGEX_GPT4O", |
| "normalizer": null, |
| "data_composition": "english", |
| "description": "GPT-4o regex, English-only data", |
| "trainer_kwargs": null |
| }, |
| "declared_run_manifest": null, |
| "registry_note": "P2 GPT-4o regex; english (D1) data; norm=none; BPE; vocab 128,260; affected by the initial_alphabet byte-drop defect (drops 32 control bytes incl. '{' and DEL); see gpt4o-english-fullbyte-bpe control; USED - panels.json main21 (camera-ready main panel)" |
| }, |
| "gpt4o-english-fullbyte-bpe": { |
| "slug": "gpt4o-english-fullbyte-bpe", |
| "runs": [ |
| "full-128k-gpt4o-english-fullbyte-bpe", |
| "full-128k-gpt4o-english-fullbyte-bpe-mathcode-scratch" |
| ], |
| "n_runs": 2, |
| "redistributable": true, |
| "loadable_with": "transformers", |
| "hub_id": null, |
| "policy_note": "trained in this project", |
| "empirical": { |
| "model_type": "BPE", |
| "n_base_tokens": 128256, |
| "n_added_tokens": 4, |
| "vocab_total": 128260, |
| "added_tokens": [ |
| "<s>", |
| "</s>", |
| "<unk>", |
| "<pad>" |
| ], |
| "normalizer": null, |
| "pre_tokenizer": { |
| "type": "Sequence", |
| "pretokenizers": [ |
| { |
| "type": "Split", |
| "pattern": { |
| "Regex": "[^\\r\\n\\p{L}\\p{N}]?[\\p{Lu}\\p{Lt}\\p{Lm}\\p{Lo}\\p{M}]*[\\p{Ll}\\p{Lm}\\p{Lo}\\p{M}]+(?i:'s|'t|'re|'ve|'m|'ll|'d)?|[^\\r\\n\\p{L}\\p{N}]?[\\p{Lu}\\p{Lt}\\p{Lm}\\p{Lo}\\p{M}]+[\\p{Ll}\\p{Lm}\\p{Lo}\\p{M}]*(?i:'s|'t|'re|'ve|'m|'ll|'d)?|\\p{N}{1,3}| ?[^\\s\\p{L}\\p{N}]+[\\r\\n/]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+" |
| }, |
| "behavior": "Isolated", |
| "invert": false |
| }, |
| { |
| "type": "ByteLevel", |
| "add_prefix_space": false, |
| "trim_offsets": true, |
| "use_regex": false |
| } |
| ] |
| }, |
| "has_post_processor": false, |
| "byte_fallback": false, |
| "bos_token": "<s>", |
| "bos_token_id": 0 |
| }, |
| "declared_experiments": { |
| "algorithm": "bpe", |
| "pretokenizer_constant": "REGEX_GPT4O", |
| "normalizer": null, |
| "data_composition": "english", |
| "description": "GPT-4o regex, English-only data, full byte alphabet seeded (control for the missing-initial_alphabet byte-drop defect; matched pair vs gpt4o-english-bpe)", |
| "trainer_kwargs": { |
| "full_byte_alphabet": true |
| } |
| }, |
| "declared_run_manifest": null, |
| "registry_note": "P2 GPT-4o regex; english (D1) data; norm=none; BPE (trainer_kwargs full_byte_alphabet=True: initial_alphabet forced to the complete 256-entry ByteLevel alphabet); vocab 128,256; isolates the initial_alphabet byte-drop defect vs gpt4o-english-bpe (209/256 -> 256/256 bytes present; 47 fewer merges); USED - panels.json pretok-controls (camera-ready)" |
| }, |
| "gpt4onl-balanced-bpe": { |
| "slug": "gpt4onl-balanced-bpe", |
| "runs": [ |
| "full-128k-gpt4onl-balanced-bpe-mathcode-scratch", |
| "full-128k-gpt4onl-balanced-bpe-seed43-mathcode-scratch", |
| "full-128k-gpt4onl-balanced-bpe-seed44-mathcode-scratch" |
| ], |
| "n_runs": 3, |
| "redistributable": true, |
| "loadable_with": "transformers", |
| "hub_id": null, |
| "policy_note": "trained in this project", |
| "empirical": { |
| "model_type": "BPE", |
| "n_base_tokens": 128256, |
| "n_added_tokens": 4, |
| "vocab_total": 128260, |
| "added_tokens": [ |
| "<s>", |
| "</s>", |
| "<unk>", |
| "<pad>" |
| ], |
| "normalizer": null, |
| "pre_tokenizer": { |
| "type": "Sequence", |
| "pretokenizers": [ |
| { |
| "type": "Split", |
| "pattern": { |
| "Regex": "[^\\r\\n\\p{L}\\p{N}]?[\\p{Lu}\\p{Lt}\\p{Lm}\\p{Lo}\\p{M}]*[\\p{Ll}\\p{Lm}\\p{Lo}\\p{M}]+(?i:'s|'t|'re|'ve|'m|'ll|'d)?|[^\\r\\n\\p{L}\\p{N}]?[\\p{Lu}\\p{Lt}\\p{Lm}\\p{Lo}\\p{M}]+[\\p{Ll}\\p{Lm}\\p{Lo}\\p{M}]*(?i:'s|'t|'re|'ve|'m|'ll|'d)?|\\p{N}{1,3}| ?[^\\s\\p{L}\\p{N}]+|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+" |
| }, |
| "behavior": "Isolated", |
| "invert": false |
| }, |
| { |
| "type": "ByteLevel", |
| "add_prefix_space": false, |
| "trim_offsets": true, |
| "use_regex": false |
| } |
| ] |
| }, |
| "has_post_processor": false, |
| "byte_fallback": false, |
| "bos_token": "<s>", |
| "bos_token_id": 0 |
| }, |
| "declared_experiments": { |
| "algorithm": "bpe", |
| "pretokenizer_constant": "REGEX_GPT4O_NONL", |
| "normalizer": null, |
| "data_composition": "balanced", |
| "description": "GPT-4o regex with the punct-arm newline-fusion tail removed (AXES-2 single-mechanism ablation vs gpt4o-balanced-bpe), balanced data, BPE", |
| "trainer_kwargs": null |
| }, |
| "declared_run_manifest": null, |
| "registry_note": "#47, gpt4onl pretok, slug `gpt4onl-balanced-bpe`. AXES-2 single-mechanism pair: non-fusing regex on the balanced composition, fused 0. FINISHED. v1: MBPP 0.158, HumanEval 0.0183, GSM8K 0.242. Fusing counterpart is the unslugged baseline `full-128k-mathcode-scratch` (gpt4o-balanced-bpe, same recipe/step). Pair v1: MBPP 0.024 vs 0.158, HumanEval 0.0854 vs 0.0183 - benchmarks disagree on sign; both have clean prompt boundaries (\u00a73a.6), not the HumanEval artifact. v2 (2026-08-04): this run MBPP 0.224/HumanEval 0.1890; baseline MBPP 0.174/HumanEval 0.1646. Not in panel." |
| }, |
| "gpt4onl-codegrad00-bpe": { |
| "slug": "gpt4onl-codegrad00-bpe", |
| "runs": [ |
| "full-128k-gpt4onl-codegrad00-bpe-mathcode-scratch" |
| ], |
| "n_runs": 1, |
| "redistributable": true, |
| "loadable_with": "transformers", |
| "hub_id": null, |
| "policy_note": "trained in this project", |
| "empirical": { |
| "model_type": "BPE", |
| "n_base_tokens": 128256, |
| "n_added_tokens": 4, |
| "vocab_total": 128260, |
| "added_tokens": [ |
| "<s>", |
| "</s>", |
| "<unk>", |
| "<pad>" |
| ], |
| "normalizer": null, |
| "pre_tokenizer": { |
| "type": "Sequence", |
| "pretokenizers": [ |
| { |
| "type": "Split", |
| "pattern": { |
| "Regex": "[^\\r\\n\\p{L}\\p{N}]?[\\p{Lu}\\p{Lt}\\p{Lm}\\p{Lo}\\p{M}]*[\\p{Ll}\\p{Lm}\\p{Lo}\\p{M}]+(?i:'s|'t|'re|'ve|'m|'ll|'d)?|[^\\r\\n\\p{L}\\p{N}]?[\\p{Lu}\\p{Lt}\\p{Lm}\\p{Lo}\\p{M}]+[\\p{Ll}\\p{Lm}\\p{Lo}\\p{M}]*(?i:'s|'t|'re|'ve|'m|'ll|'d)?|\\p{N}{1,3}| ?[^\\s\\p{L}\\p{N}]+|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+" |
| }, |
| "behavior": "Isolated", |
| "invert": false |
| }, |
| { |
| "type": "ByteLevel", |
| "add_prefix_space": false, |
| "trim_offsets": true, |
| "use_regex": false |
| } |
| ] |
| }, |
| "has_post_processor": false, |
| "byte_fallback": false, |
| "bos_token": "<s>", |
| "bos_token_id": 0 |
| }, |
| "declared_experiments": { |
| "algorithm": "bpe", |
| "pretokenizer_constant": "REGEX_GPT4O_NONL", |
| "normalizer": null, |
| "data_composition": "codegrad00", |
| "description": "GPT-4o-NoNL regex, code-fraction gradient c_w=0.00 (code omitted), BPE", |
| "trainer_kwargs": null |
| }, |
| "declared_run_manifest": null, |
| "registry_note": "#43, gpt4onl pretok, slug `gpt4onl-codegrad00-bpe`. AXES-2 code-fraction gradient, non-fusing regex (REGEX_GPT4O_NONL: punct arm loses trailing [\\r\\n/]*, else byte-identical to REGEX_GPT4O), tokenizer-corpus code fraction 0%. Fused 0 by construction. FINISHED. MBPP 0.232 (generation_spec v1), the highest of any run in the 20B mathcode family (v1-scored, like-for-like with the panel). v2 re-derivation (2026-08-04) 0.248, see \u00a73a.3. Not in panel." |
| }, |
| "gpt4onl-codegrad05-bpe": { |
| "slug": "gpt4onl-codegrad05-bpe", |
| "runs": [ |
| "full-128k-gpt4onl-codegrad05-bpe-mathcode-scratch" |
| ], |
| "n_runs": 1, |
| "redistributable": true, |
| "loadable_with": "transformers", |
| "hub_id": null, |
| "policy_note": "trained in this project", |
| "empirical": { |
| "model_type": "BPE", |
| "n_base_tokens": 128256, |
| "n_added_tokens": 4, |
| "vocab_total": 128260, |
| "added_tokens": [ |
| "<s>", |
| "</s>", |
| "<unk>", |
| "<pad>" |
| ], |
| "normalizer": null, |
| "pre_tokenizer": { |
| "type": "Sequence", |
| "pretokenizers": [ |
| { |
| "type": "Split", |
| "pattern": { |
| "Regex": "[^\\r\\n\\p{L}\\p{N}]?[\\p{Lu}\\p{Lt}\\p{Lm}\\p{Lo}\\p{M}]*[\\p{Ll}\\p{Lm}\\p{Lo}\\p{M}]+(?i:'s|'t|'re|'ve|'m|'ll|'d)?|[^\\r\\n\\p{L}\\p{N}]?[\\p{Lu}\\p{Lt}\\p{Lm}\\p{Lo}\\p{M}]+[\\p{Ll}\\p{Lm}\\p{Lo}\\p{M}]*(?i:'s|'t|'re|'ve|'m|'ll|'d)?|\\p{N}{1,3}| ?[^\\s\\p{L}\\p{N}]+|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+" |
| }, |
| "behavior": "Isolated", |
| "invert": false |
| }, |
| { |
| "type": "ByteLevel", |
| "add_prefix_space": false, |
| "trim_offsets": true, |
| "use_regex": false |
| } |
| ] |
| }, |
| "has_post_processor": false, |
| "byte_fallback": false, |
| "bos_token": "<s>", |
| "bos_token_id": 0 |
| }, |
| "declared_experiments": { |
| "algorithm": "bpe", |
| "pretokenizer_constant": "REGEX_GPT4O_NONL", |
| "normalizer": null, |
| "data_composition": "codegrad05", |
| "description": "GPT-4o-NoNL regex, code-fraction gradient c_w=0.05, BPE", |
| "trainer_kwargs": null |
| }, |
| "declared_run_manifest": null, |
| "registry_note": "#44, gpt4onl pretok, slug `gpt4onl-codegrad05-bpe`. AXES-2 code-fraction gradient, non-fusing regex (`REGEX_GPT4O_NONL`), code fraction 5%. Fused 0. FINISHED. MBPP 0.204 (v1); v2 0.206. Not in panel." |
| }, |
| "gpt4onl-codegrad30-bpe": { |
| "slug": "gpt4onl-codegrad30-bpe", |
| "runs": [ |
| "full-128k-gpt4onl-codegrad30-bpe-mathcode-scratch" |
| ], |
| "n_runs": 1, |
| "redistributable": true, |
| "loadable_with": "transformers", |
| "hub_id": null, |
| "policy_note": "trained in this project", |
| "empirical": { |
| "model_type": "BPE", |
| "n_base_tokens": 128256, |
| "n_added_tokens": 4, |
| "vocab_total": 128260, |
| "added_tokens": [ |
| "<s>", |
| "</s>", |
| "<unk>", |
| "<pad>" |
| ], |
| "normalizer": null, |
| "pre_tokenizer": { |
| "type": "Sequence", |
| "pretokenizers": [ |
| { |
| "type": "Split", |
| "pattern": { |
| "Regex": "[^\\r\\n\\p{L}\\p{N}]?[\\p{Lu}\\p{Lt}\\p{Lm}\\p{Lo}\\p{M}]*[\\p{Ll}\\p{Lm}\\p{Lo}\\p{M}]+(?i:'s|'t|'re|'ve|'m|'ll|'d)?|[^\\r\\n\\p{L}\\p{N}]?[\\p{Lu}\\p{Lt}\\p{Lm}\\p{Lo}\\p{M}]+[\\p{Ll}\\p{Lm}\\p{Lo}\\p{M}]*(?i:'s|'t|'re|'ve|'m|'ll|'d)?|\\p{N}{1,3}| ?[^\\s\\p{L}\\p{N}]+|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+" |
| }, |
| "behavior": "Isolated", |
| "invert": false |
| }, |
| { |
| "type": "ByteLevel", |
| "add_prefix_space": false, |
| "trim_offsets": true, |
| "use_regex": false |
| } |
| ] |
| }, |
| "has_post_processor": false, |
| "byte_fallback": false, |
| "bos_token": "<s>", |
| "bos_token_id": 0 |
| }, |
| "declared_experiments": { |
| "algorithm": "bpe", |
| "pretokenizer_constant": "REGEX_GPT4O_NONL", |
| "normalizer": null, |
| "data_composition": "codegrad30", |
| "description": "GPT-4o-NoNL regex, code-fraction gradient c_w=0.30, BPE", |
| "trainer_kwargs": null |
| }, |
| "declared_run_manifest": null, |
| "registry_note": "#45, gpt4onl pretok, slug `gpt4onl-codegrad30-bpe`. AXES-2 code-fraction gradient, non-fusing regex (`REGEX_GPT4O_NONL`), code fraction 30%. Fused 0. FINISHED. MBPP 0.178 (v1); v2 0.224. Not in panel." |
| }, |
| "gpt4onl-codegrad50-bpe": { |
| "slug": "gpt4onl-codegrad50-bpe", |
| "runs": [ |
| "full-128k-gpt4onl-codegrad50-bpe-mathcode-scratch" |
| ], |
| "n_runs": 1, |
| "redistributable": true, |
| "loadable_with": "transformers", |
| "hub_id": null, |
| "policy_note": "trained in this project", |
| "empirical": { |
| "model_type": "BPE", |
| "n_base_tokens": 128256, |
| "n_added_tokens": 4, |
| "vocab_total": 128260, |
| "added_tokens": [ |
| "<s>", |
| "</s>", |
| "<unk>", |
| "<pad>" |
| ], |
| "normalizer": null, |
| "pre_tokenizer": { |
| "type": "Sequence", |
| "pretokenizers": [ |
| { |
| "type": "Split", |
| "pattern": { |
| "Regex": "[^\\r\\n\\p{L}\\p{N}]?[\\p{Lu}\\p{Lt}\\p{Lm}\\p{Lo}\\p{M}]*[\\p{Ll}\\p{Lm}\\p{Lo}\\p{M}]+(?i:'s|'t|'re|'ve|'m|'ll|'d)?|[^\\r\\n\\p{L}\\p{N}]?[\\p{Lu}\\p{Lt}\\p{Lm}\\p{Lo}\\p{M}]+[\\p{Ll}\\p{Lm}\\p{Lo}\\p{M}]*(?i:'s|'t|'re|'ve|'m|'ll|'d)?|\\p{N}{1,3}| ?[^\\s\\p{L}\\p{N}]+|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+" |
| }, |
| "behavior": "Isolated", |
| "invert": false |
| }, |
| { |
| "type": "ByteLevel", |
| "add_prefix_space": false, |
| "trim_offsets": true, |
| "use_regex": false |
| } |
| ] |
| }, |
| "has_post_processor": false, |
| "byte_fallback": false, |
| "bos_token": "<s>", |
| "bos_token_id": 0 |
| }, |
| "declared_experiments": { |
| "algorithm": "bpe", |
| "pretokenizer_constant": "REGEX_GPT4O_NONL", |
| "normalizer": null, |
| "data_composition": "codegrad50", |
| "description": "GPT-4o-NoNL regex, code-fraction gradient c_w=0.50, BPE", |
| "trainer_kwargs": null |
| }, |
| "declared_run_manifest": null, |
| "registry_note": "#46, gpt4onl pretok, slug `gpt4onl-codegrad50-bpe`. AXES-2 code-fraction gradient, non-fusing regex (`REGEX_GPT4O_NONL`), code fraction 50%. Fused 0. FINISHED. MBPP 0.174 (v1); v2 0.212. Not in panel." |
| }, |
| "highmid-bpe": { |
| "slug": "highmid-bpe", |
| "runs": [ |
| "full-128k-highmid-bpe" |
| ], |
| "n_runs": 1, |
| "redistributable": true, |
| "loadable_with": "transformers", |
| "hub_id": null, |
| "policy_note": "trained in this project", |
| "empirical": { |
| "model_type": "BPE", |
| "n_base_tokens": 128256, |
| "n_added_tokens": 4, |
| "vocab_total": 128260, |
| "added_tokens": [ |
| "<s>", |
| "</s>", |
| "<unk>", |
| "<pad>" |
| ], |
| "normalizer": null, |
| "pre_tokenizer": { |
| "type": "Sequence", |
| "pretokenizers": [ |
| { |
| "type": "Split", |
| "pattern": { |
| "Regex": "[^\\r\\n\\p{L}\\p{N}]?[\\p{Lu}\\p{Lt}\\p{Lm}\\p{Lo}\\p{M}]*[\\p{Ll}\\p{Lm}\\p{Lo}\\p{M}]+(?i:'s|'t|'re|'ve|'m|'ll|'d)?|[^\\r\\n\\p{L}\\p{N}]?[\\p{Lu}\\p{Lt}\\p{Lm}\\p{Lo}\\p{M}]+[\\p{Ll}\\p{Lm}\\p{Lo}\\p{M}]*(?i:'s|'t|'re|'ve|'m|'ll|'d)?|\\p{N}{1,3}| ?[^\\s\\p{L}\\p{N}]+[\\r\\n/]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+" |
| }, |
| "behavior": "Isolated", |
| "invert": false |
| }, |
| { |
| "type": "ByteLevel", |
| "add_prefix_space": false, |
| "trim_offsets": true, |
| "use_regex": false |
| } |
| ] |
| }, |
| "has_post_processor": false, |
| "byte_fallback": false, |
| "bos_token": "<s>", |
| "bos_token_id": 0 |
| }, |
| "declared_experiments": { |
| "algorithm": "bpe", |
| "pretokenizer_constant": "REGEX_GPT4O", |
| "normalizer": null, |
| "data_composition": "highmid", |
| "description": "GPT-4o regex, high+mid-resource subset (21 langs), BPE", |
| "trainer_kwargs": null |
| }, |
| "declared_run_manifest": null, |
| "registry_note": "P2 GPT-4o regex; highmid (21 high+mid-resource langs + English + math/code) data; norm=none; BPE; vocab 128,260; intermediate-language-coverage probe (tokenizer sees 21 langs); USED - S6d.3 coverage ladder; superseded within panels.json main21 by highmid-unigram" |
| }, |
| "highmid-unigram": { |
| "slug": "highmid-unigram", |
| "runs": [ |
| "full-128k-highmid-unigram" |
| ], |
| "n_runs": 1, |
| "redistributable": true, |
| "loadable_with": "transformers", |
| "hub_id": null, |
| "policy_note": "trained in this project", |
| "empirical": { |
| "model_type": "Unigram", |
| "n_base_tokens": 128256, |
| "n_added_tokens": 4, |
| "vocab_total": 128260, |
| "added_tokens": [ |
| "<s>", |
| "</s>", |
| "<unk>", |
| "<pad>" |
| ], |
| "normalizer": null, |
| "pre_tokenizer": { |
| "type": "Sequence", |
| "pretokenizers": [ |
| { |
| "type": "Split", |
| "pattern": { |
| "Regex": "[^\\r\\n\\p{L}\\p{N}]?[\\p{Lu}\\p{Lt}\\p{Lm}\\p{Lo}\\p{M}]*[\\p{Ll}\\p{Lm}\\p{Lo}\\p{M}]+(?i:'s|'t|'re|'ve|'m|'ll|'d)?|[^\\r\\n\\p{L}\\p{N}]?[\\p{Lu}\\p{Lt}\\p{Lm}\\p{Lo}\\p{M}]+[\\p{Ll}\\p{Lm}\\p{Lo}\\p{M}]*(?i:'s|'t|'re|'ve|'m|'ll|'d)?|\\p{N}{1,3}| ?[^\\s\\p{L}\\p{N}]+[\\r\\n/]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+" |
| }, |
| "behavior": "Isolated", |
| "invert": false |
| }, |
| { |
| "type": "ByteLevel", |
| "add_prefix_space": false, |
| "trim_offsets": true, |
| "use_regex": false |
| } |
| ] |
| }, |
| "has_post_processor": false, |
| "byte_fallback": false, |
| "bos_token": "<s>", |
| "bos_token_id": 0 |
| }, |
| "declared_experiments": { |
| "algorithm": "unigram", |
| "pretokenizer_constant": "REGEX_GPT4O", |
| "normalizer": null, |
| "data_composition": "highmid", |
| "description": "GPT-4o regex, high+mid-resource subset (21 langs), UnigramLM", |
| "trainer_kwargs": null |
| }, |
| "declared_run_manifest": null, |
| "registry_note": "P2 GPT-4o regex; highmid (21 langs) data; norm=none; UnigramLM (HF, default kwargs); vocab 128,260; Unigram counterpart to highmid-bpe; USED - panels.json main21 (camera-ready main panel)" |
| }, |
| "highres-bpe": { |
| "slug": "highres-bpe", |
| "runs": [ |
| "full-128k-highres-bpe" |
| ], |
| "n_runs": 1, |
| "redistributable": true, |
| "loadable_with": "transformers", |
| "hub_id": null, |
| "policy_note": "trained in this project", |
| "empirical": { |
| "model_type": "BPE", |
| "n_base_tokens": 128256, |
| "n_added_tokens": 4, |
| "vocab_total": 128260, |
| "added_tokens": [ |
| "<s>", |
| "</s>", |
| "<unk>", |
| "<pad>" |
| ], |
| "normalizer": null, |
| "pre_tokenizer": { |
| "type": "Sequence", |
| "pretokenizers": [ |
| { |
| "type": "Split", |
| "pattern": { |
| "Regex": "[^\\r\\n\\p{L}\\p{N}]?[\\p{Lu}\\p{Lt}\\p{Lm}\\p{Lo}\\p{M}]*[\\p{Ll}\\p{Lm}\\p{Lo}\\p{M}]+(?i:'s|'t|'re|'ve|'m|'ll|'d)?|[^\\r\\n\\p{L}\\p{N}]?[\\p{Lu}\\p{Lt}\\p{Lm}\\p{Lo}\\p{M}]+[\\p{Ll}\\p{Lm}\\p{Lo}\\p{M}]*(?i:'s|'t|'re|'ve|'m|'ll|'d)?|\\p{N}{1,3}| ?[^\\s\\p{L}\\p{N}]+[\\r\\n/]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+" |
| }, |
| "behavior": "Isolated", |
| "invert": false |
| }, |
| { |
| "type": "ByteLevel", |
| "add_prefix_space": false, |
| "trim_offsets": true, |
| "use_regex": false |
| } |
| ] |
| }, |
| "has_post_processor": false, |
| "byte_fallback": false, |
| "bos_token": "<s>", |
| "bos_token_id": 0 |
| }, |
| "declared_experiments": { |
| "algorithm": "bpe", |
| "pretokenizer_constant": "REGEX_GPT4O", |
| "normalizer": null, |
| "data_composition": "highres", |
| "description": "GPT-4o regex, high-resource-only subset (6 langs), BPE", |
| "trainer_kwargs": null |
| }, |
| "declared_run_manifest": null, |
| "registry_note": "P2 GPT-4o regex; highres (6 high-resource langs + English + math/code) data; norm=none; BPE; vocab 128,260; intermediate-language-coverage probe (tokenizer sees 6 langs, LM trains on all 30); USED - S6d.3 coverage ladder (english-only/highres/highmid/balanced GPT-4o-BPE comparison); superseded within panels.json main21 by highres-unigram" |
| }, |
| "highres-unigram": { |
| "slug": "highres-unigram", |
| "runs": [ |
| "full-128k-highres-unigram" |
| ], |
| "n_runs": 1, |
| "redistributable": true, |
| "loadable_with": "transformers", |
| "hub_id": null, |
| "policy_note": "trained in this project", |
| "empirical": { |
| "model_type": "Unigram", |
| "n_base_tokens": 128256, |
| "n_added_tokens": 4, |
| "vocab_total": 128260, |
| "added_tokens": [ |
| "<s>", |
| "</s>", |
| "<unk>", |
| "<pad>" |
| ], |
| "normalizer": null, |
| "pre_tokenizer": { |
| "type": "Sequence", |
| "pretokenizers": [ |
| { |
| "type": "Split", |
| "pattern": { |
| "Regex": "[^\\r\\n\\p{L}\\p{N}]?[\\p{Lu}\\p{Lt}\\p{Lm}\\p{Lo}\\p{M}]*[\\p{Ll}\\p{Lm}\\p{Lo}\\p{M}]+(?i:'s|'t|'re|'ve|'m|'ll|'d)?|[^\\r\\n\\p{L}\\p{N}]?[\\p{Lu}\\p{Lt}\\p{Lm}\\p{Lo}\\p{M}]+[\\p{Ll}\\p{Lm}\\p{Lo}\\p{M}]*(?i:'s|'t|'re|'ve|'m|'ll|'d)?|\\p{N}{1,3}| ?[^\\s\\p{L}\\p{N}]+[\\r\\n/]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+" |
| }, |
| "behavior": "Isolated", |
| "invert": false |
| }, |
| { |
| "type": "ByteLevel", |
| "add_prefix_space": false, |
| "trim_offsets": true, |
| "use_regex": false |
| } |
| ] |
| }, |
| "has_post_processor": false, |
| "byte_fallback": false, |
| "bos_token": "<s>", |
| "bos_token_id": 0 |
| }, |
| "declared_experiments": { |
| "algorithm": "unigram", |
| "pretokenizer_constant": "REGEX_GPT4O", |
| "normalizer": null, |
| "data_composition": "highres", |
| "description": "GPT-4o regex, high-resource-only subset (6 langs), UnigramLM", |
| "trainer_kwargs": null |
| }, |
| "declared_run_manifest": null, |
| "registry_note": "P2 GPT-4o regex; highres (6 langs) data; norm=none; UnigramLM (HF, default kwargs); vocab 128,260; Unigram counterpart to highres-bpe, completes the {BPE,Unigram}x{highres,highmid,balanced} grid; USED - panels.json main21 (camera-ready main panel)" |
| }, |
| "llama3": { |
| "slug": "llama3", |
| "runs": [ |
| "full-128k-llama3-mathcode-scratch", |
| "pilot-128k-llama3-wdfix" |
| ], |
| "n_runs": 2, |
| "redistributable": false, |
| "loadable_with": "transformers", |
| "hub_id": "NousResearch/Meta-Llama-3-8B", |
| "policy_note": "These runs record the local mirror directory rather than the hub id; the file is the same off-the-shelf tokenizer.", |
| "empirical": null, |
| "declared_experiments": null, |
| "declared_run_manifest": null, |
| "registry_note": "#32, llama3 pretok, slug `llama3`. Off-the-shelf Meta-Llama-3 tokenizer, vocab 128,256. FINISHED (step 19073). MBPP 0.166 [0.134, 0.198]." |
| }, |
| "pabpe-nfc-apertus-fineweb2full-capped-tuned-hybridwindow": { |
| "slug": "pabpe-nfc-apertus-fineweb2full-capped-tuned-hybridwindow", |
| "runs": [ |
| "full-128k-pabpe-nfc-apertus-fineweb2full-capped-tuned-hybridwindow", |
| "full-128k-pabpe-nfc-apertus-fineweb2full-capped-tuned-hybridwindow-mathcode-scratch" |
| ], |
| "n_runs": 2, |
| "redistributable": true, |
| "loadable_with": "transformers", |
| "hub_id": null, |
| "policy_note": "trained in this project", |
| "empirical": { |
| "model_type": "BPE", |
| "n_base_tokens": 127831, |
| "n_added_tokens": 4, |
| "vocab_total": 127835, |
| "added_tokens": [ |
| "<s>", |
| "</s>", |
| "<unk>", |
| "<pad>" |
| ], |
| "normalizer": { |
| "type": "NFC" |
| }, |
| "pre_tokenizer": { |
| "type": "Sequence", |
| "pretokenizers": [ |
| { |
| "type": "Split", |
| "pattern": { |
| "Regex": "[^\\r\\n\\p{L}\\p{N}]?[\\p{Lu}\\p{Lt}\\p{Lm}\\p{Lo}\\p{M}]*[\\p{Ll}\\p{Lm}\\p{Lo}\\p{M}]+|[^\\r\\n\\p{L}\\p{N}]?[\\p{Lu}\\p{Lt}\\p{Lm}\\p{Lo}\\p{M}]+[\\p{Ll}\\p{Lm}\\p{Lo}\\p{M}]*|\\p{N}| ?[^\\s\\p{L}\\p{N}]{1,16}[\\r\\n]{0,16}|\\s{0,16}[\\r\\n]{1,16}|\\s{1,16}(?!\\S)|\\s{1,16}" |
| }, |
| "behavior": "Isolated", |
| "invert": false |
| }, |
| { |
| "type": "ByteLevel", |
| "add_prefix_space": false, |
| "trim_offsets": true, |
| "use_regex": false |
| } |
| ] |
| }, |
| "has_post_processor": false, |
| "byte_fallback": false, |
| "bos_token": "<s>", |
| "bos_token_id": 127831 |
| }, |
| "declared_experiments": null, |
| "declared_run_manifest": { |
| "variant": "pa_bpe_nfc_apertus_capped_hybrid_window_tuned", |
| "variant_config": { |
| "trainer": "parity-bpe", |
| "pretok": "apertus_capped_nfc", |
| "variant": "window", |
| "global_merges": 64000, |
| "window_size": 100, |
| "alpha": 2.0, |
| "signal": "ratios", |
| "grouped_config": "configs/parity_aware_config_grouped_fineweb2full_quota_tuned.json", |
| "special_tokens_override": [], |
| "decoder_add_prefix_space": true, |
| "output_group": "tokenizer-lm-toks", |
| "output_dir_name": "nfc_apertus_capped_hybrid_window_tuned", |
| "post_training_special_tokens": [ |
| "<s>", |
| "</s>", |
| "<unk>", |
| "<pad>" |
| ] |
| }, |
| "grouped_config_path": "/users/cmeister747/pa_tokenizers_branch/configs/parity_aware_config_grouped_fineweb2full_quota_tuned.json", |
| "vocab_size": 128000, |
| "min_frequency": 2, |
| "special_tokens": [], |
| "start_time": "2026-05-23T10:29:11.168479", |
| "elapsed_seconds": 7230.190345, |
| "slurm_job_id": "2353211", |
| "cli_args": [ |
| "/users/cmeister747/pa_tokenizers_branch/train_tokenizer.py", |
| "--variant", |
| "pa_bpe_nfc_apertus_capped_hybrid_window_tuned" |
| ], |
| "git": { |
| "commit": "5d8381af9ec1417b078f7dce82bac68873da915e", |
| "branch": "parity-aware-bpe", |
| "dirty": false |
| }, |
| "per_group_summary": { |
| "n_groups": 25, |
| "full_block_in": "tokenizers/pabpe-nfc-apertus-fineweb2full-capped-tuned-hybridwindow/run_manifest.json" |
| } |
| }, |
| "registry_note": "PA-BPE hybrid global+window, apertus/Mistral-Nemo pretok regex; fineweb2full, capped + tuned data; norm=NFC; PA-BPE (hybrid global+window); vocab 127,835; 2x2 candidate; the apertus-pretok arm later diagnosed with the MBPP-collapse RCA (fuses code line-endings into superwords) at 20B scale; USED - Apertus-2 development track candidate; excluded from main paper panel" |
| }, |
| "pabpe-nfc-clean-balanced": { |
| "slug": "pabpe-nfc-clean-balanced", |
| "runs": [ |
| "full-128k-pabpe-nfc-clean-balanced" |
| ], |
| "n_runs": 1, |
| "redistributable": true, |
| "loadable_with": "transformers", |
| "hub_id": null, |
| "policy_note": "trained in this project", |
| "empirical": { |
| "model_type": "BPE", |
| "n_base_tokens": 127832, |
| "n_added_tokens": 4, |
| "vocab_total": 127836, |
| "added_tokens": [ |
| "<s>", |
| "</s>", |
| "<unk>", |
| "<pad>" |
| ], |
| "normalizer": { |
| "type": "NFC" |
| }, |
| "pre_tokenizer": { |
| "type": "Sequence", |
| "pretokenizers": [ |
| { |
| "type": "Split", |
| "pattern": { |
| "Regex": "[ ]?[\\p{Lu}\\p{Lt}\\p{Lm}\\p{Lo}\\p{M}]*[\\p{Ll}\\p{Lm}\\p{Lo}\\p{M}]+|[ ]?[\\p{Lu}\\p{Lt}\\p{Lm}\\p{Lo}\\p{M}]+[\\p{Ll}\\p{Lm}\\p{Lo}\\p{M}]*|\\p{N}| ?[^\\s\\p{L}\\p{N}]+|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+" |
| }, |
| "behavior": "Isolated", |
| "invert": false |
| }, |
| { |
| "type": "ByteLevel", |
| "add_prefix_space": false, |
| "trim_offsets": true, |
| "use_regex": false |
| } |
| ] |
| }, |
| "has_post_processor": false, |
| "byte_fallback": false, |
| "bos_token": "<s>", |
| "bos_token_id": 127832 |
| }, |
| "declared_experiments": null, |
| "declared_run_manifest": { |
| "variant": "pa_bpe_nfc_clean_multi_balanced", |
| "variant_config": { |
| "trainer": "parity-bpe", |
| "pretok": "clean_multi_nfc", |
| "variant": "base", |
| "global_merges": 0, |
| "signal": "ratios", |
| "grouped_config": "configs/parity_aware_config_balanced.json", |
| "special_tokens_override": [], |
| "decoder_add_prefix_space": true, |
| "output_group": "tokenizer-lm-toks", |
| "output_dir_name": "nfc_clean_multi_balanced", |
| "post_training_special_tokens": [ |
| "<s>", |
| "</s>", |
| "<unk>", |
| "<pad>" |
| ] |
| }, |
| "grouped_config_path": "/users/cmeister747/pa_tokenizers_branch/configs/parity_aware_config_balanced.json", |
| "vocab_size": 128000, |
| "min_frequency": 2, |
| "special_tokens": [], |
| "start_time": "2026-05-16T00:20:05.420579", |
| "elapsed_seconds": 2555.150446, |
| "slurm_job_id": "2257133", |
| "cli_args": [ |
| "/users/cmeister747/pa_tokenizers_branch/train_tokenizer.py", |
| "--variant", |
| "pa_bpe_nfc_clean_multi_balanced" |
| ], |
| "git": { |
| "commit": "b2ceac78388c06deb786755b60d6c1d335cb6c25", |
| "branch": "parity-aware-bpe", |
| "dirty": false |
| }, |
| "per_group_summary": { |
| "n_groups": 34, |
| "full_block_in": "tokenizers/pabpe-nfc-clean-balanced/run_manifest.json" |
| } |
| }, |
| "registry_note": "PA-BPE (parity, base) with clean-multi regex; balanced (D2) data; norm=NFC; PA-BPE (base); vocab 127,836; same data as pabpe-nfc-gpt4-balanced (2a), different pretok regex (controlled pretok ablation); bug-fixed 2026-05-16 replacement (2b) for retired pabpe-balanced; USED - balanced-1B bootstrap panel (S6.12) + paper panel; not in panels.json main21" |
| }, |
| "pabpe-nfc-clean-balanced-hybridwindow": { |
| "slug": "pabpe-nfc-clean-balanced-hybridwindow", |
| "runs": [ |
| "full-128k-pabpe-nfc-clean-balanced-hybridwindow-mathcode-scratch" |
| ], |
| "n_runs": 1, |
| "redistributable": true, |
| "loadable_with": "transformers", |
| "hub_id": null, |
| "policy_note": "trained in this project", |
| "empirical": { |
| "model_type": "BPE", |
| "n_base_tokens": 127832, |
| "n_added_tokens": 4, |
| "vocab_total": 127836, |
| "added_tokens": [ |
| "<s>", |
| "</s>", |
| "<unk>", |
| "<pad>" |
| ], |
| "normalizer": { |
| "type": "NFC" |
| }, |
| "pre_tokenizer": { |
| "type": "Sequence", |
| "pretokenizers": [ |
| { |
| "type": "Split", |
| "pattern": { |
| "Regex": "[ ]?[\\p{Lu}\\p{Lt}\\p{Lm}\\p{Lo}\\p{M}]*[\\p{Ll}\\p{Lm}\\p{Lo}\\p{M}]+|[ ]?[\\p{Lu}\\p{Lt}\\p{Lm}\\p{Lo}\\p{M}]+[\\p{Ll}\\p{Lm}\\p{Lo}\\p{M}]*|\\p{N}| ?[^\\s\\p{L}\\p{N}]+|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+" |
| }, |
| "behavior": "Isolated", |
| "invert": false |
| }, |
| { |
| "type": "ByteLevel", |
| "add_prefix_space": false, |
| "trim_offsets": true, |
| "use_regex": false |
| } |
| ] |
| }, |
| "has_post_processor": false, |
| "byte_fallback": false, |
| "bos_token": "<s>", |
| "bos_token_id": 127832 |
| }, |
| "declared_experiments": null, |
| "declared_run_manifest": { |
| "variant": "pa_bpe_nfc_clean_multi_balanced_hybrid_window", |
| "variant_config": { |
| "trainer": "parity-bpe", |
| "pretok": "clean_multi_nfc", |
| "variant": "window", |
| "global_merges": 64000, |
| "window_size": 100, |
| "alpha": 2.0, |
| "signal": "ratios", |
| "grouped_config": "configs/parity_aware_config_balanced.json", |
| "special_tokens_override": [], |
| "decoder_add_prefix_space": true, |
| "output_group": "tokenizer-lm-toks", |
| "output_dir_name": "nfc_clean_multi_balanced_hybrid_window", |
| "post_training_special_tokens": [ |
| "<s>", |
| "</s>", |
| "<unk>", |
| "<pad>" |
| ] |
| }, |
| "grouped_config_path": "/users/cmeister747/pa_tokenizers_branch/configs/parity_aware_config_balanced.json", |
| "vocab_size": 128000, |
| "min_frequency": 2, |
| "special_tokens": [], |
| "start_time": "2026-05-16T15:19:40.285398", |
| "elapsed_seconds": 2324.797585, |
| "slurm_job_id": "2260977", |
| "cli_args": [ |
| "/users/cmeister747/pa_tokenizers_branch/train_tokenizer.py", |
| "--variant", |
| "pa_bpe_nfc_clean_multi_balanced_hybrid_window" |
| ], |
| "git": { |
| "commit": "7172cc669840d0dcce4170f52ef0430eb2a3d899", |
| "branch": "parity-aware-bpe", |
| "dirty": false |
| }, |
| "per_group_summary": { |
| "n_groups": 34, |
| "full_block_in": "tokenizers/pabpe-nfc-clean-balanced-hybridwindow/run_manifest.json" |
| } |
| }, |
| "registry_note": "#6, clean pretok, slug `pabpe-nfc-clean-balanced-hybridwindow`. PA-BPE on clean pretok (balanced, HW)." |
| }, |
| "pabpe-nfc-clean-fineweb2full": { |
| "slug": "pabpe-nfc-clean-fineweb2full", |
| "runs": [ |
| "full-128k-pabpe-nfc-clean-fineweb2full" |
| ], |
| "n_runs": 1, |
| "redistributable": true, |
| "loadable_with": "transformers", |
| "hub_id": null, |
| "policy_note": "trained in this project", |
| "empirical": { |
| "model_type": "BPE", |
| "n_base_tokens": 127831, |
| "n_added_tokens": 4, |
| "vocab_total": 127835, |
| "added_tokens": [ |
| "<s>", |
| "</s>", |
| "<unk>", |
| "<pad>" |
| ], |
| "normalizer": { |
| "type": "NFC" |
| }, |
| "pre_tokenizer": { |
| "type": "Sequence", |
| "pretokenizers": [ |
| { |
| "type": "Split", |
| "pattern": { |
| "Regex": "[ ]?[\\p{Lu}\\p{Lt}\\p{Lm}\\p{Lo}\\p{M}]*[\\p{Ll}\\p{Lm}\\p{Lo}\\p{M}]+|[ ]?[\\p{Lu}\\p{Lt}\\p{Lm}\\p{Lo}\\p{M}]+[\\p{Ll}\\p{Lm}\\p{Lo}\\p{M}]*|\\p{N}| ?[^\\s\\p{L}\\p{N}]+|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+" |
| }, |
| "behavior": "Isolated", |
| "invert": false |
| }, |
| { |
| "type": "ByteLevel", |
| "add_prefix_space": false, |
| "trim_offsets": true, |
| "use_regex": false |
| } |
| ] |
| }, |
| "has_post_processor": false, |
| "byte_fallback": false, |
| "bos_token": "<s>", |
| "bos_token_id": 127831 |
| }, |
| "declared_experiments": null, |
| "declared_run_manifest": { |
| "variant": "pa_bpe_nfc_clean_multi_fineweb2full", |
| "variant_config": { |
| "trainer": "parity-bpe", |
| "pretok": "clean_multi_nfc", |
| "variant": "base", |
| "global_merges": 0, |
| "signal": "ratios", |
| "grouped_config": "configs/parity_aware_config_grouped_fineweb2full_quota.json", |
| "special_tokens_override": [], |
| "decoder_add_prefix_space": true, |
| "output_group": "tokenizer-lm-toks", |
| "output_dir_name": "nfc_clean_multi_fineweb2full", |
| "post_training_special_tokens": [ |
| "<s>", |
| "</s>", |
| "<unk>", |
| "<pad>" |
| ] |
| }, |
| "grouped_config_path": "/users/cmeister747/pa_tokenizers_branch/configs/parity_aware_config_grouped_fineweb2full_quota.json", |
| "vocab_size": 128000, |
| "min_frequency": 2, |
| "special_tokens": [], |
| "start_time": "2026-05-22T07:43:23.883861", |
| "elapsed_seconds": 8150.928394, |
| "slurm_job_id": "2328520", |
| "cli_args": [ |
| "/users/cmeister747/pa_tokenizers_branch/train_tokenizer.py", |
| "--variant", |
| "pa_bpe_nfc_clean_multi_fineweb2full" |
| ], |
| "git": { |
| "commit": "5d8381af9ec1417b078f7dce82bac68873da915e", |
| "branch": "parity-aware-bpe", |
| "dirty": false |
| }, |
| "per_group_summary": { |
| "n_groups": 25, |
| "full_block_in": "tokenizers/pabpe-nfc-clean-fineweb2full/run_manifest.json" |
| } |
| }, |
| "registry_note": "PA-BPE (parity, base) with clean-multi regex; fineweb2full data; norm=NFC; PA-BPE (base); vocab 127,835; USED - S5b fineweb2full analysis (best all-FLORES BPB 2.848 but worst val_bpb/BLiMP/code_bpb of the six); fineweb2full data variant, excluded from main paper panel" |
| }, |
| "pabpe-nfc-clean-fineweb2full-capped-tuned-hybridwindow": { |
| "slug": "pabpe-nfc-clean-fineweb2full-capped-tuned-hybridwindow", |
| "runs": [ |
| "full-128k-pabpe-nfc-clean-fineweb2full-capped-tuned-hybridwindow", |
| "full-128k-pabpe-nfc-clean-fineweb2full-capped-tuned-hybridwindow-mathcode-scratch" |
| ], |
| "n_runs": 2, |
| "redistributable": true, |
| "loadable_with": "transformers", |
| "hub_id": null, |
| "policy_note": "trained in this project", |
| "empirical": { |
| "model_type": "BPE", |
| "n_base_tokens": 127831, |
| "n_added_tokens": 4, |
| "vocab_total": 127835, |
| "added_tokens": [ |
| "<s>", |
| "</s>", |
| "<unk>", |
| "<pad>" |
| ], |
| "normalizer": { |
| "type": "NFC" |
| }, |
| "pre_tokenizer": { |
| "type": "Sequence", |
| "pretokenizers": [ |
| { |
| "type": "Split", |
| "pattern": { |
| "Regex": "[ ]?[\\p{Lu}\\p{Lt}\\p{Lm}\\p{Lo}\\p{M}]*[\\p{Ll}\\p{Lm}\\p{Lo}\\p{M}]+|[ ]?[\\p{Lu}\\p{Lt}\\p{Lm}\\p{Lo}\\p{M}]+[\\p{Ll}\\p{Lm}\\p{Lo}\\p{M}]*|\\p{N}| ?[^\\s\\p{L}\\p{N}]{1,16}|\\s{0,16}[\\r\\n]{1,16}|\\s{1,16}(?!\\S)|\\s{1,16}" |
| }, |
| "behavior": "Isolated", |
| "invert": false |
| }, |
| { |
| "type": "ByteLevel", |
| "add_prefix_space": false, |
| "trim_offsets": true, |
| "use_regex": false |
| } |
| ] |
| }, |
| "has_post_processor": false, |
| "byte_fallback": false, |
| "bos_token": "<s>", |
| "bos_token_id": 127831 |
| }, |
| "declared_experiments": null, |
| "declared_run_manifest": { |
| "variant": "pa_bpe_nfc_clean_multi_capped_hybrid_window_tuned", |
| "variant_config": { |
| "trainer": "parity-bpe", |
| "pretok": "clean_multi_capped_nfc", |
| "variant": "window", |
| "global_merges": 64000, |
| "window_size": 100, |
| "alpha": 2.0, |
| "signal": "ratios", |
| "grouped_config": "configs/parity_aware_config_grouped_fineweb2full_quota_tuned.json", |
| "special_tokens_override": [], |
| "decoder_add_prefix_space": true, |
| "output_group": "tokenizer-lm-toks", |
| "output_dir_name": "nfc_clean_multi_capped_hybrid_window_tuned", |
| "post_training_special_tokens": [ |
| "<s>", |
| "</s>", |
| "<unk>", |
| "<pad>" |
| ] |
| }, |
| "grouped_config_path": "/users/cmeister747/pa_tokenizers_branch/configs/parity_aware_config_grouped_fineweb2full_quota_tuned.json", |
| "vocab_size": 128000, |
| "min_frequency": 2, |
| "special_tokens": [], |
| "start_time": "2026-05-23T18:26:08.559566", |
| "elapsed_seconds": 7026.863542, |
| "slurm_job_id": "2353210", |
| "cli_args": [ |
| "/users/cmeister747/pa_tokenizers_branch/train_tokenizer.py", |
| "--variant", |
| "pa_bpe_nfc_clean_multi_capped_hybrid_window_tuned" |
| ], |
| "git": { |
| "commit": "5d8381af9ec1417b078f7dce82bac68873da915e", |
| "branch": "parity-aware-bpe", |
| "dirty": false |
| }, |
| "per_group_summary": { |
| "n_groups": 25, |
| "full_block_in": "tokenizers/pabpe-nfc-clean-fineweb2full-capped-tuned-hybridwindow/run_manifest.json" |
| } |
| }, |
| "registry_note": "PA-BPE hybrid global+window, clean-multi regex; fineweb2full, capped ({1,16} repeat-run cap) + quota-tuned ('tuned'/v5 hand-tuned per-family weighting) data; norm=NFC; PA-BPE (hybrid global+window); vocab 127,835; one of 4 Apertus-2-track candidates in the original 2x2 (pretok x {apertus,clean}) design; USED - Apertus-2 development track candidate; excluded from main paper panel (fineweb2full data variant, PAPER_PANEL.md rule)" |
| }, |
| "pabpe-nfc-clean-fineweb2full-consv2-engfull-eu3-gm110k": { |
| "slug": "pabpe-nfc-clean-fineweb2full-consv2-engfull-eu3-gm110k", |
| "runs": [ |
| "full-128k-pabpe-nfc-clean-fineweb2full-consv2-engfull-eu3-gm110k", |
| "full-128k-pabpe-nfc-clean-fineweb2full-consv2-engfull-eu3-gm110k-mathcode-scratch" |
| ], |
| "n_runs": 2, |
| "redistributable": true, |
| "loadable_with": "transformers", |
| "hub_id": null, |
| "policy_note": "trained in this project", |
| "empirical": { |
| "model_type": "BPE", |
| "n_base_tokens": 131072, |
| "n_added_tokens": 124, |
| "vocab_total": 131196, |
| "added_tokens": [ |
| "<unk>", |
| "<s>", |
| "</s>", |
| "<pad>", |
| "<|system_start|>", |
| "<|system_end|>", |
| "<|developer_start|>", |
| "<|developer_end|>", |
| "<|user_start|>", |
| "<|user_end|>", |
| "<|assistant_start|>", |
| "<|assistant_end|>", |
| "<|inner_prefix|>", |
| "<|inner_suffix|>", |
| "<|tools_prefix|>", |
| "<|tools_suffix|>", |
| "<|tool_output_start|>", |
| "<|tool_output_end|>", |
| "<|image|>", |
| "<|audio|>", |
| "<reflection>", |
| "</reflection>", |
| "<think>", |
| "</think>", |
| "<SPECIAL_24>", |
| "<SPECIAL_25>", |
| "<SPECIAL_26>", |
| "<SPECIAL_27>", |
| "<SPECIAL_28>", |
| "<SPECIAL_29>", |
| "<SPECIAL_30>", |
| "<SPECIAL_31>", |
| "<SPECIAL_32>", |
| "<SPECIAL_33>", |
| "<SPECIAL_34>", |
| "<SPECIAL_35>", |
| "<SPECIAL_36>", |
| "<SPECIAL_37>", |
| "<SPECIAL_38>", |
| "<SPECIAL_39>", |
| "<SPECIAL_40>", |
| "<SPECIAL_41>", |
| "<SPECIAL_42>", |
| "<SPECIAL_43>", |
| "<SPECIAL_44>", |
| "<SPECIAL_45>", |
| "<SPECIAL_46>", |
| "<SPECIAL_47>", |
| "<SPECIAL_48>", |
| "<SPECIAL_49>", |
| "<SPECIAL_50>", |
| "<SPECIAL_51>", |
| "<SPECIAL_52>", |
| "<SPECIAL_53>", |
| "<SPECIAL_54>", |
| "<SPECIAL_55>", |
| "<SPECIAL_56>", |
| "<SPECIAL_57>", |
| "<SPECIAL_58>", |
| "<SPECIAL_59>", |
| "<SPECIAL_60>", |
| "<SPECIAL_61>", |
| "<SPECIAL_62>", |
| "<SPECIAL_63>", |
| "<SPECIAL_64>", |
| "<SPECIAL_65>", |
| "<SPECIAL_66>", |
| "<SPECIAL_67>", |
| "<SPECIAL_68>", |
| "<SPECIAL_69>", |
| "<SPECIAL_70>", |
| "<SPECIAL_71>", |
| "<SPECIAL_72>", |
| "<SPECIAL_73>", |
| "<SPECIAL_74>", |
| "<SPECIAL_75>", |
| "<SPECIAL_76>", |
| "<SPECIAL_77>", |
| "<SPECIAL_78>", |
| "<SPECIAL_79>", |
| "<SPECIAL_80>", |
| "<SPECIAL_81>", |
| "<SPECIAL_82>", |
| "<SPECIAL_83>", |
| "<SPECIAL_84>", |
| "<SPECIAL_85>", |
| "<SPECIAL_86>", |
| "<SPECIAL_87>", |
| "<SPECIAL_88>", |
| "<SPECIAL_89>", |
| "<SPECIAL_90>", |
| "<SPECIAL_91>", |
| "<SPECIAL_92>", |
| "<SPECIAL_93>", |
| "<SPECIAL_94>", |
| "<SPECIAL_95>", |
| "<SPECIAL_96>", |
| "<SPECIAL_97>", |
| "<SPECIAL_98>", |
| "<SPECIAL_99>", |
| "<SPECIAL_100>", |
| "<SPECIAL_101>", |
| "<SPECIAL_102>", |
| "<SPECIAL_103>", |
| "<SPECIAL_104>", |
| "<SPECIAL_105>", |
| "<SPECIAL_106>", |
| "<SPECIAL_107>", |
| "<SPECIAL_108>", |
| "<SPECIAL_109>", |
| "<SPECIAL_110>", |
| "<SPECIAL_111>", |
| "<SPECIAL_112>", |
| "<SPECIAL_113>", |
| "<SPECIAL_114>", |
| "<SPECIAL_115>", |
| "<SPECIAL_116>", |
| "<SPECIAL_117>", |
| "<SPECIAL_118>", |
| "<SPECIAL_119>", |
| "<SPECIAL_120>", |
| "<SPECIAL_121>", |
| "<SPECIAL_122>", |
| "<SPECIAL_123>" |
| ], |
| "normalizer": { |
| "type": "NFC" |
| }, |
| "pre_tokenizer": { |
| "type": "Sequence", |
| "pretokenizers": [ |
| { |
| "type": "Split", |
| "pattern": { |
| "Regex": "[ \\x{0F0B}\\x{0027}\\x{2019}]?(?<g>[^\\s\\p{N}])\\k<g>{7}|[ \\x{0F0B}\\x{0027}\\x{2019}]?[\\p{Lu}\\p{Lt}\\p{Lm}\\p{Lo}\\p{M}]*[\\p{Ll}\\p{Lm}\\p{Lo}\\p{M}]+|[ \\x{0F0B}\\x{0027}\\x{2019}]?[\\p{Lu}\\p{Lt}\\p{Lm}\\p{Lo}\\p{M}]+[\\p{Ll}\\p{Lm}\\p{Lo}\\p{M}]*|\\p{N}| ?[^\\s\\p{L}\\p{N}]{1,16}|\\s{0,16}[\\r\\n]{1,16}|\\s{1,16}(?!\\S)|\\s{1,16}" |
| }, |
| "behavior": "Isolated", |
| "invert": false |
| }, |
| { |
| "type": "ByteLevel", |
| "add_prefix_space": false, |
| "trim_offsets": true, |
| "use_regex": false |
| } |
| ] |
| }, |
| "has_post_processor": true, |
| "byte_fallback": false, |
| "bos_token": "<s>", |
| "bos_token_id": 1 |
| }, |
| "declared_experiments": null, |
| "declared_run_manifest": null, |
| "registry_note": "PA-BPE hybrid global+window (gm110k), clean-multi regex; fineweb2full, consv2 engfull_eu3 (full English FineWeb-1 sample + European boost + Arabic fix) data; norm=NFC; PA-BPE (hybrid global+window); vocab 131,072 + 124 specials; shipped Apertus-v2 candidate 'preliminary_enh' (English-preserving); roster #22; USED - Apertus-2 track shipped candidate; S6f Contrast B (131k arm); excluded from main paper panel" |
| }, |
| "pabpe-nfc-clean-fineweb2full-consv2-eudata-gm110k": { |
| "slug": "pabpe-nfc-clean-fineweb2full-consv2-eudata-gm110k", |
| "runs": [ |
| "full-128k-pabpe-nfc-clean-fineweb2full-consv2-eudata-gm110k" |
| ], |
| "n_runs": 1, |
| "redistributable": true, |
| "loadable_with": "transformers", |
| "hub_id": null, |
| "policy_note": "trained in this project", |
| "empirical": { |
| "model_type": "BPE", |
| "n_base_tokens": 131072, |
| "n_added_tokens": 124, |
| "vocab_total": 131196, |
| "added_tokens": [ |
| "<unk>", |
| "<s>", |
| "</s>", |
| "<pad>", |
| "<|system_start|>", |
| "<|system_end|>", |
| "<|developer_start|>", |
| "<|developer_end|>", |
| "<|user_start|>", |
| "<|user_end|>", |
| "<|assistant_start|>", |
| "<|assistant_end|>", |
| "<|inner_prefix|>", |
| "<|inner_suffix|>", |
| "<|tools_prefix|>", |
| "<|tools_suffix|>", |
| "<|tool_output_start|>", |
| "<|tool_output_end|>", |
| "<|image|>", |
| "<|audio|>", |
| "<reflection>", |
| "</reflection>", |
| "<think>", |
| "</think>", |
| "<SPECIAL_24>", |
| "<SPECIAL_25>", |
| "<SPECIAL_26>", |
| "<SPECIAL_27>", |
| "<SPECIAL_28>", |
| "<SPECIAL_29>", |
| "<SPECIAL_30>", |
| "<SPECIAL_31>", |
| "<SPECIAL_32>", |
| "<SPECIAL_33>", |
| "<SPECIAL_34>", |
| "<SPECIAL_35>", |
| "<SPECIAL_36>", |
| "<SPECIAL_37>", |
| "<SPECIAL_38>", |
| "<SPECIAL_39>", |
| "<SPECIAL_40>", |
| "<SPECIAL_41>", |
| "<SPECIAL_42>", |
| "<SPECIAL_43>", |
| "<SPECIAL_44>", |
| "<SPECIAL_45>", |
| "<SPECIAL_46>", |
| "<SPECIAL_47>", |
| "<SPECIAL_48>", |
| "<SPECIAL_49>", |
| "<SPECIAL_50>", |
| "<SPECIAL_51>", |
| "<SPECIAL_52>", |
| "<SPECIAL_53>", |
| "<SPECIAL_54>", |
| "<SPECIAL_55>", |
| "<SPECIAL_56>", |
| "<SPECIAL_57>", |
| "<SPECIAL_58>", |
| "<SPECIAL_59>", |
| "<SPECIAL_60>", |
| "<SPECIAL_61>", |
| "<SPECIAL_62>", |
| "<SPECIAL_63>", |
| "<SPECIAL_64>", |
| "<SPECIAL_65>", |
| "<SPECIAL_66>", |
| "<SPECIAL_67>", |
| "<SPECIAL_68>", |
| "<SPECIAL_69>", |
| "<SPECIAL_70>", |
| "<SPECIAL_71>", |
| "<SPECIAL_72>", |
| "<SPECIAL_73>", |
| "<SPECIAL_74>", |
| "<SPECIAL_75>", |
| "<SPECIAL_76>", |
| "<SPECIAL_77>", |
| "<SPECIAL_78>", |
| "<SPECIAL_79>", |
| "<SPECIAL_80>", |
| "<SPECIAL_81>", |
| "<SPECIAL_82>", |
| "<SPECIAL_83>", |
| "<SPECIAL_84>", |
| "<SPECIAL_85>", |
| "<SPECIAL_86>", |
| "<SPECIAL_87>", |
| "<SPECIAL_88>", |
| "<SPECIAL_89>", |
| "<SPECIAL_90>", |
| "<SPECIAL_91>", |
| "<SPECIAL_92>", |
| "<SPECIAL_93>", |
| "<SPECIAL_94>", |
| "<SPECIAL_95>", |
| "<SPECIAL_96>", |
| "<SPECIAL_97>", |
| "<SPECIAL_98>", |
| "<SPECIAL_99>", |
| "<SPECIAL_100>", |
| "<SPECIAL_101>", |
| "<SPECIAL_102>", |
| "<SPECIAL_103>", |
| "<SPECIAL_104>", |
| "<SPECIAL_105>", |
| "<SPECIAL_106>", |
| "<SPECIAL_107>", |
| "<SPECIAL_108>", |
| "<SPECIAL_109>", |
| "<SPECIAL_110>", |
| "<SPECIAL_111>", |
| "<SPECIAL_112>", |
| "<SPECIAL_113>", |
| "<SPECIAL_114>", |
| "<SPECIAL_115>", |
| "<SPECIAL_116>", |
| "<SPECIAL_117>", |
| "<SPECIAL_118>", |
| "<SPECIAL_119>", |
| "<SPECIAL_120>", |
| "<SPECIAL_121>", |
| "<SPECIAL_122>", |
| "<SPECIAL_123>" |
| ], |
| "normalizer": { |
| "type": "NFC" |
| }, |
| "pre_tokenizer": { |
| "type": "Sequence", |
| "pretokenizers": [ |
| { |
| "type": "Split", |
| "pattern": { |
| "Regex": "[ \\x{0F0B}\\x{0027}\\x{2019}]?(?<g>[^\\s])\\k<g>{7}|[ \\x{0F0B}\\x{0027}\\x{2019}]?[\\p{Lu}\\p{Lt}\\p{Lm}\\p{Lo}\\p{M}]*[\\p{Ll}\\p{Lm}\\p{Lo}\\p{M}]+|[ \\x{0F0B}\\x{0027}\\x{2019}]?[\\p{Lu}\\p{Lt}\\p{Lm}\\p{Lo}\\p{M}]+[\\p{Ll}\\p{Lm}\\p{Lo}\\p{M}]*|\\p{N}| ?[^\\s\\p{L}\\p{N}]{1,16}|\\s{0,16}[\\r\\n]{1,16}|\\s{1,16}(?!\\S)|\\s{1,16}" |
| }, |
| "behavior": "Isolated", |
| "invert": false |
| }, |
| { |
| "type": "ByteLevel", |
| "add_prefix_space": false, |
| "trim_offsets": true, |
| "use_regex": false |
| } |
| ] |
| }, |
| "has_post_processor": false, |
| "byte_fallback": false, |
| "bos_token": "<s>", |
| "bos_token_id": 1 |
| }, |
| "declared_experiments": null, |
| "declared_run_manifest": { |
| "variant": "pa_bpe_nfc_clean_multi_plus2_repcap8_capped_hybrid_window_tuned_consv2_eudata_gm110k_v131k_sp124_eng5g", |
| "variant_config": { |
| "trainer": "parity-bpe", |
| "pretok": "clean_multi_plus2_repcap8_capped_nfc", |
| "variant": "window", |
| "global_merges": 110000, |
| "window_size": 100, |
| "alpha": 2.0, |
| "signal": "ratios", |
| "vocab_size": 131072, |
| "grouped_config": "configs/parity_aware_config_grouped_fineweb2full_quota_tuned_consv2_eudata_eng5g.json", |
| "special_tokens_override": [ |
| "<unk>", |
| "<s>", |
| "</s>", |
| "<pad>", |
| "<|system_start|>", |
| "<|system_end|>", |
| "<|developer_start|>", |
| "<|developer_end|>", |
| "<|user_start|>", |
| "<|user_end|>", |
| "<|assistant_start|>", |
| "<|assistant_end|>", |
| "<|inner_prefix|>", |
| "<|inner_suffix|>", |
| "<|tools_prefix|>", |
| "<|tools_suffix|>", |
| "<|tool_output_start|>", |
| "<|tool_output_end|>", |
| "<|image|>", |
| "<|audio|>", |
| "<reflection>", |
| "</reflection>", |
| "<think>", |
| "</think>", |
| "<SPECIAL_24>", |
| "<SPECIAL_25>", |
| "<SPECIAL_26>", |
| "<SPECIAL_27>", |
| "<SPECIAL_28>", |
| "<SPECIAL_29>", |
| "<SPECIAL_30>", |
| "<SPECIAL_31>", |
| "<SPECIAL_32>", |
| "<SPECIAL_33>", |
| "<SPECIAL_34>", |
| "<SPECIAL_35>", |
| "<SPECIAL_36>", |
| "<SPECIAL_37>", |
| "<SPECIAL_38>", |
| "<SPECIAL_39>", |
| "<SPECIAL_40>", |
| "<SPECIAL_41>", |
| "<SPECIAL_42>", |
| "<SPECIAL_43>", |
| "<SPECIAL_44>", |
| "<SPECIAL_45>", |
| "<SPECIAL_46>", |
| "<SPECIAL_47>", |
| "<SPECIAL_48>", |
| "<SPECIAL_49>", |
| "<SPECIAL_50>", |
| "<SPECIAL_51>", |
| "<SPECIAL_52>", |
| "<SPECIAL_53>", |
| "<SPECIAL_54>", |
| "<SPECIAL_55>", |
| "<SPECIAL_56>", |
| "<SPECIAL_57>", |
| "<SPECIAL_58>", |
| "<SPECIAL_59>", |
| "<SPECIAL_60>", |
| "<SPECIAL_61>", |
| "<SPECIAL_62>", |
| "<SPECIAL_63>", |
| "<SPECIAL_64>", |
| "<SPECIAL_65>", |
| "<SPECIAL_66>", |
| "<SPECIAL_67>", |
| "<SPECIAL_68>", |
| "<SPECIAL_69>", |
| "<SPECIAL_70>", |
| "<SPECIAL_71>", |
| "<SPECIAL_72>", |
| "<SPECIAL_73>", |
| "<SPECIAL_74>", |
| "<SPECIAL_75>", |
| "<SPECIAL_76>", |
| "<SPECIAL_77>", |
| "<SPECIAL_78>", |
| "<SPECIAL_79>", |
| "<SPECIAL_80>", |
| "<SPECIAL_81>", |
| "<SPECIAL_82>", |
| "<SPECIAL_83>", |
| "<SPECIAL_84>", |
| "<SPECIAL_85>", |
| "<SPECIAL_86>", |
| "<SPECIAL_87>", |
| "<SPECIAL_88>", |
| "<SPECIAL_89>", |
| "<SPECIAL_90>", |
| "<SPECIAL_91>", |
| "<SPECIAL_92>", |
| "<SPECIAL_93>", |
| "<SPECIAL_94>", |
| "<SPECIAL_95>", |
| "<SPECIAL_96>", |
| "<SPECIAL_97>", |
| "<SPECIAL_98>", |
| "<SPECIAL_99>", |
| "<SPECIAL_100>", |
| "<SPECIAL_101>", |
| "<SPECIAL_102>", |
| "<SPECIAL_103>", |
| "<SPECIAL_104>", |
| "<SPECIAL_105>", |
| "<SPECIAL_106>", |
| "<SPECIAL_107>", |
| "<SPECIAL_108>", |
| "<SPECIAL_109>", |
| "<SPECIAL_110>", |
| "<SPECIAL_111>", |
| "<SPECIAL_112>", |
| "<SPECIAL_113>", |
| "<SPECIAL_114>", |
| "<SPECIAL_115>", |
| "<SPECIAL_116>", |
| "<SPECIAL_117>", |
| "<SPECIAL_118>", |
| "<SPECIAL_119>", |
| "<SPECIAL_120>", |
| "<SPECIAL_121>", |
| "<SPECIAL_122>", |
| "<SPECIAL_123>" |
| ], |
| "decoder_add_prefix_space": true, |
| "output_group": "tokenizer-lm-toks", |
| "output_dir_name": "nfc_clean_multi_plus2_repcap8_capped_hybrid_window_tuned_consv2_eudata_gm110k_v131k_sp124_eng5g" |
| }, |
| "grouped_config_path": "/users/cmeister747/pa_tokenizers_branch/configs/parity_aware_config_grouped_fineweb2full_quota_tuned_consv2_eudata_eng5g.json", |
| "vocab_size": 131072, |
| "min_frequency": 2, |
| "special_tokens": [ |
| "<unk>", |
| "<s>", |
| "</s>", |
| "<pad>", |
| "<|system_start|>", |
| "<|system_end|>", |
| "<|developer_start|>", |
| "<|developer_end|>", |
| "<|user_start|>", |
| "<|user_end|>", |
| "<|assistant_start|>", |
| "<|assistant_end|>", |
| "<|inner_prefix|>", |
| "<|inner_suffix|>", |
| "<|tools_prefix|>", |
| "<|tools_suffix|>", |
| "<|tool_output_start|>", |
| "<|tool_output_end|>", |
| "<|image|>", |
| "<|audio|>", |
| "<reflection>", |
| "</reflection>", |
| "<think>", |
| "</think>", |
| "<SPECIAL_24>", |
| "<SPECIAL_25>", |
| "<SPECIAL_26>", |
| "<SPECIAL_27>", |
| "<SPECIAL_28>", |
| "<SPECIAL_29>", |
| "<SPECIAL_30>", |
| "<SPECIAL_31>", |
| "<SPECIAL_32>", |
| "<SPECIAL_33>", |
| "<SPECIAL_34>", |
| "<SPECIAL_35>", |
| "<SPECIAL_36>", |
| "<SPECIAL_37>", |
| "<SPECIAL_38>", |
| "<SPECIAL_39>", |
| "<SPECIAL_40>", |
| "<SPECIAL_41>", |
| "<SPECIAL_42>", |
| "<SPECIAL_43>", |
| "<SPECIAL_44>", |
| "<SPECIAL_45>", |
| "<SPECIAL_46>", |
| "<SPECIAL_47>", |
| "<SPECIAL_48>", |
| "<SPECIAL_49>", |
| "<SPECIAL_50>", |
| "<SPECIAL_51>", |
| "<SPECIAL_52>", |
| "<SPECIAL_53>", |
| "<SPECIAL_54>", |
| "<SPECIAL_55>", |
| "<SPECIAL_56>", |
| "<SPECIAL_57>", |
| "<SPECIAL_58>", |
| "<SPECIAL_59>", |
| "<SPECIAL_60>", |
| "<SPECIAL_61>", |
| "<SPECIAL_62>", |
| "<SPECIAL_63>", |
| "<SPECIAL_64>", |
| "<SPECIAL_65>", |
| "<SPECIAL_66>", |
| "<SPECIAL_67>", |
| "<SPECIAL_68>", |
| "<SPECIAL_69>", |
| "<SPECIAL_70>", |
| "<SPECIAL_71>", |
| "<SPECIAL_72>", |
| "<SPECIAL_73>", |
| "<SPECIAL_74>", |
| "<SPECIAL_75>", |
| "<SPECIAL_76>", |
| "<SPECIAL_77>", |
| "<SPECIAL_78>", |
| "<SPECIAL_79>", |
| "<SPECIAL_80>", |
| "<SPECIAL_81>", |
| "<SPECIAL_82>", |
| "<SPECIAL_83>", |
| "<SPECIAL_84>", |
| "<SPECIAL_85>", |
| "<SPECIAL_86>", |
| "<SPECIAL_87>", |
| "<SPECIAL_88>", |
| "<SPECIAL_89>", |
| "<SPECIAL_90>", |
| "<SPECIAL_91>", |
| "<SPECIAL_92>", |
| "<SPECIAL_93>", |
| "<SPECIAL_94>", |
| "<SPECIAL_95>", |
| "<SPECIAL_96>", |
| "<SPECIAL_97>", |
| "<SPECIAL_98>", |
| "<SPECIAL_99>", |
| "<SPECIAL_100>", |
| "<SPECIAL_101>", |
| "<SPECIAL_102>", |
| "<SPECIAL_103>", |
| "<SPECIAL_104>", |
| "<SPECIAL_105>", |
| "<SPECIAL_106>", |
| "<SPECIAL_107>", |
| "<SPECIAL_108>", |
| "<SPECIAL_109>", |
| "<SPECIAL_110>", |
| "<SPECIAL_111>", |
| "<SPECIAL_112>", |
| "<SPECIAL_113>", |
| "<SPECIAL_114>", |
| "<SPECIAL_115>", |
| "<SPECIAL_116>", |
| "<SPECIAL_117>", |
| "<SPECIAL_118>", |
| "<SPECIAL_119>", |
| "<SPECIAL_120>", |
| "<SPECIAL_121>", |
| "<SPECIAL_122>", |
| "<SPECIAL_123>" |
| ], |
| "start_time": "2026-06-17T21:19:35.605876", |
| "elapsed_seconds": 5611.718146, |
| "slurm_job_id": "2554545", |
| "cli_args": [ |
| "/users/cmeister747/pa_tokenizers_branch/train_tokenizer.py", |
| "--variant", |
| "pa_bpe_nfc_clean_multi_plus2_repcap8_capped_hybrid_window_tuned_consv2_eudata_gm110k_v131k_sp124_eng5g" |
| ], |
| "git": { |
| "commit": "5d8381af9ec1417b078f7dce82bac68873da915e", |
| "branch": "parity-aware-bpe", |
| "dirty": true |
| }, |
| "per_group_summary": { |
| "n_groups": 25, |
| "full_block_in": "tokenizers/pabpe-nfc-clean-fineweb2full-consv2-eudata-gm110k/run_manifest.json" |
| } |
| }, |
| "registry_note": "PA-BPE hybrid global+window (gm110k), clean-multi regex; fineweb2full, consv2 + eudata data; norm=NFC; PA-BPE (hybrid global+window); vocab ~131,196; same eudata config, higher global-merges warmup; USED - S6f Contrast B (131k arm) + Apertus-2 development track; excluded from main paper panel" |
| }, |
| "pabpe-nfc-clean-fineweb2full-consv2-eudata-gm90k": { |
| "slug": "pabpe-nfc-clean-fineweb2full-consv2-eudata-gm90k", |
| "runs": [ |
| "full-128k-pabpe-nfc-clean-fineweb2full-consv2-eudata-gm90k" |
| ], |
| "n_runs": 1, |
| "redistributable": true, |
| "loadable_with": "transformers", |
| "hub_id": null, |
| "policy_note": "trained in this project", |
| "empirical": { |
| "model_type": "BPE", |
| "n_base_tokens": 131072, |
| "n_added_tokens": 124, |
| "vocab_total": 131196, |
| "added_tokens": [ |
| "<unk>", |
| "<s>", |
| "</s>", |
| "<pad>", |
| "<|system_start|>", |
| "<|system_end|>", |
| "<|developer_start|>", |
| "<|developer_end|>", |
| "<|user_start|>", |
| "<|user_end|>", |
| "<|assistant_start|>", |
| "<|assistant_end|>", |
| "<|inner_prefix|>", |
| "<|inner_suffix|>", |
| "<|tools_prefix|>", |
| "<|tools_suffix|>", |
| "<|tool_output_start|>", |
| "<|tool_output_end|>", |
| "<|image|>", |
| "<|audio|>", |
| "<reflection>", |
| "</reflection>", |
| "<think>", |
| "</think>", |
| "<SPECIAL_24>", |
| "<SPECIAL_25>", |
| "<SPECIAL_26>", |
| "<SPECIAL_27>", |
| "<SPECIAL_28>", |
| "<SPECIAL_29>", |
| "<SPECIAL_30>", |
| "<SPECIAL_31>", |
| "<SPECIAL_32>", |
| "<SPECIAL_33>", |
| "<SPECIAL_34>", |
| "<SPECIAL_35>", |
| "<SPECIAL_36>", |
| "<SPECIAL_37>", |
| "<SPECIAL_38>", |
| "<SPECIAL_39>", |
| "<SPECIAL_40>", |
| "<SPECIAL_41>", |
| "<SPECIAL_42>", |
| "<SPECIAL_43>", |
| "<SPECIAL_44>", |
| "<SPECIAL_45>", |
| "<SPECIAL_46>", |
| "<SPECIAL_47>", |
| "<SPECIAL_48>", |
| "<SPECIAL_49>", |
| "<SPECIAL_50>", |
| "<SPECIAL_51>", |
| "<SPECIAL_52>", |
| "<SPECIAL_53>", |
| "<SPECIAL_54>", |
| "<SPECIAL_55>", |
| "<SPECIAL_56>", |
| "<SPECIAL_57>", |
| "<SPECIAL_58>", |
| "<SPECIAL_59>", |
| "<SPECIAL_60>", |
| "<SPECIAL_61>", |
| "<SPECIAL_62>", |
| "<SPECIAL_63>", |
| "<SPECIAL_64>", |
| "<SPECIAL_65>", |
| "<SPECIAL_66>", |
| "<SPECIAL_67>", |
| "<SPECIAL_68>", |
| "<SPECIAL_69>", |
| "<SPECIAL_70>", |
| "<SPECIAL_71>", |
| "<SPECIAL_72>", |
| "<SPECIAL_73>", |
| "<SPECIAL_74>", |
| "<SPECIAL_75>", |
| "<SPECIAL_76>", |
| "<SPECIAL_77>", |
| "<SPECIAL_78>", |
| "<SPECIAL_79>", |
| "<SPECIAL_80>", |
| "<SPECIAL_81>", |
| "<SPECIAL_82>", |
| "<SPECIAL_83>", |
| "<SPECIAL_84>", |
| "<SPECIAL_85>", |
| "<SPECIAL_86>", |
| "<SPECIAL_87>", |
| "<SPECIAL_88>", |
| "<SPECIAL_89>", |
| "<SPECIAL_90>", |
| "<SPECIAL_91>", |
| "<SPECIAL_92>", |
| "<SPECIAL_93>", |
| "<SPECIAL_94>", |
| "<SPECIAL_95>", |
| "<SPECIAL_96>", |
| "<SPECIAL_97>", |
| "<SPECIAL_98>", |
| "<SPECIAL_99>", |
| "<SPECIAL_100>", |
| "<SPECIAL_101>", |
| "<SPECIAL_102>", |
| "<SPECIAL_103>", |
| "<SPECIAL_104>", |
| "<SPECIAL_105>", |
| "<SPECIAL_106>", |
| "<SPECIAL_107>", |
| "<SPECIAL_108>", |
| "<SPECIAL_109>", |
| "<SPECIAL_110>", |
| "<SPECIAL_111>", |
| "<SPECIAL_112>", |
| "<SPECIAL_113>", |
| "<SPECIAL_114>", |
| "<SPECIAL_115>", |
| "<SPECIAL_116>", |
| "<SPECIAL_117>", |
| "<SPECIAL_118>", |
| "<SPECIAL_119>", |
| "<SPECIAL_120>", |
| "<SPECIAL_121>", |
| "<SPECIAL_122>", |
| "<SPECIAL_123>" |
| ], |
| "normalizer": { |
| "type": "NFC" |
| }, |
| "pre_tokenizer": { |
| "type": "Sequence", |
| "pretokenizers": [ |
| { |
| "type": "Split", |
| "pattern": { |
| "Regex": "[ \\x{0F0B}\\x{0027}\\x{2019}]?(?<g>[^\\s])\\k<g>{7}|[ \\x{0F0B}\\x{0027}\\x{2019}]?[\\p{Lu}\\p{Lt}\\p{Lm}\\p{Lo}\\p{M}]*[\\p{Ll}\\p{Lm}\\p{Lo}\\p{M}]+|[ \\x{0F0B}\\x{0027}\\x{2019}]?[\\p{Lu}\\p{Lt}\\p{Lm}\\p{Lo}\\p{M}]+[\\p{Ll}\\p{Lm}\\p{Lo}\\p{M}]*|\\p{N}| ?[^\\s\\p{L}\\p{N}]{1,16}|\\s{0,16}[\\r\\n]{1,16}|\\s{1,16}(?!\\S)|\\s{1,16}" |
| }, |
| "behavior": "Isolated", |
| "invert": false |
| }, |
| { |
| "type": "ByteLevel", |
| "add_prefix_space": false, |
| "trim_offsets": true, |
| "use_regex": false |
| } |
| ] |
| }, |
| "has_post_processor": false, |
| "byte_fallback": false, |
| "bos_token": "<s>", |
| "bos_token_id": 1 |
| }, |
| "declared_experiments": null, |
| "declared_run_manifest": { |
| "variant": "pa_bpe_nfc_clean_multi_plus2_repcap8_capped_hybrid_window_tuned_consv2_eudata_gm90k_v131k_sp124_eng5g", |
| "variant_config": { |
| "trainer": "parity-bpe", |
| "pretok": "clean_multi_plus2_repcap8_capped_nfc", |
| "variant": "window", |
| "global_merges": 90000, |
| "window_size": 100, |
| "alpha": 2.0, |
| "signal": "ratios", |
| "vocab_size": 131072, |
| "grouped_config": "configs/parity_aware_config_grouped_fineweb2full_quota_tuned_consv2_eudata_eng5g.json", |
| "special_tokens_override": [ |
| "<unk>", |
| "<s>", |
| "</s>", |
| "<pad>", |
| "<|system_start|>", |
| "<|system_end|>", |
| "<|developer_start|>", |
| "<|developer_end|>", |
| "<|user_start|>", |
| "<|user_end|>", |
| "<|assistant_start|>", |
| "<|assistant_end|>", |
| "<|inner_prefix|>", |
| "<|inner_suffix|>", |
| "<|tools_prefix|>", |
| "<|tools_suffix|>", |
| "<|tool_output_start|>", |
| "<|tool_output_end|>", |
| "<|image|>", |
| "<|audio|>", |
| "<reflection>", |
| "</reflection>", |
| "<think>", |
| "</think>", |
| "<SPECIAL_24>", |
| "<SPECIAL_25>", |
| "<SPECIAL_26>", |
| "<SPECIAL_27>", |
| "<SPECIAL_28>", |
| "<SPECIAL_29>", |
| "<SPECIAL_30>", |
| "<SPECIAL_31>", |
| "<SPECIAL_32>", |
| "<SPECIAL_33>", |
| "<SPECIAL_34>", |
| "<SPECIAL_35>", |
| "<SPECIAL_36>", |
| "<SPECIAL_37>", |
| "<SPECIAL_38>", |
| "<SPECIAL_39>", |
| "<SPECIAL_40>", |
| "<SPECIAL_41>", |
| "<SPECIAL_42>", |
| "<SPECIAL_43>", |
| "<SPECIAL_44>", |
| "<SPECIAL_45>", |
| "<SPECIAL_46>", |
| "<SPECIAL_47>", |
| "<SPECIAL_48>", |
| "<SPECIAL_49>", |
| "<SPECIAL_50>", |
| "<SPECIAL_51>", |
| "<SPECIAL_52>", |
| "<SPECIAL_53>", |
| "<SPECIAL_54>", |
| "<SPECIAL_55>", |
| "<SPECIAL_56>", |
| "<SPECIAL_57>", |
| "<SPECIAL_58>", |
| "<SPECIAL_59>", |
| "<SPECIAL_60>", |
| "<SPECIAL_61>", |
| "<SPECIAL_62>", |
| "<SPECIAL_63>", |
| "<SPECIAL_64>", |
| "<SPECIAL_65>", |
| "<SPECIAL_66>", |
| "<SPECIAL_67>", |
| "<SPECIAL_68>", |
| "<SPECIAL_69>", |
| "<SPECIAL_70>", |
| "<SPECIAL_71>", |
| "<SPECIAL_72>", |
| "<SPECIAL_73>", |
| "<SPECIAL_74>", |
| "<SPECIAL_75>", |
| "<SPECIAL_76>", |
| "<SPECIAL_77>", |
| "<SPECIAL_78>", |
| "<SPECIAL_79>", |
| "<SPECIAL_80>", |
| "<SPECIAL_81>", |
| "<SPECIAL_82>", |
| "<SPECIAL_83>", |
| "<SPECIAL_84>", |
| "<SPECIAL_85>", |
| "<SPECIAL_86>", |
| "<SPECIAL_87>", |
| "<SPECIAL_88>", |
| "<SPECIAL_89>", |
| "<SPECIAL_90>", |
| "<SPECIAL_91>", |
| "<SPECIAL_92>", |
| "<SPECIAL_93>", |
| "<SPECIAL_94>", |
| "<SPECIAL_95>", |
| "<SPECIAL_96>", |
| "<SPECIAL_97>", |
| "<SPECIAL_98>", |
| "<SPECIAL_99>", |
| "<SPECIAL_100>", |
| "<SPECIAL_101>", |
| "<SPECIAL_102>", |
| "<SPECIAL_103>", |
| "<SPECIAL_104>", |
| "<SPECIAL_105>", |
| "<SPECIAL_106>", |
| "<SPECIAL_107>", |
| "<SPECIAL_108>", |
| "<SPECIAL_109>", |
| "<SPECIAL_110>", |
| "<SPECIAL_111>", |
| "<SPECIAL_112>", |
| "<SPECIAL_113>", |
| "<SPECIAL_114>", |
| "<SPECIAL_115>", |
| "<SPECIAL_116>", |
| "<SPECIAL_117>", |
| "<SPECIAL_118>", |
| "<SPECIAL_119>", |
| "<SPECIAL_120>", |
| "<SPECIAL_121>", |
| "<SPECIAL_122>", |
| "<SPECIAL_123>" |
| ], |
| "decoder_add_prefix_space": true, |
| "output_group": "tokenizer-lm-toks", |
| "output_dir_name": "nfc_clean_multi_plus2_repcap8_capped_hybrid_window_tuned_consv2_eudata_gm90k_v131k_sp124_eng5g" |
| }, |
| "grouped_config_path": "/users/cmeister747/pa_tokenizers_branch/configs/parity_aware_config_grouped_fineweb2full_quota_tuned_consv2_eudata_eng5g.json", |
| "vocab_size": 131072, |
| "min_frequency": 2, |
| "special_tokens": [ |
| "<unk>", |
| "<s>", |
| "</s>", |
| "<pad>", |
| "<|system_start|>", |
| "<|system_end|>", |
| "<|developer_start|>", |
| "<|developer_end|>", |
| "<|user_start|>", |
| "<|user_end|>", |
| "<|assistant_start|>", |
| "<|assistant_end|>", |
| "<|inner_prefix|>", |
| "<|inner_suffix|>", |
| "<|tools_prefix|>", |
| "<|tools_suffix|>", |
| "<|tool_output_start|>", |
| "<|tool_output_end|>", |
| "<|image|>", |
| "<|audio|>", |
| "<reflection>", |
| "</reflection>", |
| "<think>", |
| "</think>", |
| "<SPECIAL_24>", |
| "<SPECIAL_25>", |
| "<SPECIAL_26>", |
| "<SPECIAL_27>", |
| "<SPECIAL_28>", |
| "<SPECIAL_29>", |
| "<SPECIAL_30>", |
| "<SPECIAL_31>", |
| "<SPECIAL_32>", |
| "<SPECIAL_33>", |
| "<SPECIAL_34>", |
| "<SPECIAL_35>", |
| "<SPECIAL_36>", |
| "<SPECIAL_37>", |
| "<SPECIAL_38>", |
| "<SPECIAL_39>", |
| "<SPECIAL_40>", |
| "<SPECIAL_41>", |
| "<SPECIAL_42>", |
| "<SPECIAL_43>", |
| "<SPECIAL_44>", |
| "<SPECIAL_45>", |
| "<SPECIAL_46>", |
| "<SPECIAL_47>", |
| "<SPECIAL_48>", |
| "<SPECIAL_49>", |
| "<SPECIAL_50>", |
| "<SPECIAL_51>", |
| "<SPECIAL_52>", |
| "<SPECIAL_53>", |
| "<SPECIAL_54>", |
| "<SPECIAL_55>", |
| "<SPECIAL_56>", |
| "<SPECIAL_57>", |
| "<SPECIAL_58>", |
| "<SPECIAL_59>", |
| "<SPECIAL_60>", |
| "<SPECIAL_61>", |
| "<SPECIAL_62>", |
| "<SPECIAL_63>", |
| "<SPECIAL_64>", |
| "<SPECIAL_65>", |
| "<SPECIAL_66>", |
| "<SPECIAL_67>", |
| "<SPECIAL_68>", |
| "<SPECIAL_69>", |
| "<SPECIAL_70>", |
| "<SPECIAL_71>", |
| "<SPECIAL_72>", |
| "<SPECIAL_73>", |
| "<SPECIAL_74>", |
| "<SPECIAL_75>", |
| "<SPECIAL_76>", |
| "<SPECIAL_77>", |
| "<SPECIAL_78>", |
| "<SPECIAL_79>", |
| "<SPECIAL_80>", |
| "<SPECIAL_81>", |
| "<SPECIAL_82>", |
| "<SPECIAL_83>", |
| "<SPECIAL_84>", |
| "<SPECIAL_85>", |
| "<SPECIAL_86>", |
| "<SPECIAL_87>", |
| "<SPECIAL_88>", |
| "<SPECIAL_89>", |
| "<SPECIAL_90>", |
| "<SPECIAL_91>", |
| "<SPECIAL_92>", |
| "<SPECIAL_93>", |
| "<SPECIAL_94>", |
| "<SPECIAL_95>", |
| "<SPECIAL_96>", |
| "<SPECIAL_97>", |
| "<SPECIAL_98>", |
| "<SPECIAL_99>", |
| "<SPECIAL_100>", |
| "<SPECIAL_101>", |
| "<SPECIAL_102>", |
| "<SPECIAL_103>", |
| "<SPECIAL_104>", |
| "<SPECIAL_105>", |
| "<SPECIAL_106>", |
| "<SPECIAL_107>", |
| "<SPECIAL_108>", |
| "<SPECIAL_109>", |
| "<SPECIAL_110>", |
| "<SPECIAL_111>", |
| "<SPECIAL_112>", |
| "<SPECIAL_113>", |
| "<SPECIAL_114>", |
| "<SPECIAL_115>", |
| "<SPECIAL_116>", |
| "<SPECIAL_117>", |
| "<SPECIAL_118>", |
| "<SPECIAL_119>", |
| "<SPECIAL_120>", |
| "<SPECIAL_121>", |
| "<SPECIAL_122>", |
| "<SPECIAL_123>" |
| ], |
| "start_time": "2026-06-17T21:19:32.179033", |
| "elapsed_seconds": 5852.504629, |
| "slurm_job_id": "2554539", |
| "cli_args": [ |
| "/users/cmeister747/pa_tokenizers_branch/train_tokenizer.py", |
| "--variant", |
| "pa_bpe_nfc_clean_multi_plus2_repcap8_capped_hybrid_window_tuned_consv2_eudata_gm90k_v131k_sp124_eng5g" |
| ], |
| "git": { |
| "commit": "5d8381af9ec1417b078f7dce82bac68873da915e", |
| "branch": "parity-aware-bpe", |
| "dirty": true |
| }, |
| "per_group_summary": { |
| "n_groups": 25, |
| "full_block_in": "tokenizers/pabpe-nfc-clean-fineweb2full-consv2-eudata-gm90k/run_manifest.json" |
| } |
| }, |
| "registry_note": "PA-BPE hybrid global+window (gm90k), clean-multi regex, plus2/repcap8/capped/tuned base; fineweb2full, consv2 quota formula + eudata (European-data-quota grouped config) data; norm=NFC; PA-BPE (hybrid global+window); vocab ~131,196 (131,072 target + 124 specials); earlier eudata sweep point, superseded by the shipped consv2 candidates; USED - S6f Contrast B (PA-BPE consv2 family, 131k arm, n=12) + Apertus-2 development track; excluded from main paper panel" |
| }, |
| "pabpe-nfc-clean-fineweb2full-consv2-eudata4-gm100k": { |
| "slug": "pabpe-nfc-clean-fineweb2full-consv2-eudata4-gm100k", |
| "runs": [ |
| "full-128k-pabpe-nfc-clean-fineweb2full-consv2-eudata4-gm100k" |
| ], |
| "n_runs": 1, |
| "redistributable": true, |
| "loadable_with": "transformers", |
| "hub_id": null, |
| "policy_note": "trained in this project", |
| "empirical": { |
| "model_type": "BPE", |
| "n_base_tokens": 131072, |
| "n_added_tokens": 124, |
| "vocab_total": 131196, |
| "added_tokens": [ |
| "<unk>", |
| "<s>", |
| "</s>", |
| "<pad>", |
| "<|system_start|>", |
| "<|system_end|>", |
| "<|developer_start|>", |
| "<|developer_end|>", |
| "<|user_start|>", |
| "<|user_end|>", |
| "<|assistant_start|>", |
| "<|assistant_end|>", |
| "<|inner_prefix|>", |
| "<|inner_suffix|>", |
| "<|tools_prefix|>", |
| "<|tools_suffix|>", |
| "<|tool_output_start|>", |
| "<|tool_output_end|>", |
| "<|image|>", |
| "<|audio|>", |
| "<reflection>", |
| "</reflection>", |
| "<think>", |
| "</think>", |
| "<SPECIAL_24>", |
| "<SPECIAL_25>", |
| "<SPECIAL_26>", |
| "<SPECIAL_27>", |
| "<SPECIAL_28>", |
| "<SPECIAL_29>", |
| "<SPECIAL_30>", |
| "<SPECIAL_31>", |
| "<SPECIAL_32>", |
| "<SPECIAL_33>", |
| "<SPECIAL_34>", |
| "<SPECIAL_35>", |
| "<SPECIAL_36>", |
| "<SPECIAL_37>", |
| "<SPECIAL_38>", |
| "<SPECIAL_39>", |
| "<SPECIAL_40>", |
| "<SPECIAL_41>", |
| "<SPECIAL_42>", |
| "<SPECIAL_43>", |
| "<SPECIAL_44>", |
| "<SPECIAL_45>", |
| "<SPECIAL_46>", |
| "<SPECIAL_47>", |
| "<SPECIAL_48>", |
| "<SPECIAL_49>", |
| "<SPECIAL_50>", |
| "<SPECIAL_51>", |
| "<SPECIAL_52>", |
| "<SPECIAL_53>", |
| "<SPECIAL_54>", |
| "<SPECIAL_55>", |
| "<SPECIAL_56>", |
| "<SPECIAL_57>", |
| "<SPECIAL_58>", |
| "<SPECIAL_59>", |
| "<SPECIAL_60>", |
| "<SPECIAL_61>", |
| "<SPECIAL_62>", |
| "<SPECIAL_63>", |
| "<SPECIAL_64>", |
| "<SPECIAL_65>", |
| "<SPECIAL_66>", |
| "<SPECIAL_67>", |
| "<SPECIAL_68>", |
| "<SPECIAL_69>", |
| "<SPECIAL_70>", |
| "<SPECIAL_71>", |
| "<SPECIAL_72>", |
| "<SPECIAL_73>", |
| "<SPECIAL_74>", |
| "<SPECIAL_75>", |
| "<SPECIAL_76>", |
| "<SPECIAL_77>", |
| "<SPECIAL_78>", |
| "<SPECIAL_79>", |
| "<SPECIAL_80>", |
| "<SPECIAL_81>", |
| "<SPECIAL_82>", |
| "<SPECIAL_83>", |
| "<SPECIAL_84>", |
| "<SPECIAL_85>", |
| "<SPECIAL_86>", |
| "<SPECIAL_87>", |
| "<SPECIAL_88>", |
| "<SPECIAL_89>", |
| "<SPECIAL_90>", |
| "<SPECIAL_91>", |
| "<SPECIAL_92>", |
| "<SPECIAL_93>", |
| "<SPECIAL_94>", |
| "<SPECIAL_95>", |
| "<SPECIAL_96>", |
| "<SPECIAL_97>", |
| "<SPECIAL_98>", |
| "<SPECIAL_99>", |
| "<SPECIAL_100>", |
| "<SPECIAL_101>", |
| "<SPECIAL_102>", |
| "<SPECIAL_103>", |
| "<SPECIAL_104>", |
| "<SPECIAL_105>", |
| "<SPECIAL_106>", |
| "<SPECIAL_107>", |
| "<SPECIAL_108>", |
| "<SPECIAL_109>", |
| "<SPECIAL_110>", |
| "<SPECIAL_111>", |
| "<SPECIAL_112>", |
| "<SPECIAL_113>", |
| "<SPECIAL_114>", |
| "<SPECIAL_115>", |
| "<SPECIAL_116>", |
| "<SPECIAL_117>", |
| "<SPECIAL_118>", |
| "<SPECIAL_119>", |
| "<SPECIAL_120>", |
| "<SPECIAL_121>", |
| "<SPECIAL_122>", |
| "<SPECIAL_123>" |
| ], |
| "normalizer": { |
| "type": "NFC" |
| }, |
| "pre_tokenizer": { |
| "type": "Sequence", |
| "pretokenizers": [ |
| { |
| "type": "Split", |
| "pattern": { |
| "Regex": "[ \\x{0F0B}\\x{0027}\\x{2019}]?(?<g>[^\\s])\\k<g>{7}|[ \\x{0F0B}\\x{0027}\\x{2019}]?[\\p{Lu}\\p{Lt}\\p{Lm}\\p{Lo}\\p{M}]*[\\p{Ll}\\p{Lm}\\p{Lo}\\p{M}]+|[ \\x{0F0B}\\x{0027}\\x{2019}]?[\\p{Lu}\\p{Lt}\\p{Lm}\\p{Lo}\\p{M}]+[\\p{Ll}\\p{Lm}\\p{Lo}\\p{M}]*|\\p{N}| ?[^\\s\\p{L}\\p{N}]{1,16}|\\s{0,16}[\\r\\n]{1,16}|\\s{1,16}(?!\\S)|\\s{1,16}" |
| }, |
| "behavior": "Isolated", |
| "invert": false |
| }, |
| { |
| "type": "ByteLevel", |
| "add_prefix_space": false, |
| "trim_offsets": true, |
| "use_regex": false |
| } |
| ] |
| }, |
| "has_post_processor": false, |
| "byte_fallback": false, |
| "bos_token": "<s>", |
| "bos_token_id": 1 |
| }, |
| "declared_experiments": null, |
| "declared_run_manifest": { |
| "variant": "pa_bpe_nfc_clean_multi_plus2_repcap8_capped_hybrid_window_tuned_consv2_eudata4_gm100k_v131k_sp124_eng5g", |
| "variant_config": { |
| "trainer": "parity-bpe", |
| "pretok": "clean_multi_plus2_repcap8_capped_nfc", |
| "variant": "window", |
| "global_merges": 100000, |
| "window_size": 100, |
| "alpha": 2.0, |
| "signal": "ratios", |
| "vocab_size": 131072, |
| "grouped_config": "configs/parity_aware_config_grouped_fineweb2full_quota_tuned_consv2_eudata4_eng5g.json", |
| "special_tokens_override": [ |
| "<unk>", |
| "<s>", |
| "</s>", |
| "<pad>", |
| "<|system_start|>", |
| "<|system_end|>", |
| "<|developer_start|>", |
| "<|developer_end|>", |
| "<|user_start|>", |
| "<|user_end|>", |
| "<|assistant_start|>", |
| "<|assistant_end|>", |
| "<|inner_prefix|>", |
| "<|inner_suffix|>", |
| "<|tools_prefix|>", |
| "<|tools_suffix|>", |
| "<|tool_output_start|>", |
| "<|tool_output_end|>", |
| "<|image|>", |
| "<|audio|>", |
| "<reflection>", |
| "</reflection>", |
| "<think>", |
| "</think>", |
| "<SPECIAL_24>", |
| "<SPECIAL_25>", |
| "<SPECIAL_26>", |
| "<SPECIAL_27>", |
| "<SPECIAL_28>", |
| "<SPECIAL_29>", |
| "<SPECIAL_30>", |
| "<SPECIAL_31>", |
| "<SPECIAL_32>", |
| "<SPECIAL_33>", |
| "<SPECIAL_34>", |
| "<SPECIAL_35>", |
| "<SPECIAL_36>", |
| "<SPECIAL_37>", |
| "<SPECIAL_38>", |
| "<SPECIAL_39>", |
| "<SPECIAL_40>", |
| "<SPECIAL_41>", |
| "<SPECIAL_42>", |
| "<SPECIAL_43>", |
| "<SPECIAL_44>", |
| "<SPECIAL_45>", |
| "<SPECIAL_46>", |
| "<SPECIAL_47>", |
| "<SPECIAL_48>", |
| "<SPECIAL_49>", |
| "<SPECIAL_50>", |
| "<SPECIAL_51>", |
| "<SPECIAL_52>", |
| "<SPECIAL_53>", |
| "<SPECIAL_54>", |
| "<SPECIAL_55>", |
| "<SPECIAL_56>", |
| "<SPECIAL_57>", |
| "<SPECIAL_58>", |
| "<SPECIAL_59>", |
| "<SPECIAL_60>", |
| "<SPECIAL_61>", |
| "<SPECIAL_62>", |
| "<SPECIAL_63>", |
| "<SPECIAL_64>", |
| "<SPECIAL_65>", |
| "<SPECIAL_66>", |
| "<SPECIAL_67>", |
| "<SPECIAL_68>", |
| "<SPECIAL_69>", |
| "<SPECIAL_70>", |
| "<SPECIAL_71>", |
| "<SPECIAL_72>", |
| "<SPECIAL_73>", |
| "<SPECIAL_74>", |
| "<SPECIAL_75>", |
| "<SPECIAL_76>", |
| "<SPECIAL_77>", |
| "<SPECIAL_78>", |
| "<SPECIAL_79>", |
| "<SPECIAL_80>", |
| "<SPECIAL_81>", |
| "<SPECIAL_82>", |
| "<SPECIAL_83>", |
| "<SPECIAL_84>", |
| "<SPECIAL_85>", |
| "<SPECIAL_86>", |
| "<SPECIAL_87>", |
| "<SPECIAL_88>", |
| "<SPECIAL_89>", |
| "<SPECIAL_90>", |
| "<SPECIAL_91>", |
| "<SPECIAL_92>", |
| "<SPECIAL_93>", |
| "<SPECIAL_94>", |
| "<SPECIAL_95>", |
| "<SPECIAL_96>", |
| "<SPECIAL_97>", |
| "<SPECIAL_98>", |
| "<SPECIAL_99>", |
| "<SPECIAL_100>", |
| "<SPECIAL_101>", |
| "<SPECIAL_102>", |
| "<SPECIAL_103>", |
| "<SPECIAL_104>", |
| "<SPECIAL_105>", |
| "<SPECIAL_106>", |
| "<SPECIAL_107>", |
| "<SPECIAL_108>", |
| "<SPECIAL_109>", |
| "<SPECIAL_110>", |
| "<SPECIAL_111>", |
| "<SPECIAL_112>", |
| "<SPECIAL_113>", |
| "<SPECIAL_114>", |
| "<SPECIAL_115>", |
| "<SPECIAL_116>", |
| "<SPECIAL_117>", |
| "<SPECIAL_118>", |
| "<SPECIAL_119>", |
| "<SPECIAL_120>", |
| "<SPECIAL_121>", |
| "<SPECIAL_122>", |
| "<SPECIAL_123>" |
| ], |
| "decoder_add_prefix_space": true, |
| "output_group": "tokenizer-lm-toks", |
| "output_dir_name": "nfc_clean_multi_plus2_repcap8_capped_hybrid_window_tuned_consv2_eudata4_gm100k_v131k_sp124_eng5g" |
| }, |
| "grouped_config_path": "/users/cmeister747/pa_tokenizers_branch/configs/parity_aware_config_grouped_fineweb2full_quota_tuned_consv2_eudata4_eng5g.json", |
| "vocab_size": 131072, |
| "min_frequency": 2, |
| "special_tokens": [ |
| "<unk>", |
| "<s>", |
| "</s>", |
| "<pad>", |
| "<|system_start|>", |
| "<|system_end|>", |
| "<|developer_start|>", |
| "<|developer_end|>", |
| "<|user_start|>", |
| "<|user_end|>", |
| "<|assistant_start|>", |
| "<|assistant_end|>", |
| "<|inner_prefix|>", |
| "<|inner_suffix|>", |
| "<|tools_prefix|>", |
| "<|tools_suffix|>", |
| "<|tool_output_start|>", |
| "<|tool_output_end|>", |
| "<|image|>", |
| "<|audio|>", |
| "<reflection>", |
| "</reflection>", |
| "<think>", |
| "</think>", |
| "<SPECIAL_24>", |
| "<SPECIAL_25>", |
| "<SPECIAL_26>", |
| "<SPECIAL_27>", |
| "<SPECIAL_28>", |
| "<SPECIAL_29>", |
| "<SPECIAL_30>", |
| "<SPECIAL_31>", |
| "<SPECIAL_32>", |
| "<SPECIAL_33>", |
| "<SPECIAL_34>", |
| "<SPECIAL_35>", |
| "<SPECIAL_36>", |
| "<SPECIAL_37>", |
| "<SPECIAL_38>", |
| "<SPECIAL_39>", |
| "<SPECIAL_40>", |
| "<SPECIAL_41>", |
| "<SPECIAL_42>", |
| "<SPECIAL_43>", |
| "<SPECIAL_44>", |
| "<SPECIAL_45>", |
| "<SPECIAL_46>", |
| "<SPECIAL_47>", |
| "<SPECIAL_48>", |
| "<SPECIAL_49>", |
| "<SPECIAL_50>", |
| "<SPECIAL_51>", |
| "<SPECIAL_52>", |
| "<SPECIAL_53>", |
| "<SPECIAL_54>", |
| "<SPECIAL_55>", |
| "<SPECIAL_56>", |
| "<SPECIAL_57>", |
| "<SPECIAL_58>", |
| "<SPECIAL_59>", |
| "<SPECIAL_60>", |
| "<SPECIAL_61>", |
| "<SPECIAL_62>", |
| "<SPECIAL_63>", |
| "<SPECIAL_64>", |
| "<SPECIAL_65>", |
| "<SPECIAL_66>", |
| "<SPECIAL_67>", |
| "<SPECIAL_68>", |
| "<SPECIAL_69>", |
| "<SPECIAL_70>", |
| "<SPECIAL_71>", |
| "<SPECIAL_72>", |
| "<SPECIAL_73>", |
| "<SPECIAL_74>", |
| "<SPECIAL_75>", |
| "<SPECIAL_76>", |
| "<SPECIAL_77>", |
| "<SPECIAL_78>", |
| "<SPECIAL_79>", |
| "<SPECIAL_80>", |
| "<SPECIAL_81>", |
| "<SPECIAL_82>", |
| "<SPECIAL_83>", |
| "<SPECIAL_84>", |
| "<SPECIAL_85>", |
| "<SPECIAL_86>", |
| "<SPECIAL_87>", |
| "<SPECIAL_88>", |
| "<SPECIAL_89>", |
| "<SPECIAL_90>", |
| "<SPECIAL_91>", |
| "<SPECIAL_92>", |
| "<SPECIAL_93>", |
| "<SPECIAL_94>", |
| "<SPECIAL_95>", |
| "<SPECIAL_96>", |
| "<SPECIAL_97>", |
| "<SPECIAL_98>", |
| "<SPECIAL_99>", |
| "<SPECIAL_100>", |
| "<SPECIAL_101>", |
| "<SPECIAL_102>", |
| "<SPECIAL_103>", |
| "<SPECIAL_104>", |
| "<SPECIAL_105>", |
| "<SPECIAL_106>", |
| "<SPECIAL_107>", |
| "<SPECIAL_108>", |
| "<SPECIAL_109>", |
| "<SPECIAL_110>", |
| "<SPECIAL_111>", |
| "<SPECIAL_112>", |
| "<SPECIAL_113>", |
| "<SPECIAL_114>", |
| "<SPECIAL_115>", |
| "<SPECIAL_116>", |
| "<SPECIAL_117>", |
| "<SPECIAL_118>", |
| "<SPECIAL_119>", |
| "<SPECIAL_120>", |
| "<SPECIAL_121>", |
| "<SPECIAL_122>", |
| "<SPECIAL_123>" |
| ], |
| "start_time": "2026-06-17T21:34:19.615240", |
| "elapsed_seconds": 5219.069353, |
| "slurm_job_id": "2554820", |
| "cli_args": [ |
| "/users/cmeister747/pa_tokenizers_branch/train_tokenizer.py", |
| "--variant", |
| "pa_bpe_nfc_clean_multi_plus2_repcap8_capped_hybrid_window_tuned_consv2_eudata4_gm100k_v131k_sp124_eng5g" |
| ], |
| "git": { |
| "commit": "5d8381af9ec1417b078f7dce82bac68873da915e", |
| "branch": "parity-aware-bpe", |
| "dirty": true |
| }, |
| "per_group_summary": { |
| "n_groups": 25, |
| "full_block_in": "tokenizers/pabpe-nfc-clean-fineweb2full-consv2-eudata4-gm100k/run_manifest.json" |
| } |
| }, |
| "registry_note": "PA-BPE hybrid global+window (gm100k), clean-multi regex; fineweb2full, consv2 + eudata4 variant data; norm=NFC; PA-BPE (hybrid global+window); vocab ~131,196; USED - S6f Contrast B (131k arm) + Apertus-2 development track; excluded from main paper panel" |
| }, |
| "pabpe-nfc-clean-fineweb2full-consv2-eusino-v2c-frde-kr120-gm130k-v200k": { |
| "slug": "pabpe-nfc-clean-fineweb2full-consv2-eusino-v2c-frde-kr120-gm130k-v200k", |
| "runs": [ |
| "full-128k-pabpe-nfc-clean-fineweb2full-consv2-eusino-v2c-frde-kr120-gm130k-v200k", |
| "full-128k-pabpe-nfc-clean-fineweb2full-consv2-eusino-v2c-frde-kr120-gm130k-v200k-mathcode-scratch" |
| ], |
| "n_runs": 2, |
| "redistributable": true, |
| "loadable_with": "transformers", |
| "hub_id": null, |
| "policy_note": "trained in this project", |
| "empirical": { |
| "model_type": "BPE", |
| "n_base_tokens": 200000, |
| "n_added_tokens": 124, |
| "vocab_total": 200124, |
| "added_tokens": [ |
| "<unk>", |
| "<s>", |
| "</s>", |
| "<pad>", |
| "<|system_start|>", |
| "<|system_end|>", |
| "<|developer_start|>", |
| "<|developer_end|>", |
| "<|user_start|>", |
| "<|user_end|>", |
| "<|assistant_start|>", |
| "<|assistant_end|>", |
| "<|inner_prefix|>", |
| "<|inner_suffix|>", |
| "<|tools_prefix|>", |
| "<|tools_suffix|>", |
| "<|tool_output_start|>", |
| "<|tool_output_end|>", |
| "<|image|>", |
| "<|audio|>", |
| "<reflection>", |
| "</reflection>", |
| "<think>", |
| "</think>", |
| "<pii-iban>", |
| "<pii-email>", |
| "<pii-ip>", |
| "<SPECIAL_27>", |
| "<SPECIAL_28>", |
| "<SPECIAL_29>", |
| "<SPECIAL_30>", |
| "<SPECIAL_31>", |
| "<SPECIAL_32>", |
| "<SPECIAL_33>", |
| "<SPECIAL_34>", |
| "<SPECIAL_35>", |
| "<SPECIAL_36>", |
| "<SPECIAL_37>", |
| "<SPECIAL_38>", |
| "<SPECIAL_39>", |
| "<SPECIAL_40>", |
| "<SPECIAL_41>", |
| "<SPECIAL_42>", |
| "<SPECIAL_43>", |
| "<SPECIAL_44>", |
| "<SPECIAL_45>", |
| "<SPECIAL_46>", |
| "<SPECIAL_47>", |
| "<SPECIAL_48>", |
| "<SPECIAL_49>", |
| "<SPECIAL_50>", |
| "<SPECIAL_51>", |
| "<SPECIAL_52>", |
| "<SPECIAL_53>", |
| "<SPECIAL_54>", |
| "<SPECIAL_55>", |
| "<SPECIAL_56>", |
| "<SPECIAL_57>", |
| "<SPECIAL_58>", |
| "<SPECIAL_59>", |
| "<SPECIAL_60>", |
| "<SPECIAL_61>", |
| "<SPECIAL_62>", |
| "<SPECIAL_63>", |
| "<SPECIAL_64>", |
| "<SPECIAL_65>", |
| "<SPECIAL_66>", |
| "<SPECIAL_67>", |
| "<SPECIAL_68>", |
| "<SPECIAL_69>", |
| "<SPECIAL_70>", |
| "<SPECIAL_71>", |
| "<SPECIAL_72>", |
| "<SPECIAL_73>", |
| "<SPECIAL_74>", |
| "<SPECIAL_75>", |
| "<SPECIAL_76>", |
| "<SPECIAL_77>", |
| "<SPECIAL_78>", |
| "<SPECIAL_79>", |
| "<SPECIAL_80>", |
| "<SPECIAL_81>", |
| "<SPECIAL_82>", |
| "<SPECIAL_83>", |
| "<SPECIAL_84>", |
| "<SPECIAL_85>", |
| "<SPECIAL_86>", |
| "<SPECIAL_87>", |
| "<SPECIAL_88>", |
| "<SPECIAL_89>", |
| "<SPECIAL_90>", |
| "<SPECIAL_91>", |
| "<SPECIAL_92>", |
| "<SPECIAL_93>", |
| "<SPECIAL_94>", |
| "<SPECIAL_95>", |
| "<SPECIAL_96>", |
| "<SPECIAL_97>", |
| "<SPECIAL_98>", |
| "<SPECIAL_99>", |
| "<SPECIAL_100>", |
| "<SPECIAL_101>", |
| "<SPECIAL_102>", |
| "<SPECIAL_103>", |
| "<SPECIAL_104>", |
| "<SPECIAL_105>", |
| "<SPECIAL_106>", |
| "<SPECIAL_107>", |
| "<SPECIAL_108>", |
| "<SPECIAL_109>", |
| "<SPECIAL_110>", |
| "<SPECIAL_111>", |
| "<SPECIAL_112>", |
| "<SPECIAL_113>", |
| "<SPECIAL_114>", |
| "<SPECIAL_115>", |
| "<SPECIAL_116>", |
| "<SPECIAL_117>", |
| "<SPECIAL_118>", |
| "<SPECIAL_119>", |
| "<SPECIAL_120>", |
| "<SPECIAL_121>", |
| "<SPECIAL_122>", |
| "<SPECIAL_123>" |
| ], |
| "normalizer": { |
| "type": "NFC" |
| }, |
| "pre_tokenizer": { |
| "type": "Sequence", |
| "pretokenizers": [ |
| { |
| "type": "Split", |
| "pattern": { |
| "Regex": "[ \\x{0F0B}\\x{0027}\\x{2019}]?(?<g>[^\\s\\p{N}])\\k<g>{7}|[ \\x{0F0B}\\x{0027}\\x{2019}]?[\\p{Lu}\\p{Lt}\\p{Lm}\\p{Lo}\\p{M}]*[\\p{Ll}\\p{Lm}\\p{Lo}\\p{M}]+|[ \\x{0F0B}\\x{0027}\\x{2019}]?[\\p{Lu}\\p{Lt}\\p{Lm}\\p{Lo}\\p{M}]+[\\p{Ll}\\p{Lm}\\p{Lo}\\p{M}]*|\\p{N}| ?[^\\s\\p{L}\\p{N}]{1,16}|\\s{0,16}[\\r\\n]{1,16}|\\s{1,16}(?!\\S)|\\s{1,16}" |
| }, |
| "behavior": "Isolated", |
| "invert": false |
| }, |
| { |
| "type": "ByteLevel", |
| "add_prefix_space": false, |
| "trim_offsets": true, |
| "use_regex": false |
| } |
| ] |
| }, |
| "has_post_processor": false, |
| "byte_fallback": false, |
| "bos_token": "<s>", |
| "bos_token_id": 1 |
| }, |
| "declared_experiments": null, |
| "declared_run_manifest": { |
| "variant": "pa_bpe_nfc_clean_multi_plus2_repcap8_capped_hybrid_window_tuned_consv2_eusino_v2c_frde_kr120_gm130k_v200k_sp124_eng5g", |
| "variant_config": { |
| "trainer": "parity-bpe", |
| "pretok": "clean_multi_plus2_repcap8_capped_nfc", |
| "variant": "window", |
| "global_merges": 130000, |
| "window_size": 100, |
| "alpha": 2.0, |
| "signal": "ratios", |
| "vocab_size": 200000, |
| "grouped_config": "configs/parity_aware_config_grouped_fineweb2full_quota_tuned_consv2_eusino_v2c_frde_kr120_eng5g.json", |
| "special_tokens_override": [ |
| "<unk>", |
| "<s>", |
| "</s>", |
| "<pad>", |
| "<|system_start|>", |
| "<|system_end|>", |
| "<|developer_start|>", |
| "<|developer_end|>", |
| "<|user_start|>", |
| "<|user_end|>", |
| "<|assistant_start|>", |
| "<|assistant_end|>", |
| "<|inner_prefix|>", |
| "<|inner_suffix|>", |
| "<|tools_prefix|>", |
| "<|tools_suffix|>", |
| "<|tool_output_start|>", |
| "<|tool_output_end|>", |
| "<|image|>", |
| "<|audio|>", |
| "<reflection>", |
| "</reflection>", |
| "<think>", |
| "</think>", |
| "<pii-iban>", |
| "<pii-email>", |
| "<pii-ip>", |
| "<SPECIAL_27>", |
| "<SPECIAL_28>", |
| "<SPECIAL_29>", |
| "<SPECIAL_30>", |
| "<SPECIAL_31>", |
| "<SPECIAL_32>", |
| "<SPECIAL_33>", |
| "<SPECIAL_34>", |
| "<SPECIAL_35>", |
| "<SPECIAL_36>", |
| "<SPECIAL_37>", |
| "<SPECIAL_38>", |
| "<SPECIAL_39>", |
| "<SPECIAL_40>", |
| "<SPECIAL_41>", |
| "<SPECIAL_42>", |
| "<SPECIAL_43>", |
| "<SPECIAL_44>", |
| "<SPECIAL_45>", |
| "<SPECIAL_46>", |
| "<SPECIAL_47>", |
| "<SPECIAL_48>", |
| "<SPECIAL_49>", |
| "<SPECIAL_50>", |
| "<SPECIAL_51>", |
| "<SPECIAL_52>", |
| "<SPECIAL_53>", |
| "<SPECIAL_54>", |
| "<SPECIAL_55>", |
| "<SPECIAL_56>", |
| "<SPECIAL_57>", |
| "<SPECIAL_58>", |
| "<SPECIAL_59>", |
| "<SPECIAL_60>", |
| "<SPECIAL_61>", |
| "<SPECIAL_62>", |
| "<SPECIAL_63>", |
| "<SPECIAL_64>", |
| "<SPECIAL_65>", |
| "<SPECIAL_66>", |
| "<SPECIAL_67>", |
| "<SPECIAL_68>", |
| "<SPECIAL_69>", |
| "<SPECIAL_70>", |
| "<SPECIAL_71>", |
| "<SPECIAL_72>", |
| "<SPECIAL_73>", |
| "<SPECIAL_74>", |
| "<SPECIAL_75>", |
| "<SPECIAL_76>", |
| "<SPECIAL_77>", |
| "<SPECIAL_78>", |
| "<SPECIAL_79>", |
| "<SPECIAL_80>", |
| "<SPECIAL_81>", |
| "<SPECIAL_82>", |
| "<SPECIAL_83>", |
| "<SPECIAL_84>", |
| "<SPECIAL_85>", |
| "<SPECIAL_86>", |
| "<SPECIAL_87>", |
| "<SPECIAL_88>", |
| "<SPECIAL_89>", |
| "<SPECIAL_90>", |
| "<SPECIAL_91>", |
| "<SPECIAL_92>", |
| "<SPECIAL_93>", |
| "<SPECIAL_94>", |
| "<SPECIAL_95>", |
| "<SPECIAL_96>", |
| "<SPECIAL_97>", |
| "<SPECIAL_98>", |
| "<SPECIAL_99>", |
| "<SPECIAL_100>", |
| "<SPECIAL_101>", |
| "<SPECIAL_102>", |
| "<SPECIAL_103>", |
| "<SPECIAL_104>", |
| "<SPECIAL_105>", |
| "<SPECIAL_106>", |
| "<SPECIAL_107>", |
| "<SPECIAL_108>", |
| "<SPECIAL_109>", |
| "<SPECIAL_110>", |
| "<SPECIAL_111>", |
| "<SPECIAL_112>", |
| "<SPECIAL_113>", |
| "<SPECIAL_114>", |
| "<SPECIAL_115>", |
| "<SPECIAL_116>", |
| "<SPECIAL_117>", |
| "<SPECIAL_118>", |
| "<SPECIAL_119>", |
| "<SPECIAL_120>", |
| "<SPECIAL_121>", |
| "<SPECIAL_122>", |
| "<SPECIAL_123>" |
| ], |
| "decoder_add_prefix_space": true, |
| "output_group": "tokenizer-lm-toks", |
| "output_dir_name": "nfc_clean_multi_plus2_repcap8_capped_hybrid_window_tuned_consv2_eusino_v2c_frde_kr120_gm130k_v200k_sp124_eng5g" |
| }, |
| "grouped_config_path": "/users/cmeister747/pa_tokenizers_branch/configs/parity_aware_config_grouped_fineweb2full_quota_tuned_consv2_eusino_v2c_frde_kr120_eng5g.json", |
| "vocab_size": 200000, |
| "min_frequency": 2, |
| "special_tokens": [ |
| "<unk>", |
| "<s>", |
| "</s>", |
| "<pad>", |
| "<|system_start|>", |
| "<|system_end|>", |
| "<|developer_start|>", |
| "<|developer_end|>", |
| "<|user_start|>", |
| "<|user_end|>", |
| "<|assistant_start|>", |
| "<|assistant_end|>", |
| "<|inner_prefix|>", |
| "<|inner_suffix|>", |
| "<|tools_prefix|>", |
| "<|tools_suffix|>", |
| "<|tool_output_start|>", |
| "<|tool_output_end|>", |
| "<|image|>", |
| "<|audio|>", |
| "<reflection>", |
| "</reflection>", |
| "<think>", |
| "</think>", |
| "<pii-iban>", |
| "<pii-email>", |
| "<pii-ip>", |
| "<SPECIAL_27>", |
| "<SPECIAL_28>", |
| "<SPECIAL_29>", |
| "<SPECIAL_30>", |
| "<SPECIAL_31>", |
| "<SPECIAL_32>", |
| "<SPECIAL_33>", |
| "<SPECIAL_34>", |
| "<SPECIAL_35>", |
| "<SPECIAL_36>", |
| "<SPECIAL_37>", |
| "<SPECIAL_38>", |
| "<SPECIAL_39>", |
| "<SPECIAL_40>", |
| "<SPECIAL_41>", |
| "<SPECIAL_42>", |
| "<SPECIAL_43>", |
| "<SPECIAL_44>", |
| "<SPECIAL_45>", |
| "<SPECIAL_46>", |
| "<SPECIAL_47>", |
| "<SPECIAL_48>", |
| "<SPECIAL_49>", |
| "<SPECIAL_50>", |
| "<SPECIAL_51>", |
| "<SPECIAL_52>", |
| "<SPECIAL_53>", |
| "<SPECIAL_54>", |
| "<SPECIAL_55>", |
| "<SPECIAL_56>", |
| "<SPECIAL_57>", |
| "<SPECIAL_58>", |
| "<SPECIAL_59>", |
| "<SPECIAL_60>", |
| "<SPECIAL_61>", |
| "<SPECIAL_62>", |
| "<SPECIAL_63>", |
| "<SPECIAL_64>", |
| "<SPECIAL_65>", |
| "<SPECIAL_66>", |
| "<SPECIAL_67>", |
| "<SPECIAL_68>", |
| "<SPECIAL_69>", |
| "<SPECIAL_70>", |
| "<SPECIAL_71>", |
| "<SPECIAL_72>", |
| "<SPECIAL_73>", |
| "<SPECIAL_74>", |
| "<SPECIAL_75>", |
| "<SPECIAL_76>", |
| "<SPECIAL_77>", |
| "<SPECIAL_78>", |
| "<SPECIAL_79>", |
| "<SPECIAL_80>", |
| "<SPECIAL_81>", |
| "<SPECIAL_82>", |
| "<SPECIAL_83>", |
| "<SPECIAL_84>", |
| "<SPECIAL_85>", |
| "<SPECIAL_86>", |
| "<SPECIAL_87>", |
| "<SPECIAL_88>", |
| "<SPECIAL_89>", |
| "<SPECIAL_90>", |
| "<SPECIAL_91>", |
| "<SPECIAL_92>", |
| "<SPECIAL_93>", |
| "<SPECIAL_94>", |
| "<SPECIAL_95>", |
| "<SPECIAL_96>", |
| "<SPECIAL_97>", |
| "<SPECIAL_98>", |
| "<SPECIAL_99>", |
| "<SPECIAL_100>", |
| "<SPECIAL_101>", |
| "<SPECIAL_102>", |
| "<SPECIAL_103>", |
| "<SPECIAL_104>", |
| "<SPECIAL_105>", |
| "<SPECIAL_106>", |
| "<SPECIAL_107>", |
| "<SPECIAL_108>", |
| "<SPECIAL_109>", |
| "<SPECIAL_110>", |
| "<SPECIAL_111>", |
| "<SPECIAL_112>", |
| "<SPECIAL_113>", |
| "<SPECIAL_114>", |
| "<SPECIAL_115>", |
| "<SPECIAL_116>", |
| "<SPECIAL_117>", |
| "<SPECIAL_118>", |
| "<SPECIAL_119>", |
| "<SPECIAL_120>", |
| "<SPECIAL_121>", |
| "<SPECIAL_122>", |
| "<SPECIAL_123>" |
| ], |
| "start_time": "2026-06-22T10:31:53.137663", |
| "elapsed_seconds": 6071.042492, |
| "slurm_job_id": "2592411", |
| "cli_args": [ |
| "train_tokenizer.py", |
| "--variant", |
| "pa_bpe_nfc_clean_multi_plus2_repcap8_capped_hybrid_window_tuned_consv2_eusino_v2c_frde_kr120_gm130k_v200k_sp124_eng5g" |
| ], |
| "git": { |
| "commit": "5d8381af9ec1417b078f7dce82bac68873da915e", |
| "branch": "parity-aware-bpe", |
| "dirty": true |
| }, |
| "per_group_summary": { |
| "n_groups": 25, |
| "full_block_in": "tokenizers/pabpe-nfc-clean-fineweb2full-consv2-eusino-v2c-frde-kr120-gm130k-v200k/run_manifest.json" |
| } |
| }, |
| "registry_note": "PA-BPE hybrid global+window (gm130k), clean-multi regex; fineweb2full, consv2 eusino_v2c + French/German boost + kr120 (+120MB Korean) data; norm=NFC; PA-BPE (hybrid global+window); vocab 200,064; the shipped preliminary_mul_200k recipe; roster #38; USED - S6f Contrast B (200k arm, n=2) + Apertus-2 track shipped candidate; excluded from main paper panel" |
| }, |
| "pabpe-nfc-clean-fineweb2full-consv2-eusino-v2c-gm120k-v200k": { |
| "slug": "pabpe-nfc-clean-fineweb2full-consv2-eusino-v2c-gm120k-v200k", |
| "runs": [ |
| "full-128k-pabpe-nfc-clean-fineweb2full-consv2-eusino-v2c-gm120k-v200k", |
| "full-128k-pabpe-nfc-clean-fineweb2full-consv2-eusino-v2c-gm120k-v200k-mathcode-scratch" |
| ], |
| "n_runs": 2, |
| "redistributable": true, |
| "loadable_with": "transformers", |
| "hub_id": null, |
| "policy_note": "trained in this project", |
| "empirical": { |
| "model_type": "BPE", |
| "n_base_tokens": 200000, |
| "n_added_tokens": 124, |
| "vocab_total": 200124, |
| "added_tokens": [ |
| "<unk>", |
| "<s>", |
| "</s>", |
| "<pad>", |
| "<|system_start|>", |
| "<|system_end|>", |
| "<|developer_start|>", |
| "<|developer_end|>", |
| "<|user_start|>", |
| "<|user_end|>", |
| "<|assistant_start|>", |
| "<|assistant_end|>", |
| "<|inner_prefix|>", |
| "<|inner_suffix|>", |
| "<|tools_prefix|>", |
| "<|tools_suffix|>", |
| "<|tool_output_start|>", |
| "<|tool_output_end|>", |
| "<|image|>", |
| "<|audio|>", |
| "<reflection>", |
| "</reflection>", |
| "<think>", |
| "</think>", |
| "<SPECIAL_24>", |
| "<SPECIAL_25>", |
| "<SPECIAL_26>", |
| "<SPECIAL_27>", |
| "<SPECIAL_28>", |
| "<SPECIAL_29>", |
| "<SPECIAL_30>", |
| "<SPECIAL_31>", |
| "<SPECIAL_32>", |
| "<SPECIAL_33>", |
| "<SPECIAL_34>", |
| "<SPECIAL_35>", |
| "<SPECIAL_36>", |
| "<SPECIAL_37>", |
| "<SPECIAL_38>", |
| "<SPECIAL_39>", |
| "<SPECIAL_40>", |
| "<SPECIAL_41>", |
| "<SPECIAL_42>", |
| "<SPECIAL_43>", |
| "<SPECIAL_44>", |
| "<SPECIAL_45>", |
| "<SPECIAL_46>", |
| "<SPECIAL_47>", |
| "<SPECIAL_48>", |
| "<SPECIAL_49>", |
| "<SPECIAL_50>", |
| "<SPECIAL_51>", |
| "<SPECIAL_52>", |
| "<SPECIAL_53>", |
| "<SPECIAL_54>", |
| "<SPECIAL_55>", |
| "<SPECIAL_56>", |
| "<SPECIAL_57>", |
| "<SPECIAL_58>", |
| "<SPECIAL_59>", |
| "<SPECIAL_60>", |
| "<SPECIAL_61>", |
| "<SPECIAL_62>", |
| "<SPECIAL_63>", |
| "<SPECIAL_64>", |
| "<SPECIAL_65>", |
| "<SPECIAL_66>", |
| "<SPECIAL_67>", |
| "<SPECIAL_68>", |
| "<SPECIAL_69>", |
| "<SPECIAL_70>", |
| "<SPECIAL_71>", |
| "<SPECIAL_72>", |
| "<SPECIAL_73>", |
| "<SPECIAL_74>", |
| "<SPECIAL_75>", |
| "<SPECIAL_76>", |
| "<SPECIAL_77>", |
| "<SPECIAL_78>", |
| "<SPECIAL_79>", |
| "<SPECIAL_80>", |
| "<SPECIAL_81>", |
| "<SPECIAL_82>", |
| "<SPECIAL_83>", |
| "<SPECIAL_84>", |
| "<SPECIAL_85>", |
| "<SPECIAL_86>", |
| "<SPECIAL_87>", |
| "<SPECIAL_88>", |
| "<SPECIAL_89>", |
| "<SPECIAL_90>", |
| "<SPECIAL_91>", |
| "<SPECIAL_92>", |
| "<SPECIAL_93>", |
| "<SPECIAL_94>", |
| "<SPECIAL_95>", |
| "<SPECIAL_96>", |
| "<SPECIAL_97>", |
| "<SPECIAL_98>", |
| "<SPECIAL_99>", |
| "<SPECIAL_100>", |
| "<SPECIAL_101>", |
| "<SPECIAL_102>", |
| "<SPECIAL_103>", |
| "<SPECIAL_104>", |
| "<SPECIAL_105>", |
| "<SPECIAL_106>", |
| "<SPECIAL_107>", |
| "<SPECIAL_108>", |
| "<SPECIAL_109>", |
| "<SPECIAL_110>", |
| "<SPECIAL_111>", |
| "<SPECIAL_112>", |
| "<SPECIAL_113>", |
| "<SPECIAL_114>", |
| "<SPECIAL_115>", |
| "<SPECIAL_116>", |
| "<SPECIAL_117>", |
| "<SPECIAL_118>", |
| "<SPECIAL_119>", |
| "<SPECIAL_120>", |
| "<SPECIAL_121>", |
| "<SPECIAL_122>", |
| "<SPECIAL_123>" |
| ], |
| "normalizer": { |
| "type": "NFC" |
| }, |
| "pre_tokenizer": { |
| "type": "Sequence", |
| "pretokenizers": [ |
| { |
| "type": "Split", |
| "pattern": { |
| "Regex": "[ \\x{0F0B}\\x{0027}\\x{2019}]?(?<g>[^\\s\\p{N}])\\k<g>{7}|[ \\x{0F0B}\\x{0027}\\x{2019}]?[\\p{Lu}\\p{Lt}\\p{Lm}\\p{Lo}\\p{M}]*[\\p{Ll}\\p{Lm}\\p{Lo}\\p{M}]+|[ \\x{0F0B}\\x{0027}\\x{2019}]?[\\p{Lu}\\p{Lt}\\p{Lm}\\p{Lo}\\p{M}]+[\\p{Ll}\\p{Lm}\\p{Lo}\\p{M}]*|\\p{N}| ?[^\\s\\p{L}\\p{N}]{1,16}|\\s{0,16}[\\r\\n]{1,16}|\\s{1,16}(?!\\S)|\\s{1,16}" |
| }, |
| "behavior": "Isolated", |
| "invert": false |
| }, |
| { |
| "type": "ByteLevel", |
| "add_prefix_space": false, |
| "trim_offsets": true, |
| "use_regex": false |
| } |
| ] |
| }, |
| "has_post_processor": true, |
| "byte_fallback": false, |
| "bos_token": "<s>", |
| "bos_token_id": 1 |
| }, |
| "declared_experiments": null, |
| "declared_run_manifest": null, |
| "registry_note": "PA-BPE hybrid global+window (gm120k), clean-multi regex; fineweb2full, consv2 eusino_v2c (European+Sinotibetan rebalance) data; norm=NFC; PA-BPE (hybrid global+window); vocab 200,000; the 200k predecessor LM evaluated for shipped candidate 'preliminary_mul_200k'; roster #25; USED - S6f Contrast B (200k arm, n=2) + Apertus-2 track shipped candidate; excluded from main paper panel" |
| }, |
| "pabpe-nfc-clean-fineweb2full-consv2-frde2-gm110k": { |
| "slug": "pabpe-nfc-clean-fineweb2full-consv2-frde2-gm110k", |
| "runs": [ |
| "full-128k-pabpe-nfc-clean-fineweb2full-consv2-frde2-gm110k", |
| "full-128k-pabpe-nfc-clean-fineweb2full-consv2-frde2-gm110k-mathcode-scratch" |
| ], |
| "n_runs": 2, |
| "redistributable": true, |
| "loadable_with": "transformers", |
| "hub_id": null, |
| "policy_note": "trained in this project", |
| "empirical": { |
| "model_type": "BPE", |
| "n_base_tokens": 131072, |
| "n_added_tokens": 124, |
| "vocab_total": 131196, |
| "added_tokens": [ |
| "<unk>", |
| "<s>", |
| "</s>", |
| "<pad>", |
| "<|system_start|>", |
| "<|system_end|>", |
| "<|developer_start|>", |
| "<|developer_end|>", |
| "<|user_start|>", |
| "<|user_end|>", |
| "<|assistant_start|>", |
| "<|assistant_end|>", |
| "<|inner_prefix|>", |
| "<|inner_suffix|>", |
| "<|tools_prefix|>", |
| "<|tools_suffix|>", |
| "<|tool_output_start|>", |
| "<|tool_output_end|>", |
| "<|image|>", |
| "<|audio|>", |
| "<reflection>", |
| "</reflection>", |
| "<think>", |
| "</think>", |
| "<SPECIAL_24>", |
| "<SPECIAL_25>", |
| "<SPECIAL_26>", |
| "<SPECIAL_27>", |
| "<SPECIAL_28>", |
| "<SPECIAL_29>", |
| "<SPECIAL_30>", |
| "<SPECIAL_31>", |
| "<SPECIAL_32>", |
| "<SPECIAL_33>", |
| "<SPECIAL_34>", |
| "<SPECIAL_35>", |
| "<SPECIAL_36>", |
| "<SPECIAL_37>", |
| "<SPECIAL_38>", |
| "<SPECIAL_39>", |
| "<SPECIAL_40>", |
| "<SPECIAL_41>", |
| "<SPECIAL_42>", |
| "<SPECIAL_43>", |
| "<SPECIAL_44>", |
| "<SPECIAL_45>", |
| "<SPECIAL_46>", |
| "<SPECIAL_47>", |
| "<SPECIAL_48>", |
| "<SPECIAL_49>", |
| "<SPECIAL_50>", |
| "<SPECIAL_51>", |
| "<SPECIAL_52>", |
| "<SPECIAL_53>", |
| "<SPECIAL_54>", |
| "<SPECIAL_55>", |
| "<SPECIAL_56>", |
| "<SPECIAL_57>", |
| "<SPECIAL_58>", |
| "<SPECIAL_59>", |
| "<SPECIAL_60>", |
| "<SPECIAL_61>", |
| "<SPECIAL_62>", |
| "<SPECIAL_63>", |
| "<SPECIAL_64>", |
| "<SPECIAL_65>", |
| "<SPECIAL_66>", |
| "<SPECIAL_67>", |
| "<SPECIAL_68>", |
| "<SPECIAL_69>", |
| "<SPECIAL_70>", |
| "<SPECIAL_71>", |
| "<SPECIAL_72>", |
| "<SPECIAL_73>", |
| "<SPECIAL_74>", |
| "<SPECIAL_75>", |
| "<SPECIAL_76>", |
| "<SPECIAL_77>", |
| "<SPECIAL_78>", |
| "<SPECIAL_79>", |
| "<SPECIAL_80>", |
| "<SPECIAL_81>", |
| "<SPECIAL_82>", |
| "<SPECIAL_83>", |
| "<SPECIAL_84>", |
| "<SPECIAL_85>", |
| "<SPECIAL_86>", |
| "<SPECIAL_87>", |
| "<SPECIAL_88>", |
| "<SPECIAL_89>", |
| "<SPECIAL_90>", |
| "<SPECIAL_91>", |
| "<SPECIAL_92>", |
| "<SPECIAL_93>", |
| "<SPECIAL_94>", |
| "<SPECIAL_95>", |
| "<SPECIAL_96>", |
| "<SPECIAL_97>", |
| "<SPECIAL_98>", |
| "<SPECIAL_99>", |
| "<SPECIAL_100>", |
| "<SPECIAL_101>", |
| "<SPECIAL_102>", |
| "<SPECIAL_103>", |
| "<SPECIAL_104>", |
| "<SPECIAL_105>", |
| "<SPECIAL_106>", |
| "<SPECIAL_107>", |
| "<SPECIAL_108>", |
| "<SPECIAL_109>", |
| "<SPECIAL_110>", |
| "<SPECIAL_111>", |
| "<SPECIAL_112>", |
| "<SPECIAL_113>", |
| "<SPECIAL_114>", |
| "<SPECIAL_115>", |
| "<SPECIAL_116>", |
| "<SPECIAL_117>", |
| "<SPECIAL_118>", |
| "<SPECIAL_119>", |
| "<SPECIAL_120>", |
| "<SPECIAL_121>", |
| "<SPECIAL_122>", |
| "<SPECIAL_123>" |
| ], |
| "normalizer": { |
| "type": "NFC" |
| }, |
| "pre_tokenizer": { |
| "type": "Sequence", |
| "pretokenizers": [ |
| { |
| "type": "Split", |
| "pattern": { |
| "Regex": "[ \\x{0F0B}\\x{0027}\\x{2019}]?(?<g>[^\\s\\p{N}])\\k<g>{7}|[ \\x{0F0B}\\x{0027}\\x{2019}]?[\\p{Lu}\\p{Lt}\\p{Lm}\\p{Lo}\\p{M}]*[\\p{Ll}\\p{Lm}\\p{Lo}\\p{M}]+|[ \\x{0F0B}\\x{0027}\\x{2019}]?[\\p{Lu}\\p{Lt}\\p{Lm}\\p{Lo}\\p{M}]+[\\p{Ll}\\p{Lm}\\p{Lo}\\p{M}]*|\\p{N}| ?[^\\s\\p{L}\\p{N}]{1,16}|\\s{0,16}[\\r\\n]{1,16}|\\s{1,16}(?!\\S)|\\s{1,16}" |
| }, |
| "behavior": "Isolated", |
| "invert": false |
| }, |
| { |
| "type": "ByteLevel", |
| "add_prefix_space": false, |
| "trim_offsets": true, |
| "use_regex": false |
| } |
| ] |
| }, |
| "has_post_processor": true, |
| "byte_fallback": false, |
| "bos_token": "<s>", |
| "bos_token_id": 1 |
| }, |
| "declared_experiments": null, |
| "declared_run_manifest": null, |
| "registry_note": "PA-BPE hybrid global+window (gm110k), clean-multi regex; fineweb2full, consv2 frde2 (more French/German, less Chinese) data; norm=NFC; PA-BPE (hybrid global+window); vocab 131,072 + 124 specials; shipped Apertus-v2 candidate 'preliminary_euh' (EU-dense, least fair of the four); roster #23; USED - Apertus-2 track shipped candidate; S6f Contrast B (131k arm); excluded from main paper panel" |
| }, |
| "pabpe-nfc-clean-fineweb2full-consv2-plus3-repcap8": { |
| "slug": "pabpe-nfc-clean-fineweb2full-consv2-plus3-repcap8", |
| "runs": [ |
| "full-128k-pabpe-nfc-clean-fineweb2full-consv2-plus3-repcap8", |
| "full-128k-pabpe-nfc-clean-fineweb2full-consv2-plus3-repcap8-mathcode-scratch" |
| ], |
| "n_runs": 2, |
| "redistributable": true, |
| "loadable_with": "transformers", |
| "hub_id": null, |
| "policy_note": "trained in this project", |
| "empirical": { |
| "model_type": "BPE", |
| "n_base_tokens": 131017, |
| "n_added_tokens": 124, |
| "vocab_total": 131141, |
| "added_tokens": [ |
| "<unk>", |
| "<s>", |
| "</s>", |
| "<pad>", |
| "<|system_start|>", |
| "<|system_end|>", |
| "<|developer_start|>", |
| "<|developer_end|>", |
| "<|user_start|>", |
| "<|user_end|>", |
| "<|assistant_start|>", |
| "<|assistant_end|>", |
| "<|inner_prefix|>", |
| "<|inner_suffix|>", |
| "<|tools_prefix|>", |
| "<|tools_suffix|>", |
| "<|tool_output_start|>", |
| "<|tool_output_end|>", |
| "<|image|>", |
| "<|audio|>", |
| "<reflection>", |
| "</reflection>", |
| "<think>", |
| "</think>", |
| "<SPECIAL_24>", |
| "<SPECIAL_25>", |
| "<SPECIAL_26>", |
| "<SPECIAL_27>", |
| "<SPECIAL_28>", |
| "<SPECIAL_29>", |
| "<SPECIAL_30>", |
| "<SPECIAL_31>", |
| "<SPECIAL_32>", |
| "<SPECIAL_33>", |
| "<SPECIAL_34>", |
| "<SPECIAL_35>", |
| "<SPECIAL_36>", |
| "<SPECIAL_37>", |
| "<SPECIAL_38>", |
| "<SPECIAL_39>", |
| "<SPECIAL_40>", |
| "<SPECIAL_41>", |
| "<SPECIAL_42>", |
| "<SPECIAL_43>", |
| "<SPECIAL_44>", |
| "<SPECIAL_45>", |
| "<SPECIAL_46>", |
| "<SPECIAL_47>", |
| "<SPECIAL_48>", |
| "<SPECIAL_49>", |
| "<SPECIAL_50>", |
| "<SPECIAL_51>", |
| "<SPECIAL_52>", |
| "<SPECIAL_53>", |
| "<SPECIAL_54>", |
| "<SPECIAL_55>", |
| "<SPECIAL_56>", |
| "<SPECIAL_57>", |
| "<SPECIAL_58>", |
| "<SPECIAL_59>", |
| "<SPECIAL_60>", |
| "<SPECIAL_61>", |
| "<SPECIAL_62>", |
| "<SPECIAL_63>", |
| "<SPECIAL_64>", |
| "<SPECIAL_65>", |
| "<SPECIAL_66>", |
| "<SPECIAL_67>", |
| "<SPECIAL_68>", |
| "<SPECIAL_69>", |
| "<SPECIAL_70>", |
| "<SPECIAL_71>", |
| "<SPECIAL_72>", |
| "<SPECIAL_73>", |
| "<SPECIAL_74>", |
| "<SPECIAL_75>", |
| "<SPECIAL_76>", |
| "<SPECIAL_77>", |
| "<SPECIAL_78>", |
| "<SPECIAL_79>", |
| "<SPECIAL_80>", |
| "<SPECIAL_81>", |
| "<SPECIAL_82>", |
| "<SPECIAL_83>", |
| "<SPECIAL_84>", |
| "<SPECIAL_85>", |
| "<SPECIAL_86>", |
| "<SPECIAL_87>", |
| "<SPECIAL_88>", |
| "<SPECIAL_89>", |
| "<SPECIAL_90>", |
| "<SPECIAL_91>", |
| "<SPECIAL_92>", |
| "<SPECIAL_93>", |
| "<SPECIAL_94>", |
| "<SPECIAL_95>", |
| "<SPECIAL_96>", |
| "<SPECIAL_97>", |
| "<SPECIAL_98>", |
| "<SPECIAL_99>", |
| "<SPECIAL_100>", |
| "<SPECIAL_101>", |
| "<SPECIAL_102>", |
| "<SPECIAL_103>", |
| "<SPECIAL_104>", |
| "<SPECIAL_105>", |
| "<SPECIAL_106>", |
| "<SPECIAL_107>", |
| "<SPECIAL_108>", |
| "<SPECIAL_109>", |
| "<SPECIAL_110>", |
| "<SPECIAL_111>", |
| "<SPECIAL_112>", |
| "<SPECIAL_113>", |
| "<SPECIAL_114>", |
| "<SPECIAL_115>", |
| "<SPECIAL_116>", |
| "<SPECIAL_117>", |
| "<SPECIAL_118>", |
| "<SPECIAL_119>", |
| "<SPECIAL_120>", |
| "<SPECIAL_121>", |
| "<SPECIAL_122>", |
| "<SPECIAL_123>" |
| ], |
| "normalizer": { |
| "type": "NFC" |
| }, |
| "pre_tokenizer": { |
| "type": "Sequence", |
| "pretokenizers": [ |
| { |
| "type": "Split", |
| "pattern": { |
| "Regex": "[ \\x{0F0B}\\x{0027}\\x{2019}]?(?<g>[^\\s])\\k<g>{7}|[ \\x{0F0B}\\x{0027}\\x{2019}]?(?:(?!(?<u1>.)\\k<u1>{8})[\\p{Lu}\\p{Lt}\\p{Lm}\\p{Lo}\\p{M}])*(?:(?!(?<l1>.)\\k<l1>{8})[\\p{Ll}\\p{Lm}\\p{Lo}\\p{M}])+(?:[\\x{0027}\\x{2019}](?!\\p{L}))?|[ \\x{0F0B}\\x{0027}\\x{2019}]?(?:(?!(?<u2>.)\\k<u2>{8})[\\p{Lu}\\p{Lt}\\p{Lm}\\p{Lo}\\p{M}])+(?:(?!(?<l2>.)\\k<l2>{8})[\\p{Ll}\\p{Lm}\\p{Lo}\\p{M}])*(?:[\\x{0027}\\x{2019}](?!\\p{L}))?|\\p{N}| ?(?:(?!(?<p>.)\\k<p>{8})[^\\s\\p{L}\\p{N}]){1,16}|\\s{0,16}[\\r\\n]{1,16}|\\s{1,16}(?!\\S)|\\s{1,16}" |
| }, |
| "behavior": "Isolated", |
| "invert": false |
| }, |
| { |
| "type": "ByteLevel", |
| "add_prefix_space": false, |
| "trim_offsets": true, |
| "use_regex": false |
| } |
| ] |
| }, |
| "has_post_processor": false, |
| "byte_fallback": false, |
| "bos_token": "<s>", |
| "bos_token_id": 1 |
| }, |
| "declared_experiments": null, |
| "declared_run_manifest": null, |
| "registry_note": "PA-BPE hybrid global+window, clean-multi plus3 regex + repcap8 (8+ char run cap); fineweb2full, consv2 baseline (reparam ratios) data; norm=NFC; PA-BPE (hybrid global+window); vocab 131,017 (55-token ParityBpe shortfall vs 131,072 target); shipped Apertus-v2 candidate 'preliminary_mul' (balanced baseline, fairest of the four); roster #24; USED - Apertus-2 track shipped candidate; S6f Contrast B (131k arm); excluded from main paper panel" |
| }, |
| "pabpe-nfc-clean-fineweb2full-consv2-prelim-mul-v131072": { |
| "slug": "pabpe-nfc-clean-fineweb2full-consv2-prelim-mul-v131072", |
| "runs": [ |
| "full-128k-pabpe-nfc-clean-fineweb2full-consv2-prelim-mul-v131072", |
| "full-128k-pabpe-nfc-clean-fineweb2full-consv2-prelim-mul-v131072-mathcode-scratch" |
| ], |
| "n_runs": 2, |
| "redistributable": true, |
| "loadable_with": "transformers", |
| "hub_id": null, |
| "policy_note": "trained in this project", |
| "empirical": { |
| "model_type": "BPE", |
| "n_base_tokens": 131072, |
| "n_added_tokens": 124, |
| "vocab_total": 131196, |
| "added_tokens": [ |
| "<unk>", |
| "<s>", |
| "</s>", |
| "<pad>", |
| "<|system_start|>", |
| "<|system_end|>", |
| "<|developer_start|>", |
| "<|developer_end|>", |
| "<|user_start|>", |
| "<|user_end|>", |
| "<|assistant_start|>", |
| "<|assistant_end|>", |
| "<|inner_prefix|>", |
| "<|inner_suffix|>", |
| "<|tools_prefix|>", |
| "<|tools_suffix|>", |
| "<|tool_output_start|>", |
| "<|tool_output_end|>", |
| "<|image|>", |
| "<|audio|>", |
| "<reflection>", |
| "</reflection>", |
| "<think>", |
| "</think>", |
| "<iban-pii>", |
| "<email-pii>", |
| "<ip-pii>", |
| "<SPECIAL_27>", |
| "<SPECIAL_28>", |
| "<SPECIAL_29>", |
| "<SPECIAL_30>", |
| "<SPECIAL_31>", |
| "<SPECIAL_32>", |
| "<SPECIAL_33>", |
| "<SPECIAL_34>", |
| "<SPECIAL_35>", |
| "<SPECIAL_36>", |
| "<SPECIAL_37>", |
| "<SPECIAL_38>", |
| "<SPECIAL_39>", |
| "<SPECIAL_40>", |
| "<SPECIAL_41>", |
| "<SPECIAL_42>", |
| "<SPECIAL_43>", |
| "<SPECIAL_44>", |
| "<SPECIAL_45>", |
| "<SPECIAL_46>", |
| "<SPECIAL_47>", |
| "<SPECIAL_48>", |
| "<SPECIAL_49>", |
| "<SPECIAL_50>", |
| "<SPECIAL_51>", |
| "<SPECIAL_52>", |
| "<SPECIAL_53>", |
| "<SPECIAL_54>", |
| "<SPECIAL_55>", |
| "<SPECIAL_56>", |
| "<SPECIAL_57>", |
| "<SPECIAL_58>", |
| "<SPECIAL_59>", |
| "<SPECIAL_60>", |
| "<SPECIAL_61>", |
| "<SPECIAL_62>", |
| "<SPECIAL_63>", |
| "<SPECIAL_64>", |
| "<SPECIAL_65>", |
| "<SPECIAL_66>", |
| "<SPECIAL_67>", |
| "<SPECIAL_68>", |
| "<SPECIAL_69>", |
| "<SPECIAL_70>", |
| "<SPECIAL_71>", |
| "<SPECIAL_72>", |
| "<SPECIAL_73>", |
| "<SPECIAL_74>", |
| "<SPECIAL_75>", |
| "<SPECIAL_76>", |
| "<SPECIAL_77>", |
| "<SPECIAL_78>", |
| "<SPECIAL_79>", |
| "<SPECIAL_80>", |
| "<SPECIAL_81>", |
| "<SPECIAL_82>", |
| "<SPECIAL_83>", |
| "<SPECIAL_84>", |
| "<SPECIAL_85>", |
| "<SPECIAL_86>", |
| "<SPECIAL_87>", |
| "<SPECIAL_88>", |
| "<SPECIAL_89>", |
| "<SPECIAL_90>", |
| "<SPECIAL_91>", |
| "<SPECIAL_92>", |
| "<SPECIAL_93>", |
| "<SPECIAL_94>", |
| "<SPECIAL_95>", |
| "<SPECIAL_96>", |
| "<SPECIAL_97>", |
| "<SPECIAL_98>", |
| "<SPECIAL_99>", |
| "<SPECIAL_100>", |
| "<SPECIAL_101>", |
| "<SPECIAL_102>", |
| "<SPECIAL_103>", |
| "<SPECIAL_104>", |
| "<SPECIAL_105>", |
| "<SPECIAL_106>", |
| "<SPECIAL_107>", |
| "<SPECIAL_108>", |
| "<SPECIAL_109>", |
| "<SPECIAL_110>", |
| "<SPECIAL_111>", |
| "<SPECIAL_112>", |
| "<SPECIAL_113>", |
| "<SPECIAL_114>", |
| "<SPECIAL_115>", |
| "<SPECIAL_116>", |
| "<SPECIAL_117>", |
| "<SPECIAL_118>", |
| "<SPECIAL_119>", |
| "<SPECIAL_120>", |
| "<SPECIAL_121>", |
| "<SPECIAL_122>", |
| "<SPECIAL_123>" |
| ], |
| "normalizer": { |
| "type": "NFC" |
| }, |
| "pre_tokenizer": { |
| "type": "Sequence", |
| "pretokenizers": [ |
| { |
| "type": "Split", |
| "pattern": { |
| "Regex": "[ \\x{0F0B}\\x{0027}\\x{2019}]?(?<g>[^\\s])\\k<g>{7}|[ \\x{0F0B}\\x{0027}\\x{2019}]?[\\p{Lu}\\p{Lt}\\p{Lm}\\p{Lo}\\p{M}]*[\\p{Ll}\\p{Lm}\\p{Lo}\\p{M}]+(?:[\\x{0027}\\x{2019}](?!\\p{L}))?|[ \\x{0F0B}\\x{0027}\\x{2019}]?[\\p{Lu}\\p{Lt}\\p{Lm}\\p{Lo}\\p{M}]+[\\p{Ll}\\p{Lm}\\p{Lo}\\p{M}]*(?:[\\x{0027}\\x{2019}](?!\\p{L}))?|\\p{N}| ?[^\\s\\p{L}\\p{N}]{1,16}|\\s{0,16}[\\r\\n]{1,16}|\\s{1,16}(?!\\S)|\\s{1,16}" |
| }, |
| "behavior": "Isolated", |
| "invert": false |
| }, |
| { |
| "type": "ByteLevel", |
| "add_prefix_space": false, |
| "trim_offsets": true, |
| "use_regex": false |
| } |
| ] |
| }, |
| "has_post_processor": true, |
| "byte_fallback": false, |
| "bos_token": "<s>", |
| "bos_token_id": 1 |
| }, |
| "declared_experiments": null, |
| "declared_run_manifest": null, |
| "registry_note": "PA-BPE hybrid global+window, clean-multi plus3 + repcap8; fineweb2full, consv2 reparam (preliminary_mul) data; norm=NFC; PA-BPE (hybrid global+window); vocab 131,072; the preliminary_mul build registered under an explicit slug (distinct md5 from the roster #24 twin); USED - S6h code analysis is not applicable (20B-only); Apertus-2 track; excluded from main paper panel" |
| }, |
| "pabpe-nfc-clean-fineweb2full-hybridwindow": { |
| "slug": "pabpe-nfc-clean-fineweb2full-hybridwindow", |
| "runs": [ |
| "full-128k-pabpe-nfc-clean-fineweb2full-hybridwindow" |
| ], |
| "n_runs": 1, |
| "redistributable": true, |
| "loadable_with": "transformers", |
| "hub_id": null, |
| "policy_note": "trained in this project", |
| "empirical": { |
| "model_type": "BPE", |
| "n_base_tokens": 127831, |
| "n_added_tokens": 4, |
| "vocab_total": 127835, |
| "added_tokens": [ |
| "<s>", |
| "</s>", |
| "<unk>", |
| "<pad>" |
| ], |
| "normalizer": { |
| "type": "NFC" |
| }, |
| "pre_tokenizer": { |
| "type": "Sequence", |
| "pretokenizers": [ |
| { |
| "type": "Split", |
| "pattern": { |
| "Regex": "[ ]?[\\p{Lu}\\p{Lt}\\p{Lm}\\p{Lo}\\p{M}]*[\\p{Ll}\\p{Lm}\\p{Lo}\\p{M}]+|[ ]?[\\p{Lu}\\p{Lt}\\p{Lm}\\p{Lo}\\p{M}]+[\\p{Ll}\\p{Lm}\\p{Lo}\\p{M}]*|\\p{N}| ?[^\\s\\p{L}\\p{N}]+|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+" |
| }, |
| "behavior": "Isolated", |
| "invert": false |
| }, |
| { |
| "type": "ByteLevel", |
| "add_prefix_space": false, |
| "trim_offsets": true, |
| "use_regex": false |
| } |
| ] |
| }, |
| "has_post_processor": false, |
| "byte_fallback": false, |
| "bos_token": "<s>", |
| "bos_token_id": 127831 |
| }, |
| "declared_experiments": null, |
| "declared_run_manifest": { |
| "variant": "pa_bpe_nfc_clean_multi_fineweb2full_hybrid_window", |
| "variant_config": { |
| "trainer": "parity-bpe", |
| "pretok": "clean_multi_nfc", |
| "variant": "window", |
| "global_merges": 64000, |
| "window_size": 100, |
| "alpha": 2.0, |
| "signal": "ratios", |
| "grouped_config": "configs/parity_aware_config_grouped_fineweb2full_quota.json", |
| "special_tokens_override": [], |
| "decoder_add_prefix_space": true, |
| "output_group": "tokenizer-lm-toks", |
| "output_dir_name": "nfc_clean_multi_fineweb2full_hybrid_window", |
| "post_training_special_tokens": [ |
| "<s>", |
| "</s>", |
| "<unk>", |
| "<pad>" |
| ] |
| }, |
| "grouped_config_path": "/users/cmeister747/pa_tokenizers_branch/configs/parity_aware_config_grouped_fineweb2full_quota.json", |
| "vocab_size": 128000, |
| "min_frequency": 2, |
| "special_tokens": [], |
| "start_time": "2026-05-22T07:43:23.362987", |
| "elapsed_seconds": 7228.839727, |
| "slurm_job_id": "2328521", |
| "cli_args": [ |
| "/users/cmeister747/pa_tokenizers_branch/train_tokenizer.py", |
| "--variant", |
| "pa_bpe_nfc_clean_multi_fineweb2full_hybrid_window" |
| ], |
| "git": { |
| "commit": "5d8381af9ec1417b078f7dce82bac68873da915e", |
| "branch": "parity-aware-bpe", |
| "dirty": false |
| }, |
| "per_group_summary": { |
| "n_groups": 25, |
| "full_block_in": "tokenizers/pabpe-nfc-clean-fineweb2full-hybridwindow/run_manifest.json" |
| } |
| }, |
| "registry_note": "PA-BPE hybrid global+window (gm=64000,W=100,a=2.0) with clean-multi regex; fineweb2full data; norm=NFC; PA-BPE (hybrid global+window); vocab 127,835; hybrid-window pulls val_bpb/BLiMP/code_bpb back toward BPE-like at the cost of most of the all-FLORES gain; USED - S5b fineweb2full analysis; fineweb2full data variant, excluded from main paper panel" |
| }, |
| "pabpe-nfc-clean-fineweb2full-plus2-capped-tuned-hybridwindow": { |
| "slug": "pabpe-nfc-clean-fineweb2full-plus2-capped-tuned-hybridwindow", |
| "runs": [ |
| "full-128k-pabpe-nfc-clean-fineweb2full-plus2-capped-tuned-hybridwindow", |
| "full-128k-pabpe-nfc-clean-fineweb2full-plus2-capped-tuned-hybridwindow-mathcode-scratch" |
| ], |
| "n_runs": 2, |
| "redistributable": true, |
| "loadable_with": "transformers", |
| "hub_id": null, |
| "policy_note": "trained in this project", |
| "empirical": { |
| "model_type": "BPE", |
| "n_base_tokens": 127831, |
| "n_added_tokens": 4, |
| "vocab_total": 127835, |
| "added_tokens": [ |
| "<s>", |
| "</s>", |
| "<unk>", |
| "<pad>" |
| ], |
| "normalizer": { |
| "type": "NFC" |
| }, |
| "pre_tokenizer": { |
| "type": "Sequence", |
| "pretokenizers": [ |
| { |
| "type": "Split", |
| "pattern": { |
| "Regex": "[ \\x{0F0B}\\x{0027}\\x{2019}]?[\\p{Lu}\\p{Lt}\\p{Lm}\\p{Lo}\\p{M}]*[\\p{Ll}\\p{Lm}\\p{Lo}\\p{M}]+|[ \\x{0F0B}\\x{0027}\\x{2019}]?[\\p{Lu}\\p{Lt}\\p{Lm}\\p{Lo}\\p{M}]+[\\p{Ll}\\p{Lm}\\p{Lo}\\p{M}]*|\\p{N}| ?[^\\s\\p{L}\\p{N}]{1,16}|\\s{0,16}[\\r\\n]{1,16}|\\s{1,16}(?!\\S)|\\s{1,16}" |
| }, |
| "behavior": "Isolated", |
| "invert": false |
| }, |
| { |
| "type": "ByteLevel", |
| "add_prefix_space": false, |
| "trim_offsets": true, |
| "use_regex": false |
| } |
| ] |
| }, |
| "has_post_processor": false, |
| "byte_fallback": false, |
| "bos_token": "<s>", |
| "bos_token_id": 127831 |
| }, |
| "declared_experiments": null, |
| "declared_run_manifest": { |
| "variant": "pa_bpe_nfc_clean_multi_plus2_capped_hybrid_window_tuned", |
| "variant_config": { |
| "trainer": "parity-bpe", |
| "pretok": "clean_multi_plus2_capped_nfc", |
| "variant": "window", |
| "global_merges": 64000, |
| "window_size": 100, |
| "alpha": 2.0, |
| "signal": "ratios", |
| "grouped_config": "configs/parity_aware_config_grouped_fineweb2full_quota_tuned.json", |
| "special_tokens_override": [], |
| "decoder_add_prefix_space": true, |
| "output_group": "tokenizer-lm-toks", |
| "output_dir_name": "nfc_clean_multi_plus2_capped_hybrid_window_tuned", |
| "post_training_special_tokens": [ |
| "<s>", |
| "</s>", |
| "<unk>", |
| "<pad>" |
| ] |
| }, |
| "grouped_config_path": "/users/cmeister747/pa_tokenizers_branch/configs/parity_aware_config_grouped_fineweb2full_quota_tuned.json", |
| "vocab_size": 128000, |
| "min_frequency": 2, |
| "special_tokens": [], |
| "start_time": "2026-06-02T17:49:32.228735", |
| "elapsed_seconds": 7130.059512, |
| "slurm_job_id": "2458438", |
| "cli_args": [ |
| "/users/cmeister747/pa_tokenizers_branch/train_tokenizer.py", |
| "--variant", |
| "pa_bpe_nfc_clean_multi_plus2_capped_hybrid_window_tuned" |
| ], |
| "git": { |
| "commit": "5d8381af9ec1417b078f7dce82bac68873da915e", |
| "branch": "parity-aware-bpe", |
| "dirty": false |
| }, |
| "per_group_summary": { |
| "n_groups": 25, |
| "full_block_in": "tokenizers/pabpe-nfc-clean-fineweb2full-plus2-capped-tuned-hybridwindow/run_manifest.json" |
| } |
| }, |
| "registry_note": "PA-BPE hybrid global+window, clean-multi plus2 regex; fineweb2full, capped + tuned (v5) data; norm=NFC; PA-BPE (hybrid global+window); vocab ~131,196; plus2 tuned-data baseline; USED - S6f Contrast B (131k arm) + Apertus-2 track; excluded from main paper panel" |
| }, |
| "pabpe-nfc-clean-fineweb2full-plus2-capped-tuned-hybridwindow-consv2-tailcuts-gm90k-v130-sp-eng5g": { |
| "slug": "pabpe-nfc-clean-fineweb2full-plus2-capped-tuned-hybridwindow-consv2-tailcuts-gm90k-v130-sp-eng5g", |
| "runs": [ |
| "full-128k-pabpe-nfc-clean-fineweb2full-plus2-capped-tuned-hybridwindow-consv2-tailcuts-gm90k-v130-sp-eng5g", |
| "full-128k-pabpe-nfc-clean-fineweb2full-plus2-capped-tuned-hybridwindow-consv2-tailcuts-gm90k-v130-sp-eng5g-mathcode-scratch" |
| ], |
| "n_runs": 2, |
| "redistributable": true, |
| "loadable_with": "transformers", |
| "hub_id": null, |
| "policy_note": "trained in this project", |
| "empirical": { |
| "model_type": "BPE", |
| "n_base_tokens": 130929, |
| "n_added_tokens": 99, |
| "vocab_total": 131028, |
| "added_tokens": [ |
| "<unk>", |
| "<s>", |
| "</s>", |
| "<pad>", |
| "[/INST]", |
| "[AVAILABLE_TOOLS]", |
| "[/AVAILABLE_TOOLS]", |
| "[TOOL_RESULTS]", |
| "[/TOOL_RESULTS]", |
| "[TOOL_CALLS]", |
| "<SPECIAL_10>", |
| "[PREFIX]", |
| "[MIDDLE]", |
| "[SUFFIX]", |
| "\\begin{", |
| "\\end{", |
| "\\text{", |
| "\\boxed{", |
| "<filename>", |
| "<gh_stars>", |
| "<issue_start>", |
| "<issue_comment>", |
| "<issue_closed>", |
| "<jupyter_start>", |
| "<jupyter_text>", |
| "<jupyter_code>", |
| "<jupyter_output>", |
| "<empty_output>", |
| "<commit_before>", |
| "<commit_msg>", |
| "<commit_after>", |
| "<reponame>", |
| "<think>", |
| "</think>", |
| "<answer>", |
| "</answer>", |
| "<iban-pii>", |
| "<email-pii>", |
| "<ip-pii>", |
| "<file_sep>", |
| "<code_to_intermediate>", |
| "<intermediate_to_code>", |
| "<pr>", |
| "<pr_status>", |
| "<pr_is_merged>", |
| "<pr_base>", |
| "<pr_file>", |
| "<pr_base_code>", |
| "<pr_diff>", |
| "<pr_diff_hunk>", |
| "<pr_comment>", |
| "<pr_event_id>", |
| "<pr_review>", |
| "<pr_review_state>", |
| "<pr_review_comment>", |
| "<pr_in_reply_to_review_id>", |
| "<pr_in_reply_to_comment_id>", |
| "<pr_diff_hunk_comment_line>", |
| "<|fim_begin|>", |
| "<|fim_hole|>", |
| "<|fim_end|>", |
| "<|system_start|>", |
| "<|system_end|>", |
| "<|developer_start|>", |
| "<|developer_end|>", |
| "<|user_start|>", |
| "<|user_end|>", |
| "<|assistant_start|>", |
| "<|assistant_end|>", |
| "<|inner_prefix|>", |
| "<|inner_suffix|>", |
| "<|tools_prefix|>", |
| "<|tools_suffix|>", |
| "<|tool_output_start|>", |
| "<|tool_output_end|>", |
| "<|image|>", |
| "<|audio|>", |
| "<SPECIAL_77>", |
| "<SPECIAL_78>", |
| "<SPECIAL_79>", |
| "<SPECIAL_80>", |
| "<SPECIAL_81>", |
| "<SPECIAL_82>", |
| "<SPECIAL_83>", |
| "<SPECIAL_84>", |
| "<SPECIAL_85>", |
| "<SPECIAL_86>", |
| "<SPECIAL_87>", |
| "<SPECIAL_88>", |
| "<SPECIAL_89>", |
| "<SPECIAL_90>", |
| "<SPECIAL_91>", |
| "<SPECIAL_92>", |
| "<SPECIAL_93>", |
| "<SPECIAL_94>", |
| "<SPECIAL_95>", |
| "<SPECIAL_96>", |
| "<SPECIAL_97>", |
| "<SPECIAL_98>" |
| ], |
| "normalizer": { |
| "type": "NFC" |
| }, |
| "pre_tokenizer": { |
| "type": "Sequence", |
| "pretokenizers": [ |
| { |
| "type": "Split", |
| "pattern": { |
| "Regex": "[ \\x{0F0B}\\x{0027}\\x{2019}]?[\\p{Lu}\\p{Lt}\\p{Lm}\\p{Lo}\\p{M}]*[\\p{Ll}\\p{Lm}\\p{Lo}\\p{M}]+|[ \\x{0F0B}\\x{0027}\\x{2019}]?[\\p{Lu}\\p{Lt}\\p{Lm}\\p{Lo}\\p{M}]+[\\p{Ll}\\p{Lm}\\p{Lo}\\p{M}]*|\\p{N}| ?[^\\s\\p{L}\\p{N}]{1,16}|\\s{0,16}[\\r\\n]{1,16}|\\s{1,16}(?!\\S)|\\s{1,16}" |
| }, |
| "behavior": "Isolated", |
| "invert": false |
| }, |
| { |
| "type": "ByteLevel", |
| "add_prefix_space": false, |
| "trim_offsets": true, |
| "use_regex": false |
| } |
| ] |
| }, |
| "has_post_processor": false, |
| "byte_fallback": false, |
| "bos_token": "<s>", |
| "bos_token_id": 1 |
| }, |
| "declared_experiments": null, |
| "declared_run_manifest": { |
| "variant": "pa_bpe_nfc_clean_multi_plus2_capped_hybrid_window_tuned_consv2_tailcuts_gm90k_v130_sp_eng5g", |
| "variant_config": { |
| "trainer": "parity-bpe", |
| "pretok": "clean_multi_plus2_capped_nfc", |
| "variant": "window", |
| "global_merges": 90000, |
| "window_size": 100, |
| "alpha": 2.0, |
| "signal": "ratios", |
| "vocab_size": 130999, |
| "grouped_config": "configs/parity_aware_config_grouped_fineweb2full_quota_tuned_consv2_tailcuts_eng5g.json", |
| "special_tokens_override": [ |
| "<unk>", |
| "<s>", |
| "</s>", |
| "<pad>", |
| "[/INST]", |
| "[AVAILABLE_TOOLS]", |
| "[/AVAILABLE_TOOLS]", |
| "[TOOL_RESULTS]", |
| "[/TOOL_RESULTS]", |
| "[TOOL_CALLS]", |
| "<SPECIAL_10>", |
| "[PREFIX]", |
| "[MIDDLE]", |
| "[SUFFIX]", |
| "\\begin{", |
| "\\end{", |
| "\\text{", |
| "\\boxed{", |
| "<filename>", |
| "<gh_stars>", |
| "<issue_start>", |
| "<issue_comment>", |
| "<issue_closed>", |
| "<jupyter_start>", |
| "<jupyter_text>", |
| "<jupyter_code>", |
| "<jupyter_output>", |
| "<empty_output>", |
| "<commit_before>", |
| "<commit_msg>", |
| "<commit_after>", |
| "<reponame>", |
| "<think>", |
| "</think>", |
| "<answer>", |
| "</answer>", |
| "<iban-pii>", |
| "<email-pii>", |
| "<ip-pii>", |
| "<file_sep>", |
| "<code_to_intermediate>", |
| "<intermediate_to_code>", |
| "<pr>", |
| "<pr_status>", |
| "<pr_is_merged>", |
| "<pr_base>", |
| "<pr_file>", |
| "<pr_base_code>", |
| "<pr_diff>", |
| "<pr_diff_hunk>", |
| "<pr_comment>", |
| "<pr_event_id>", |
| "<pr_review>", |
| "<pr_review_state>", |
| "<pr_review_comment>", |
| "<pr_in_reply_to_review_id>", |
| "<pr_in_reply_to_comment_id>", |
| "<pr_diff_hunk_comment_line>", |
| "<|fim_begin|>", |
| "<|fim_hole|>", |
| "<|fim_end|>", |
| "<|system_start|>", |
| "<|system_end|>", |
| "<|developer_start|>", |
| "<|developer_end|>", |
| "<|user_start|>", |
| "<|user_end|>", |
| "<|assistant_start|>", |
| "<|assistant_end|>", |
| "<|inner_prefix|>", |
| "<|inner_suffix|>", |
| "<|tools_prefix|>", |
| "<|tools_suffix|>", |
| "<|tool_output_start|>", |
| "<|tool_output_end|>", |
| "<|image|>", |
| "<|audio|>", |
| "<SPECIAL_77>", |
| "<SPECIAL_78>", |
| "<SPECIAL_79>", |
| "<SPECIAL_80>", |
| "<SPECIAL_81>", |
| "<SPECIAL_82>", |
| "<SPECIAL_83>", |
| "<SPECIAL_84>", |
| "<SPECIAL_85>", |
| "<SPECIAL_86>", |
| "<SPECIAL_87>", |
| "<SPECIAL_88>", |
| "<SPECIAL_89>", |
| "<SPECIAL_90>", |
| "<SPECIAL_91>", |
| "<SPECIAL_92>", |
| "<SPECIAL_93>", |
| "<SPECIAL_94>", |
| "<SPECIAL_95>", |
| "<SPECIAL_96>", |
| "<SPECIAL_97>", |
| "<SPECIAL_98>" |
| ], |
| "decoder_add_prefix_space": true, |
| "output_group": "tokenizer-lm-toks", |
| "output_dir_name": "nfc_clean_multi_plus2_capped_hybrid_window_tuned_consv2_tailcuts_gm90k_v130_sp_eng5g" |
| }, |
| "grouped_config_path": "/users/cmeister747/pa_tokenizers_branch/configs/parity_aware_config_grouped_fineweb2full_quota_tuned_consv2_tailcuts_eng5g.json", |
| "vocab_size": 130999, |
| "min_frequency": 2, |
| "special_tokens": [ |
| "<unk>", |
| "<s>", |
| "</s>", |
| "<pad>", |
| "[/INST]", |
| "[AVAILABLE_TOOLS]", |
| "[/AVAILABLE_TOOLS]", |
| "[TOOL_RESULTS]", |
| "[/TOOL_RESULTS]", |
| "[TOOL_CALLS]", |
| "<SPECIAL_10>", |
| "[PREFIX]", |
| "[MIDDLE]", |
| "[SUFFIX]", |
| "\\begin{", |
| "\\end{", |
| "\\text{", |
| "\\boxed{", |
| "<filename>", |
| "<gh_stars>", |
| "<issue_start>", |
| "<issue_comment>", |
| "<issue_closed>", |
| "<jupyter_start>", |
| "<jupyter_text>", |
| "<jupyter_code>", |
| "<jupyter_output>", |
| "<empty_output>", |
| "<commit_before>", |
| "<commit_msg>", |
| "<commit_after>", |
| "<reponame>", |
| "<think>", |
| "</think>", |
| "<answer>", |
| "</answer>", |
| "<iban-pii>", |
| "<email-pii>", |
| "<ip-pii>", |
| "<file_sep>", |
| "<code_to_intermediate>", |
| "<intermediate_to_code>", |
| "<pr>", |
| "<pr_status>", |
| "<pr_is_merged>", |
| "<pr_base>", |
| "<pr_file>", |
| "<pr_base_code>", |
| "<pr_diff>", |
| "<pr_diff_hunk>", |
| "<pr_comment>", |
| "<pr_event_id>", |
| "<pr_review>", |
| "<pr_review_state>", |
| "<pr_review_comment>", |
| "<pr_in_reply_to_review_id>", |
| "<pr_in_reply_to_comment_id>", |
| "<pr_diff_hunk_comment_line>", |
| "<|fim_begin|>", |
| "<|fim_hole|>", |
| "<|fim_end|>", |
| "<|system_start|>", |
| "<|system_end|>", |
| "<|developer_start|>", |
| "<|developer_end|>", |
| "<|user_start|>", |
| "<|user_end|>", |
| "<|assistant_start|>", |
| "<|assistant_end|>", |
| "<|inner_prefix|>", |
| "<|inner_suffix|>", |
| "<|tools_prefix|>", |
| "<|tools_suffix|>", |
| "<|tool_output_start|>", |
| "<|tool_output_end|>", |
| "<|image|>", |
| "<|audio|>", |
| "<SPECIAL_77>", |
| "<SPECIAL_78>", |
| "<SPECIAL_79>", |
| "<SPECIAL_80>", |
| "<SPECIAL_81>", |
| "<SPECIAL_82>", |
| "<SPECIAL_83>", |
| "<SPECIAL_84>", |
| "<SPECIAL_85>", |
| "<SPECIAL_86>", |
| "<SPECIAL_87>", |
| "<SPECIAL_88>", |
| "<SPECIAL_89>", |
| "<SPECIAL_90>", |
| "<SPECIAL_91>", |
| "<SPECIAL_92>", |
| "<SPECIAL_93>", |
| "<SPECIAL_94>", |
| "<SPECIAL_95>", |
| "<SPECIAL_96>", |
| "<SPECIAL_97>", |
| "<SPECIAL_98>" |
| ], |
| "start_time": "2026-06-05T18:34:53.428167", |
| "elapsed_seconds": 6850.657703, |
| "slurm_job_id": "2476236", |
| "cli_args": [ |
| "/users/cmeister747/pa_tokenizers_branch/train_tokenizer.py", |
| "--variant", |
| "pa_bpe_nfc_clean_multi_plus2_capped_hybrid_window_tuned_consv2_tailcuts_gm90k_v130_sp_eng5g" |
| ], |
| "git": { |
| "commit": "5d8381af9ec1417b078f7dce82bac68873da915e", |
| "branch": "parity-aware-bpe", |
| "dirty": false |
| }, |
| "per_group_summary": { |
| "n_groups": 25, |
| "full_block_in": "tokenizers/pabpe-nfc-clean-fineweb2full-plus2-capped-tuned-hybridwindow-consv2-tailcuts-gm90k-v130-sp-eng5g/run_manifest.json" |
| } |
| }, |
| "registry_note": "PA-BPE hybrid global+window (gm90k), clean-multi plus2 regex; fineweb2full, consv2 + tailcuts (6 families demoted to ratio 1.0) + eng5g (+5GB English) data; norm=NFC; PA-BPE (hybrid global+window); vocab ~131,196 (~131k + full Apertus chat-template specials, ~99 tokens); Apertus-2 development ablation 'PA-Clean-plus2-A8'; USED - S6f Contrast B (131k arm) + Apertus-2 track ablation; excluded from main paper panel" |
| }, |
| "pabpe-nfc-clean-fineweb2full-plus3-capped-tuned-hybridwindow-consv2": { |
| "slug": "pabpe-nfc-clean-fineweb2full-plus3-capped-tuned-hybridwindow-consv2", |
| "runs": [ |
| "full-128k-pabpe-nfc-clean-fineweb2full-plus3-capped-tuned-hybridwindow-consv2", |
| "full-128k-pabpe-nfc-clean-fineweb2full-plus3-capped-tuned-hybridwindow-consv2-mathcode-scratch" |
| ], |
| "n_runs": 2, |
| "redistributable": true, |
| "loadable_with": "transformers", |
| "hub_id": null, |
| "policy_note": "trained in this project", |
| "empirical": { |
| "model_type": "BPE", |
| "n_base_tokens": 127831, |
| "n_added_tokens": 4, |
| "vocab_total": 127835, |
| "added_tokens": [ |
| "<s>", |
| "</s>", |
| "<unk>", |
| "<pad>" |
| ], |
| "normalizer": { |
| "type": "NFC" |
| }, |
| "pre_tokenizer": { |
| "type": "Sequence", |
| "pretokenizers": [ |
| { |
| "type": "Split", |
| "pattern": { |
| "Regex": "[ \\x{0F0B}\\x{0027}\\x{2019}]?[\\p{Lu}\\p{Lt}\\p{Lm}\\p{Lo}\\p{M}]*[\\p{Ll}\\p{Lm}\\p{Lo}\\p{M}]+(?:[\\x{0027}\\x{2019}](?!\\p{L}))?|[ \\x{0F0B}\\x{0027}\\x{2019}]?[\\p{Lu}\\p{Lt}\\p{Lm}\\p{Lo}\\p{M}]+[\\p{Ll}\\p{Lm}\\p{Lo}\\p{M}]*(?:[\\x{0027}\\x{2019}](?!\\p{L}))?|\\p{N}| ?[^\\s\\p{L}\\p{N}]{1,16}|\\s{0,16}[\\r\\n]{1,16}|\\s{1,16}(?!\\S)|\\s{1,16}" |
| }, |
| "behavior": "Isolated", |
| "invert": false |
| }, |
| { |
| "type": "ByteLevel", |
| "add_prefix_space": false, |
| "trim_offsets": true, |
| "use_regex": false |
| } |
| ] |
| }, |
| "has_post_processor": false, |
| "byte_fallback": false, |
| "bos_token": "<s>", |
| "bos_token_id": 127831 |
| }, |
| "declared_experiments": null, |
| "declared_run_manifest": { |
| "variant": "pa_bpe_nfc_clean_multi_plus3_capped_hybrid_window_tuned_consv2", |
| "variant_config": { |
| "trainer": "parity-bpe", |
| "pretok": "clean_multi_plus3_capped_nfc", |
| "variant": "window", |
| "global_merges": 64000, |
| "window_size": 100, |
| "alpha": 2.0, |
| "signal": "ratios", |
| "grouped_config": "configs/parity_aware_config_grouped_fineweb2full_quota_tuned_consv2.json", |
| "special_tokens_override": [], |
| "decoder_add_prefix_space": true, |
| "output_group": "tokenizer-lm-toks", |
| "output_dir_name": "nfc_clean_multi_plus3_capped_hybrid_window_tuned_consv2", |
| "post_training_special_tokens": [ |
| "<s>", |
| "</s>", |
| "<unk>", |
| "<pad>" |
| ] |
| }, |
| "grouped_config_path": "/users/cmeister747/pa_tokenizers_branch/configs/parity_aware_config_grouped_fineweb2full_quota_tuned_consv2.json", |
| "vocab_size": 128000, |
| "min_frequency": 2, |
| "special_tokens": [], |
| "start_time": "2026-06-04T00:07:42.220508", |
| "elapsed_seconds": 7096.755123, |
| "slurm_job_id": "2466634", |
| "cli_args": [ |
| "/users/cmeister747/pa_tokenizers_branch/train_tokenizer.py", |
| "--variant", |
| "pa_bpe_nfc_clean_multi_plus3_capped_hybrid_window_tuned_consv2" |
| ], |
| "git": { |
| "commit": "5d8381af9ec1417b078f7dce82bac68873da915e", |
| "branch": "parity-aware-bpe", |
| "dirty": false |
| }, |
| "per_group_summary": { |
| "n_groups": 25, |
| "full_block_in": "tokenizers/pabpe-nfc-clean-fineweb2full-plus3-capped-tuned-hybridwindow-consv2/run_manifest.json" |
| } |
| }, |
| "registry_note": "PA-BPE hybrid global+window, clean-multi plus3 regex; fineweb2full, consv2 (no tailcuts) data; norm=NFC; PA-BPE (hybrid global+window); vocab ~131,196; plus3 consv2 baseline; USED - S6f Contrast B (131k arm) + Apertus-2 track; excluded from main paper panel" |
| }, |
| "pabpe-nfc-clean-fineweb2full-plus3-capped-tuned-hybridwindow-consv2-tailcuts-gm70k-v130-sp-eng5g": { |
| "slug": "pabpe-nfc-clean-fineweb2full-plus3-capped-tuned-hybridwindow-consv2-tailcuts-gm70k-v130-sp-eng5g", |
| "runs": [ |
| "full-128k-pabpe-nfc-clean-fineweb2full-plus3-capped-tuned-hybridwindow-consv2-tailcuts-gm70k-v130-sp-eng5g", |
| "full-128k-pabpe-nfc-clean-fineweb2full-plus3-capped-tuned-hybridwindow-consv2-tailcuts-gm70k-v130-sp-eng5g-mathcode-scratch" |
| ], |
| "n_runs": 2, |
| "redistributable": true, |
| "loadable_with": "transformers", |
| "hub_id": null, |
| "policy_note": "trained in this project", |
| "empirical": { |
| "model_type": "BPE", |
| "n_base_tokens": 130929, |
| "n_added_tokens": 99, |
| "vocab_total": 131028, |
| "added_tokens": [ |
| "<unk>", |
| "<s>", |
| "</s>", |
| "<pad>", |
| "[/INST]", |
| "[AVAILABLE_TOOLS]", |
| "[/AVAILABLE_TOOLS]", |
| "[TOOL_RESULTS]", |
| "[/TOOL_RESULTS]", |
| "[TOOL_CALLS]", |
| "<SPECIAL_10>", |
| "[PREFIX]", |
| "[MIDDLE]", |
| "[SUFFIX]", |
| "\\begin{", |
| "\\end{", |
| "\\text{", |
| "\\boxed{", |
| "<filename>", |
| "<gh_stars>", |
| "<issue_start>", |
| "<issue_comment>", |
| "<issue_closed>", |
| "<jupyter_start>", |
| "<jupyter_text>", |
| "<jupyter_code>", |
| "<jupyter_output>", |
| "<empty_output>", |
| "<commit_before>", |
| "<commit_msg>", |
| "<commit_after>", |
| "<reponame>", |
| "<think>", |
| "</think>", |
| "<answer>", |
| "</answer>", |
| "<iban-pii>", |
| "<email-pii>", |
| "<ip-pii>", |
| "<file_sep>", |
| "<code_to_intermediate>", |
| "<intermediate_to_code>", |
| "<pr>", |
| "<pr_status>", |
| "<pr_is_merged>", |
| "<pr_base>", |
| "<pr_file>", |
| "<pr_base_code>", |
| "<pr_diff>", |
| "<pr_diff_hunk>", |
| "<pr_comment>", |
| "<pr_event_id>", |
| "<pr_review>", |
| "<pr_review_state>", |
| "<pr_review_comment>", |
| "<pr_in_reply_to_review_id>", |
| "<pr_in_reply_to_comment_id>", |
| "<pr_diff_hunk_comment_line>", |
| "<|fim_begin|>", |
| "<|fim_hole|>", |
| "<|fim_end|>", |
| "<|system_start|>", |
| "<|system_end|>", |
| "<|developer_start|>", |
| "<|developer_end|>", |
| "<|user_start|>", |
| "<|user_end|>", |
| "<|assistant_start|>", |
| "<|assistant_end|>", |
| "<|inner_prefix|>", |
| "<|inner_suffix|>", |
| "<|tools_prefix|>", |
| "<|tools_suffix|>", |
| "<|tool_output_start|>", |
| "<|tool_output_end|>", |
| "<|image|>", |
| "<|audio|>", |
| "<SPECIAL_77>", |
| "<SPECIAL_78>", |
| "<SPECIAL_79>", |
| "<SPECIAL_80>", |
| "<SPECIAL_81>", |
| "<SPECIAL_82>", |
| "<SPECIAL_83>", |
| "<SPECIAL_84>", |
| "<SPECIAL_85>", |
| "<SPECIAL_86>", |
| "<SPECIAL_87>", |
| "<SPECIAL_88>", |
| "<SPECIAL_89>", |
| "<SPECIAL_90>", |
| "<SPECIAL_91>", |
| "<SPECIAL_92>", |
| "<SPECIAL_93>", |
| "<SPECIAL_94>", |
| "<SPECIAL_95>", |
| "<SPECIAL_96>", |
| "<SPECIAL_97>", |
| "<SPECIAL_98>" |
| ], |
| "normalizer": { |
| "type": "NFC" |
| }, |
| "pre_tokenizer": { |
| "type": "Sequence", |
| "pretokenizers": [ |
| { |
| "type": "Split", |
| "pattern": { |
| "Regex": "[ \\x{0F0B}\\x{0027}\\x{2019}]?[\\p{Lu}\\p{Lt}\\p{Lm}\\p{Lo}\\p{M}]*[\\p{Ll}\\p{Lm}\\p{Lo}\\p{M}]+(?:[\\x{0027}\\x{2019}](?!\\p{L}))?|[ \\x{0F0B}\\x{0027}\\x{2019}]?[\\p{Lu}\\p{Lt}\\p{Lm}\\p{Lo}\\p{M}]+[\\p{Ll}\\p{Lm}\\p{Lo}\\p{M}]*(?:[\\x{0027}\\x{2019}](?!\\p{L}))?|\\p{N}| ?[^\\s\\p{L}\\p{N}]{1,16}|\\s{0,16}[\\r\\n]{1,16}|\\s{1,16}(?!\\S)|\\s{1,16}" |
| }, |
| "behavior": "Isolated", |
| "invert": false |
| }, |
| { |
| "type": "ByteLevel", |
| "add_prefix_space": false, |
| "trim_offsets": true, |
| "use_regex": false |
| } |
| ] |
| }, |
| "has_post_processor": false, |
| "byte_fallback": false, |
| "bos_token": "<s>", |
| "bos_token_id": 1 |
| }, |
| "declared_experiments": null, |
| "declared_run_manifest": { |
| "variant": "pa_bpe_nfc_clean_multi_plus3_capped_hybrid_window_tuned_consv2_tailcuts_gm70k_v130_sp_eng5g", |
| "variant_config": { |
| "trainer": "parity-bpe", |
| "pretok": "clean_multi_plus3_capped_nfc", |
| "variant": "window", |
| "global_merges": 70000, |
| "window_size": 100, |
| "alpha": 2.0, |
| "signal": "ratios", |
| "vocab_size": 130999, |
| "grouped_config": "configs/parity_aware_config_grouped_fineweb2full_quota_tuned_consv2_tailcuts_eng5g.json", |
| "special_tokens_override": [ |
| "<unk>", |
| "<s>", |
| "</s>", |
| "<pad>", |
| "[/INST]", |
| "[AVAILABLE_TOOLS]", |
| "[/AVAILABLE_TOOLS]", |
| "[TOOL_RESULTS]", |
| "[/TOOL_RESULTS]", |
| "[TOOL_CALLS]", |
| "<SPECIAL_10>", |
| "[PREFIX]", |
| "[MIDDLE]", |
| "[SUFFIX]", |
| "\\begin{", |
| "\\end{", |
| "\\text{", |
| "\\boxed{", |
| "<filename>", |
| "<gh_stars>", |
| "<issue_start>", |
| "<issue_comment>", |
| "<issue_closed>", |
| "<jupyter_start>", |
| "<jupyter_text>", |
| "<jupyter_code>", |
| "<jupyter_output>", |
| "<empty_output>", |
| "<commit_before>", |
| "<commit_msg>", |
| "<commit_after>", |
| "<reponame>", |
| "<think>", |
| "</think>", |
| "<answer>", |
| "</answer>", |
| "<iban-pii>", |
| "<email-pii>", |
| "<ip-pii>", |
| "<file_sep>", |
| "<code_to_intermediate>", |
| "<intermediate_to_code>", |
| "<pr>", |
| "<pr_status>", |
| "<pr_is_merged>", |
| "<pr_base>", |
| "<pr_file>", |
| "<pr_base_code>", |
| "<pr_diff>", |
| "<pr_diff_hunk>", |
| "<pr_comment>", |
| "<pr_event_id>", |
| "<pr_review>", |
| "<pr_review_state>", |
| "<pr_review_comment>", |
| "<pr_in_reply_to_review_id>", |
| "<pr_in_reply_to_comment_id>", |
| "<pr_diff_hunk_comment_line>", |
| "<|fim_begin|>", |
| "<|fim_hole|>", |
| "<|fim_end|>", |
| "<|system_start|>", |
| "<|system_end|>", |
| "<|developer_start|>", |
| "<|developer_end|>", |
| "<|user_start|>", |
| "<|user_end|>", |
| "<|assistant_start|>", |
| "<|assistant_end|>", |
| "<|inner_prefix|>", |
| "<|inner_suffix|>", |
| "<|tools_prefix|>", |
| "<|tools_suffix|>", |
| "<|tool_output_start|>", |
| "<|tool_output_end|>", |
| "<|image|>", |
| "<|audio|>", |
| "<SPECIAL_77>", |
| "<SPECIAL_78>", |
| "<SPECIAL_79>", |
| "<SPECIAL_80>", |
| "<SPECIAL_81>", |
| "<SPECIAL_82>", |
| "<SPECIAL_83>", |
| "<SPECIAL_84>", |
| "<SPECIAL_85>", |
| "<SPECIAL_86>", |
| "<SPECIAL_87>", |
| "<SPECIAL_88>", |
| "<SPECIAL_89>", |
| "<SPECIAL_90>", |
| "<SPECIAL_91>", |
| "<SPECIAL_92>", |
| "<SPECIAL_93>", |
| "<SPECIAL_94>", |
| "<SPECIAL_95>", |
| "<SPECIAL_96>", |
| "<SPECIAL_97>", |
| "<SPECIAL_98>" |
| ], |
| "decoder_add_prefix_space": true, |
| "output_group": "tokenizer-lm-toks", |
| "output_dir_name": "nfc_clean_multi_plus3_capped_hybrid_window_tuned_consv2_tailcuts_gm70k_v130_sp_eng5g" |
| }, |
| "grouped_config_path": "/users/cmeister747/pa_tokenizers_branch/configs/parity_aware_config_grouped_fineweb2full_quota_tuned_consv2_tailcuts_eng5g.json", |
| "vocab_size": 130999, |
| "min_frequency": 2, |
| "special_tokens": [ |
| "<unk>", |
| "<s>", |
| "</s>", |
| "<pad>", |
| "[/INST]", |
| "[AVAILABLE_TOOLS]", |
| "[/AVAILABLE_TOOLS]", |
| "[TOOL_RESULTS]", |
| "[/TOOL_RESULTS]", |
| "[TOOL_CALLS]", |
| "<SPECIAL_10>", |
| "[PREFIX]", |
| "[MIDDLE]", |
| "[SUFFIX]", |
| "\\begin{", |
| "\\end{", |
| "\\text{", |
| "\\boxed{", |
| "<filename>", |
| "<gh_stars>", |
| "<issue_start>", |
| "<issue_comment>", |
| "<issue_closed>", |
| "<jupyter_start>", |
| "<jupyter_text>", |
| "<jupyter_code>", |
| "<jupyter_output>", |
| "<empty_output>", |
| "<commit_before>", |
| "<commit_msg>", |
| "<commit_after>", |
| "<reponame>", |
| "<think>", |
| "</think>", |
| "<answer>", |
| "</answer>", |
| "<iban-pii>", |
| "<email-pii>", |
| "<ip-pii>", |
| "<file_sep>", |
| "<code_to_intermediate>", |
| "<intermediate_to_code>", |
| "<pr>", |
| "<pr_status>", |
| "<pr_is_merged>", |
| "<pr_base>", |
| "<pr_file>", |
| "<pr_base_code>", |
| "<pr_diff>", |
| "<pr_diff_hunk>", |
| "<pr_comment>", |
| "<pr_event_id>", |
| "<pr_review>", |
| "<pr_review_state>", |
| "<pr_review_comment>", |
| "<pr_in_reply_to_review_id>", |
| "<pr_in_reply_to_comment_id>", |
| "<pr_diff_hunk_comment_line>", |
| "<|fim_begin|>", |
| "<|fim_hole|>", |
| "<|fim_end|>", |
| "<|system_start|>", |
| "<|system_end|>", |
| "<|developer_start|>", |
| "<|developer_end|>", |
| "<|user_start|>", |
| "<|user_end|>", |
| "<|assistant_start|>", |
| "<|assistant_end|>", |
| "<|inner_prefix|>", |
| "<|inner_suffix|>", |
| "<|tools_prefix|>", |
| "<|tools_suffix|>", |
| "<|tool_output_start|>", |
| "<|tool_output_end|>", |
| "<|image|>", |
| "<|audio|>", |
| "<SPECIAL_77>", |
| "<SPECIAL_78>", |
| "<SPECIAL_79>", |
| "<SPECIAL_80>", |
| "<SPECIAL_81>", |
| "<SPECIAL_82>", |
| "<SPECIAL_83>", |
| "<SPECIAL_84>", |
| "<SPECIAL_85>", |
| "<SPECIAL_86>", |
| "<SPECIAL_87>", |
| "<SPECIAL_88>", |
| "<SPECIAL_89>", |
| "<SPECIAL_90>", |
| "<SPECIAL_91>", |
| "<SPECIAL_92>", |
| "<SPECIAL_93>", |
| "<SPECIAL_94>", |
| "<SPECIAL_95>", |
| "<SPECIAL_96>", |
| "<SPECIAL_97>", |
| "<SPECIAL_98>" |
| ], |
| "start_time": "2026-06-05T15:57:20.559966", |
| "elapsed_seconds": 7154.459067, |
| "slurm_job_id": "2475069", |
| "cli_args": [ |
| "/users/cmeister747/pa_tokenizers_branch/train_tokenizer.py", |
| "--variant", |
| "pa_bpe_nfc_clean_multi_plus3_capped_hybrid_window_tuned_consv2_tailcuts_gm70k_v130_sp_eng5g" |
| ], |
| "git": { |
| "commit": "5d8381af9ec1417b078f7dce82bac68873da915e", |
| "branch": "parity-aware-bpe", |
| "dirty": false |
| }, |
| "per_group_summary": { |
| "n_groups": 25, |
| "full_block_in": "tokenizers/pabpe-nfc-clean-fineweb2full-plus3-capped-tuned-hybridwindow-consv2-tailcuts-gm70k-v130-sp-eng5g/run_manifest.json" |
| } |
| }, |
| "registry_note": "PA-BPE hybrid global+window (gm70k), clean-multi plus3 regex; fineweb2full, consv2 + tailcuts + eng5g data; norm=NFC; PA-BPE (hybrid global+window); vocab ~131,196; Apertus-2 development ablation 'PA-Clean-plus3-A6'; USED - S6f Contrast B (131k arm) + Apertus-2 track ablation; excluded from main paper panel" |
| }, |
| "pabpe-nfc-clean-fineweb2full-plus3-capped-tuned-hybridwindow-consv2-tailcuts-gm90k-v130-sp-eng5g": { |
| "slug": "pabpe-nfc-clean-fineweb2full-plus3-capped-tuned-hybridwindow-consv2-tailcuts-gm90k-v130-sp-eng5g", |
| "runs": [ |
| "full-128k-pabpe-nfc-clean-fineweb2full-plus3-capped-tuned-hybridwindow-consv2-tailcuts-gm90k-v130-sp-eng5g", |
| "full-128k-pabpe-nfc-clean-fineweb2full-plus3-capped-tuned-hybridwindow-consv2-tailcuts-gm90k-v130-sp-eng5g-mathcode-scratch" |
| ], |
| "n_runs": 2, |
| "redistributable": true, |
| "loadable_with": "transformers", |
| "hub_id": null, |
| "policy_note": "trained in this project", |
| "empirical": { |
| "model_type": "BPE", |
| "n_base_tokens": 130929, |
| "n_added_tokens": 99, |
| "vocab_total": 131028, |
| "added_tokens": [ |
| "<unk>", |
| "<s>", |
| "</s>", |
| "<pad>", |
| "[/INST]", |
| "[AVAILABLE_TOOLS]", |
| "[/AVAILABLE_TOOLS]", |
| "[TOOL_RESULTS]", |
| "[/TOOL_RESULTS]", |
| "[TOOL_CALLS]", |
| "<SPECIAL_10>", |
| "[PREFIX]", |
| "[MIDDLE]", |
| "[SUFFIX]", |
| "\\begin{", |
| "\\end{", |
| "\\text{", |
| "\\boxed{", |
| "<filename>", |
| "<gh_stars>", |
| "<issue_start>", |
| "<issue_comment>", |
| "<issue_closed>", |
| "<jupyter_start>", |
| "<jupyter_text>", |
| "<jupyter_code>", |
| "<jupyter_output>", |
| "<empty_output>", |
| "<commit_before>", |
| "<commit_msg>", |
| "<commit_after>", |
| "<reponame>", |
| "<think>", |
| "</think>", |
| "<answer>", |
| "</answer>", |
| "<iban-pii>", |
| "<email-pii>", |
| "<ip-pii>", |
| "<file_sep>", |
| "<code_to_intermediate>", |
| "<intermediate_to_code>", |
| "<pr>", |
| "<pr_status>", |
| "<pr_is_merged>", |
| "<pr_base>", |
| "<pr_file>", |
| "<pr_base_code>", |
| "<pr_diff>", |
| "<pr_diff_hunk>", |
| "<pr_comment>", |
| "<pr_event_id>", |
| "<pr_review>", |
| "<pr_review_state>", |
| "<pr_review_comment>", |
| "<pr_in_reply_to_review_id>", |
| "<pr_in_reply_to_comment_id>", |
| "<pr_diff_hunk_comment_line>", |
| "<|fim_begin|>", |
| "<|fim_hole|>", |
| "<|fim_end|>", |
| "<|system_start|>", |
| "<|system_end|>", |
| "<|developer_start|>", |
| "<|developer_end|>", |
| "<|user_start|>", |
| "<|user_end|>", |
| "<|assistant_start|>", |
| "<|assistant_end|>", |
| "<|inner_prefix|>", |
| "<|inner_suffix|>", |
| "<|tools_prefix|>", |
| "<|tools_suffix|>", |
| "<|tool_output_start|>", |
| "<|tool_output_end|>", |
| "<|image|>", |
| "<|audio|>", |
| "<SPECIAL_77>", |
| "<SPECIAL_78>", |
| "<SPECIAL_79>", |
| "<SPECIAL_80>", |
| "<SPECIAL_81>", |
| "<SPECIAL_82>", |
| "<SPECIAL_83>", |
| "<SPECIAL_84>", |
| "<SPECIAL_85>", |
| "<SPECIAL_86>", |
| "<SPECIAL_87>", |
| "<SPECIAL_88>", |
| "<SPECIAL_89>", |
| "<SPECIAL_90>", |
| "<SPECIAL_91>", |
| "<SPECIAL_92>", |
| "<SPECIAL_93>", |
| "<SPECIAL_94>", |
| "<SPECIAL_95>", |
| "<SPECIAL_96>", |
| "<SPECIAL_97>", |
| "<SPECIAL_98>" |
| ], |
| "normalizer": { |
| "type": "NFC" |
| }, |
| "pre_tokenizer": { |
| "type": "Sequence", |
| "pretokenizers": [ |
| { |
| "type": "Split", |
| "pattern": { |
| "Regex": "[ \\x{0F0B}\\x{0027}\\x{2019}]?[\\p{Lu}\\p{Lt}\\p{Lm}\\p{Lo}\\p{M}]*[\\p{Ll}\\p{Lm}\\p{Lo}\\p{M}]+(?:[\\x{0027}\\x{2019}](?!\\p{L}))?|[ \\x{0F0B}\\x{0027}\\x{2019}]?[\\p{Lu}\\p{Lt}\\p{Lm}\\p{Lo}\\p{M}]+[\\p{Ll}\\p{Lm}\\p{Lo}\\p{M}]*(?:[\\x{0027}\\x{2019}](?!\\p{L}))?|\\p{N}| ?[^\\s\\p{L}\\p{N}]{1,16}|\\s{0,16}[\\r\\n]{1,16}|\\s{1,16}(?!\\S)|\\s{1,16}" |
| }, |
| "behavior": "Isolated", |
| "invert": false |
| }, |
| { |
| "type": "ByteLevel", |
| "add_prefix_space": false, |
| "trim_offsets": true, |
| "use_regex": false |
| } |
| ] |
| }, |
| "has_post_processor": false, |
| "byte_fallback": false, |
| "bos_token": "<s>", |
| "bos_token_id": 1 |
| }, |
| "declared_experiments": null, |
| "declared_run_manifest": { |
| "variant": "pa_bpe_nfc_clean_multi_plus3_capped_hybrid_window_tuned_consv2_tailcuts_gm90k_v130_sp_eng5g", |
| "variant_config": { |
| "trainer": "parity-bpe", |
| "pretok": "clean_multi_plus3_capped_nfc", |
| "variant": "window", |
| "global_merges": 90000, |
| "window_size": 100, |
| "alpha": 2.0, |
| "signal": "ratios", |
| "vocab_size": 130999, |
| "grouped_config": "configs/parity_aware_config_grouped_fineweb2full_quota_tuned_consv2_tailcuts_eng5g.json", |
| "special_tokens_override": [ |
| "<unk>", |
| "<s>", |
| "</s>", |
| "<pad>", |
| "[/INST]", |
| "[AVAILABLE_TOOLS]", |
| "[/AVAILABLE_TOOLS]", |
| "[TOOL_RESULTS]", |
| "[/TOOL_RESULTS]", |
| "[TOOL_CALLS]", |
| "<SPECIAL_10>", |
| "[PREFIX]", |
| "[MIDDLE]", |
| "[SUFFIX]", |
| "\\begin{", |
| "\\end{", |
| "\\text{", |
| "\\boxed{", |
| "<filename>", |
| "<gh_stars>", |
| "<issue_start>", |
| "<issue_comment>", |
| "<issue_closed>", |
| "<jupyter_start>", |
| "<jupyter_text>", |
| "<jupyter_code>", |
| "<jupyter_output>", |
| "<empty_output>", |
| "<commit_before>", |
| "<commit_msg>", |
| "<commit_after>", |
| "<reponame>", |
| "<think>", |
| "</think>", |
| "<answer>", |
| "</answer>", |
| "<iban-pii>", |
| "<email-pii>", |
| "<ip-pii>", |
| "<file_sep>", |
| "<code_to_intermediate>", |
| "<intermediate_to_code>", |
| "<pr>", |
| "<pr_status>", |
| "<pr_is_merged>", |
| "<pr_base>", |
| "<pr_file>", |
| "<pr_base_code>", |
| "<pr_diff>", |
| "<pr_diff_hunk>", |
| "<pr_comment>", |
| "<pr_event_id>", |
| "<pr_review>", |
| "<pr_review_state>", |
| "<pr_review_comment>", |
| "<pr_in_reply_to_review_id>", |
| "<pr_in_reply_to_comment_id>", |
| "<pr_diff_hunk_comment_line>", |
| "<|fim_begin|>", |
| "<|fim_hole|>", |
| "<|fim_end|>", |
| "<|system_start|>", |
| "<|system_end|>", |
| "<|developer_start|>", |
| "<|developer_end|>", |
| "<|user_start|>", |
| "<|user_end|>", |
| "<|assistant_start|>", |
| "<|assistant_end|>", |
| "<|inner_prefix|>", |
| "<|inner_suffix|>", |
| "<|tools_prefix|>", |
| "<|tools_suffix|>", |
| "<|tool_output_start|>", |
| "<|tool_output_end|>", |
| "<|image|>", |
| "<|audio|>", |
| "<SPECIAL_77>", |
| "<SPECIAL_78>", |
| "<SPECIAL_79>", |
| "<SPECIAL_80>", |
| "<SPECIAL_81>", |
| "<SPECIAL_82>", |
| "<SPECIAL_83>", |
| "<SPECIAL_84>", |
| "<SPECIAL_85>", |
| "<SPECIAL_86>", |
| "<SPECIAL_87>", |
| "<SPECIAL_88>", |
| "<SPECIAL_89>", |
| "<SPECIAL_90>", |
| "<SPECIAL_91>", |
| "<SPECIAL_92>", |
| "<SPECIAL_93>", |
| "<SPECIAL_94>", |
| "<SPECIAL_95>", |
| "<SPECIAL_96>", |
| "<SPECIAL_97>", |
| "<SPECIAL_98>" |
| ], |
| "decoder_add_prefix_space": true, |
| "output_group": "tokenizer-lm-toks", |
| "output_dir_name": "nfc_clean_multi_plus3_capped_hybrid_window_tuned_consv2_tailcuts_gm90k_v130_sp_eng5g" |
| }, |
| "grouped_config_path": "/users/cmeister747/pa_tokenizers_branch/configs/parity_aware_config_grouped_fineweb2full_quota_tuned_consv2_tailcuts_eng5g.json", |
| "vocab_size": 130999, |
| "min_frequency": 2, |
| "special_tokens": [ |
| "<unk>", |
| "<s>", |
| "</s>", |
| "<pad>", |
| "[/INST]", |
| "[AVAILABLE_TOOLS]", |
| "[/AVAILABLE_TOOLS]", |
| "[TOOL_RESULTS]", |
| "[/TOOL_RESULTS]", |
| "[TOOL_CALLS]", |
| "<SPECIAL_10>", |
| "[PREFIX]", |
| "[MIDDLE]", |
| "[SUFFIX]", |
| "\\begin{", |
| "\\end{", |
| "\\text{", |
| "\\boxed{", |
| "<filename>", |
| "<gh_stars>", |
| "<issue_start>", |
| "<issue_comment>", |
| "<issue_closed>", |
| "<jupyter_start>", |
| "<jupyter_text>", |
| "<jupyter_code>", |
| "<jupyter_output>", |
| "<empty_output>", |
| "<commit_before>", |
| "<commit_msg>", |
| "<commit_after>", |
| "<reponame>", |
| "<think>", |
| "</think>", |
| "<answer>", |
| "</answer>", |
| "<iban-pii>", |
| "<email-pii>", |
| "<ip-pii>", |
| "<file_sep>", |
| "<code_to_intermediate>", |
| "<intermediate_to_code>", |
| "<pr>", |
| "<pr_status>", |
| "<pr_is_merged>", |
| "<pr_base>", |
| "<pr_file>", |
| "<pr_base_code>", |
| "<pr_diff>", |
| "<pr_diff_hunk>", |
| "<pr_comment>", |
| "<pr_event_id>", |
| "<pr_review>", |
| "<pr_review_state>", |
| "<pr_review_comment>", |
| "<pr_in_reply_to_review_id>", |
| "<pr_in_reply_to_comment_id>", |
| "<pr_diff_hunk_comment_line>", |
| "<|fim_begin|>", |
| "<|fim_hole|>", |
| "<|fim_end|>", |
| "<|system_start|>", |
| "<|system_end|>", |
| "<|developer_start|>", |
| "<|developer_end|>", |
| "<|user_start|>", |
| "<|user_end|>", |
| "<|assistant_start|>", |
| "<|assistant_end|>", |
| "<|inner_prefix|>", |
| "<|inner_suffix|>", |
| "<|tools_prefix|>", |
| "<|tools_suffix|>", |
| "<|tool_output_start|>", |
| "<|tool_output_end|>", |
| "<|image|>", |
| "<|audio|>", |
| "<SPECIAL_77>", |
| "<SPECIAL_78>", |
| "<SPECIAL_79>", |
| "<SPECIAL_80>", |
| "<SPECIAL_81>", |
| "<SPECIAL_82>", |
| "<SPECIAL_83>", |
| "<SPECIAL_84>", |
| "<SPECIAL_85>", |
| "<SPECIAL_86>", |
| "<SPECIAL_87>", |
| "<SPECIAL_88>", |
| "<SPECIAL_89>", |
| "<SPECIAL_90>", |
| "<SPECIAL_91>", |
| "<SPECIAL_92>", |
| "<SPECIAL_93>", |
| "<SPECIAL_94>", |
| "<SPECIAL_95>", |
| "<SPECIAL_96>", |
| "<SPECIAL_97>", |
| "<SPECIAL_98>" |
| ], |
| "start_time": "2026-06-05T15:57:20.375628", |
| "elapsed_seconds": 7320.610917, |
| "slurm_job_id": "2475071", |
| "cli_args": [ |
| "/users/cmeister747/pa_tokenizers_branch/train_tokenizer.py", |
| "--variant", |
| "pa_bpe_nfc_clean_multi_plus3_capped_hybrid_window_tuned_consv2_tailcuts_gm90k_v130_sp_eng5g" |
| ], |
| "git": { |
| "commit": "5d8381af9ec1417b078f7dce82bac68873da915e", |
| "branch": "parity-aware-bpe", |
| "dirty": false |
| }, |
| "per_group_summary": { |
| "n_groups": 25, |
| "full_block_in": "tokenizers/pabpe-nfc-clean-fineweb2full-plus3-capped-tuned-hybridwindow-consv2-tailcuts-gm90k-v130-sp-eng5g/run_manifest.json" |
| } |
| }, |
| "registry_note": "PA-BPE hybrid global+window (gm90k), clean-multi plus3 regex; fineweb2full, consv2 + tailcuts + eng5g data; norm=NFC; PA-BPE (hybrid global+window); vocab ~131,196; Apertus-2 development ablation 'PA-Clean-plus3-A8'; USED - S6f Contrast B (131k arm) + Apertus-2 track ablation; excluded from main paper panel" |
| }, |
| "pabpe-nfc-clean-fw2full-plus3-cap-hw-consv2-gm90k-tailcuts-eng5g-repcap8": { |
| "slug": "pabpe-nfc-clean-fw2full-plus3-cap-hw-consv2-gm90k-tailcuts-eng5g-repcap8", |
| "runs": [ |
| "full-128k-pabpe-nfc-clean-fw2full-plus3-cap-hw-consv2-gm90k-tailcuts-eng5g-repcap8", |
| "full-128k-pabpe-nfc-clean-fw2full-plus3-cap-hw-consv2-gm90k-tailcuts-eng5g-repcap8-mathcode-scratch" |
| ], |
| "n_runs": 2, |
| "redistributable": true, |
| "loadable_with": "transformers", |
| "hub_id": null, |
| "policy_note": "trained in this project", |
| "empirical": { |
| "model_type": "BPE", |
| "n_base_tokens": 131017, |
| "n_added_tokens": 124, |
| "vocab_total": 131141, |
| "added_tokens": [ |
| "<unk>", |
| "<s>", |
| "</s>", |
| "<pad>", |
| "<|system_start|>", |
| "<|system_end|>", |
| "<|developer_start|>", |
| "<|developer_end|>", |
| "<|user_start|>", |
| "<|user_end|>", |
| "<|assistant_start|>", |
| "<|assistant_end|>", |
| "<|inner_prefix|>", |
| "<|inner_suffix|>", |
| "<|tools_prefix|>", |
| "<|tools_suffix|>", |
| "<|tool_output_start|>", |
| "<|tool_output_end|>", |
| "<|image|>", |
| "<|audio|>", |
| "<reflection>", |
| "</reflection>", |
| "<think>", |
| "</think>", |
| "<SPECIAL_24>", |
| "<SPECIAL_25>", |
| "<SPECIAL_26>", |
| "<SPECIAL_27>", |
| "<SPECIAL_28>", |
| "<SPECIAL_29>", |
| "<SPECIAL_30>", |
| "<SPECIAL_31>", |
| "<SPECIAL_32>", |
| "<SPECIAL_33>", |
| "<SPECIAL_34>", |
| "<SPECIAL_35>", |
| "<SPECIAL_36>", |
| "<SPECIAL_37>", |
| "<SPECIAL_38>", |
| "<SPECIAL_39>", |
| "<SPECIAL_40>", |
| "<SPECIAL_41>", |
| "<SPECIAL_42>", |
| "<SPECIAL_43>", |
| "<SPECIAL_44>", |
| "<SPECIAL_45>", |
| "<SPECIAL_46>", |
| "<SPECIAL_47>", |
| "<SPECIAL_48>", |
| "<SPECIAL_49>", |
| "<SPECIAL_50>", |
| "<SPECIAL_51>", |
| "<SPECIAL_52>", |
| "<SPECIAL_53>", |
| "<SPECIAL_54>", |
| "<SPECIAL_55>", |
| "<SPECIAL_56>", |
| "<SPECIAL_57>", |
| "<SPECIAL_58>", |
| "<SPECIAL_59>", |
| "<SPECIAL_60>", |
| "<SPECIAL_61>", |
| "<SPECIAL_62>", |
| "<SPECIAL_63>", |
| "<SPECIAL_64>", |
| "<SPECIAL_65>", |
| "<SPECIAL_66>", |
| "<SPECIAL_67>", |
| "<SPECIAL_68>", |
| "<SPECIAL_69>", |
| "<SPECIAL_70>", |
| "<SPECIAL_71>", |
| "<SPECIAL_72>", |
| "<SPECIAL_73>", |
| "<SPECIAL_74>", |
| "<SPECIAL_75>", |
| "<SPECIAL_76>", |
| "<SPECIAL_77>", |
| "<SPECIAL_78>", |
| "<SPECIAL_79>", |
| "<SPECIAL_80>", |
| "<SPECIAL_81>", |
| "<SPECIAL_82>", |
| "<SPECIAL_83>", |
| "<SPECIAL_84>", |
| "<SPECIAL_85>", |
| "<SPECIAL_86>", |
| "<SPECIAL_87>", |
| "<SPECIAL_88>", |
| "<SPECIAL_89>", |
| "<SPECIAL_90>", |
| "<SPECIAL_91>", |
| "<SPECIAL_92>", |
| "<SPECIAL_93>", |
| "<SPECIAL_94>", |
| "<SPECIAL_95>", |
| "<SPECIAL_96>", |
| "<SPECIAL_97>", |
| "<SPECIAL_98>", |
| "<SPECIAL_99>", |
| "<SPECIAL_100>", |
| "<SPECIAL_101>", |
| "<SPECIAL_102>", |
| "<SPECIAL_103>", |
| "<SPECIAL_104>", |
| "<SPECIAL_105>", |
| "<SPECIAL_106>", |
| "<SPECIAL_107>", |
| "<SPECIAL_108>", |
| "<SPECIAL_109>", |
| "<SPECIAL_110>", |
| "<SPECIAL_111>", |
| "<SPECIAL_112>", |
| "<SPECIAL_113>", |
| "<SPECIAL_114>", |
| "<SPECIAL_115>", |
| "<SPECIAL_116>", |
| "<SPECIAL_117>", |
| "<SPECIAL_118>", |
| "<SPECIAL_119>", |
| "<SPECIAL_120>", |
| "<SPECIAL_121>", |
| "<SPECIAL_122>", |
| "<SPECIAL_123>" |
| ], |
| "normalizer": { |
| "type": "NFC" |
| }, |
| "pre_tokenizer": { |
| "type": "Sequence", |
| "pretokenizers": [ |
| { |
| "type": "Split", |
| "pattern": { |
| "Regex": "[ \\x{0F0B}\\x{0027}\\x{2019}]?(?<g>[^\\s])\\k<g>{7}|[ \\x{0F0B}\\x{0027}\\x{2019}]?(?:(?!(?<u1>.)\\k<u1>{8})[\\p{Lu}\\p{Lt}\\p{Lm}\\p{Lo}\\p{M}])*(?:(?!(?<l1>.)\\k<l1>{8})[\\p{Ll}\\p{Lm}\\p{Lo}\\p{M}])+(?:[\\x{0027}\\x{2019}](?!\\p{L}))?|[ \\x{0F0B}\\x{0027}\\x{2019}]?(?:(?!(?<u2>.)\\k<u2>{8})[\\p{Lu}\\p{Lt}\\p{Lm}\\p{Lo}\\p{M}])+(?:(?!(?<l2>.)\\k<l2>{8})[\\p{Ll}\\p{Lm}\\p{Lo}\\p{M}])*(?:[\\x{0027}\\x{2019}](?!\\p{L}))?|\\p{N}| ?(?:(?!(?<p>.)\\k<p>{8})[^\\s\\p{L}\\p{N}]){1,16}|\\s{0,16}[\\r\\n]{1,16}|\\s{1,16}(?!\\S)|\\s{1,16}" |
| }, |
| "behavior": "Isolated", |
| "invert": false |
| }, |
| { |
| "type": "ByteLevel", |
| "add_prefix_space": false, |
| "trim_offsets": true, |
| "use_regex": false |
| } |
| ] |
| }, |
| "has_post_processor": false, |
| "byte_fallback": false, |
| "bos_token": "<s>", |
| "bos_token_id": 1 |
| }, |
| "declared_experiments": null, |
| "declared_run_manifest": { |
| "variant": "pa_bpe_nfc_clean_multi_plus3_repcap8_capped_hybrid_window_tuned_consv2_tailcuts_gm90k_v131k_sp124_eng5g_fr", |
| "variant_config": { |
| "trainer": "parity-bpe", |
| "pretok": "clean_multi_plus3_repcap8_capped_nfc", |
| "variant": "window", |
| "global_merges": 90000, |
| "window_size": 100, |
| "alpha": 2.0, |
| "signal": "ratios", |
| "vocab_size": 131072, |
| "grouped_config": "configs/parity_aware_config_grouped_fineweb2full_quota_tuned_consv2_tailcuts_eng5g.json", |
| "special_tokens_override": [ |
| "<unk>", |
| "<s>", |
| "</s>", |
| "<pad>", |
| "<|system_start|>", |
| "<|system_end|>", |
| "<|developer_start|>", |
| "<|developer_end|>", |
| "<|user_start|>", |
| "<|user_end|>", |
| "<|assistant_start|>", |
| "<|assistant_end|>", |
| "<|inner_prefix|>", |
| "<|inner_suffix|>", |
| "<|tools_prefix|>", |
| "<|tools_suffix|>", |
| "<|tool_output_start|>", |
| "<|tool_output_end|>", |
| "<|image|>", |
| "<|audio|>", |
| "<reflection>", |
| "</reflection>", |
| "<think>", |
| "</think>", |
| "<SPECIAL_24>", |
| "<SPECIAL_25>", |
| "<SPECIAL_26>", |
| "<SPECIAL_27>", |
| "<SPECIAL_28>", |
| "<SPECIAL_29>", |
| "<SPECIAL_30>", |
| "<SPECIAL_31>", |
| "<SPECIAL_32>", |
| "<SPECIAL_33>", |
| "<SPECIAL_34>", |
| "<SPECIAL_35>", |
| "<SPECIAL_36>", |
| "<SPECIAL_37>", |
| "<SPECIAL_38>", |
| "<SPECIAL_39>", |
| "<SPECIAL_40>", |
| "<SPECIAL_41>", |
| "<SPECIAL_42>", |
| "<SPECIAL_43>", |
| "<SPECIAL_44>", |
| "<SPECIAL_45>", |
| "<SPECIAL_46>", |
| "<SPECIAL_47>", |
| "<SPECIAL_48>", |
| "<SPECIAL_49>", |
| "<SPECIAL_50>", |
| "<SPECIAL_51>", |
| "<SPECIAL_52>", |
| "<SPECIAL_53>", |
| "<SPECIAL_54>", |
| "<SPECIAL_55>", |
| "<SPECIAL_56>", |
| "<SPECIAL_57>", |
| "<SPECIAL_58>", |
| "<SPECIAL_59>", |
| "<SPECIAL_60>", |
| "<SPECIAL_61>", |
| "<SPECIAL_62>", |
| "<SPECIAL_63>", |
| "<SPECIAL_64>", |
| "<SPECIAL_65>", |
| "<SPECIAL_66>", |
| "<SPECIAL_67>", |
| "<SPECIAL_68>", |
| "<SPECIAL_69>", |
| "<SPECIAL_70>", |
| "<SPECIAL_71>", |
| "<SPECIAL_72>", |
| "<SPECIAL_73>", |
| "<SPECIAL_74>", |
| "<SPECIAL_75>", |
| "<SPECIAL_76>", |
| "<SPECIAL_77>", |
| "<SPECIAL_78>", |
| "<SPECIAL_79>", |
| "<SPECIAL_80>", |
| "<SPECIAL_81>", |
| "<SPECIAL_82>", |
| "<SPECIAL_83>", |
| "<SPECIAL_84>", |
| "<SPECIAL_85>", |
| "<SPECIAL_86>", |
| "<SPECIAL_87>", |
| "<SPECIAL_88>", |
| "<SPECIAL_89>", |
| "<SPECIAL_90>", |
| "<SPECIAL_91>", |
| "<SPECIAL_92>", |
| "<SPECIAL_93>", |
| "<SPECIAL_94>", |
| "<SPECIAL_95>", |
| "<SPECIAL_96>", |
| "<SPECIAL_97>", |
| "<SPECIAL_98>", |
| "<SPECIAL_99>", |
| "<SPECIAL_100>", |
| "<SPECIAL_101>", |
| "<SPECIAL_102>", |
| "<SPECIAL_103>", |
| "<SPECIAL_104>", |
| "<SPECIAL_105>", |
| "<SPECIAL_106>", |
| "<SPECIAL_107>", |
| "<SPECIAL_108>", |
| "<SPECIAL_109>", |
| "<SPECIAL_110>", |
| "<SPECIAL_111>", |
| "<SPECIAL_112>", |
| "<SPECIAL_113>", |
| "<SPECIAL_114>", |
| "<SPECIAL_115>", |
| "<SPECIAL_116>", |
| "<SPECIAL_117>", |
| "<SPECIAL_118>", |
| "<SPECIAL_119>", |
| "<SPECIAL_120>", |
| "<SPECIAL_121>", |
| "<SPECIAL_122>", |
| "<SPECIAL_123>" |
| ], |
| "decoder_add_prefix_space": true, |
| "output_group": "tokenizer-lm-toks", |
| "output_dir_name": "nfc_clean_multi_plus3_repcap8_capped_hybrid_window_tuned_consv2_tailcuts_gm90k_v131k_sp124_eng5g_fr" |
| }, |
| "grouped_config_path": "/users/cmeister747/pa_tokenizers_branch/configs/parity_aware_config_grouped_fineweb2full_quota_tuned_consv2_tailcuts_eng5g.json", |
| "vocab_size": 131072, |
| "min_frequency": 2, |
| "special_tokens": [ |
| "<unk>", |
| "<s>", |
| "</s>", |
| "<pad>", |
| "<|system_start|>", |
| "<|system_end|>", |
| "<|developer_start|>", |
| "<|developer_end|>", |
| "<|user_start|>", |
| "<|user_end|>", |
| "<|assistant_start|>", |
| "<|assistant_end|>", |
| "<|inner_prefix|>", |
| "<|inner_suffix|>", |
| "<|tools_prefix|>", |
| "<|tools_suffix|>", |
| "<|tool_output_start|>", |
| "<|tool_output_end|>", |
| "<|image|>", |
| "<|audio|>", |
| "<reflection>", |
| "</reflection>", |
| "<think>", |
| "</think>", |
| "<SPECIAL_24>", |
| "<SPECIAL_25>", |
| "<SPECIAL_26>", |
| "<SPECIAL_27>", |
| "<SPECIAL_28>", |
| "<SPECIAL_29>", |
| "<SPECIAL_30>", |
| "<SPECIAL_31>", |
| "<SPECIAL_32>", |
| "<SPECIAL_33>", |
| "<SPECIAL_34>", |
| "<SPECIAL_35>", |
| "<SPECIAL_36>", |
| "<SPECIAL_37>", |
| "<SPECIAL_38>", |
| "<SPECIAL_39>", |
| "<SPECIAL_40>", |
| "<SPECIAL_41>", |
| "<SPECIAL_42>", |
| "<SPECIAL_43>", |
| "<SPECIAL_44>", |
| "<SPECIAL_45>", |
| "<SPECIAL_46>", |
| "<SPECIAL_47>", |
| "<SPECIAL_48>", |
| "<SPECIAL_49>", |
| "<SPECIAL_50>", |
| "<SPECIAL_51>", |
| "<SPECIAL_52>", |
| "<SPECIAL_53>", |
| "<SPECIAL_54>", |
| "<SPECIAL_55>", |
| "<SPECIAL_56>", |
| "<SPECIAL_57>", |
| "<SPECIAL_58>", |
| "<SPECIAL_59>", |
| "<SPECIAL_60>", |
| "<SPECIAL_61>", |
| "<SPECIAL_62>", |
| "<SPECIAL_63>", |
| "<SPECIAL_64>", |
| "<SPECIAL_65>", |
| "<SPECIAL_66>", |
| "<SPECIAL_67>", |
| "<SPECIAL_68>", |
| "<SPECIAL_69>", |
| "<SPECIAL_70>", |
| "<SPECIAL_71>", |
| "<SPECIAL_72>", |
| "<SPECIAL_73>", |
| "<SPECIAL_74>", |
| "<SPECIAL_75>", |
| "<SPECIAL_76>", |
| "<SPECIAL_77>", |
| "<SPECIAL_78>", |
| "<SPECIAL_79>", |
| "<SPECIAL_80>", |
| "<SPECIAL_81>", |
| "<SPECIAL_82>", |
| "<SPECIAL_83>", |
| "<SPECIAL_84>", |
| "<SPECIAL_85>", |
| "<SPECIAL_86>", |
| "<SPECIAL_87>", |
| "<SPECIAL_88>", |
| "<SPECIAL_89>", |
| "<SPECIAL_90>", |
| "<SPECIAL_91>", |
| "<SPECIAL_92>", |
| "<SPECIAL_93>", |
| "<SPECIAL_94>", |
| "<SPECIAL_95>", |
| "<SPECIAL_96>", |
| "<SPECIAL_97>", |
| "<SPECIAL_98>", |
| "<SPECIAL_99>", |
| "<SPECIAL_100>", |
| "<SPECIAL_101>", |
| "<SPECIAL_102>", |
| "<SPECIAL_103>", |
| "<SPECIAL_104>", |
| "<SPECIAL_105>", |
| "<SPECIAL_106>", |
| "<SPECIAL_107>", |
| "<SPECIAL_108>", |
| "<SPECIAL_109>", |
| "<SPECIAL_110>", |
| "<SPECIAL_111>", |
| "<SPECIAL_112>", |
| "<SPECIAL_113>", |
| "<SPECIAL_114>", |
| "<SPECIAL_115>", |
| "<SPECIAL_116>", |
| "<SPECIAL_117>", |
| "<SPECIAL_118>", |
| "<SPECIAL_119>", |
| "<SPECIAL_120>", |
| "<SPECIAL_121>", |
| "<SPECIAL_122>", |
| "<SPECIAL_123>" |
| ], |
| "start_time": "2026-06-08T17:50:22.080123", |
| "elapsed_seconds": 7204.537718, |
| "slurm_job_id": "2497771", |
| "cli_args": [ |
| "/users/cmeister747/pa_tokenizers_branch/train_tokenizer.py", |
| "--variant", |
| "pa_bpe_nfc_clean_multi_plus3_repcap8_capped_hybrid_window_tuned_consv2_tailcuts_gm90k_v131k_sp124_eng5g_fr" |
| ], |
| "git": { |
| "commit": "5d8381af9ec1417b078f7dce82bac68873da915e", |
| "branch": "parity-aware-bpe", |
| "dirty": false |
| }, |
| "per_group_summary": { |
| "n_groups": 25, |
| "full_block_in": "tokenizers/pabpe-nfc-clean-fw2full-plus3-cap-hw-consv2-gm90k-tailcuts-eng5g-repcap8/run_manifest.json" |
| } |
| }, |
| "registry_note": "PA-BPE hybrid global+window (gm90k), clean-multi plus3 regex + repcap8; fineweb2full, consv2 + tailcuts + eng5g data; norm=NFC; PA-BPE (hybrid global+window); vocab ~131,141 (131,017 + 124 specials; Apertus precedent for >128k vocab); Apertus-2 development ablation 'PA-Clean-plus3-repcap8fr-A8'; USED - S6f Contrast B (131k arm) + Apertus-2 track ablation; excluded from main paper panel" |
| }, |
| "pabpe-nfc-gpt4-balanced": { |
| "slug": "pabpe-nfc-gpt4-balanced", |
| "runs": [ |
| "full-128k-pabpe-nfc-gpt4-balanced", |
| "full-128k-pabpe-nfc-gpt4-balanced-mathcode-scratch", |
| "pilot-128k-pabpe-nfc-gpt4-balanced", |
| "small-128k-pabpe-nfc-gpt4-balanced", |
| "tiny-128k-pabpe-nfc-gpt4-balanced" |
| ], |
| "n_runs": 5, |
| "redistributable": true, |
| "loadable_with": "transformers", |
| "hub_id": null, |
| "policy_note": "trained in this project", |
| "empirical": { |
| "model_type": "BPE", |
| "n_base_tokens": 127822, |
| "n_added_tokens": 4, |
| "vocab_total": 127826, |
| "added_tokens": [ |
| "<s>", |
| "</s>", |
| "<unk>", |
| "<pad>" |
| ], |
| "normalizer": { |
| "type": "NFC" |
| }, |
| "pre_tokenizer": { |
| "type": "Sequence", |
| "pretokenizers": [ |
| { |
| "type": "Split", |
| "pattern": { |
| "Regex": "[^\\r\\n\\p{L}\\p{N}]?[\\p{Lu}\\p{Lt}\\p{Lm}\\p{Lo}\\p{M}]*[\\p{Ll}\\p{Lm}\\p{Lo}\\p{M}]+(?i:'s|'t|'re|'ve|'m|'ll|'d)?|[^\\r\\n\\p{L}\\p{N}]?[\\p{Lu}\\p{Lt}\\p{Lm}\\p{Lo}\\p{M}]+[\\p{Ll}\\p{Lm}\\p{Lo}\\p{M}]*(?i:'s|'t|'re|'ve|'m|'ll|'d)?|\\p{N}{1,3}| ?[^\\s\\p{L}\\p{N}]+[\\r\\n/]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+" |
| }, |
| "behavior": "Isolated", |
| "invert": false |
| }, |
| { |
| "type": "ByteLevel", |
| "add_prefix_space": false, |
| "trim_offsets": true, |
| "use_regex": false |
| } |
| ] |
| }, |
| "has_post_processor": false, |
| "byte_fallback": false, |
| "bos_token": "<s>", |
| "bos_token_id": 127822 |
| }, |
| "declared_experiments": null, |
| "declared_run_manifest": { |
| "variant": "pa_bpe_nfc_gpt4_balanced", |
| "variant_config": { |
| "trainer": "parity-bpe", |
| "pretok": "gpt4o_nfc", |
| "variant": "base", |
| "global_merges": 0, |
| "signal": "ratios", |
| "grouped_config": "configs/parity_aware_config_balanced.json", |
| "special_tokens_override": [], |
| "decoder_add_prefix_space": true, |
| "output_group": "tokenizer-lm-toks", |
| "output_dir_name": "nfc_gpt4_balanced", |
| "post_training_special_tokens": [ |
| "<s>", |
| "</s>", |
| "<unk>", |
| "<pad>" |
| ] |
| }, |
| "grouped_config_path": "/users/cmeister747/pa_tokenizers_branch/configs/parity_aware_config_balanced.json", |
| "vocab_size": 128000, |
| "min_frequency": 2, |
| "special_tokens": [], |
| "start_time": "2026-05-15T21:37:46.656038", |
| "elapsed_seconds": 2543.241044, |
| "slurm_job_id": "2256968", |
| "cli_args": [ |
| "/users/cmeister747/pa_tokenizers_branch/train_tokenizer.py", |
| "--variant", |
| "pa_bpe_nfc_gpt4_balanced" |
| ], |
| "git": { |
| "commit": "b2ceac78388c06deb786755b60d6c1d335cb6c25", |
| "branch": "parity-aware-bpe", |
| "dirty": false |
| }, |
| "per_group_summary": { |
| "n_groups": 34, |
| "full_block_in": "tokenizers/pabpe-nfc-gpt4-balanced/run_manifest.json" |
| } |
| }, |
| "registry_note": "PA-BPE (parity, grouped=25 script-family units) with GPT-4o regex; balanced (D2) data; norm=NFC; PA-BPE (ParityBpeTrainer); vocab 127,826; bug-fixed 2026-05-16 re-export of the retired pabpe-balanced (2a); replaces the newline-stripping-bug original; USED - panels.json main21 (camera-ready main panel)" |
| }, |
| "pabpe-nfc-gpt4-balanced-hybridwindow": { |
| "slug": "pabpe-nfc-gpt4-balanced-hybridwindow", |
| "runs": [ |
| "full-128k-pabpe-nfc-gpt4-balanced-hybridwindow" |
| ], |
| "n_runs": 1, |
| "redistributable": true, |
| "loadable_with": "transformers", |
| "hub_id": null, |
| "policy_note": "trained in this project", |
| "empirical": { |
| "model_type": "BPE", |
| "n_base_tokens": 127822, |
| "n_added_tokens": 4, |
| "vocab_total": 127826, |
| "added_tokens": [ |
| "<s>", |
| "</s>", |
| "<unk>", |
| "<pad>" |
| ], |
| "normalizer": { |
| "type": "NFC" |
| }, |
| "pre_tokenizer": { |
| "type": "Sequence", |
| "pretokenizers": [ |
| { |
| "type": "Split", |
| "pattern": { |
| "Regex": "[^\\r\\n\\p{L}\\p{N}]?[\\p{Lu}\\p{Lt}\\p{Lm}\\p{Lo}\\p{M}]*[\\p{Ll}\\p{Lm}\\p{Lo}\\p{M}]+(?i:'s|'t|'re|'ve|'m|'ll|'d)?|[^\\r\\n\\p{L}\\p{N}]?[\\p{Lu}\\p{Lt}\\p{Lm}\\p{Lo}\\p{M}]+[\\p{Ll}\\p{Lm}\\p{Lo}\\p{M}]*(?i:'s|'t|'re|'ve|'m|'ll|'d)?|\\p{N}{1,3}| ?[^\\s\\p{L}\\p{N}]+[\\r\\n/]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+" |
| }, |
| "behavior": "Isolated", |
| "invert": false |
| }, |
| { |
| "type": "ByteLevel", |
| "add_prefix_space": false, |
| "trim_offsets": true, |
| "use_regex": false |
| } |
| ] |
| }, |
| "has_post_processor": false, |
| "byte_fallback": false, |
| "bos_token": "<s>", |
| "bos_token_id": 127822 |
| }, |
| "declared_experiments": null, |
| "declared_run_manifest": { |
| "variant": "pa_bpe_nfc_gpt4_balanced_hybrid_window", |
| "variant_config": { |
| "trainer": "parity-bpe", |
| "pretok": "gpt4o_nfc", |
| "variant": "window", |
| "global_merges": 64000, |
| "window_size": 100, |
| "alpha": 2.0, |
| "signal": "ratios", |
| "grouped_config": "configs/parity_aware_config_balanced.json", |
| "special_tokens_override": [], |
| "decoder_add_prefix_space": true, |
| "output_group": "tokenizer-lm-toks", |
| "output_dir_name": "nfc_gpt4_balanced_hybrid_window", |
| "post_training_special_tokens": [ |
| "<s>", |
| "</s>", |
| "<unk>", |
| "<pad>" |
| ] |
| }, |
| "grouped_config_path": "/users/cmeister747/pa_tokenizers_branch/configs/parity_aware_config_balanced.json", |
| "vocab_size": 128000, |
| "min_frequency": 2, |
| "special_tokens": [], |
| "start_time": "2026-05-16T15:19:37.996073", |
| "elapsed_seconds": 2379.645128, |
| "slurm_job_id": "2260974", |
| "cli_args": [ |
| "/users/cmeister747/pa_tokenizers_branch/train_tokenizer.py", |
| "--variant", |
| "pa_bpe_nfc_gpt4_balanced_hybrid_window" |
| ], |
| "git": { |
| "commit": "7172cc669840d0dcce4170f52ef0430eb2a3d899", |
| "branch": "parity-aware-bpe", |
| "dirty": false |
| }, |
| "per_group_summary": { |
| "n_groups": 34, |
| "full_block_in": "tokenizers/pabpe-nfc-gpt4-balanced-hybridwindow/run_manifest.json" |
| } |
| }, |
| "registry_note": "PA-BPE hybrid global+window (global_merges=64000, window=100, alpha=2.0, signal=ratios) with GPT-4o regex; balanced (D2) data; norm=NFC; PA-BPE (hybrid global+window); vocab 127,826; tests whether the hybrid-window scheme improves over base PA-BPE at fixed data/pretok; USED - balanced-1B bootstrap panel (S6.12) + paper panel; not in panels.json main21" |
| }, |
| "pabpe-nfc-gpt4-fineweb2full": { |
| "slug": "pabpe-nfc-gpt4-fineweb2full", |
| "runs": [ |
| "full-128k-pabpe-nfc-gpt4-fineweb2full" |
| ], |
| "n_runs": 1, |
| "redistributable": true, |
| "loadable_with": "transformers", |
| "hub_id": null, |
| "policy_note": "trained in this project", |
| "empirical": { |
| "model_type": "BPE", |
| "n_base_tokens": 127821, |
| "n_added_tokens": 4, |
| "vocab_total": 127825, |
| "added_tokens": [ |
| "<s>", |
| "</s>", |
| "<unk>", |
| "<pad>" |
| ], |
| "normalizer": { |
| "type": "NFC" |
| }, |
| "pre_tokenizer": { |
| "type": "Sequence", |
| "pretokenizers": [ |
| { |
| "type": "Split", |
| "pattern": { |
| "Regex": "[^\\r\\n\\p{L}\\p{N}]?[\\p{Lu}\\p{Lt}\\p{Lm}\\p{Lo}\\p{M}]*[\\p{Ll}\\p{Lm}\\p{Lo}\\p{M}]+(?i:'s|'t|'re|'ve|'m|'ll|'d)?|[^\\r\\n\\p{L}\\p{N}]?[\\p{Lu}\\p{Lt}\\p{Lm}\\p{Lo}\\p{M}]+[\\p{Ll}\\p{Lm}\\p{Lo}\\p{M}]*(?i:'s|'t|'re|'ve|'m|'ll|'d)?|\\p{N}{1,3}| ?[^\\s\\p{L}\\p{N}]+[\\r\\n/]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+" |
| }, |
| "behavior": "Isolated", |
| "invert": false |
| }, |
| { |
| "type": "ByteLevel", |
| "add_prefix_space": false, |
| "trim_offsets": true, |
| "use_regex": false |
| } |
| ] |
| }, |
| "has_post_processor": false, |
| "byte_fallback": false, |
| "bos_token": "<s>", |
| "bos_token_id": 127821 |
| }, |
| "declared_experiments": null, |
| "declared_run_manifest": { |
| "variant": "pa_bpe_nfc_gpt4_fineweb2full", |
| "variant_config": { |
| "trainer": "parity-bpe", |
| "pretok": "gpt4o_nfc", |
| "variant": "base", |
| "global_merges": 0, |
| "signal": "ratios", |
| "grouped_config": "configs/parity_aware_config_grouped_fineweb2full_quota.json", |
| "special_tokens_override": [], |
| "decoder_add_prefix_space": true, |
| "output_group": "tokenizer-lm-toks", |
| "output_dir_name": "nfc_gpt4_fineweb2full", |
| "post_training_special_tokens": [ |
| "<s>", |
| "</s>", |
| "<unk>", |
| "<pad>" |
| ] |
| }, |
| "grouped_config_path": "/users/cmeister747/pa_tokenizers_branch/configs/parity_aware_config_grouped_fineweb2full_quota.json", |
| "vocab_size": 128000, |
| "min_frequency": 2, |
| "special_tokens": [], |
| "start_time": "2026-05-22T07:43:23.652650", |
| "elapsed_seconds": 8217.121987, |
| "slurm_job_id": "2328518", |
| "cli_args": [ |
| "/users/cmeister747/pa_tokenizers_branch/train_tokenizer.py", |
| "--variant", |
| "pa_bpe_nfc_gpt4_fineweb2full" |
| ], |
| "git": { |
| "commit": "5d8381af9ec1417b078f7dce82bac68873da915e", |
| "branch": "parity-aware-bpe", |
| "dirty": false |
| }, |
| "per_group_summary": { |
| "n_groups": 25, |
| "full_block_in": "tokenizers/pabpe-nfc-gpt4-fineweb2full/run_manifest.json" |
| } |
| }, |
| "registry_note": "PA-BPE (parity, base) with GPT-4o regex; fineweb2full data; norm=NFC; PA-BPE (base); vocab 127,825; USED - S5b fineweb2full analysis; fineweb2full data variant, excluded from main paper panel" |
| }, |
| "pabpe-nfc-gpt4-fineweb2full-hybridwindow": { |
| "slug": "pabpe-nfc-gpt4-fineweb2full-hybridwindow", |
| "runs": [ |
| "full-128k-pabpe-nfc-gpt4-fineweb2full-hybridwindow" |
| ], |
| "n_runs": 1, |
| "redistributable": true, |
| "loadable_with": "transformers", |
| "hub_id": null, |
| "policy_note": "trained in this project", |
| "empirical": { |
| "model_type": "BPE", |
| "n_base_tokens": 127821, |
| "n_added_tokens": 4, |
| "vocab_total": 127825, |
| "added_tokens": [ |
| "<s>", |
| "</s>", |
| "<unk>", |
| "<pad>" |
| ], |
| "normalizer": { |
| "type": "NFC" |
| }, |
| "pre_tokenizer": { |
| "type": "Sequence", |
| "pretokenizers": [ |
| { |
| "type": "Split", |
| "pattern": { |
| "Regex": "[^\\r\\n\\p{L}\\p{N}]?[\\p{Lu}\\p{Lt}\\p{Lm}\\p{Lo}\\p{M}]*[\\p{Ll}\\p{Lm}\\p{Lo}\\p{M}]+(?i:'s|'t|'re|'ve|'m|'ll|'d)?|[^\\r\\n\\p{L}\\p{N}]?[\\p{Lu}\\p{Lt}\\p{Lm}\\p{Lo}\\p{M}]+[\\p{Ll}\\p{Lm}\\p{Lo}\\p{M}]*(?i:'s|'t|'re|'ve|'m|'ll|'d)?|\\p{N}{1,3}| ?[^\\s\\p{L}\\p{N}]+[\\r\\n/]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+" |
| }, |
| "behavior": "Isolated", |
| "invert": false |
| }, |
| { |
| "type": "ByteLevel", |
| "add_prefix_space": false, |
| "trim_offsets": true, |
| "use_regex": false |
| } |
| ] |
| }, |
| "has_post_processor": false, |
| "byte_fallback": false, |
| "bos_token": "<s>", |
| "bos_token_id": 127821 |
| }, |
| "declared_experiments": null, |
| "declared_run_manifest": { |
| "variant": "pa_bpe_nfc_gpt4_fineweb2full_hybrid_window", |
| "variant_config": { |
| "trainer": "parity-bpe", |
| "pretok": "gpt4o_nfc", |
| "variant": "window", |
| "global_merges": 64000, |
| "window_size": 100, |
| "alpha": 2.0, |
| "signal": "ratios", |
| "grouped_config": "configs/parity_aware_config_grouped_fineweb2full_quota.json", |
| "special_tokens_override": [], |
| "decoder_add_prefix_space": true, |
| "output_group": "tokenizer-lm-toks", |
| "output_dir_name": "nfc_gpt4_fineweb2full_hybrid_window", |
| "post_training_special_tokens": [ |
| "<s>", |
| "</s>", |
| "<unk>", |
| "<pad>" |
| ] |
| }, |
| "grouped_config_path": "/users/cmeister747/pa_tokenizers_branch/configs/parity_aware_config_grouped_fineweb2full_quota.json", |
| "vocab_size": 128000, |
| "min_frequency": 2, |
| "special_tokens": [], |
| "start_time": "2026-05-22T07:43:23.607801", |
| "elapsed_seconds": 7587.888544, |
| "slurm_job_id": "2328519", |
| "cli_args": [ |
| "/users/cmeister747/pa_tokenizers_branch/train_tokenizer.py", |
| "--variant", |
| "pa_bpe_nfc_gpt4_fineweb2full_hybrid_window" |
| ], |
| "git": { |
| "commit": "5d8381af9ec1417b078f7dce82bac68873da915e", |
| "branch": "parity-aware-bpe", |
| "dirty": false |
| }, |
| "per_group_summary": { |
| "n_groups": 25, |
| "full_block_in": "tokenizers/pabpe-nfc-gpt4-fineweb2full-hybridwindow/run_manifest.json" |
| } |
| }, |
| "registry_note": "PA-BPE hybrid global+window with GPT-4o regex; fineweb2full data; norm=NFC; PA-BPE (hybrid global+window); vocab 127,825; USED - S5b fineweb2full analysis; fineweb2full data variant, excluded from main paper panel" |
| }, |
| "punct-allmulti-equal-bpe": { |
| "slug": "punct-allmulti-equal-bpe", |
| "runs": [ |
| "full-128k-punct-allmulti-equal-bpe" |
| ], |
| "n_runs": 1, |
| "redistributable": true, |
| "loadable_with": "transformers", |
| "hub_id": null, |
| "policy_note": "trained in this project", |
| "empirical": { |
| "model_type": "BPE", |
| "n_base_tokens": 128256, |
| "n_added_tokens": 4, |
| "vocab_total": 128260, |
| "added_tokens": [ |
| "<s>", |
| "</s>", |
| "<unk>", |
| "<pad>" |
| ], |
| "normalizer": null, |
| "pre_tokenizer": { |
| "type": "Sequence", |
| "pretokenizers": [ |
| { |
| "type": "Punctuation", |
| "behavior": "Isolated" |
| }, |
| { |
| "type": "ByteLevel", |
| "add_prefix_space": false, |
| "trim_offsets": true, |
| "use_regex": true |
| } |
| ] |
| }, |
| "has_post_processor": false, |
| "byte_fallback": false, |
| "bos_token": "<s>", |
| "bos_token_id": 0 |
| }, |
| "declared_experiments": { |
| "algorithm": "bpe", |
| "pretokenizer_constant": "Punctuation+ByteLevel", |
| "normalizer": null, |
| "data_composition": "allmulti_equal", |
| "description": "Punctuation+ByteLevel pretok, equal-per-language data (repeat-sampled), BPE. Pairs with punct-balanced-bpe.", |
| "trainer_kwargs": null |
| }, |
| "declared_run_manifest": null, |
| "registry_note": "P1 Punctuation+ByteLevel; allmulti_equal data; norm=none; BPE; vocab 128,256; pairs with punct-balanced-bpe; USED - S6d.4 (rho=-0.375, weakest of the four - punct also compresses the tail least)" |
| }, |
| "punct-balanced-bpe": { |
| "slug": "punct-balanced-bpe", |
| "runs": [ |
| "full-128k-punct-balanced-bpe", |
| "full-128k-punct-balanced-bpe-mathcode-scratch", |
| "full-128k-punct-balanced-bpe-seed42", |
| "full-128k-punct-balanced-bpe-seed42-mathcode-scratch", |
| "full-128k-punct-balanced-bpe-seed43-mathcode-scratch", |
| "full-128k-punct-balanced-bpe-seed44-mathcode-scratch", |
| "pilot-128k-punct-balanced-bpe-wdfix", |
| "small-128k-punct-balanced-bpe", |
| "tiny-128k-punct-balanced-bpe" |
| ], |
| "n_runs": 9, |
| "redistributable": true, |
| "loadable_with": "transformers", |
| "hub_id": null, |
| "policy_note": "trained in this project", |
| "empirical": { |
| "model_type": "BPE", |
| "n_base_tokens": 128256, |
| "n_added_tokens": 4, |
| "vocab_total": 128260, |
| "added_tokens": [ |
| "<s>", |
| "</s>", |
| "<unk>", |
| "<pad>" |
| ], |
| "normalizer": null, |
| "pre_tokenizer": { |
| "type": "Sequence", |
| "pretokenizers": [ |
| { |
| "type": "Punctuation", |
| "behavior": "Isolated" |
| }, |
| { |
| "type": "ByteLevel", |
| "add_prefix_space": false, |
| "trim_offsets": true, |
| "use_regex": true |
| } |
| ] |
| }, |
| "has_post_processor": false, |
| "byte_fallback": false, |
| "bos_token": "<s>", |
| "bos_token_id": 0 |
| }, |
| "declared_experiments": { |
| "algorithm": "bpe", |
| "pretokenizer_constant": "Punctuation+ByteLevel", |
| "normalizer": null, |
| "data_composition": "balanced", |
| "description": "Punctuation + ByteLevel pretok, balanced data", |
| "trainer_kwargs": null |
| }, |
| "declared_run_manifest": null, |
| "registry_note": "P1 Punctuation+ByteLevel; balanced (D2) data; norm=none; BPE; vocab 128,260; USED - panels.json main21 (camera-ready main panel)" |
| }, |
| "punct-balancedhalf-superbpe-t64k-copy": { |
| "slug": "punct-balancedhalf-superbpe-t64k-copy", |
| "runs": [ |
| "full-128k-punct-balancedhalf-superbpe-t64k-copy" |
| ], |
| "n_runs": 1, |
| "redistributable": true, |
| "loadable_with": "transformers", |
| "hub_id": null, |
| "policy_note": "trained in this project", |
| "empirical": { |
| "model_type": "BPE", |
| "n_base_tokens": 128000, |
| "n_added_tokens": 4, |
| "vocab_total": 128004, |
| "added_tokens": [ |
| "<s>", |
| "</s>", |
| "<unk>", |
| "<pad>" |
| ], |
| "normalizer": null, |
| "pre_tokenizer": { |
| "type": "Sequence", |
| "pretokenizers": [ |
| { |
| "type": "Split", |
| "pattern": { |
| "Regex": "\\s+(?!\\S)|\\s*[\\r\\n]+" |
| }, |
| "behavior": "Isolated", |
| "invert": false |
| }, |
| { |
| "type": "ByteLevel", |
| "add_prefix_space": false, |
| "trim_offsets": true, |
| "use_regex": false |
| } |
| ] |
| }, |
| "has_post_processor": false, |
| "byte_fallback": false, |
| "bos_token": "<s>", |
| "bos_token_id": 128000 |
| }, |
| "declared_experiments": null, |
| "declared_run_manifest": null, |
| "registry_note": "SuperBPE stage-1 Punct(Isolated)+ByteLevel base (64k transition) / stage-2 minimal regex (allows superwords to span punctuation); balanced-half (D2 halved) data; norm=none (inherited from punct base); SuperBPE; vocab 128,004; the only SuperBPE variant EXEMPT from the S9.5 combining-mark bug (its stage-2 regex has no [^\\s\\p{L}\\p{N}] alternative); USED - balanced-1B bootstrap panel (S6.12) + paper panel; not in panels.json main21" |
| }, |
| "punct-english-bpe": { |
| "slug": "punct-english-bpe", |
| "runs": [ |
| "full-128k-punct-english-bpe", |
| "full-128k-punct-english-bpe-mathcode-scratch", |
| "full-128k-punct-english-bpe-seed42", |
| "pilot-128k-punct-english-bpe-wdfix", |
| "small-128k-punct-english-bpe", |
| "tiny-128k-punct-english-bpe" |
| ], |
| "n_runs": 6, |
| "redistributable": true, |
| "loadable_with": "transformers", |
| "hub_id": null, |
| "policy_note": "trained in this project", |
| "empirical": { |
| "model_type": "BPE", |
| "n_base_tokens": 128256, |
| "n_added_tokens": 4, |
| "vocab_total": 128260, |
| "added_tokens": [ |
| "<s>", |
| "</s>", |
| "<unk>", |
| "<pad>" |
| ], |
| "normalizer": null, |
| "pre_tokenizer": { |
| "type": "Sequence", |
| "pretokenizers": [ |
| { |
| "type": "Punctuation", |
| "behavior": "Isolated" |
| }, |
| { |
| "type": "ByteLevel", |
| "add_prefix_space": false, |
| "trim_offsets": true, |
| "use_regex": true |
| } |
| ] |
| }, |
| "has_post_processor": false, |
| "byte_fallback": false, |
| "bos_token": "<s>", |
| "bos_token_id": 0 |
| }, |
| "declared_experiments": { |
| "algorithm": "bpe", |
| "pretokenizer_constant": "Punctuation+ByteLevel", |
| "normalizer": null, |
| "data_composition": "english", |
| "description": "Punctuation + ByteLevel pretok, English-only data", |
| "trainer_kwargs": null |
| }, |
| "declared_run_manifest": null, |
| "registry_note": "P1 Punctuation+ByteLevel; english (D1) data; norm=none; BPE; vocab 128,260; USED - panels.json main21 (camera-ready main panel)" |
| }, |
| "rightalign-balanced-bpe": { |
| "slug": "rightalign-balanced-bpe", |
| "runs": [ |
| "full-128k-rightalign-balanced-bpe", |
| "full-128k-rightalign-balanced-bpe-mathcode-scratch", |
| "full-128k-rightalign-balanced-bpe-seed42", |
| "pilot-128k-rightalign-balanced-bpe-wdfix", |
| "small-128k-rightalign-balanced-bpe", |
| "tiny-128k-rightalign-balanced-bpe" |
| ], |
| "n_runs": 6, |
| "redistributable": true, |
| "loadable_with": "transformers", |
| "hub_id": null, |
| "policy_note": "trained in this project", |
| "empirical": { |
| "model_type": "BPE", |
| "n_base_tokens": 128256, |
| "n_added_tokens": 4, |
| "vocab_total": 128260, |
| "added_tokens": [ |
| "<s>", |
| "</s>", |
| "<unk>", |
| "<pad>" |
| ], |
| "normalizer": null, |
| "pre_tokenizer": { |
| "type": "Sequence", |
| "pretokenizers": [ |
| { |
| "type": "Split", |
| "pattern": { |
| "Regex": "[^\\r\\n\\p{L}\\p{N}]?[\\p{Lu}\\p{Lt}\\p{Lm}\\p{Lo}\\p{M}]*[\\p{Ll}\\p{Lm}\\p{Lo}\\p{M}]+(?i:'s|'t|'re|'ve|'m|'ll|'d)?|[^\\r\\n\\p{L}\\p{N}]?[\\p{Lu}\\p{Lt}\\p{Lm}\\p{Lo}\\p{M}]+[\\p{Ll}\\p{Lm}\\p{Lo}\\p{M}]*(?i:'s|'t|'re|'ve|'m|'ll|'d)?|\\p{N}{1,3}(?=(?:\\p{N}{3})*(?:\\P{N}|$))| ?[^\\s\\p{L}\\p{N}]+[\\r\\n/]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+" |
| }, |
| "behavior": "Isolated", |
| "invert": false |
| }, |
| { |
| "type": "ByteLevel", |
| "add_prefix_space": false, |
| "trim_offsets": true, |
| "use_regex": false |
| } |
| ] |
| }, |
| "has_post_processor": false, |
| "byte_fallback": false, |
| "bos_token": "<s>", |
| "bos_token_id": 0 |
| }, |
| "declared_experiments": { |
| "algorithm": "bpe", |
| "pretokenizer_constant": "REGEX_GPT4O_RIGHTALIGN", |
| "normalizer": null, |
| "data_composition": "balanced", |
| "description": "GPT-4o regex with right-aligned 3-digit parsing, balanced data", |
| "trainer_kwargs": null |
| }, |
| "declared_run_manifest": null, |
| "registry_note": "P4 right-aligned digits (REGEX_GPT4O_RIGHTALIGN); balanced (D2) data; norm=none; BPE; vocab 128,260; USED - panels.json main21 (camera-ready main panel)" |
| }, |
| "rightalign-balanced-nfc-bpe": { |
| "slug": "rightalign-balanced-nfc-bpe", |
| "runs": [ |
| "full-128k-rightalign-balanced-nfc-bpe", |
| "full-128k-rightalign-balanced-nfc-bpe-mathcode-scratch", |
| "full-128k-rightalign-balanced-nfc-bpe-seed42", |
| "pilot-128k-rightalign-balanced-nfc-bpe-wdfix", |
| "small-128k-rightalign-balanced-nfc-bpe", |
| "tiny-128k-rightalign-balanced-nfc-bpe" |
| ], |
| "n_runs": 6, |
| "redistributable": true, |
| "loadable_with": "transformers", |
| "hub_id": null, |
| "policy_note": "trained in this project", |
| "empirical": { |
| "model_type": "BPE", |
| "n_base_tokens": 128256, |
| "n_added_tokens": 4, |
| "vocab_total": 128260, |
| "added_tokens": [ |
| "<s>", |
| "</s>", |
| "<unk>", |
| "<pad>" |
| ], |
| "normalizer": { |
| "type": "NFC" |
| }, |
| "pre_tokenizer": { |
| "type": "Sequence", |
| "pretokenizers": [ |
| { |
| "type": "Split", |
| "pattern": { |
| "Regex": "[^\\r\\n\\p{L}\\p{N}]?[\\p{Lu}\\p{Lt}\\p{Lm}\\p{Lo}\\p{M}]*[\\p{Ll}\\p{Lm}\\p{Lo}\\p{M}]+(?i:'s|'t|'re|'ve|'m|'ll|'d)?|[^\\r\\n\\p{L}\\p{N}]?[\\p{Lu}\\p{Lt}\\p{Lm}\\p{Lo}\\p{M}]+[\\p{Ll}\\p{Lm}\\p{Lo}\\p{M}]*(?i:'s|'t|'re|'ve|'m|'ll|'d)?|\\p{N}{1,3}(?=(?:\\p{N}{3})*(?:\\P{N}|$))| ?[^\\s\\p{L}\\p{N}]+[\\r\\n/]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+" |
| }, |
| "behavior": "Isolated", |
| "invert": false |
| }, |
| { |
| "type": "ByteLevel", |
| "add_prefix_space": false, |
| "trim_offsets": true, |
| "use_regex": false |
| } |
| ] |
| }, |
| "has_post_processor": false, |
| "byte_fallback": false, |
| "bos_token": "<s>", |
| "bos_token_id": 0 |
| }, |
| "declared_experiments": { |
| "algorithm": "bpe", |
| "pretokenizer_constant": "REGEX_GPT4O_RIGHTALIGN", |
| "normalizer": { |
| "type": "NFC" |
| }, |
| "data_composition": "balanced", |
| "description": "Right-aligned digits, balanced data, NFC normalization", |
| "trainer_kwargs": null |
| }, |
| "declared_run_manifest": null, |
| "registry_note": "P4 right-aligned digits; balanced (D2) data; norm=NFC; BPE; vocab 128,260; USED - panels.json main21 (camera-ready main panel)" |
| }, |
| "rightalign-balanced-unigram": { |
| "slug": "rightalign-balanced-unigram", |
| "runs": [ |
| "full-128k-rightalign-balanced-unigram", |
| "full-128k-rightalign-balanced-unigram-mathcode-scratch", |
| "full-128k-rightalign-balanced-unigram-seed42", |
| "pilot-128k-rightalign-balanced-unigram-wdfix", |
| "small-128k-rightalign-balanced-unigram", |
| "tiny-128k-rightalign-balanced-unigram" |
| ], |
| "n_runs": 6, |
| "redistributable": true, |
| "loadable_with": "transformers", |
| "hub_id": null, |
| "policy_note": "trained in this project", |
| "empirical": { |
| "model_type": "Unigram", |
| "n_base_tokens": 128256, |
| "n_added_tokens": 4, |
| "vocab_total": 128260, |
| "added_tokens": [ |
| "<s>", |
| "</s>", |
| "<unk>", |
| "<pad>" |
| ], |
| "normalizer": null, |
| "pre_tokenizer": { |
| "type": "Sequence", |
| "pretokenizers": [ |
| { |
| "type": "Split", |
| "pattern": { |
| "Regex": "[^\\r\\n\\p{L}\\p{N}]?[\\p{Lu}\\p{Lt}\\p{Lm}\\p{Lo}\\p{M}]*[\\p{Ll}\\p{Lm}\\p{Lo}\\p{M}]+(?i:'s|'t|'re|'ve|'m|'ll|'d)?|[^\\r\\n\\p{L}\\p{N}]?[\\p{Lu}\\p{Lt}\\p{Lm}\\p{Lo}\\p{M}]+[\\p{Ll}\\p{Lm}\\p{Lo}\\p{M}]*(?i:'s|'t|'re|'ve|'m|'ll|'d)?|\\p{N}{1,3}(?=(?:\\p{N}{3})*(?:\\P{N}|$))| ?[^\\s\\p{L}\\p{N}]+[\\r\\n/]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+" |
| }, |
| "behavior": "Isolated", |
| "invert": false |
| }, |
| { |
| "type": "ByteLevel", |
| "add_prefix_space": false, |
| "trim_offsets": true, |
| "use_regex": false |
| } |
| ] |
| }, |
| "has_post_processor": false, |
| "byte_fallback": false, |
| "bos_token": "<s>", |
| "bos_token_id": 0 |
| }, |
| "declared_experiments": { |
| "algorithm": "unigram", |
| "pretokenizer_constant": "REGEX_GPT4O_RIGHTALIGN", |
| "normalizer": null, |
| "data_composition": "balanced", |
| "description": "Right-aligned digits, balanced data, UnigramLM algorithm", |
| "trainer_kwargs": null |
| }, |
| "declared_run_manifest": null, |
| "registry_note": "P4 right-aligned digits; balanced (D2) data; norm=none; UnigramLM (HF, default kwargs); vocab 128,260; USED - panels.json main21 (camera-ready main panel)" |
| }, |
| "scripttok-bpe-scriptenc_gpt4o_cb": { |
| "slug": "scripttok-bpe-scriptenc_gpt4o_cb", |
| "runs": [ |
| "full-128k-scripttok-bpe-scriptenc_gpt4o_cb", |
| "full-128k-scripttok-bpe-scriptenc_gpt4o_cb-mathcode-scratch" |
| ], |
| "n_runs": 2, |
| "redistributable": true, |
| "loadable_with": "script_tok", |
| "hub_id": null, |
| "policy_note": "trained in this project; the SCRIPT-encoding format needs the script_tok loader, stock AutoTokenizer cannot read it", |
| "empirical": { |
| "format": "tokenizer.json.gz", |
| "note": "not a tokenizers-library file; not introspected here" |
| }, |
| "declared_experiments": null, |
| "declared_run_manifest": null, |
| "registry_note": "script_tok BPE, scriptenc_gpt4o_cb SCRIPT-encoding pretok; balanced (D2, script_tok's own fit sampler) data; norm=SCRIPT-encoding normalization; BPE (script_tok); vocab 128,256; requires pre-tokenized .npz shards; USED - panels.json structure-pretok (camera-ready)" |
| }, |
| "scripttok-mingram-scriptenc_cb": { |
| "slug": "scripttok-mingram-scriptenc_cb", |
| "runs": [ |
| "full-128k-scripttok-mingram-scriptenc_cb", |
| "full-128k-scripttok-mingram-scriptenc_cb-mathcode-scratch" |
| ], |
| "n_runs": 2, |
| "redistributable": true, |
| "loadable_with": "script_tok", |
| "hub_id": null, |
| "policy_note": "trained in this project; the SCRIPT-encoding format needs the script_tok loader, stock AutoTokenizer cannot read it", |
| "empirical": { |
| "format": "tokenizer.json.gz", |
| "note": "not a tokenizers-library file; not introspected here" |
| }, |
| "declared_experiments": null, |
| "declared_run_manifest": null, |
| "registry_note": "script_tok MinGram, scriptenc_cb SCRIPT-encoding pretok (-n 126339), PUA/unassigned code points stripped; balanced (D2, script_tok's own fit sampler/composition/seed) data; norm=SCRIPT-encoding normalization (designed, applied identically train/eval); MinGram; vocab 128,256 (1,916 atomic + 126,339 learned + synthesized BOS); requires pre-tokenized .npz shards; forward-fusing (newline+following indentation fuse into one atomic pretoken, later diagnosed as the MBPP/HumanEval-collapse mechanism at 20B, S3a.1/3a.6/S6h); USED - panels.json main21 (camera-ready main panel)" |
| }, |
| "scripttok-mingram-scriptenc_cb_nl": { |
| "slug": "scripttok-mingram-scriptenc_cb_nl", |
| "runs": [ |
| "full-128k-scripttok-mingram-scriptenc_cb_nl", |
| "full-128k-scripttok-mingram-scriptenc_cb_nl-mathcode-scratch" |
| ], |
| "n_runs": 2, |
| "redistributable": true, |
| "loadable_with": "script_tok", |
| "hub_id": null, |
| "policy_note": "trained in this project; the SCRIPT-encoding format needs the script_tok loader, stock AutoTokenizer cannot read it", |
| "empirical": { |
| "format": "tokenizer.json.gz", |
| "note": "not a tokenizers-library file; not introspected here" |
| }, |
| "declared_experiments": null, |
| "declared_run_manifest": null, |
| "registry_note": "script_tok MinGram, scriptenc_cb_nl (split_line_breaks=True: newline isolated from indentation, additive opt-in); balanced (D2, balanced_tlm10g_cb_nl corpus); SCRIPT-encoding normalization; MinGram; vocab 128,256 (1,916 atomic+126,339 learned+BOS); line-break-splitting ablation vs scripttok-mingram-scriptenc_cb (newline-fusion mechanism). USED - S3a.6/S3a.7/S6h newline-fusion analysis (std-1B twin of 20B pair, finished 2026-08-08); design-variant ablation, not a panels.json member." |
| }, |
| "superbpe-apertus-fineweb2full-capped-hybridwindow": { |
| "slug": "superbpe-apertus-fineweb2full-capped-hybridwindow", |
| "runs": [ |
| "full-128k-superbpe-apertus-fineweb2full-capped-hybridwindow", |
| "full-128k-superbpe-apertus-fineweb2full-capped-hybridwindow-mathcode-scratch" |
| ], |
| "n_runs": 2, |
| "redistributable": true, |
| "loadable_with": "transformers", |
| "hub_id": null, |
| "policy_note": "trained in this project", |
| "empirical": { |
| "model_type": "BPE", |
| "n_base_tokens": 128000, |
| "n_added_tokens": 4, |
| "vocab_total": 128004, |
| "added_tokens": [ |
| "<s>", |
| "</s>", |
| "<unk>", |
| "<pad>" |
| ], |
| "normalizer": { |
| "type": "NFC" |
| }, |
| "pre_tokenizer": { |
| "type": "Sequence", |
| "pretokenizers": [ |
| { |
| "type": "Split", |
| "pattern": { |
| "Regex": "\\p{N}| ?(?:(?:[^\\s\\p{L}\\p{N}\\p{M}]\\p{M}*){1,16}[\\r\\n]{1,16}|(?:[^\\s\\p{L}\\p{N}\\p{M}]\\p{M}*){2,16})|\\s{0,16}[\\r\\n]{1,16}|\\s{1,16}(?!\\S)" |
| }, |
| "behavior": "Isolated", |
| "invert": false |
| }, |
| { |
| "type": "ByteLevel", |
| "add_prefix_space": false, |
| "trim_offsets": true, |
| "use_regex": false |
| } |
| ] |
| }, |
| "has_post_processor": false, |
| "byte_fallback": false, |
| "bos_token": "<s>", |
| "bos_token_id": 128000 |
| }, |
| "declared_experiments": null, |
| "declared_run_manifest": null, |
| "registry_note": "SuperBPE on the apertus-pretok capped hybrid-window base; fineweb2full, capped data; norm=NFC; SuperBPE; vocab 128,004; 2x2 candidate; USED - Apertus-2 development track candidate; excluded from main paper panel" |
| }, |
| "superbpe-clean-c2-balanced": { |
| "slug": "superbpe-clean-c2-balanced", |
| "runs": [ |
| "full-128k-superbpe-clean-c2-balanced", |
| "full-128k-superbpe-clean-c2-balanced-mathcode-scratch" |
| ], |
| "n_runs": 2, |
| "redistributable": true, |
| "loadable_with": "transformers", |
| "hub_id": null, |
| "policy_note": "trained in this project", |
| "empirical": { |
| "model_type": "BPE", |
| "n_base_tokens": 128000, |
| "n_added_tokens": 4, |
| "vocab_total": 128004, |
| "added_tokens": [ |
| "<s>", |
| "</s>", |
| "<unk>", |
| "<pad>" |
| ], |
| "normalizer": { |
| "type": "NFC" |
| }, |
| "pre_tokenizer": { |
| "type": "Sequence", |
| "pretokenizers": [ |
| { |
| "type": "Split", |
| "pattern": { |
| "Regex": "\\p{N}| ?[^\\s\\p{L}\\p{N}\\p{M}]\\p{M}*(?:[^\\s\\p{L}\\p{N}\\p{M}]\\p{M}*)*|\\s*[\\r\\n]+|\\s+(?!\\S)" |
| }, |
| "behavior": "Isolated", |
| "invert": false |
| }, |
| { |
| "type": "ByteLevel", |
| "add_prefix_space": false, |
| "trim_offsets": true, |
| "use_regex": false |
| } |
| ] |
| }, |
| "has_post_processor": false, |
| "byte_fallback": false, |
| "bos_token": "<s>", |
| "bos_token_id": 128000 |
| }, |
| "declared_experiments": null, |
| "declared_run_manifest": null, |
| "registry_note": "SuperBPE stage-1 clean-multi (PA-BPE clean base) / stage-2 clean_multi C2 reduced regex; balanced (D2) data; norm=NFC (inherited from PA-BPE clean base); SuperBPE; vocab 128,004; Phase-5 clean_multi_c2_t90k stage-2 preset; USED - balanced-1B bootstrap panel (S6.12) + paper panel; not in panels.json main21" |
| }, |
| "superbpe-clean-c3-balanced": { |
| "slug": "superbpe-clean-c3-balanced", |
| "runs": [ |
| "full-128k-superbpe-clean-c3-balanced", |
| "full-128k-superbpe-clean-c3-balanced-mathcode-scratch" |
| ], |
| "n_runs": 2, |
| "redistributable": true, |
| "loadable_with": "transformers", |
| "hub_id": null, |
| "policy_note": "trained in this project", |
| "empirical": { |
| "model_type": "BPE", |
| "n_base_tokens": 128000, |
| "n_added_tokens": 4, |
| "vocab_total": 128004, |
| "added_tokens": [ |
| "<s>", |
| "</s>", |
| "<unk>", |
| "<pad>" |
| ], |
| "normalizer": { |
| "type": "NFC" |
| }, |
| "pre_tokenizer": { |
| "type": "Sequence", |
| "pretokenizers": [ |
| { |
| "type": "Split", |
| "pattern": { |
| "Regex": "\\p{N}{1,3}| ?[^\\s\\p{L}\\p{N}\\p{M}]\\p{M}*(?:[^\\s\\p{L}\\p{N}\\p{M}]\\p{M}*)+|\\s*[\\r\\n]+|\\s+(?!\\S)" |
| }, |
| "behavior": "Isolated", |
| "invert": false |
| }, |
| { |
| "type": "ByteLevel", |
| "add_prefix_space": false, |
| "trim_offsets": true, |
| "use_regex": false |
| } |
| ] |
| }, |
| "has_post_processor": false, |
| "byte_fallback": false, |
| "bos_token": "<s>", |
| "bos_token_id": 128000 |
| }, |
| "declared_experiments": null, |
| "declared_run_manifest": null, |
| "registry_note": "SuperBPE stage-1 clean-multi / stage-2 clean_multi C3 reduced regex; balanced (D2) data; norm=NFC; SuperBPE; vocab 128,004; Phase-5 clean_multi_c3_t90k stage-2 preset; USED - balanced-1B bootstrap panel (S6.12) + paper panel; not in panels.json main21" |
| }, |
| "superbpe-clean-fineweb2full-capped-hybridwindow": { |
| "slug": "superbpe-clean-fineweb2full-capped-hybridwindow", |
| "runs": [ |
| "full-128k-superbpe-clean-fineweb2full-capped-hybridwindow", |
| "full-128k-superbpe-clean-fineweb2full-capped-hybridwindow-mathcode-scratch" |
| ], |
| "n_runs": 2, |
| "redistributable": true, |
| "loadable_with": "transformers", |
| "hub_id": null, |
| "policy_note": "trained in this project", |
| "empirical": { |
| "model_type": "BPE", |
| "n_base_tokens": 128000, |
| "n_added_tokens": 4, |
| "vocab_total": 128004, |
| "added_tokens": [ |
| "<s>", |
| "</s>", |
| "<unk>", |
| "<pad>" |
| ], |
| "normalizer": { |
| "type": "NFC" |
| }, |
| "pre_tokenizer": { |
| "type": "Sequence", |
| "pretokenizers": [ |
| { |
| "type": "Split", |
| "pattern": { |
| "Regex": "\\p{N}| ?(?:[^\\s\\p{L}\\p{N}\\p{M}]\\p{M}*){1,16}|\\s{0,16}[\\r\\n]{1,16}|\\s{1,16}(?!\\S)" |
| }, |
| "behavior": "Isolated", |
| "invert": false |
| }, |
| { |
| "type": "ByteLevel", |
| "add_prefix_space": false, |
| "trim_offsets": true, |
| "use_regex": false |
| } |
| ] |
| }, |
| "has_post_processor": false, |
| "byte_fallback": false, |
| "bos_token": "<s>", |
| "bos_token_id": 128000 |
| }, |
| "declared_experiments": null, |
| "declared_run_manifest": null, |
| "registry_note": "SuperBPE on the clean-multi-pretok capped hybrid-window base; fineweb2full, capped data; norm=NFC; SuperBPE; vocab 128,004; 2x2 candidate (t90k/v128k); USED - Apertus-2 development track candidate + S6h code analysis roster (20B twin only, see MC table); excluded from main paper panel" |
| }, |
| "superbpe-clean-fineweb2full-capped-hybridwindow-t110k-v130k": { |
| "slug": "superbpe-clean-fineweb2full-capped-hybridwindow-t110k-v130k", |
| "runs": [ |
| "full-128k-superbpe-clean-fineweb2full-capped-hybridwindow-t110k-v130k", |
| "full-128k-superbpe-clean-fineweb2full-capped-hybridwindow-t110k-v130k-mathcode-scratch" |
| ], |
| "n_runs": 2, |
| "redistributable": true, |
| "loadable_with": "transformers", |
| "hub_id": null, |
| "policy_note": "trained in this project", |
| "empirical": { |
| "model_type": "BPE", |
| "n_base_tokens": 130000, |
| "n_added_tokens": 4, |
| "vocab_total": 130004, |
| "added_tokens": [ |
| "<s>", |
| "</s>", |
| "<unk>", |
| "<pad>" |
| ], |
| "normalizer": { |
| "type": "NFC" |
| }, |
| "pre_tokenizer": { |
| "type": "Sequence", |
| "pretokenizers": [ |
| { |
| "type": "Split", |
| "pattern": { |
| "Regex": "\\p{N}| ?(?:[^\\s\\p{L}\\p{N}\\p{M}]\\p{M}*){1,16}|\\s{0,16}[\\r\\n]{1,16}|\\s{1,16}(?!\\S)" |
| }, |
| "behavior": "Isolated", |
| "invert": false |
| }, |
| { |
| "type": "ByteLevel", |
| "add_prefix_space": false, |
| "trim_offsets": true, |
| "use_regex": false |
| } |
| ] |
| }, |
| "has_post_processor": false, |
| "byte_fallback": false, |
| "bos_token": "<s>", |
| "bos_token_id": 130000 |
| }, |
| "declared_experiments": null, |
| "declared_run_manifest": null, |
| "registry_note": "SuperBPE, clean-multi pretok base; fineweb2full, capped data; norm=NFC; SuperBPE; vocab ~130,000; SuperBPE stage-2 transition-point ablation (t110k vs the 2x2 candidate's t90k); USED - Apertus-2 development track ablation; excluded from main paper panel" |
| }, |
| "superbpe-clean-fineweb2full-plus2-cappedv2-hybridwindow-t110k-v130k-consv2": { |
| "slug": "superbpe-clean-fineweb2full-plus2-cappedv2-hybridwindow-t110k-v130k-consv2", |
| "runs": [ |
| "full-128k-superbpe-clean-fineweb2full-plus2-cappedv2-hybridwindow-t110k-v130k-consv2", |
| "full-128k-superbpe-clean-fineweb2full-plus2-cappedv2-hybridwindow-t110k-v130k-consv2-mathcode-scratch" |
| ], |
| "n_runs": 2, |
| "redistributable": true, |
| "loadable_with": "transformers", |
| "hub_id": null, |
| "policy_note": "trained in this project", |
| "empirical": { |
| "model_type": "BPE", |
| "n_base_tokens": 130000, |
| "n_added_tokens": 4, |
| "vocab_total": 130004, |
| "added_tokens": [ |
| "<s>", |
| "</s>", |
| "<unk>", |
| "<pad>" |
| ], |
| "normalizer": { |
| "type": "NFC" |
| }, |
| "pre_tokenizer": { |
| "type": "Sequence", |
| "pretokenizers": [ |
| { |
| "type": "Split", |
| "pattern": { |
| "Regex": "\\p{N}| ?(?:(?:[^\\s\\p{L}\\p{N}\\p{M}]\\p{M}*){2,16}|[^\\s\\p{L}\\p{N}\\p{M}\\x{0027}\\x{2019}\\x{0F0B}]\\p{M}*)|\\s{0,16}[\\r\\n]{1,16}|\\s{1,16}(?!\\S)" |
| }, |
| "behavior": "Isolated", |
| "invert": false |
| }, |
| { |
| "type": "ByteLevel", |
| "add_prefix_space": false, |
| "trim_offsets": true, |
| "use_regex": false |
| } |
| ] |
| }, |
| "has_post_processor": false, |
| "byte_fallback": false, |
| "bos_token": "<s>", |
| "bos_token_id": 130000 |
| }, |
| "declared_experiments": null, |
| "declared_run_manifest": null, |
| "registry_note": "SuperBPE on the clean-multi plus2 (v2) capped-v2 hybrid-window base, stage-2 transition t110k; fineweb2full, consv2 data; norm=NFC; SuperBPE; vocab ~130,000; Apertus-2 development ablation 'SuperBPE-plus2v2-cv2-t110k'; USED - Apertus-2 track ablation; excluded from main paper panel" |
| }, |
| "superbpe-gpt4-fineweb2full-hybridwindow": { |
| "slug": "superbpe-gpt4-fineweb2full-hybridwindow", |
| "runs": [ |
| "full-128k-superbpe-gpt4-fineweb2full-hybridwindow-mathcode-scratch" |
| ], |
| "n_runs": 1, |
| "redistributable": true, |
| "loadable_with": "transformers", |
| "hub_id": null, |
| "policy_note": "trained in this project", |
| "empirical": { |
| "model_type": "BPE", |
| "n_base_tokens": 128000, |
| "n_added_tokens": 4, |
| "vocab_total": 128004, |
| "added_tokens": [ |
| "<s>", |
| "</s>", |
| "<unk>", |
| "<pad>" |
| ], |
| "normalizer": { |
| "type": "NFC" |
| }, |
| "pre_tokenizer": { |
| "type": "Sequence", |
| "pretokenizers": [ |
| { |
| "type": "Split", |
| "pattern": { |
| "Regex": "\\p{N}{1,3}| ?(?:[^\\s\\p{L}\\p{N}\\p{M}]\\p{M}*(?:[^\\s\\p{L}\\p{N}\\p{M}]\\p{M}*)*[\\r\\n/]+|[^\\s\\p{L}\\p{N}\\p{M}]\\p{M}*(?:[^\\s\\p{L}\\p{N}\\p{M}]\\p{M}*)+)|\\s*[\\r\\n]+|\\s+(?!\\S)" |
| }, |
| "behavior": "Isolated", |
| "invert": false |
| }, |
| { |
| "type": "ByteLevel", |
| "add_prefix_space": false, |
| "trim_offsets": true, |
| "use_regex": false |
| } |
| ] |
| }, |
| "has_post_processor": false, |
| "byte_fallback": false, |
| "bos_token": "<s>", |
| "bos_token_id": 128000 |
| }, |
| "declared_experiments": null, |
| "declared_run_manifest": null, |
| "registry_note": "#12, gpt4 pretok, slug `superbpe-gpt4-fineweb2full-hybridwindow`. SuperBPE on older GPT-4 regex (uncapped, {1,3}-digit)." |
| }, |
| "superbpe-gpt4o-balanced": { |
| "slug": "superbpe-gpt4o-balanced", |
| "runs": [ |
| "full-128k-superbpe-gpt4o-balanced", |
| "full-128k-superbpe-gpt4o-balanced-mathcode-scratch", |
| "pilot-128k-superbpe-gpt4o-balanced", |
| "small-128k-superbpe-gpt4o-balanced", |
| "tiny-128k-superbpe-gpt4o-balanced" |
| ], |
| "n_runs": 5, |
| "redistributable": true, |
| "loadable_with": "transformers", |
| "hub_id": null, |
| "policy_note": "trained in this project", |
| "empirical": { |
| "model_type": "BPE", |
| "n_base_tokens": 128000, |
| "n_added_tokens": 4, |
| "vocab_total": 128004, |
| "added_tokens": [ |
| "<s>", |
| "</s>", |
| "<unk>", |
| "<pad>" |
| ], |
| "normalizer": null, |
| "pre_tokenizer": { |
| "type": "Sequence", |
| "pretokenizers": [ |
| { |
| "type": "Split", |
| "pattern": { |
| "Regex": "\\p{N}{1,3}| ?(?:[^\\s\\p{L}\\p{N}\\p{M}]\\p{M}*(?:[^\\s\\p{L}\\p{N}\\p{M}]\\p{M}*)*[\\r\\n/]+|[^\\s\\p{L}\\p{N}\\p{M}]\\p{M}*(?:[^\\s\\p{L}\\p{N}\\p{M}]\\p{M}*)+)|\\s*[\\r\\n]+|\\s+(?!\\S)" |
| }, |
| "behavior": "Isolated", |
| "invert": false |
| }, |
| { |
| "type": "ByteLevel", |
| "add_prefix_space": false, |
| "trim_offsets": true, |
| "use_regex": false |
| } |
| ] |
| }, |
| "has_post_processor": false, |
| "byte_fallback": false, |
| "bos_token": "<s>", |
| "bos_token_id": 128000 |
| }, |
| "declared_experiments": null, |
| "declared_run_manifest": null, |
| "registry_note": "SuperBPE stage-1 GPT-4o regex (90k inherited merges) / stage-2 reduced GPT-4o regex (38k superword merges); base = gpt4o-balanced-bpe; balanced (D2, halved for stage-2: balanced_half.json) data; norm=none (inherited from gpt4o-balanced-bpe base); SuperBPE; vocab 128,004; Phase-5 fixed retrain (2026-05-18) of the combining-mark-bug Run 1; USED - panels.json main21 (camera-ready main panel)" |
| }, |
| "superbpe-pabpe-nfc-gpt4-balanced": { |
| "slug": "superbpe-pabpe-nfc-gpt4-balanced", |
| "runs": [ |
| "full-128k-superbpe-pabpe-nfc-gpt4-balanced" |
| ], |
| "n_runs": 1, |
| "redistributable": true, |
| "loadable_with": "transformers", |
| "hub_id": null, |
| "policy_note": "trained in this project", |
| "empirical": { |
| "model_type": "BPE", |
| "n_base_tokens": 128000, |
| "n_added_tokens": 4, |
| "vocab_total": 128004, |
| "added_tokens": [ |
| "<s>", |
| "</s>", |
| "<unk>", |
| "<pad>" |
| ], |
| "normalizer": { |
| "type": "NFC" |
| }, |
| "pre_tokenizer": { |
| "type": "Sequence", |
| "pretokenizers": [ |
| { |
| "type": "Split", |
| "pattern": { |
| "Regex": "\\p{N}{1,3}| ?(?:[^\\s\\p{L}\\p{N}\\p{M}]\\p{M}*(?:[^\\s\\p{L}\\p{N}\\p{M}]\\p{M}*)*[\\r\\n/]+|[^\\s\\p{L}\\p{N}\\p{M}]\\p{M}*(?:[^\\s\\p{L}\\p{N}\\p{M}]\\p{M}*)+)|\\s*[\\r\\n]+|\\s+(?!\\S)" |
| }, |
| "behavior": "Isolated", |
| "invert": false |
| }, |
| { |
| "type": "ByteLevel", |
| "add_prefix_space": false, |
| "trim_offsets": true, |
| "use_regex": false |
| } |
| ] |
| }, |
| "has_post_processor": false, |
| "byte_fallback": false, |
| "bos_token": "<s>", |
| "bos_token_id": 128000 |
| }, |
| "declared_experiments": null, |
| "declared_run_manifest": null, |
| "registry_note": "SuperBPE stage-1 GPT-4o regex / stage-2 reduced GPT-4o regex; base = PA-BPE nfc_gpt4_balanced; balanced (D2, halved) data; norm=NFC (inherited from PA-BPE base); SuperBPE; vocab 128,004; Phase-5 fixed retrain of Run 2; USED - balanced-1B bootstrap panel (S6.12) + paper panel; not in panels.json main21" |
| }, |
| "superbpe-pabpe-nfc-gpt4-t64k": { |
| "slug": "superbpe-pabpe-nfc-gpt4-t64k", |
| "runs": [ |
| "full-128k-superbpe-pabpe-nfc-gpt4-t64k" |
| ], |
| "n_runs": 1, |
| "redistributable": true, |
| "loadable_with": "transformers", |
| "hub_id": null, |
| "policy_note": "trained in this project", |
| "empirical": { |
| "model_type": "BPE", |
| "n_base_tokens": 128000, |
| "n_added_tokens": 4, |
| "vocab_total": 128004, |
| "added_tokens": [ |
| "<s>", |
| "</s>", |
| "<unk>", |
| "<pad>" |
| ], |
| "normalizer": { |
| "type": "NFC" |
| }, |
| "pre_tokenizer": { |
| "type": "Sequence", |
| "pretokenizers": [ |
| { |
| "type": "Split", |
| "pattern": { |
| "Regex": "\\p{N}{1,3}| ?(?:[^\\s\\p{L}\\p{N}\\p{M}]\\p{M}*(?:[^\\s\\p{L}\\p{N}\\p{M}]\\p{M}*)*[\\r\\n/]+|[^\\s\\p{L}\\p{N}\\p{M}]\\p{M}*(?:[^\\s\\p{L}\\p{N}\\p{M}]\\p{M}*)+)|\\s*[\\r\\n]+|\\s+(?!\\S)" |
| }, |
| "behavior": "Isolated", |
| "invert": false |
| }, |
| { |
| "type": "ByteLevel", |
| "add_prefix_space": false, |
| "trim_offsets": true, |
| "use_regex": false |
| } |
| ] |
| }, |
| "has_post_processor": false, |
| "byte_fallback": false, |
| "bos_token": "<s>", |
| "bos_token_id": 128000 |
| }, |
| "declared_experiments": null, |
| "declared_run_manifest": null, |
| "registry_note": "SuperBPE stage-1 GPT-4o regex (64k transition, more aggressive superword fusing) / stage-2 reduced GPT-4o regex; base = PA-BPE nfc_gpt4_balanced; balanced (D2, halved) data; norm=NFC; SuperBPE; vocab 128,004; Phase-5 fixed retrain of Run 4 (t64k transition point vs t90k); USED - balanced-1B bootstrap panel (S6.12) + paper panel; not in panels.json main21" |
| }, |
| "swiss-ai/Apertus-70B-2509": { |
| "slug": "swiss-ai/Apertus-70B-2509", |
| "runs": [ |
| "full-128k-apertus", |
| "full-128k-apertus-seed42", |
| "small-128k-apertus", |
| "tiny-128k-apertus" |
| ], |
| "n_runs": 4, |
| "redistributable": false, |
| "loadable_with": "transformers", |
| "hub_id": "swiss-ai/Apertus-70B-2509", |
| "policy_note": "The papers call this tokenizer Mistral-Nemo: the Apertus and Mistral-Nemo tokenizers are the same artifact.", |
| "empirical": null, |
| "declared_experiments": null, |
| "declared_run_manifest": null, |
| "registry_note": "Apertus/Mistral-Nemo (off-the-shelf); n/a (off-the-shelf, Apertus-70B-2509 pretraining mix) data; norm=n/a; off-the-shelf BPE; vocab 131,072; USED - panels.json main21 (camera-ready main panel)" |
| }, |
| "whitespace-balanced-bpe": { |
| "slug": "whitespace-balanced-bpe", |
| "runs": [ |
| "full-128k-whitespace-balanced-bpe", |
| "full-128k-whitespace-balanced-bpe-mathcode-scratch" |
| ], |
| "n_runs": 2, |
| "redistributable": true, |
| "loadable_with": "transformers", |
| "hub_id": null, |
| "policy_note": "trained in this project", |
| "empirical": { |
| "model_type": "BPE", |
| "n_base_tokens": 128256, |
| "n_added_tokens": 4, |
| "vocab_total": 128260, |
| "added_tokens": [ |
| "<s>", |
| "</s>", |
| "<unk>", |
| "<pad>" |
| ], |
| "normalizer": null, |
| "pre_tokenizer": { |
| "type": "Sequence", |
| "pretokenizers": [ |
| { |
| "type": "Split", |
| "pattern": { |
| "Regex": "\\s+" |
| }, |
| "behavior": "MergedWithNext", |
| "invert": false |
| }, |
| { |
| "type": "ByteLevel", |
| "add_prefix_space": false, |
| "trim_offsets": true, |
| "use_regex": false |
| } |
| ] |
| }, |
| "has_post_processor": false, |
| "byte_fallback": false, |
| "bos_token": "<s>", |
| "bos_token_id": 0 |
| }, |
| "declared_experiments": { |
| "algorithm": "bpe", |
| "pretokenizer_constant": "Split(\\s+)+ByteLevel", |
| "normalizer": null, |
| "data_composition": "balanced", |
| "description": "Split(\\s+, MergedWithNext) + ByteLevel pretok, balanced data, BPE", |
| "trainer_kwargs": null |
| }, |
| "declared_run_manifest": null, |
| "registry_note": "WhitespaceSplit(preserve)+ByteLevel (minimal-structure anchor); balanced (D2) data; norm=none; BPE; vocab 128,260; lower-end pretok anchor; forward-fusing (whitespace+following word one pretoken); USED - panels.json pretok-controls (camera-ready)" |
| }, |
| "whitespace-balanced-unigram": { |
| "slug": "whitespace-balanced-unigram", |
| "runs": [ |
| "full-128k-whitespace-balanced-unigram" |
| ], |
| "n_runs": 1, |
| "redistributable": true, |
| "loadable_with": "transformers", |
| "hub_id": null, |
| "policy_note": "trained in this project", |
| "empirical": { |
| "model_type": "Unigram", |
| "n_base_tokens": 128256, |
| "n_added_tokens": 0, |
| "vocab_total": 128256, |
| "added_tokens": [], |
| "normalizer": null, |
| "pre_tokenizer": { |
| "type": "Metaspace", |
| "replacement": "\u2581", |
| "prepend_scheme": "always", |
| "split": true |
| }, |
| "has_post_processor": false, |
| "byte_fallback": true, |
| "bos_token": "<s>", |
| "bos_token_id": 0 |
| }, |
| "declared_experiments": { |
| "algorithm": "unigram", |
| "pretokenizer_constant": "Metaspace", |
| "normalizer": null, |
| "data_composition": "balanced", |
| "description": "Metaspace + ByteFallback, balanced data, UnigramLM via sentencepiece (SPM-native)", |
| "trainer_kwargs": { |
| "backend": "spm", |
| "mode": "native", |
| "shrinking_factor": 0.7, |
| "num_sub_iterations": 3, |
| "max_sentencepiece_length": 64 |
| } |
| }, |
| "declared_run_manifest": null, |
| "registry_note": "Metaspace(replacement=U+2581)+ByteFallback (SPM native mode); balanced (D2) data; norm=none; UnigramLM (Google sentencepiece, mode=native; tuned kwargs matched to UNIGRAM_TUNED_KWARGS); vocab 128,260; algorithm-axis Unigram counterpart to whitespace-balanced-bpe; trained via SPM not HF (HF UnigramTrainer NaN-panics on WhitespaceSplit+CJK); USED - S5e AXES-2 whitespace comparison + balanced-1B bootstrap panel (S6.12); not in panels.json groups" |
| }, |
| "whitespace-multiheavy-bpe": { |
| "slug": "whitespace-multiheavy-bpe", |
| "runs": [ |
| "full-128k-whitespace-multiheavy-bpe" |
| ], |
| "n_runs": 1, |
| "redistributable": true, |
| "loadable_with": "transformers", |
| "hub_id": null, |
| "policy_note": "trained in this project", |
| "empirical": { |
| "model_type": "BPE", |
| "n_base_tokens": 128256, |
| "n_added_tokens": 4, |
| "vocab_total": 128260, |
| "added_tokens": [ |
| "<s>", |
| "</s>", |
| "<unk>", |
| "<pad>" |
| ], |
| "normalizer": null, |
| "pre_tokenizer": { |
| "type": "Sequence", |
| "pretokenizers": [ |
| { |
| "type": "Split", |
| "pattern": { |
| "Regex": "\\s+" |
| }, |
| "behavior": "MergedWithNext", |
| "invert": false |
| }, |
| { |
| "type": "ByteLevel", |
| "add_prefix_space": false, |
| "trim_offsets": true, |
| "use_regex": false |
| } |
| ] |
| }, |
| "has_post_processor": false, |
| "byte_fallback": false, |
| "bos_token": "<s>", |
| "bos_token_id": 0 |
| }, |
| "declared_experiments": { |
| "algorithm": "bpe", |
| "pretokenizer_constant": "Split(\\s+)+ByteLevel", |
| "normalizer": null, |
| "data_composition": "multiheavy", |
| "description": "Split(\\s+, MergedWithNext) + ByteLevel pretok, multiheavy data (multilingual budget doubled to 0.60), BPE", |
| "trainer_kwargs": null |
| }, |
| "declared_run_manifest": null, |
| "registry_note": "WhitespaceSplit(preserve)+ByteLevel; multiheavy (30 FineWeb-2 lang weights doubled 0.30->0.60; english 0.175, math/code 0.0875 each) data; norm=none; BPE; vocab 128,256; data-compensation test: does doubling the multilingual budget offset whitespace-pretok's multilingual damage? (S5e: no, MBPP stays 0.002); USED - panels.json pretok-controls (camera-ready)" |
| } |
| } |
| } |
|
|
