| extends: configs/base.yaml | |
| phase: phase5 | |
| # Phase 5 moves the primary lever from data to model capacity: 345M -> ~730M | |
| # (36L/20H/1280E). Rationale (2026-07-22): phase-4 held the model fixed and | |
| # grew unique tokens 2.5x, moving historical-test BPB only 3.53% | |
| # (1.0654 -> 1.0278) with a small train/val gap (0.127 nats) and no | |
| # overfitting at 2.58 epochs. The phase-3 "data-bound" diagnosis is resolved; | |
| # 345M is approaching this corpus's data-scaling knee. Data collection | |
| # (below) is kept as a non-blocking supporting lever, not the main bet. | |
| # See PLAN.md phase-5 section for the full 3-model comparison and budget math. | |
| data: | |
| ndl_metadata_dir: data/raw/ndl/metadata | |
| ndl_metadata_glob: data/raw/ndl/metadata/books_*.zip | |
| ndl_metadata_urls: | |
| - https://dl.ndl.go.jp/static/files/dataset/dataset_202602_t_internet_01.zip | |
| - https://dl.ndl.go.jp/static/files/dataset/dataset_202602_t_internet_02.zip | |
| - https://dl.ndl.go.jp/static/files/dataset/dataset_202602_t_internet_03.zip | |
| - https://dl.ndl.go.jp/static/files/dataset/dataset_202602_t_internet_04.zip | |
| - https://dl.ndl.go.jp/static/files/dataset/dataset_202602_t_internet_05.zip | |
| # Same frozen 241,430-candidate pool and selection order as phases 3-4. The | |
| # head 186,000 zips are already on disk (phase-4 case B) and skipped by | |
| # checksum; this run exhausts the remaining ~55,430 (case C, the pool ceiling). | |
| ndl_selection_manifest: data/processed/ndl/selection.jsonl | |
| ndl_download_manifest: data/processed/phase5/ndl_downloads.jsonl | |
| ndl_raw_dir: data/raw/ndl/books | |
| ndl_clean_dir: data/clean/ndl | |
| ndl_clean_manifest: data/processed/phase5/ndl_clean.jsonl | |
| # Reuse phase-4's manifest (already a superset carrying phases 1-3 forward) | |
| # so ndl-clean only processes the newly downloaded candidates. | |
| ndl_clean_reuse_manifest: data/processed/phase4/ndl_clean.jsonl | |
| ndl_clean_report: phase5_ndl_cleaning.json | |
| ndl_quality_sample_report: phase5_ndl_quality_sample.jsonl | |
| # Exclude phase-4 documents from the fresh 200-page human review sample. | |
| ndl_quality_exclude_manifest: data/processed/phase4/ndl_clean.jsonl | |
| # Case C: exhaust the frozen pool. RESULT (2026-07-24, completed): only | |
| # 8,202/55,430 new candidates accepted (14.8%, vs phase-4's 46.3%) for an | |
| # estimated ~239M new train tokens -- below the +0.3B retokenize threshold. | |
| # Per PLAN.md 5.2/5.3 this held data at phase-4's frozen .bin/tokenizer (see | |
| # reports/phase5_corpus.json); the newly cleaned docs remain on disk for a | |
| # possible future source expansion but are not merged into training. | |
| ndl_target_documents: 241430 | |
| ndl_download_delay_seconds: 0.2 | |
| ndl_download_retries: 3 | |
| ndl_max_download_seconds: 600 | |
| # Held at phase-4's frozen split manifest (see tokenizer/tokenized notes | |
| # below): evaluate() reads this for test-set clean_path byte sizes, so it | |
| # must point at real data, not an unwritten phase5-only path. | |
| split_manifest: data/processed/phase4/splits.jsonl | |
| ndl_max_abnormal_ngram_rate: 0.38 | |
| ndl_max_latin_rate: 0.005 | |
| tokenizer: | |
| # DECISION (2026-07-24, see reports/phase5_corpus.json): the residual pool's | |
| # deepest ~55,430 candidates yielded only ~8,202 accepted docs / ~239M | |
| # estimated new train tokens (~8.7% on top of phase-4's 2.742B) -- below the | |
| # +0.3B retokenize threshold from PLAN.md 5.2/5.3. Data is held at phase-4's | |
| # frozen tokenizer/tokens; point directly at the phase-4 artifacts rather | |
| # than duplicating ~5.5GB of .bin data. Verified 2026-07-24: tokenizer.json | |
| # SHA-256 c442ae590eed9f9da33bd3bdda3bcd991dde74b3e611fde021b31a6ed4545616 | |
| # matches reports/phase4_evaluation.json. | |
| vocab_size: 16384 | |
| directory: artifacts/phase4/tokenizer | |
| max_train_chars: 200000000 | |
| tokenized: | |
| # Held at phase-4's frozen tokens (see tokenizer.directory note above). | |
| # train.bin verified 2026-07-24: 5,484,978,736 bytes = 2,742,489,368 uint16 | |
| # tokens, matching reports/phase4_training.json. | |
| directory: data/processed/phase4/tokenized | |
| model: | |
| # Case B from PLAN.md phase-5 comparison: ~730M params, 2.1x phase-4's 345M. | |
| # head_dim = 1280/20 = 64 (GPT-2 style, same ratio as phase 3-4's config). | |
| # Verify exact count with `tcja parameter-count --vocab-size 16384`. | |
| n_layer: 36 | |
| n_head: 20 | |
| n_embd: 1280 | |
| block_size: 1024 | |
| dropout: 0.0 | |
| training: | |
| device: cuda | |
| dtype: bfloat16 | |
| # Provisional: 730M will not fit the L40S 46GB at phase-4's batch32/accum8 | |
| # shapes (345M already used 44.3/47.8GB there). First candidate GPU is A100 | |
| # 80GB. tokens_per_step kept at 262,144 (batch16 x accum16 x block1024). | |
| # CONFIRM ON FLIGHT (PLAN.md 5.5) before any paid run; shrink batch/accum | |
| # (e.g. 8x32) if measured VRAM headroom < 10%. | |
| batch_size: 16 | |
| gradient_accumulation_steps: 16 | |
| # FINALIZED (2026-07-24, PLAN.md 5.4): data held at phase-4's 2,742,489,368 | |
| # train tokens (see tokenized.directory above) x 3 epochs = 8,227,468,104 | |
| # processed tokens / 262,144 tok/step = 31,383.9 -> 31,400 iters (~3.00 | |
| # epochs). 3 epochs keeps tokens/param at ~11.3 (730,164,480 params), | |
| # within the data-constrained-scaling tolerance cited in PLAN.md. Batch/accum | |
| # shapes above remain provisional pending the GPU flight (PLAN.md 5.5). | |
| max_iters: 31400 | |
| learning_rate: 0.00025 | |
| min_lr: 0.000025 | |
| warmup_iters: 300 | |
| lr_decay_iters: 31400 | |
| eval_interval: 500 | |
| eval_iters: 50 | |
| checkpoint_interval: 500 | |
| grad_clip: 1.0 | |
| out_dir: artifacts/phase5 | |
| budget: | |
| # PROVISIONAL upper bound from PLAN.md phase-5 section; confirm via flight | |
| # (PLAN.md 5.5) before starting any paid GPU run. | |
| minimum_train_tokens: 2700000000 | |
| maximum_hours: 90 | |
| maximum_usd: 130 | |
| gpu_hourly_usd: 1.39 | |
| huggingface: | |
| # DECISION (2026-07-24, reports/phase5_corpus.json): data was held at | |
| # phase-4's frozen corpus, so v3 is a model-only release. Point the model | |
| # card's dataset link at the existing v2 dataset repo instead of the | |
| # placeholder v3 dataset repo in base.yaml (which is never published). | |
| dataset_repos: | |
| phase5: "trtd56/TimeCapsuleLLM-ja-corpus-v2" | |