{ "checkpoint_config": { "activation": "gelu", "actual_precision": "bf16", "adamw_betas": [ 0.9, 0.95 ], "adamw_eps": 1e-08, "architecture": "gpt2", "attention_kernel_policy": "auto", "batch_size": 2, "benchmark": { "enable_central_tensorboard": true, "enable_local_tensorboard": true, "enabled": false, "output_path": "/mnt/apps/llm-nanochat/artifacts/runs/20260715_resume-gpt2medium-gpt2preln-k20-optimizeronly-cpt14700-step34000-d1700-webwiki/throughput_benchmark.json", "warmup_steps": 0 }, "bias": true, "block_type": "gpt2_prelayernorm", "causal_mask_mode": "buffered_upper_triangular", "checkpoint_dir": "/mnt/apps/llm-nanochat/checkpoints/20260715_resume-gpt2medium-gpt2preln-k20-optimizeronly-cpt14700-step34000-d1700-webwiki", "clip_grad_norm": 1.0, "compile": { "backend": null, "compile_setup_sec": 0.0, "diagnostic": null, "dynamic": false, "enabled": false, "error_policy": "raise", "fullgraph": false, "mode": null, "requested": false, "status": "disabled" }, "dataset": { "storage_mode": "indexed_jsonl" }, "dataset_dir": "/mnt/apps/llm-nanochat/datasets/202605141153_fineweb50_wiki50_50en_50it_score100_2500context_5Btokens_tok_20260515_en50it50_webwiki_stratified_500M", "decay_shape": "inverse_proportional", "decay_steps": 1700, "decay_steps_source": "explicit", "deterministic_algorithms": false, "device": "cuda", "dim": 1024, "dropout": 0.0, "effective_global_step": 34200, "effective_layer_count": 24, "effective_learning_rate": 3.309994670030697e-05, "final_lr": 1e-05, "final_lr_source": "explicit", "fp8_backend": null, "grad_accum_steps": 48, "init_from": "/mnt/apps/llm-nanochat/checkpoints/20260703_continual-pretraining-gpt2medium-gpt2preln-k20-step14700-lr5e5-w500-s18500-d2000-final1e5-webwiki/step_34000.pt", "init_strategy": "gpt2_std_0.02_residual_scale", "learning_rate": 4.7832243797285384e-05, "local_max_steps": 35700, "local_step": 200, "logging": { "enable_central_tensorboard": true, "enable_local_tensorboard": true, "metrics_flush_every_steps": 1, "metrics_writer": "persistent_jsonl_handle" }, "lr": 4.7832243797285384e-05, "lr_schedule": "wsd-decay-only", "max_seq_len": 2500, "max_steps": 35700, "n_heads": 16, "n_layers": 24, "norm_order": "preln", "norm_type": "layernorm", "optimizer": { "backend": "torch", "betas": [ 0.9, 0.95 ], "eps": 1e-08, "implementation": "torch.optim.AdamW", "learning_rate": 4.7832243797285384e-05, "state_precision": "full_precision", "type": "adamw", "weight_decay": 0.1 }, "optimizer_backend": "torch", "optimizer_implementation": "torch.optim.AdamW", "optimizer_reset": false, "optimizer_state_precision": "full_precision", "optimizer_type": "adamw", "peak_lr": 4.7832243797285384e-05, "positional_encoding": "learned_absolute", "repro": { "attention_kernel_policy": "auto", "cublas_workspace_config": null, "cudnn_benchmark": true, "cudnn_deterministic": false, "deterministic_algorithms": false, "flash_sdp_enabled": true, "math_sdp_enabled": true, "mem_efficient_sdp_enabled": true, "pythonhashseed": "1337", "seed": 1337 }, "requested_precision": "bf16", "resume_checkpoint_step": 34000, "resume_from": "/mnt/apps/llm-nanochat/checkpoints/20260703_continual-pretraining-gpt2medium-gpt2preln-k20-step14700-lr5e5-w500-s18500-d2000-final1e5-webwiki/step_34000.pt", "resume_mode": "optimizer_only", "resume_step": 34000, "save_every_steps": 100, "save_final_checkpoint": true, "schedule_total_steps": 35700, "scheduler": { "decay_shape": "inverse_proportional", "decay_steps": 1700, "decay_steps_source": "explicit", "final_lr": 1e-05, "final_lr_source": "explicit", "peak_lr": 4.7832243797285384e-05, "resume_step": 34000, "schedule_type": "wsd-decay-only", "stable_steps": 0, "target_final_step": 35700, "total_steps": 35700, "warmup_steps": 0 }, "scheduler_reset": true, "seed": 1337, "source_global_step": 34000, "stable_steps": 0, "target_final_step": 35700, "target_global_step": 35700, "tie_word_embeddings": true, "train_cache_ram_bytes": 1073741824, "train_cache_ram_mb": 1024, "vocab_size": 32000, "warmup_steps": 0, "weight_decay": 0.1 }, "checkpoint_path": "/mnt/apps/llm-nanochat/checkpoints/20260715_resume-gpt2medium-gpt2preln-k20-optimizeronly-cpt14700-step34000-d1700-webwiki/step_34200.pt", "config_path": "/mnt/apps/llm-nanochat/hf_exports/1gpu-llm-medium-v2/config.json", "exported_at": "2026-07-16T13:41:05.693620+00:00", "format": "llm-nanochat-safetensors-export", "global_step": 34200, "metadata_path": "/mnt/apps/llm-nanochat/hf_exports/1gpu-llm-medium-v2/step_34200.safetensors.json", "model_config": { "activation": "gelu", "architecture": "gpt2", "bias": true, "block_type": "gpt2_prelayernorm", "causal_mask_mode": "buffered_upper_triangular", "dim": 1024, "dropout": 0.0, "init_strategy": "gpt2_std_0.02_residual_scale", "max_seq_len": 2500, "n_heads": 16, "n_layers": 24, "norm_order": "preln", "norm_type": "layernorm", "positional_encoding": "learned_absolute", "tie_word_embeddings": true, "vocab_size": 32000 }, "num_parameters": 370439424, "num_tensors": 294, "provenance": { "checkpoint_dir": "/mnt/apps/llm-nanochat/checkpoints/20260715_resume-gpt2medium-gpt2preln-k20-optimizeronly-cpt14700-step34000-d1700-webwiki", "checkpoint_name": "step_34200.pt", "checkpoint_path": "/mnt/apps/llm-nanochat/checkpoints/20260715_resume-gpt2medium-gpt2preln-k20-optimizeronly-cpt14700-step34000-d1700-webwiki/step_34200.pt", "global_step": 34200, "packed_dataset_config_path": null, "run_dir": "/mnt/apps/llm-nanochat/checkpoints", "tokenizer_dir": "/mnt/apps/llm-nanochat/tokenizers/tokenizer_20260515_en50it50_webwiki_stratified_500M", "training_config_path": null }, "safetensors_path": "/mnt/apps/llm-nanochat/hf_exports/1gpu-llm-medium-v2/step_34200.safetensors", "source_checkpoint_path": "/mnt/apps/llm-nanochat/checkpoints/20260715_resume-gpt2medium-gpt2preln-k20-optimizeronly-cpt14700-step34000-d1700-webwiki/step_34200.pt", "source_global_step": 34200, "tensor_names": [ "token_emb.weight", "pos_emb.weight", "blocks.0.ln_1.weight", "blocks.0.ln_1.bias", "blocks.0.ln_2.weight", "blocks.0.ln_2.bias", "blocks.0.attn.qkv.weight", "blocks.0.attn.qkv.bias", "blocks.0.attn.out_proj.weight", "blocks.0.attn.out_proj.bias", "blocks.0.mlp.fc.weight", "blocks.0.mlp.fc.bias", "blocks.0.mlp.proj.weight", "blocks.0.mlp.proj.bias", "blocks.1.ln_1.weight", "blocks.1.ln_1.bias", "blocks.1.ln_2.weight", "blocks.1.ln_2.bias", "blocks.1.attn.qkv.weight", "blocks.1.attn.qkv.bias", "blocks.1.attn.out_proj.weight", "blocks.1.attn.out_proj.bias", "blocks.1.mlp.fc.weight", "blocks.1.mlp.fc.bias", "blocks.1.mlp.proj.weight", "blocks.1.mlp.proj.bias", "blocks.2.ln_1.weight", "blocks.2.ln_1.bias", "blocks.2.ln_2.weight", "blocks.2.ln_2.bias", "blocks.2.attn.qkv.weight", "blocks.2.attn.qkv.bias", "blocks.2.attn.out_proj.weight", "blocks.2.attn.out_proj.bias", "blocks.2.mlp.fc.weight", "blocks.2.mlp.fc.bias", "blocks.2.mlp.proj.weight", "blocks.2.mlp.proj.bias", "blocks.3.ln_1.weight", "blocks.3.ln_1.bias", "blocks.3.ln_2.weight", "blocks.3.ln_2.bias", "blocks.3.attn.qkv.weight", "blocks.3.attn.qkv.bias", "blocks.3.attn.out_proj.weight", "blocks.3.attn.out_proj.bias", "blocks.3.mlp.fc.weight", "blocks.3.mlp.fc.bias", "blocks.3.mlp.proj.weight", "blocks.3.mlp.proj.bias", "blocks.4.ln_1.weight", "blocks.4.ln_1.bias", "blocks.4.ln_2.weight", "blocks.4.ln_2.bias", "blocks.4.attn.qkv.weight", "blocks.4.attn.qkv.bias", "blocks.4.attn.out_proj.weight", "blocks.4.attn.out_proj.bias", "blocks.4.mlp.fc.weight", "blocks.4.mlp.fc.bias", "blocks.4.mlp.proj.weight", "blocks.4.mlp.proj.bias", "blocks.5.ln_1.weight", "blocks.5.ln_1.bias", "blocks.5.ln_2.weight", "blocks.5.ln_2.bias", "blocks.5.attn.qkv.weight", "blocks.5.attn.qkv.bias", "blocks.5.attn.out_proj.weight", "blocks.5.attn.out_proj.bias", "blocks.5.mlp.fc.weight", "blocks.5.mlp.fc.bias", "blocks.5.mlp.proj.weight", "blocks.5.mlp.proj.bias", "blocks.6.ln_1.weight", "blocks.6.ln_1.bias", "blocks.6.ln_2.weight", "blocks.6.ln_2.bias", "blocks.6.attn.qkv.weight", "blocks.6.attn.qkv.bias", "blocks.6.attn.out_proj.weight", "blocks.6.attn.out_proj.bias", "blocks.6.mlp.fc.weight", "blocks.6.mlp.fc.bias", "blocks.6.mlp.proj.weight", "blocks.6.mlp.proj.bias", "blocks.7.ln_1.weight", "blocks.7.ln_1.bias", "blocks.7.ln_2.weight", "blocks.7.ln_2.bias", "blocks.7.attn.qkv.weight", "blocks.7.attn.qkv.bias", "blocks.7.attn.out_proj.weight", "blocks.7.attn.out_proj.bias", "blocks.7.mlp.fc.weight", "blocks.7.mlp.fc.bias", "blocks.7.mlp.proj.weight", "blocks.7.mlp.proj.bias", "blocks.8.ln_1.weight", "blocks.8.ln_1.bias", "blocks.8.ln_2.weight", "blocks.8.ln_2.bias", "blocks.8.attn.qkv.weight", "blocks.8.attn.qkv.bias", "blocks.8.attn.out_proj.weight", "blocks.8.attn.out_proj.bias", "blocks.8.mlp.fc.weight", "blocks.8.mlp.fc.bias", "blocks.8.mlp.proj.weight", "blocks.8.mlp.proj.bias", "blocks.9.ln_1.weight", "blocks.9.ln_1.bias", "blocks.9.ln_2.weight", "blocks.9.ln_2.bias", "blocks.9.attn.qkv.weight", "blocks.9.attn.qkv.bias", "blocks.9.attn.out_proj.weight", "blocks.9.attn.out_proj.bias", "blocks.9.mlp.fc.weight", "blocks.9.mlp.fc.bias", "blocks.9.mlp.proj.weight", "blocks.9.mlp.proj.bias", "blocks.10.ln_1.weight", "blocks.10.ln_1.bias", "blocks.10.ln_2.weight", "blocks.10.ln_2.bias", "blocks.10.attn.qkv.weight", "blocks.10.attn.qkv.bias", "blocks.10.attn.out_proj.weight", "blocks.10.attn.out_proj.bias", "blocks.10.mlp.fc.weight", "blocks.10.mlp.fc.bias", "blocks.10.mlp.proj.weight", "blocks.10.mlp.proj.bias", "blocks.11.ln_1.weight", "blocks.11.ln_1.bias", "blocks.11.ln_2.weight", "blocks.11.ln_2.bias", "blocks.11.attn.qkv.weight", "blocks.11.attn.qkv.bias", "blocks.11.attn.out_proj.weight", "blocks.11.attn.out_proj.bias", "blocks.11.mlp.fc.weight", "blocks.11.mlp.fc.bias", "blocks.11.mlp.proj.weight", "blocks.11.mlp.proj.bias", "blocks.12.ln_1.weight", "blocks.12.ln_1.bias", "blocks.12.ln_2.weight", "blocks.12.ln_2.bias", "blocks.12.attn.qkv.weight", "blocks.12.attn.qkv.bias", "blocks.12.attn.out_proj.weight", "blocks.12.attn.out_proj.bias", "blocks.12.mlp.fc.weight", "blocks.12.mlp.fc.bias", "blocks.12.mlp.proj.weight", "blocks.12.mlp.proj.bias", "blocks.13.ln_1.weight", "blocks.13.ln_1.bias", "blocks.13.ln_2.weight", "blocks.13.ln_2.bias", "blocks.13.attn.qkv.weight", "blocks.13.attn.qkv.bias", "blocks.13.attn.out_proj.weight", "blocks.13.attn.out_proj.bias", "blocks.13.mlp.fc.weight", "blocks.13.mlp.fc.bias", "blocks.13.mlp.proj.weight", "blocks.13.mlp.proj.bias", "blocks.14.ln_1.weight", "blocks.14.ln_1.bias", "blocks.14.ln_2.weight", "blocks.14.ln_2.bias", "blocks.14.attn.qkv.weight", "blocks.14.attn.qkv.bias", "blocks.14.attn.out_proj.weight", "blocks.14.attn.out_proj.bias", "blocks.14.mlp.fc.weight", "blocks.14.mlp.fc.bias", "blocks.14.mlp.proj.weight", "blocks.14.mlp.proj.bias", "blocks.15.ln_1.weight", "blocks.15.ln_1.bias", "blocks.15.ln_2.weight", "blocks.15.ln_2.bias", "blocks.15.attn.qkv.weight", "blocks.15.attn.qkv.bias", "blocks.15.attn.out_proj.weight", "blocks.15.attn.out_proj.bias", "blocks.15.mlp.fc.weight", "blocks.15.mlp.fc.bias", "blocks.15.mlp.proj.weight", "blocks.15.mlp.proj.bias", "blocks.16.ln_1.weight", "blocks.16.ln_1.bias", "blocks.16.ln_2.weight", "blocks.16.ln_2.bias", "blocks.16.attn.qkv.weight", "blocks.16.attn.qkv.bias", "blocks.16.attn.out_proj.weight", "blocks.16.attn.out_proj.bias", "blocks.16.mlp.fc.weight", "blocks.16.mlp.fc.bias", "blocks.16.mlp.proj.weight", "blocks.16.mlp.proj.bias", "blocks.17.ln_1.weight", "blocks.17.ln_1.bias", "blocks.17.ln_2.weight", "blocks.17.ln_2.bias", "blocks.17.attn.qkv.weight", "blocks.17.attn.qkv.bias", "blocks.17.attn.out_proj.weight", "blocks.17.attn.out_proj.bias", "blocks.17.mlp.fc.weight", "blocks.17.mlp.fc.bias", "blocks.17.mlp.proj.weight", "blocks.17.mlp.proj.bias", "blocks.18.ln_1.weight", "blocks.18.ln_1.bias", "blocks.18.ln_2.weight", "blocks.18.ln_2.bias", "blocks.18.attn.qkv.weight", "blocks.18.attn.qkv.bias", "blocks.18.attn.out_proj.weight", "blocks.18.attn.out_proj.bias", "blocks.18.mlp.fc.weight", "blocks.18.mlp.fc.bias", "blocks.18.mlp.proj.weight", "blocks.18.mlp.proj.bias", "blocks.19.ln_1.weight", "blocks.19.ln_1.bias", "blocks.19.ln_2.weight", "blocks.19.ln_2.bias", "blocks.19.attn.qkv.weight", "blocks.19.attn.qkv.bias", "blocks.19.attn.out_proj.weight", "blocks.19.attn.out_proj.bias", "blocks.19.mlp.fc.weight", "blocks.19.mlp.fc.bias", "blocks.19.mlp.proj.weight", "blocks.19.mlp.proj.bias", "blocks.20.ln_1.weight", "blocks.20.ln_1.bias", "blocks.20.ln_2.weight", "blocks.20.ln_2.bias", "blocks.20.attn.qkv.weight", "blocks.20.attn.qkv.bias", "blocks.20.attn.out_proj.weight", "blocks.20.attn.out_proj.bias", "blocks.20.mlp.fc.weight", "blocks.20.mlp.fc.bias", "blocks.20.mlp.proj.weight", "blocks.20.mlp.proj.bias", "blocks.21.ln_1.weight", "blocks.21.ln_1.bias", "blocks.21.ln_2.weight", "blocks.21.ln_2.bias", "blocks.21.attn.qkv.weight", "blocks.21.attn.qkv.bias", "blocks.21.attn.out_proj.weight", "blocks.21.attn.out_proj.bias", "blocks.21.mlp.fc.weight", "blocks.21.mlp.fc.bias", "blocks.21.mlp.proj.weight", "blocks.21.mlp.proj.bias", "blocks.22.ln_1.weight", "blocks.22.ln_1.bias", "blocks.22.ln_2.weight", "blocks.22.ln_2.bias", "blocks.22.attn.qkv.weight", "blocks.22.attn.qkv.bias", "blocks.22.attn.out_proj.weight", "blocks.22.attn.out_proj.bias", "blocks.22.mlp.fc.weight", "blocks.22.mlp.fc.bias", "blocks.22.mlp.proj.weight", "blocks.22.mlp.proj.bias", "blocks.23.ln_1.weight", "blocks.23.ln_1.bias", "blocks.23.ln_2.weight", "blocks.23.ln_2.bias", "blocks.23.attn.qkv.weight", "blocks.23.attn.qkv.bias", "blocks.23.attn.out_proj.weight", "blocks.23.attn.out_proj.bias", "blocks.23.mlp.fc.weight", "blocks.23.mlp.fc.bias", "blocks.23.mlp.proj.weight", "blocks.23.mlp.proj.bias", "ln_f.weight", "ln_f.bias", "head.weight", "head.bias" ], "tokenizer_bundle": { "special_tokens_map.json": "/mnt/apps/llm-nanochat/hf_exports/1gpu-llm-medium-v2/special_tokens_map.json", "tokenizer.json": "/mnt/apps/llm-nanochat/hf_exports/1gpu-llm-medium-v2/tokenizer.json", "tokenizer_config.json": "/mnt/apps/llm-nanochat/hf_exports/1gpu-llm-medium-v2/tokenizer_config.json", "tokenizer_meta.json": "/mnt/apps/llm-nanochat/hf_exports/1gpu-llm-medium-v2/tokenizer_meta.json" }, "tokenizer_reference": { "packed_dataset_config_path": null, "tokenizer_dir": "/mnt/apps/llm-nanochat/tokenizers/tokenizer_20260515_en50it50_webwiki_stratified_500M", "training_config_path": null }, "transformers_config": { "activation_function": "gelu", "architecture": "gpt2", "architectures": [ "GPT2LMHeadModel" ], "attn_pdrop": 0.0, "block_type": "gpt2_prelayernorm", "causal_mask_mode": "buffered_upper_triangular", "embd_pdrop": 0.0, "init_strategy": "gpt2_std_0.02_residual_scale", "layer_norm_epsilon": 1e-05, "model_type": "gpt2", "n_ctx": 2500, "n_embd": 1024, "n_head": 16, "n_layer": 24, "n_positions": 2500, "norm_order": "preln", "norm_type": "layernorm", "positional_encoding": "learned_absolute", "resid_pdrop": 0.0, "tie_word_embeddings": true, "use_cache": true, "vocab_size": 32000 }, "transformers_export_notes": { "dropped_lm_head_bias_for_gpt2_compat": true, "dropped_lm_head_bias_max_abs": 1.3327739238739014, "dropped_lm_head_bias_mean_abs": 0.046846117824316025 }, "transformers_safetensors_path": "/mnt/apps/llm-nanochat/hf_exports/1gpu-llm-medium-v2/model.safetensors" }