add gpt_d8_next3_r40

gpt_d8_next3_r40/base_checkpoints/d8/meta_008960.json

@@ -0,0 +1,69 @@
+{
+  "step": 8960,
+  "model_config": {
+    "sequence_len": 512,
+    "vocab_size": 4096,
+    "n_layer": 8,
+    "n_head": 4,
+    "n_kv_head": 4,
+    "n_embd": 512,
+    "target_shift": 3
+  },
+  "user_config": {
+    "run": "gpt_d8_next3_r40",
+    "device_type": "",
+    "model_architecture": "Karpathy_gpt2",
+    "model_type": "next_token_ar",
+    "target_shift": 3,
+    "depth": 8,
+    "max_seq_len": 512,
+    "block_size": 8,
+    "prefix_pure_tokens": 1,
+    "is_causal": true,
+    "noise_total_steps": 16,
+    "pdlm_stage": "stage2",
+    "bd3lm_compute_matched": true,
+    "mtp_loss_beta": 0.8,
+    "n_future_tokens": 4,
+    "mtp_loss_weight": 1.0,
+    "soft_p_within": 1.0,
+    "noise_count": 64,
+    "loss_weight_mode": "manual",
+    "loss_weight_warmup_steps": 10,
+    "stage1_target_mode": "pure",
+    "debug": false,
+    "num_iterations": -1,
+    "target_flops": -1.0,
+    "target_param_data_ratio": 40,
+    "device_batch_size": 64,
+    "total_batch_size": 131072,
+    "embedding_lr": 0.2,
+    "unembedding_lr": 0.004,
+    "weight_decay": 0.0,
+    "matrix_lr": 0.02,
+    "grad_clip": 1.0,
+    "warmup_ratio": 0.0,
+    "warmdown_ratio": 0.2,
+    "final_lr_frac": 0.0,
+    "resume_from_step": -1,
+    "eval_every": 2500,
+    "eval_num_batches": 20,
+    "eval_num_batches_final": 100,
+    "save_every": -1,
+    "gradient_track_every": 0,
+    "gradient_block_size": 4,
+    "model_tag": ""
+  },
+  "device_batch_size": 64,
+  "max_seq_len": 512,
+  "dataloader_state_dict": {
+    "pq_idx": 9,
+    "rg_idx": 183,
+    "epoch": 1
+  },
+  "loop_state": {
+    "smooth_train_loss": 3.365408381108792,
+    "total_training_time": 1785.0645592212677,
+    "total_effective_tokens": 1174405120
+  }
+}

gpt_d8_next3_r40/base_checkpoints/d8/model_008960.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:847c537ed8eaab4f14c4267184de459eaadfd6295c59d38568bc6d3559bbe1e0
+size 113266917

gpt_d8_next3_r40/base_checkpoints/d8/optim_008960_rank0.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a3c4a52bdb68943c4883082fe182d5d9b0f63e0ddea5f7ceded3e50f56fd25e8
+size 125847061

gpt_d8_next3_r40/report/base-model-training.md

@@ -0,0 +1,62 @@
+## Base model training
+timestamp: 2026-03-02 00:27:50
+
+- run: gpt_d8_next3_r40
+- device_type: 
+- model_architecture: Karpathy_gpt2
+- model_type: next_token_ar
+- target_shift: 3
+- depth: 8
+- max_seq_len: 512
+- block_size: 8
+- prefix_pure_tokens: 1
+- is_causal: True
+- noise_total_steps: 16
+- pdlm_stage: stage2
+- bd3lm_compute_matched: True
+- mtp_loss_beta: 0.8000
+- n_future_tokens: 4
+- mtp_loss_weight: 1.0000
+- soft_p_within: 1.0000
+- noise_count: 64
+- loss_weight_mode: manual
+- loss_weight_warmup_steps: 10
+- stage1_target_mode: pure
+- debug: False
+- num_iterations: -1
+- target_flops: -1.0000
+- target_param_data_ratio: 40
+- device_batch_size: 64
+- total_batch_size: 131,072
+- embedding_lr: 0.2000
+- unembedding_lr: 0.0040
+- weight_decay: 0.0000
+- matrix_lr: 0.0200
+- grad_clip: 1.0000
+- warmup_ratio: 0.0000
+- warmdown_ratio: 0.2000
+- final_lr_frac: 0.0000
+- resume_from_step: -1
+- eval_every: 2500
+- eval_num_batches: 20
+- eval_num_batches_final: 100
+- save_every: -1
+- gradient_track_every: 0
+- gradient_block_size: 4
+- model_tag: 
+- Number of parameters: 29,360,128
+- Number of FLOPs per token: 1.887437e+08
+- Calculated number of iterations: 8960
+- Number of training tokens: 1,174,405,120
+- Tokens : Params ratio: 40.0000
+- DDP world size: 1
+- warmup_ratio: 0.0000
+- warmdown_ratio: 0.2000
+- final_lr_frac: 0.0000
+- MFU %: 12.61%
+- Total training flops: 2.216615e+17
+- Total training time: 29.75m
+- Peak memory usage: 5354.51MiB
+- Total effective tokens: 1,174,405,120
+- Actual effective ratio: 1.0000
+

gpt_d8_next3_r40/tokenizer/token_bytes.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:17f468b0626e587a9948f73588629d679198a6f38fe834875017b222ec2443d7
+size 17961

gpt_d8_next3_r40/tokenizer/tokenizer.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:893aa318d37e34b8852086f8fcdffcd9d45242706dc6f06dc5d315499aa5a633
+size 45915