Test-Subject-3 / training_meta.json
CyanMonkey's picture
round 14 step 7000 (24.56%)
9b4717b verified
Raw History Blame Contribute Delete
5.46 kB
{
"format": "oleander-training-meta/1",
"generated_at": "2026-11-15T09:39:11Z",
"kind": "round 14",
"step": 7000,
"tokens_seen": 17301504000,
"pushed_at": "2026-11-15T09:39:11Z",
"params_M": 149.976,
"architecture": "Oleander-150M",
"features": [
"GDN2 (gated delta net 2) x21: 9h x 64, conv4",
"gated GQA x7: 9 Q / 3 KV heads, head_dim 64",
"QK-Norm (offset form)",
"partial RoPE (rotary_dim 32, theta 100000)",
"value residual (v_mix)",
"SwiGLU",
"RMSNorm",
"AttnRes (block attention residuals)",
"tied embeddings",
"TST (token superposition training, bag 6)"
],
"optimizer": "muon_adamw",
"val_loss": 6.348879925906658,
"val_ppl": 571.8518337425736,
"run": {
"model_name": "Oleander-150M",
"architecture_detail": "Oleander-150M (28L = 7 x [GDN2 \u00b7 GDN2 \u00b7 GDN2 \u00b7 GQA] \u00b7 d=576 \u00b7 SwiGLU(1600) \u00b7 tied 32,770 vocab)",
"session_id": "20261115_063000_ca8087",
"repo_id": "CyanMonkey/Test-Subject-3",
"started_at": "2026-11-15T06:35:32Z",
"session_elapsed_s": 11018.2,
"last_update_at": "2026-11-15T09:39:10Z",
"steps_per_second": 0.13902,
"compute_tokens_per_second": 76859.5,
"eta": "1d 18h 57m",
"checkpoint_kind": "round 14"
},
"progress": {
"step": 7000,
"max_steps": 28500,
"step_fraction": 0.245614,
"percent_complete": 24.561,
"tokens_fraction": 0.192982,
"percent_tokens": 19.298,
"phase": "tst",
"phase_fraction": 0.777778,
"tst_switch_step": 9000,
"tst_switch_applied": false
},
"tokens": {
"compute_tokens_seen": 2883584000,
"compute_tokens_total": 14942208000,
"raw_tokens_seen": 17301504000,
"raw_tokens_total": 38535168000,
"raw_over_compute_ratio": 6.0,
"samples_seen": 16896000,
"rows_consumed": 16896000,
"compute_tokens_per_step": 524288,
"cluster": {
"n_workers": 1,
"compute_tokens_seen_all_workers": 2883584000,
"raw_tokens_seen_all_workers": 17301504000,
"compute_tokens_total_planned_all_workers": 14942208000,
"raw_tokens_total_planned_all_workers": 38535168000
}
},
"metrics": {
"train_loss": 5.2985029220581055,
"val_loss": 6.348879925906658,
"val_ppl": 571.8518337425736,
"grad_norm": 0.21302621066570282,
"lr_mult": 1.0
},
"optimization": {
"name": "muon_adamw",
"description": "MuonEq-R (muoneq_r, lr 0.02, 5 NS steps, momentum 0.95) + AdamW (lr 0.0015, betas (0.9, 0.95))",
"muon": {
"variant": "muoneq_r",
"lr": 0.02,
"momentum": 0.95,
"nesterov": true,
"ns_steps": 5,
"weight_decay": 0.01
},
"adamw": {
"lr": 0.0015,
"betas": [
0.9,
0.95
],
"eps": 1e-08,
"weight_decay": 0.0
},
"grad_clip": 1.0,
"schedule": {
"name": "WSD",
"warmup_steps": 100,
"stable_until": 0.825,
"decay_shape": "1-sqrt",
"min_lr_ratio": 0.0,
"decay_start_step": 23512
},
"tst": {
"enabled": true,
"bag_size": 6,
"phase_fraction": 0.3,
"switch_step": 9000
}
},
"model": {
"params": 149976139,
"params_M": 149.976,
"num_layers": 28,
"hidden_size": 576,
"group_pattern": [
"gdn2",
"gdn2",
"gdn2",
"gqa"
],
"num_groups": 7,
"mlp_hidden": 1600,
"vocab_size": 32770,
"max_seq_len": 1024,
"attn_res": true,
"residual_dtype": "bfloat16",
"tie_embeddings": true,
"features": [
"GDN2 (gated delta net 2) x21: 9h x 64, conv4",
"gated GQA x7: 9 Q / 3 KV heads, head_dim 64",
"QK-Norm (offset form)",
"partial RoPE (rotary_dim 32, theta 100000)",
"value residual (v_mix)",
"SwiGLU",
"RMSNorm",
"AttnRes (block attention residuals)",
"tied embeddings",
"TST (token superposition training, bag 6)"
],
"export_dtype": "float32"
},
"data": {
"dataset_repo_id": "CyanMonkey/NextGen-1-32-1024",
"train_split": "train",
"tokenizer_id": "AxiomicLabs/GPT-X2.5-135M",
"n_train_rows": 14753933,
"n_val_rows": 2048,
"seq_len": 1024,
"micro_batch": 32,
"grad_accum_steps": 16,
"doc_masking": "auto",
"shuffle_block_rows": 192,
"seed": 42,
"epochs_consumed_this_worker": 1.1452
},
"hardware": {
"device": "cuda",
"torch": "2.11.0+cu130",
"amp_dtype": "torch.bfloat16",
"preset": "full",
"python": "3.13.11",
"transformers": "5.14.1",
"gpu": "NVIDIA RTX PRO 6000 Blackwell Server Edition",
"vram_gb": 101.98,
"peak_gb": 39.36
},
"checkpoint": {
"kind": "round 14",
"weights_file": "model.safetensors",
"dtype": "float32",
"config_file": "config.json",
"auto_map": {
"AutoConfig": "configuration_oleander.OleanderConfig",
"AutoModelForCausalLM": "modeling_oleander.OleanderForCausalLM"
},
"modeling_files": [],
"tokenizer_files": [
"tokenizer.json",
"tokenizer_config.json",
"special_tokens_map.json",
"vocab.json",
"merges.txt",
"added_tokens.json",
"chat_template.jinja",
"tokenizer.model"
]
},
"diloco": {
"enabled": true,
"worker_id": 0,
"n_workers": 1,
"inner_steps": 500,
"outer_lr": 0.7,
"outer_momentum": 0.9,
"is_leader": true,
"support_repo_id": "CyanMonkey/TestSubject-3-support"
},
"extra": {
"diloco_round": 14
}
}