{ "format": "oleander-training-meta/1", "generated_at": "2026-11-15T09:39:11Z", "kind": "round 14", "step": 7000, "tokens_seen": 17301504000, "pushed_at": "2026-11-15T09:39:11Z", "params_M": 149.976, "architecture": "Oleander-150M", "features": [ "GDN2 (gated delta net 2) x21: 9h x 64, conv4", "gated GQA x7: 9 Q / 3 KV heads, head_dim 64", "QK-Norm (offset form)", "partial RoPE (rotary_dim 32, theta 100000)", "value residual (v_mix)", "SwiGLU", "RMSNorm", "AttnRes (block attention residuals)", "tied embeddings", "TST (token superposition training, bag 6)" ], "optimizer": "muon_adamw", "val_loss": 6.348879925906658, "val_ppl": 571.8518337425736, "run": { "model_name": "Oleander-150M", "architecture_detail": "Oleander-150M (28L = 7 x [GDN2 \u00b7 GDN2 \u00b7 GDN2 \u00b7 GQA] \u00b7 d=576 \u00b7 SwiGLU(1600) \u00b7 tied 32,770 vocab)", "session_id": "20261115_063000_ca8087", "repo_id": "CyanMonkey/Test-Subject-3", "started_at": "2026-11-15T06:35:32Z", "session_elapsed_s": 11018.2, "last_update_at": "2026-11-15T09:39:10Z", "steps_per_second": 0.13902, "compute_tokens_per_second": 76859.5, "eta": "1d 18h 57m", "checkpoint_kind": "round 14" }, "progress": { "step": 7000, "max_steps": 28500, "step_fraction": 0.245614, "percent_complete": 24.561, "tokens_fraction": 0.192982, "percent_tokens": 19.298, "phase": "tst", "phase_fraction": 0.777778, "tst_switch_step": 9000, "tst_switch_applied": false }, "tokens": { "compute_tokens_seen": 2883584000, "compute_tokens_total": 14942208000, "raw_tokens_seen": 17301504000, "raw_tokens_total": 38535168000, "raw_over_compute_ratio": 6.0, "samples_seen": 16896000, "rows_consumed": 16896000, "compute_tokens_per_step": 524288, "cluster": { "n_workers": 1, "compute_tokens_seen_all_workers": 2883584000, "raw_tokens_seen_all_workers": 17301504000, "compute_tokens_total_planned_all_workers": 14942208000, "raw_tokens_total_planned_all_workers": 38535168000 } }, "metrics": { "train_loss": 5.2985029220581055, "val_loss": 6.348879925906658, "val_ppl": 571.8518337425736, "grad_norm": 0.21302621066570282, "lr_mult": 1.0 }, "optimization": { "name": "muon_adamw", "description": "MuonEq-R (muoneq_r, lr 0.02, 5 NS steps, momentum 0.95) + AdamW (lr 0.0015, betas (0.9, 0.95))", "muon": { "variant": "muoneq_r", "lr": 0.02, "momentum": 0.95, "nesterov": true, "ns_steps": 5, "weight_decay": 0.01 }, "adamw": { "lr": 0.0015, "betas": [ 0.9, 0.95 ], "eps": 1e-08, "weight_decay": 0.0 }, "grad_clip": 1.0, "schedule": { "name": "WSD", "warmup_steps": 100, "stable_until": 0.825, "decay_shape": "1-sqrt", "min_lr_ratio": 0.0, "decay_start_step": 23512 }, "tst": { "enabled": true, "bag_size": 6, "phase_fraction": 0.3, "switch_step": 9000 } }, "model": { "params": 149976139, "params_M": 149.976, "num_layers": 28, "hidden_size": 576, "group_pattern": [ "gdn2", "gdn2", "gdn2", "gqa" ], "num_groups": 7, "mlp_hidden": 1600, "vocab_size": 32770, "max_seq_len": 1024, "attn_res": true, "residual_dtype": "bfloat16", "tie_embeddings": true, "features": [ "GDN2 (gated delta net 2) x21: 9h x 64, conv4", "gated GQA x7: 9 Q / 3 KV heads, head_dim 64", "QK-Norm (offset form)", "partial RoPE (rotary_dim 32, theta 100000)", "value residual (v_mix)", "SwiGLU", "RMSNorm", "AttnRes (block attention residuals)", "tied embeddings", "TST (token superposition training, bag 6)" ], "export_dtype": "float32" }, "data": { "dataset_repo_id": "CyanMonkey/NextGen-1-32-1024", "train_split": "train", "tokenizer_id": "AxiomicLabs/GPT-X2.5-135M", "n_train_rows": 14753933, "n_val_rows": 2048, "seq_len": 1024, "micro_batch": 32, "grad_accum_steps": 16, "doc_masking": "auto", "shuffle_block_rows": 192, "seed": 42, "epochs_consumed_this_worker": 1.1452 }, "hardware": { "device": "cuda", "torch": "2.11.0+cu130", "amp_dtype": "torch.bfloat16", "preset": "full", "python": "3.13.11", "transformers": "5.14.1", "gpu": "NVIDIA RTX PRO 6000 Blackwell Server Edition", "vram_gb": 101.98, "peak_gb": 39.36 }, "checkpoint": { "kind": "round 14", "weights_file": "model.safetensors", "dtype": "float32", "config_file": "config.json", "auto_map": { "AutoConfig": "configuration_oleander.OleanderConfig", "AutoModelForCausalLM": "modeling_oleander.OleanderForCausalLM" }, "modeling_files": [], "tokenizer_files": [ "tokenizer.json", "tokenizer_config.json", "special_tokens_map.json", "vocab.json", "merges.txt", "added_tokens.json", "chat_template.jinja", "tokenizer.model" ] }, "diloco": { "enabled": true, "worker_id": 0, "n_workers": 1, "inner_steps": 500, "outer_lr": 0.7, "outer_momentum": 0.9, "is_leader": true, "support_repo_id": "CyanMonkey/TestSubject-3-support" }, "extra": { "diloco_round": 14 } }