baseline: run config

baseline/summary.json

@@ -0,0 +1,58 @@
+{
+ "train_loss": [],
+ "val_loss": [],
+ "val_pp": [],
+ "val_acc": [],
+ "args": {
+  "seed": 0,
+  "data_seed": 1337,
+  "device": "cuda:0",
+  "dtype": "bfloat16",
+  "compile": true,
+  "vocab_size": 50257,
+  "block_size": 1024,
+  "n_layer": 12,
+  "n_embd": 768,
+  "n_head": 12,
+  "kv_lora_rank": 192,
+  "q_lora_rank": 256,
+  "rope_dim": 64,
+  "n_experts": 32,
+  "n_experts_per_token": 2,
+  "expert_intermediate_size": 296,
+  "shared_expert_intermediate_size": 296,
+  "mtp_num_heads": 0,
+  "dropout": 0.0,
+  "bias": true,
+  "gate_bias": true,
+  "gate_score_func": "softmax",
+  "aux_loss_weight": 0.0,
+  "mtp_loss_weight": 0.3,
+  "opt": "adamw",
+  "lr": 0.0003,
+  "weight_decay": 0.1,
+  "beta1": 0.9,
+  "beta2": 0.95,
+  "grad_clip": 1.0,
+  "batch_size": 32,
+  "acc_steps": 8,
+  "scheduler": "cos",
+  "iterations": 12800,
+  "warmup_steps": 1280,
+  "min_lr_ratio": 0.1,
+  "wsd_fract_decay": 0.1,
+  "wsd_final_lr_scale": 0.0,
+  "decay_type": "cosine",
+  "eval_interval": 200,
+  "eval_batches": 32,
+  "full_eval_at": [],
+  "log_interval": 50,
+  "diag_interval": 200,
+  "diag_batches": 4,
+  "permanent_ckpt_interval": 3200,
+  "latest_ckpt_interval": 500,
+  "resume_from": null,
+  "auto_resume": true,
+  "wandb": true
+ }
+}