Upload config.json with huggingface_hub

config.json

@@ -0,0 +1,56 @@
+{
+  "data_dir": "../data/fineweb-edu-10B",
+  "batch_size": 80,
+  "seq_len": 1024,
+  "max_iters": 16000,
+  "gradient_accumulation_steps": 4,
+  "optimizer": "muon",
+  "learning_rate": 0.0006,
+  "min_lr": 6e-05,
+  "weight_decay": 0.1,
+  "beta1": 0.9,
+  "beta2": 0.95,
+  "grad_clip": 1.0,
+  "warmup_iters": 0,
+  "lr_decay_iters": 16000,
+  "muon_lr": 0.006,
+  "muon_min_lr": 0.0,
+  "adamw_lr": 0.006,
+  "adamw_min_lr": 0.0,
+  "muon_momentum": 0.95,
+  "muon_nesterov": true,
+  "muon_ns_steps": 5,
+  "muon_weight_decay": 0.0,
+  "lr_schedule": "linear_warmdown",
+  "warmdown_frac": 0.4,
+  "device": "cuda",
+  "dtype": "bfloat16",
+  "compile": true,
+  "n_layer": 12,
+  "n_head": 8,
+  "n_embd": 1024,
+  "block_size": 1024,
+  "vocab_size": 50304,
+  "sparsity_mode": "none",
+  "rmsnorm_affine": true,
+  "cayley_levels": [],
+  "cayley_per_parent_budget": false,
+  "cayley_seed": 42,
+  "cayley_backend": "auto",
+  "cayley_locations": [
+    "resid_mid"
+  ],
+  "cayley_score_standardize": false,
+  "cayley_richardson_refine": false,
+  "cayley_matching_pursuit": false,
+  "tied_block_init": false,
+  "dead_threshold_c": 0.1,
+  "resume": false,
+  "stop_val_loss": 3.173,
+  "out_dir": "../out/vanilla-v5-parity",
+  "eval_interval": 250,
+  "eval_iters": 100,
+  "log_interval": 10,
+  "wandb_project": "sparse-nanogpt",
+  "wandb_run_name": "vanilla-v5-parity"
+}