Upload config.json with huggingface_hub

config.json

@@ -0,0 +1,86 @@
+{
+  "data_dir": "../data/fineweb-edu-10B",
+  "batch_size": 80,
+  "seq_len": 1024,
+  "max_iters": 16000,
+  "gradient_accumulation_steps": 1,
+  "optimizer": "muon",
+  "learning_rate": 0.0006,
+  "min_lr": 6e-05,
+  "weight_decay": 0.1,
+  "beta1": 0.9,
+  "beta2": 0.95,
+  "grad_clip": 1.0,
+  "warmup_iters": 2000,
+  "lr_decay_iters": 16000,
+  "muon_lr": 0.006,
+  "muon_min_lr": 0.002,
+  "adamw_lr": 0.006,
+  "adamw_min_lr": 0.00036,
+  "muon_momentum": 0.95,
+  "muon_nesterov": true,
+  "muon_ns_steps": 5,
+  "muon_weight_decay": 0.0,
+  "lr_schedule": "linear_warmdown",
+  "warmdown_frac": 0.2,
+  "device": "cuda",
+  "dtype": "bfloat16",
+  "compile": true,
+  "n_layer": 12,
+  "n_head": 8,
+  "n_embd": 1024,
+  "block_size": 1024,
+  "vocab_size": 50304,
+  "sparsity_mode": "none",
+  "k1": 8,
+  "k2": 64,
+  "m2": null,
+  "basis_k": 128,
+  "r": 32,
+  "hash_seed": 42,
+  "deeptopk_backend": "auto",
+  "rmsnorm_affine": true,
+  "wf_f_min_ratio": 0.0,
+  "wf_quantile_lr": 0.1,
+  "wf_ema_beta": 0.9,
+  "weight_sparsify": false,
+  "weight_sparsify_k": 32,
+  "grad_sparsify_k": 4,
+  "l1l1_grad_proj": false,
+  "sgd_lr": 0.01,
+  "sgd_min_lr": 0.001,
+  "sgd_momentum": 0.9,
+  "sgd_weight_decay": 0.0,
+  "cayley_levels": [],
+  "cayley_per_parent_budget": false,
+  "cayley_wf_f_min_ratio": 0.0,
+  "cayley_wf_ema_beta": 0.99,
+  "cayley_wf_bisect_iters": 15,
+  "cayley_wf_warmup_batches": 10,
+  "cayley_wf_batch_size": 64,
+  "cayley_seed": 42,
+  "cayley_backend": "auto",
+  "cayley_locations": [
+    "resid_mid"
+  ],
+  "cayley_score_standardize": false,
+  "cayley_richardson_refine": false,
+  "cayley_matching_pursuit": false,
+  "cayley_sparse_mlp": false,
+  "sparse_mlp_fan_out": 64,
+  "sparse_mlp_K_out": 0,
+  "sparse_mlp_activation": "relu",
+  "sparse_mlp_chunk_size": 1024,
+  "sparse_mlp_norm_preserve": true,
+  "sparse_mlp_backend": "auto",
+  "sparse_mlp_target_hidden_size": 4,
+  "tied_block_init": false,
+  "dead_threshold_c": 0.1,
+  "resume": false,
+  "out_dir": "../out/vanilla-10b",
+  "eval_interval": 500,
+  "eval_iters": 100,
+  "log_interval": 10,
+  "wandb_project": "sparse-nanogpt",
+  "wandb_run_name": "vanilla-10b"
+}