File size: 1,515 Bytes
564ee09 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 | {
"steps": 20000,
"batch_size": 512,
"grad_accum": 4,
"seq_len": 2048,
"eval_seq_len": 2048,
"d_model": 128,
"n_layers": 6,
"n_heads": 4,
"n_kv_heads": 2,
"chunk": 16,
"cma_heads": 2,
"expand": 2,
"cma_identity_prob": 0.9,
"lr": 0.001,
"muon_momentum": 0.95,
"muon_ns_steps": 5,
"muon_adjust_lr_fn": "match_rms_adamw",
"warmup": 500,
"wd": 0.01,
"grad_clip": 5.0,
"log_every": 5,
"early_eval_step": 1000,
"diag_every": 1000,
"eval_every": 2000,
"val_batch_size": 32,
"val_batches": 0,
"val_stride": 1024,
"infer_tokens": 512,
"infer_repeat_penalty": 1.2,
"infer_prompt": "The process of photosynthesis",
"lm_eval_tasks": "arc_easy,arc_challenge,hellaswag,piqa",
"lm_eval_batch_size": "auto",
"lm_eval_device": "cuda",
"lm_eval_dtype": "bfloat16",
"lm_eval_softmax_dtype": "float32",
"lm_eval_expected_version": "0.4.12",
"lm_eval_retries": 3,
"lm_eval_export_dir": "CMA_1M_Mini_lm_eval_hf",
"lm_eval_output_dir": "lm_eval_results_CMA_1M_Mini",
"arithmark_batch_size": 16,
"arithmark_data_path": "benchmark_cache/arithmark_2.0.jsonl",
"arithmark_force_download": false,
"recipe_version": "CMA_1M_mini_byte_c16_h2_e2_muon_w500_cosine20k_b512_ga4_ctx2k_async_v7",
"hf_repo_id": "User01110/CMA-1M-Mini",
"hf_repo_private": false,
"hub_upload_retries": 3,
"tokenizer_name": "local-byte-level",
"tokenizer_revision": "byte-v1",
"data_seed": 1337,
"shuffle_buffer": 50000,
"tokenize_batch_size": 64,
"compile": true
}
|