| { | |
| "step": 69, | |
| "val_bpb": 0.6465639004175056, | |
| "model_config": { | |
| "sequence_len": 2048, | |
| "vocab_size": 32768, | |
| "n_layer": 34, | |
| "n_head": 17, | |
| "n_kv_head": 17, | |
| "n_embd": 2176, | |
| "window_pattern": "L" | |
| }, | |
| "user_config": { | |
| "run": "corpus_sft_d34_physics_expanded", | |
| "device_type": "", | |
| "dtype": "bfloat16", | |
| "model_tag": "d34", | |
| "model_step": null, | |
| "checkpoints_dir": "physics_sft_expanded_checkpoints", | |
| "output_dir": "instruct_checkpoints_physics_expanded", | |
| "num_iterations": -1, | |
| "max_seq_len": 2048, | |
| "device_batch_size": 4, | |
| "total_batch_size": 524288, | |
| "embedding_lr": 0.3, | |
| "unembedding_lr": 0.004, | |
| "matrix_lr": 0.02, | |
| "weight_decay": 0.0, | |
| "init_lr_frac": 1.0, | |
| "eval_every": 150, | |
| "eval_tokens": 10485760, | |
| "train_data": "instruct_data/v2_corpus/all_train.jsonl", | |
| "val_data": "instruct_data/v2_corpus/all_val.jsonl", | |
| "dry_run": false | |
| } | |
| } |