| { | |
| "step": 765, | |
| "val_bpb": 0.4384375044226997, | |
| "model_config": { | |
| "sequence_len": 2048, | |
| "vocab_size": 65536, | |
| "n_layer": 20, | |
| "n_head": 10, | |
| "n_kv_head": 10, | |
| "n_embd": 1280 | |
| }, | |
| "user_config": { | |
| "run": "nanochat-h100-diloco-base-mid-standard-f", | |
| "max_iterations": -1, | |
| "dtype": "bfloat16", | |
| "max_seq_len": 2048, | |
| "device_batch_size": 32, | |
| "unembedding_lr": 0.004, | |
| "embedding_lr": 0.2, | |
| "matrix_lr": 0.02, | |
| "init_lr_frac": 1.0, | |
| "weight_decay": 0.0, | |
| "eval_every": 150, | |
| "eval_tokens": 10485760, | |
| "total_batch_size": 524288, | |
| "use_diloco": 0, | |
| "base_diloco": 1, | |
| "diloco_H": 30, | |
| "diloco_outer_lr": 0.8, | |
| "diloco_outer_momentum": 0.9, | |
| "dry_run": 0 | |
| }, | |
| "device_batch_size": 32, | |
| "max_seq_len": 2048, | |
| "use_diloco": 0, | |
| "base_diloco": 1 | |
| } |