| { | |
| "step": 87000, | |
| "val_bpb": 0.8657516205114915, | |
| "model_config": { | |
| "sequence_len": 2048, | |
| "vocab_size": 32768, | |
| "n_layer": 12, | |
| "n_head": 6, | |
| "n_kv_head": 6, | |
| "n_embd": 768, | |
| "window_pattern": "L" | |
| }, | |
| "user_config": { | |
| "run": "rtx4070-d12-chinchilla", | |
| "tracker": "wandb", | |
| "mlflow_uri": null, | |
| "mlflow_experiment": "nanochat", | |
| "device_type": "", | |
| "fp8": false, | |
| "fp8_recipe": "tensorwise", | |
| "depth": 12, | |
| "aspect_ratio": 64, | |
| "head_dim": 128, | |
| "max_seq_len": 2048, | |
| "window_pattern": "L", | |
| "num_iterations": 87000, | |
| "target_flops": -1.0, | |
| "target_param_data_ratio": 12, | |
| "device_batch_size": 8, | |
| "total_batch_size": 65536, | |
| "embedding_lr": 0.3, | |
| "unembedding_lr": 0.008, | |
| "weight_decay": 0.28, | |
| "matrix_lr": 0.02, | |
| "scalar_lr": 0.5, | |
| "warmup_steps": 40, | |
| "warmdown_ratio": 0.65, | |
| "final_lr_frac": 0.05, | |
| "resume_from_step": 10000, | |
| "eval_every": 2000, | |
| "eval_tokens": 524288, | |
| "core_metric_every": 10000, | |
| "core_metric_max_per_task": 200, | |
| "sample_every": 5000, | |
| "save_every": 10000, | |
| "model_tag": null | |
| }, | |
| "device_batch_size": 8, | |
| "max_seq_len": 2048, | |
| "total_batch_size": 65536, | |
| "dataloader_state_dict": { | |
| "pq_idx": 10, | |
| "rg_idx": 77, | |
| "epoch": 1 | |
| }, | |
| "loop_state": { | |
| "min_val_bpb": 0.8657516205114915, | |
| "smooth_train_loss": 2.7484221461371146, | |
| "total_training_time": 102370.698939085 | |
| } | |
| } |