| { | |
| "data": "fineweb_edu_200M_v1024.uint16.bin", | |
| "base": "LowOnMind-300k", | |
| "derived_from": "DedeProGames/DynamicMind-Mini", | |
| "dataset": "HuggingFaceFW/fineweb-edu:sample-10BT", | |
| "tokenizer": "byte-level BPE 1024", | |
| "params": 296960, | |
| "seq_len": 512, | |
| "batch_size": 64, | |
| "grad_accum": 1, | |
| "max_steps": 6103, | |
| "total_tokens": 199983104, | |
| "lr": 0.002, | |
| "min_lr": 0.0002, | |
| "warmup_steps": 250, | |
| "weight_decay": 0.1, | |
| "grad_clip": 1.0, | |
| "val_tokens": 2000000, | |
| "eval_every": 500, | |
| "eval_batches": 40, | |
| "device": "cuda", | |
| "dtype": "bfloat16", | |
| "compile": true, | |
| "seed": 1337, | |
| "train_minutes": 42.6, | |
| "final_val_loss": 3.2982, | |
| "final_val_ppl": 27.063, | |
| "history": [ | |
| { | |
| "step": 500, | |
| "val_loss": 3.8883, | |
| "val_ppl": 48.827 | |
| }, | |
| { | |
| "step": 1000, | |
| "val_loss": 3.621, | |
| "val_ppl": 37.375 | |
| }, | |
| { | |
| "step": 1500, | |
| "val_loss": 3.5287, | |
| "val_ppl": 34.079 | |
| }, | |
| { | |
| "step": 2000, | |
| "val_loss": 3.4824, | |
| "val_ppl": 32.537 | |
| }, | |
| { | |
| "step": 2500, | |
| "val_loss": 3.4508, | |
| "val_ppl": 31.524 | |
| }, | |
| { | |
| "step": 3000, | |
| "val_loss": 3.4226, | |
| "val_ppl": 30.65 | |
| }, | |
| { | |
| "step": 3500, | |
| "val_loss": 3.3997, | |
| "val_ppl": 29.955 | |
| }, | |
| { | |
| "step": 4000, | |
| "val_loss": 3.3725, | |
| "val_ppl": 29.151 | |
| }, | |
| { | |
| "step": 4500, | |
| "val_loss": 3.3489, | |
| "val_ppl": 28.471 | |
| }, | |
| { | |
| "step": 5000, | |
| "val_loss": 3.3408, | |
| "val_ppl": 28.241 | |
| }, | |
| { | |
| "step": 5500, | |
| "val_loss": 3.3165, | |
| "val_ppl": 27.565 | |
| }, | |
| { | |
| "step": 6000, | |
| "val_loss": 3.3251, | |
| "val_ppl": 27.801 | |
| }, | |
| { | |
| "step": 6103, | |
| "val_loss": 3.3125, | |
| "val_ppl": 27.454 | |
| } | |
| ] | |
| } |