| { |
| "params": 950608, |
| "args": { |
| "data": "data/tinystories_full.txt", |
| "output": "checkpoints/vanilla_1m_v1.pt", |
| "steps": 30000, |
| "seq_len": 256, |
| "batch_size": 64, |
| "accum_steps": 4, |
| "lr": 0.0003, |
| "min_lr": 3e-05, |
| "warmup": 1000, |
| "weight_decay": 0.1, |
| "d_model": 152, |
| "n_layers": 3, |
| "n_heads": 4, |
| "d_ff": 608, |
| "max_seq": 256, |
| "bf16": true, |
| "eval_every": 1000, |
| "seed": 0 |
| }, |
| "log": [ |
| { |
| "step": 1000, |
| "train_loss": 2.0875988006591797, |
| "val_loss": 2.0943055227398872, |
| "val_ppl": 8.119799995221573, |
| "lr": 0.0003 |
| }, |
| { |
| "step": 2000, |
| "train_loss": 1.5252898037433624, |
| "val_loss": 1.5066693723201752, |
| "val_ppl": 4.511679019275092, |
| "lr": 0.0002992086242158385 |
| }, |
| { |
| "step": 3000, |
| "train_loss": 1.3099323511123657, |
| "val_loss": 1.3194083347916603, |
| "val_ppl": 3.7412071801680677, |
| "lr": 0.00029684377502086165 |
| }, |
| { |
| "step": 4000, |
| "train_loss": 1.2161387205123901, |
| "val_loss": 1.2286550998687744, |
| "val_ppl": 3.4166314169360987, |
| "lr": 0.0002929331781096783 |
| }, |
| { |
| "step": 5000, |
| "train_loss": 1.1787906289100647, |
| "val_loss": 1.1772918552160263, |
| "val_ppl": 3.2455728094700103, |
| "lr": 0.00028752268165557917 |
| }, |
| { |
| "step": 6000, |
| "train_loss": 1.1403338611125946, |
| "val_loss": 1.1352313607931137, |
| "val_ppl": 3.1118934297571132, |
| "lr": 0.0002806757187826245 |
| }, |
| { |
| "step": 7000, |
| "train_loss": 1.1162661612033844, |
| "val_loss": 1.1075621414929628, |
| "val_ppl": 3.0269700675173796, |
| "lr": 0.00027247256387026185 |
| }, |
| { |
| "step": 8000, |
| "train_loss": 1.0829694867134094, |
| "val_loss": 1.0843632984906435, |
| "val_ppl": 2.9575561386746556, |
| "lr": 0.0002630093914096226 |
| }, |
| { |
| "step": 9000, |
| "train_loss": 1.0747118294239044, |
| "val_loss": 1.0635895021259785, |
| "val_ppl": 2.8967502410992467, |
| "lr": 0.0002523971484455467 |
| }, |
| { |
| "step": 10000, |
| "train_loss": 1.0519791841506958, |
| "val_loss": 1.0476661436259747, |
| "val_ppl": 2.85098954738486, |
| "lr": 0.0002407602538239216 |
| }, |
| { |
| "step": 11000, |
| "train_loss": 1.0250678956508636, |
| "val_loss": 1.0324134565889835, |
| "val_ppl": 2.807834249846705, |
| "lr": 0.00022823513949447164 |
| }, |
| { |
| "step": 12000, |
| "train_loss": 1.0199836790561676, |
| "val_loss": 1.023882026784122, |
| "val_ppl": 2.783981303587245, |
| "lr": 0.00021496865097088842 |
| }, |
| { |
| "step": 13000, |
| "train_loss": 1.0101815909147263, |
| "val_loss": 1.0102009763941169, |
| "val_ppl": 2.7461528714618, |
| "lr": 0.0002011163257014448 |
| }, |
| { |
| "step": 14000, |
| "train_loss": 1.0113594383001328, |
| "val_loss": 1.0001213569194078, |
| "val_ppl": 2.7186117307853896, |
| "lr": 0.00018684056953462323 |
| }, |
| { |
| "step": 15000, |
| "train_loss": 0.98267862200737, |
| "val_loss": 0.9921664940193295, |
| "val_ppl": 2.697071336220516, |
| "lr": 0.00017230875265903135 |
| }, |
| { |
| "step": 16000, |
| "train_loss": 0.995794028043747, |
| "val_loss": 0.9845060091465712, |
| "val_ppl": 2.6764893965183, |
| "lr": 0.00015769124734096862 |
| }, |
| { |
| "step": 17000, |
| "train_loss": 0.962462991476059, |
| "val_loss": 0.9766457295045257, |
| "val_ppl": 2.655533907298061, |
| "lr": 0.00014315943046537674 |
| }, |
| { |
| "step": 18000, |
| "train_loss": 0.9672404527664185, |
| "val_loss": 0.9714991142973304, |
| "val_ppl": 2.6419020052744058, |
| "lr": 0.0001288836742985552 |
| }, |
| { |
| "step": 19000, |
| "train_loss": 0.9653829336166382, |
| "val_loss": 0.9648234033957124, |
| "val_ppl": 2.624324168813844, |
| "lr": 0.00011503134902911152 |
| }, |
| { |
| "step": 20000, |
| "train_loss": 0.9600358754396439, |
| "val_loss": 0.959049197845161, |
| "val_ppl": 2.6092144469334535, |
| "lr": 0.00010176486050552833 |
| }, |
| { |
| "step": 21000, |
| "train_loss": 0.9566726982593536, |
| "val_loss": 0.9548654137179255, |
| "val_ppl": 2.598320861041842, |
| "lr": 8.923974617607838e-05 |
| }, |
| { |
| "step": 22000, |
| "train_loss": 0.9502571374177933, |
| "val_loss": 0.9499085610732436, |
| "val_ppl": 2.5854732356090246, |
| "lr": 7.760285155445327e-05 |
| }, |
| { |
| "step": 23000, |
| "train_loss": 0.9525800943374634, |
| "val_loss": 0.9469442367553711, |
| "val_ppl": 2.5778204027666733, |
| "lr": 6.699060859037736e-05 |
| }, |
| { |
| "step": 24000, |
| "train_loss": 0.9471650272607803, |
| "val_loss": 0.9441628893837333, |
| "val_ppl": 2.57066055039882, |
| "lr": 5.7527436129738084e-05 |
| }, |
| { |
| "step": 25000, |
| "train_loss": 0.9476055055856705, |
| "val_loss": 0.9407382626086473, |
| "val_ppl": 2.561872054696453, |
| "lr": 4.9324281217375474e-05 |
| }, |
| { |
| "step": 26000, |
| "train_loss": 0.9304470866918564, |
| "val_loss": 0.9391492558643222, |
| "val_ppl": 2.5578044553007495, |
| "lr": 4.247731834442082e-05 |
| }, |
| { |
| "step": 27000, |
| "train_loss": 0.9319835901260376, |
| "val_loss": 0.936947762966156, |
| "val_ppl": 2.5521796607019356, |
| "lr": 3.7066821890321684e-05 |
| }, |
| { |
| "step": 28000, |
| "train_loss": 0.933847963809967, |
| "val_loss": 0.9346829485148191, |
| "val_ppl": 2.5464059879406724, |
| "lr": 3.31562249791383e-05 |
| }, |
| { |
| "step": 29000, |
| "train_loss": 0.936771810054779, |
| "val_loss": 0.9336990155279636, |
| "val_ppl": 2.5439017273055704, |
| "lr": 3.0791375784161455e-05 |
| } |
| ], |
| "final_val": 0.9317306941375136, |
| "best_val": 0.9336990155279636 |
| } |
