| { |
| "params": 944640, |
| "args": { |
| "data": "data/tinystories_full.txt", |
| "output": "checkpoints/atome_1m_v1.pt", |
| "steps": 30000, |
| "seq_len": 256, |
| "batch_size": 64, |
| "accum_steps": 4, |
| "lr": 0.0003, |
| "min_lr": 3e-05, |
| "warmup": 1000, |
| "weight_decay": 0.1, |
| "d_model": 256, |
| "n_layers": 8, |
| "d_head": 64, |
| "top_k": 4, |
| "bf16": true, |
| "eval_every": 1000, |
| "seed": 0 |
| }, |
| "log": [ |
| { |
| "step": 1000, |
| "train_loss": 1.689065933227539, |
| "val_loss": 1.6851140782237053, |
| "val_ppl": 5.3930661286628725, |
| "lr": 0.0003 |
| }, |
| { |
| "step": 2000, |
| "train_loss": 1.475701928138733, |
| "val_loss": 1.4368714336305857, |
| "val_ppl": 4.207511724416042, |
| "lr": 0.0002992086242158385 |
| }, |
| { |
| "step": 3000, |
| "train_loss": 1.3402614891529083, |
| "val_loss": 1.355498529970646, |
| "val_ppl": 3.8786941199889884, |
| "lr": 0.00029684377502086165 |
| }, |
| { |
| "step": 4000, |
| "train_loss": 1.2906470894813538, |
| "val_loss": 1.298057682812214, |
| "val_ppl": 3.662176646542712, |
| "lr": 0.0002929331781096783 |
| }, |
| { |
| "step": 5000, |
| "train_loss": 1.2640663385391235, |
| "val_loss": 1.2564894184470177, |
| "val_ppl": 3.513066906295889, |
| "lr": 0.00028752268165557917 |
| }, |
| { |
| "step": 6000, |
| "train_loss": 1.205640196800232, |
| "val_loss": 1.2161348164081573, |
| "val_ppl": 3.374120900293555, |
| "lr": 0.0002806757187826245 |
| }, |
| { |
| "step": 7000, |
| "train_loss": 1.1917544305324554, |
| "val_loss": 1.1835042145103216, |
| "val_ppl": 3.2657982326287116, |
| "lr": 0.00027247256387026185 |
| }, |
| { |
| "step": 8000, |
| "train_loss": 1.1544596254825592, |
| "val_loss": 1.1677243299782276, |
| "val_ppl": 3.2146687829705525, |
| "lr": 0.0002630093914096226 |
| }, |
| { |
| "step": 9000, |
| "train_loss": 1.1510637402534485, |
| "val_loss": 1.1527819111943245, |
| "val_ppl": 3.166990953913901, |
| "lr": 0.0002523971484455467 |
| }, |
| { |
| "step": 10000, |
| "train_loss": 1.140123575925827, |
| "val_loss": 1.1461433116346598, |
| "val_ppl": 3.146036201225796, |
| "lr": 0.0002407602538239216 |
| }, |
| { |
| "step": 11000, |
| "train_loss": 1.1275735795497894, |
| "val_loss": 1.131921675056219, |
| "val_ppl": 3.1016110655411038, |
| "lr": 0.00022823513949447164 |
| }, |
| { |
| "step": 12000, |
| "train_loss": 1.1099890172481537, |
| "val_loss": 1.112453417852521, |
| "val_ppl": 3.041812083259338, |
| "lr": 0.00021496865097088842 |
| }, |
| { |
| "step": 13000, |
| "train_loss": 1.1127586960792542, |
| "val_loss": 1.112892348319292, |
| "val_ppl": 3.043147520317438, |
| "lr": 0.0002011163257014448 |
| }, |
| { |
| "step": 14000, |
| "train_loss": 1.0873990654945374, |
| "val_loss": 1.1024821121245623, |
| "val_ppl": 3.0116319626741244, |
| "lr": 0.00018684056953462323 |
| }, |
| { |
| "step": 15000, |
| "train_loss": 1.0949949026107788, |
| "val_loss": 1.1003286074846983, |
| "val_ppl": 3.0051533776041945, |
| "lr": 0.00017230875265903135 |
| }, |
| { |
| "step": 16000, |
| "train_loss": 1.092372715473175, |
| "val_loss": 1.0886210184544325, |
| "val_ppl": 2.9701754301311736, |
| "lr": 0.00015769124734096862 |
| }, |
| { |
| "step": 17000, |
| "train_loss": 1.0719301402568817, |
| "val_loss": 1.087962357327342, |
| "val_ppl": 2.968219735175533, |
| "lr": 0.00014315943046537674 |
| }, |
| { |
| "step": 18000, |
| "train_loss": 1.0894330739974976, |
| "val_loss": 1.0875801891088486, |
| "val_ppl": 2.9670855926576603, |
| "lr": 0.0001288836742985552 |
| }, |
| { |
| "step": 19000, |
| "train_loss": 1.0676527321338654, |
| "val_loss": 1.0716162715107203, |
| "val_ppl": 2.920095354830056, |
| "lr": 0.00011503134902911152 |
| }, |
| { |
| "step": 20000, |
| "train_loss": 1.0742259323596954, |
| "val_loss": 1.0812196973711252, |
| "val_ppl": 2.948273360207015, |
| "lr": 0.00010176486050552833 |
| }, |
| { |
| "step": 21000, |
| "train_loss": 1.0726729929447174, |
| "val_loss": 1.0718515273183584, |
| "val_ppl": 2.9207824050342435, |
| "lr": 8.923974617607838e-05 |
| }, |
| { |
| "step": 22000, |
| "train_loss": 1.0701198875904083, |
| "val_loss": 1.0739975553005934, |
| "val_ppl": 2.927057216357621, |
| "lr": 7.760285155445327e-05 |
| }, |
| { |
| "step": 23000, |
| "train_loss": 1.0675779581069946, |
| "val_loss": 1.0646078549325466, |
| "val_ppl": 2.899701657373658, |
| "lr": 6.699060859037736e-05 |
| }, |
| { |
| "step": 24000, |
| "train_loss": 1.0793527662754059, |
| "val_loss": 1.0707154776901007, |
| "val_ppl": 2.917466135348921, |
| "lr": 5.7527436129738084e-05 |
| }, |
| { |
| "step": 25000, |
| "train_loss": 1.0686360597610474, |
| "val_loss": 1.067691769450903, |
| "val_ppl": 2.9086578924472115, |
| "lr": 4.9324281217375474e-05 |
| }, |
| { |
| "step": 26000, |
| "train_loss": 1.079252928495407, |
| "val_loss": 1.064154027029872, |
| "val_ppl": 2.8983859904178786, |
| "lr": 4.247731834442082e-05 |
| }, |
| { |
| "step": 27000, |
| "train_loss": 1.0666958093643188, |
| "val_loss": 1.0639245696365833, |
| "val_ppl": 2.8977210106189566, |
| "lr": 3.7066821890321684e-05 |
| }, |
| { |
| "step": 28000, |
| "train_loss": 1.065284639596939, |
| "val_loss": 1.0690924655646086, |
| "val_ppl": 2.912734892906038, |
| "lr": 3.31562249791383e-05 |
| }, |
| { |
| "step": 29000, |
| "train_loss": 1.06133571267128, |
| "val_loss": 1.0545352958142757, |
| "val_ppl": 2.8706408450794916, |
| "lr": 3.0791375784161455e-05 |
| } |
| ], |
| "final_val": 1.0572172198444605, |
| "best_val": 1.0545352958142757 |
| } |
