| transformer | 17,440,512 params | vocab 8192 | mps |
| step 1 | loss 9.0659 | gnorm 5.66 | 1s |
| step 25 | loss 8.4237 | gnorm 2.84 | 21s |
| step 50 | loss 8.0053 | gnorm 2.37 | 36s |
| step 75 | loss 7.5894 | gnorm 2.29 | 51s |
| step 100 | loss 7.2227 | gnorm 1.95 | 66s |
| step 125 | loss 6.5297 | gnorm 1.79 | 83s |
| step 150 | loss 6.0340 | gnorm 1.76 | 99s |
| step 175 | loss 5.3586 | gnorm 1.44 | 117s |
| step 200 | loss 5.1159 | gnorm 1.48 | 133s |
| step 225 | loss 5.0118 | gnorm 1.45 | 148s |
| step 250 | loss 4.6414 | gnorm 1.30 | 163s |
| val 4.7656 | ppl 117.40 | 1M tokens |
| step 275 | loss 4.6810 | gnorm 1.50 | 182s |
| step 300 | loss 4.5977 | gnorm 1.46 | 197s |
| step 325 | loss 4.3595 | gnorm 1.41 | 212s |
| step 350 | loss 4.4616 | gnorm 1.52 | 228s |
| step 375 | loss 4.0337 | gnorm 1.43 | 244s |
| step 400 | loss 4.1460 | gnorm 1.46 | 259s |
| step 425 | loss 4.1583 | gnorm 1.43 | 275s |
| step 450 | loss 4.2407 | gnorm 1.49 | 290s |
| step 475 | loss 4.1234 | gnorm 1.59 | 305s |
| step 500 | loss 4.1939 | gnorm 1.50 | 320s |
| val 4.0422 | ppl 56.95 | 1M tokens |
| step 525 | loss 3.9165 | gnorm 1.65 | 338s |
| step 550 | loss 4.0645 | gnorm 1.59 | 353s |
| step 575 | loss 3.9202 | gnorm 1.70 | 369s |
| step 600 | loss 3.7699 | gnorm 1.48 | 384s |
| step 625 | loss 3.6540 | gnorm 1.65 | 399s |
| step 650 | loss 3.8476 | gnorm 1.67 | 414s |
| step 675 | loss 3.7533 | gnorm 1.69 | 430s |
| step 700 | loss 4.0363 | gnorm 1.59 | 446s |
| step 725 | loss 3.8794 | gnorm 1.61 | 462s |
| step 750 | loss 3.7619 | gnorm 1.65 | 478s |
| val 3.7400 | ppl 42.10 | 2M tokens |
| step 775 | loss 3.6882 | gnorm 1.73 | 498s |
| step 800 | loss 3.7385 | gnorm 1.74 | 515s |
| step 825 | loss 3.4740 | gnorm 1.78 | 531s |
| step 850 | loss 3.6808 | gnorm 1.80 | 547s |
| step 875 | loss 3.6482 | gnorm 1.77 | 562s |
| step 900 | loss 3.4743 | gnorm 1.75 | 577s |
| step 925 | loss 3.6641 | gnorm 1.71 | 593s |
| step 950 | loss 3.4985 | gnorm 1.95 | 608s |
| step 975 | loss 3.4978 | gnorm 1.86 | 623s |
| step 1000 | loss 3.6681 | gnorm 1.93 | 638s |
| val 3.5336 | ppl 34.25 | 2M tokens |
| step 1025 | loss 3.3692 | gnorm 1.85 | 656s |
| step 1050 | loss 3.7098 | gnorm 1.80 | 674s |
| step 1075 | loss 3.4980 | gnorm 1.89 | 692s |
| step 1100 | loss 3.6340 | gnorm 1.88 | 708s |
| step 1125 | loss 3.4360 | gnorm 1.74 | 727s |
| step 1150 | loss 3.2896 | gnorm 1.88 | 744s |
| step 1175 | loss 3.6475 | gnorm 2.15 | 765s |
| step 1200 | loss 3.4324 | gnorm 2.00 | 784s |
| step 1225 | loss 3.3737 | gnorm 1.80 | 802s |
| step 1250 | loss 3.4515 | gnorm 1.92 | 818s |
| val 3.3742 | ppl 29.20 | 3M tokens |
| step 1275 | loss 3.3330 | gnorm 1.96 | 837s |
| step 1300 | loss 3.2365 | gnorm 1.89 | 853s |
| step 1325 | loss 3.1443 | gnorm 1.79 | 869s |
| step 1350 | loss 3.0539 | gnorm 1.84 | 885s |
| step 1375 | loss 3.1282 | gnorm 1.84 | 901s |
| step 1400 | loss 3.3304 | gnorm 1.85 | 917s |
| step 1425 | loss 3.4560 | gnorm 2.02 | 934s |
| step 1450 | loss 3.0201 | gnorm 1.82 | 953s |
| step 1475 | loss 3.2094 | gnorm 1.87 | 972s |
| step 1500 | loss 3.1902 | gnorm 2.00 | 992s |
| val 3.2581 | ppl 26.00 | 3M tokens |
| step 1525 | loss 3.4144 | gnorm 1.97 | 1012s |
| step 1550 | loss 2.9986 | gnorm 1.92 | 1027s |
| step 1575 | loss 3.1162 | gnorm 1.97 | 1045s |
| step 1600 | loss 2.9763 | gnorm 1.97 | 1061s |
| step 1625 | loss 3.2165 | gnorm 1.76 | 1076s |
| step 1650 | loss 3.2712 | gnorm 1.91 | 1090s |
| step 1675 | loss 3.0484 | gnorm 2.04 | 1107s |
| step 1700 | loss 3.1147 | gnorm 2.01 | 1122s |
| step 1725 | loss 3.0797 | gnorm 1.95 | 1140s |
| step 1750 | loss 3.1958 | gnorm 1.92 | 1155s |
| val 3.1551 | ppl 23.46 | 4M tokens |
| step 1775 | loss 3.2327 | gnorm 1.99 | 1176s |
| step 1800 | loss 3.1617 | gnorm 2.09 | 1190s |
| step 1825 | loss 3.2977 | gnorm 1.95 | 1207s |
| step 1850 | loss 3.1577 | gnorm 2.26 | 1224s |
| step 1875 | loss 3.3915 | gnorm 2.02 | 1239s |
| step 1900 | loss 3.3305 | gnorm 1.93 | 1254s |
| step 1925 | loss 2.9534 | gnorm 2.16 | 1269s |
| step 1950 | loss 3.1981 | gnorm 2.04 | 1284s |
| step 1975 | loss 2.8632 | gnorm 1.96 | 1299s |
| step 2000 | loss 3.3176 | gnorm 2.07 | 1314s |
| val 3.0857 | ppl 21.88 | 4M tokens |
| step 2025 | loss 2.8043 | gnorm 1.89 | 1332s |
| step 2050 | loss 2.7409 | gnorm 2.03 | 1347s |
| step 2075 | loss 3.0667 | gnorm 2.06 | 1363s |
| step 2100 | loss 3.2567 | gnorm 2.02 | 1377s |
| step 2125 | loss 3.1960 | gnorm 2.09 | 1392s |
| step 2150 | loss 2.9177 | gnorm 2.08 | 1407s |
| step 2175 | loss 3.0648 | gnorm 2.19 | 1422s |
| step 2200 | loss 3.1265 | gnorm 2.02 | 1438s |
| step 2225 | loss 3.0281 | gnorm 2.10 | 1453s |
| step 2250 | loss 3.0515 | gnorm 1.95 | 1468s |
| val 3.0140 | ppl 20.37 | 5M tokens |
| step 2275 | loss 3.2566 | gnorm 2.13 | 1486s |
| step 2300 | loss 2.8999 | gnorm 2.08 | 1505s |
| step 2325 | loss 3.0517 | gnorm 2.23 | 1524s |
| step 2350 | loss 2.9837 | gnorm 2.06 | 1539s |
| step 2375 | loss 2.9317 | gnorm 2.04 | 1555s |
| step 2400 | loss 3.0361 | gnorm 2.26 | 1571s |
| step 2425 | loss 3.0551 | gnorm 2.24 | 1586s |
| step 2450 | loss 2.8550 | gnorm 2.21 | 1601s |
| step 2475 | loss 2.9036 | gnorm 2.09 | 1617s |
| step 2500 | loss 3.1739 | gnorm 2.19 | 1633s |
| val 2.9613 | ppl 19.32 | 5M tokens |
| step 2525 | loss 2.7797 | gnorm 2.35 | 1652s |
| step 2550 | loss 2.8502 | gnorm 2.07 | 1668s |
| step 2575 | loss 3.1404 | gnorm 2.30 | 1683s |
| step 2600 | loss 2.8394 | gnorm 2.22 | 1700s |
| step 2625 | loss 2.8314 | gnorm 2.18 | 1717s |
| step 2650 | loss 3.0105 | gnorm 2.20 | 1733s |
| step 2675 | loss 2.9467 | gnorm 2.24 | 1748s |
| step 2700 | loss 3.0323 | gnorm 2.19 | 1764s |
| step 2725 | loss 2.7853 | gnorm 2.20 | 1779s |
| step 2750 | loss 3.0100 | gnorm 2.21 | 1794s |
| val 2.9078 | ppl 18.32 | 6M tokens |
| step 2775 | loss 2.8640 | gnorm 2.03 | 1812s |
| step 2800 | loss 3.0415 | gnorm 2.24 | 1827s |
| step 2825 | loss 3.0081 | gnorm 2.26 | 1843s |
| step 2850 | loss 2.9286 | gnorm 2.36 | 1858s |
| step 2875 | loss 3.0645 | gnorm 2.04 | 1873s |
| step 2900 | loss 2.8814 | gnorm 2.10 | 1888s |
| step 2925 | loss 2.9102 | gnorm 2.16 | 1903s |
| step 2950 | loss 2.9170 | gnorm 2.10 | 1918s |
| step 2975 | loss 2.7310 | gnorm 2.13 | 1933s |
| step 3000 | loss 2.8980 | gnorm 2.19 | 1948s |
| val 2.8735 | ppl 17.70 | 6M tokens |
| step 3025 | loss 3.1517 | gnorm 2.18 | 1966s |
| step 3050 | loss 2.8395 | gnorm 2.21 | 1982s |
| step 3075 | loss 2.6157 | gnorm 2.09 | 1997s |
| step 3100 | loss 2.7318 | gnorm 2.17 | 2012s |
| step 3125 | loss 2.7952 | gnorm 2.28 | 2027s |
| step 3150 | loss 2.8145 | gnorm 2.35 | 2042s |
| step 3175 | loss 3.0589 | gnorm 2.37 | 2057s |
| step 3200 | loss 3.0978 | gnorm 2.40 | 2072s |
| step 3225 | loss 3.0286 | gnorm 2.26 | 2087s |
| step 3250 | loss 2.9351 | gnorm 2.35 | 2102s |
| val 2.8314 | ppl 16.97 | 7M tokens |
| step 3275 | loss 2.7162 | gnorm 2.53 | 2121s |
| step 3300 | loss 2.7243 | gnorm 2.38 | 2136s |
| step 3325 | loss 2.7972 | gnorm 2.46 | 2151s |
| step 3350 | loss 3.0039 | gnorm 2.37 | 2166s |
| step 3375 | loss 3.0588 | gnorm 2.30 | 2181s |
| step 3400 | loss 2.8216 | gnorm 2.14 | 2196s |
| step 3425 | loss 2.8228 | gnorm 2.26 | 2211s |
| step 3450 | loss 2.8083 | gnorm 2.28 | 2227s |
| step 3475 | loss 2.9651 | gnorm 2.26 | 2242s |
| step 3500 | loss 2.9724 | gnorm 2.36 | 2257s |
| val 2.8072 | ppl 16.56 | 7M tokens |
| step 3525 | loss 2.9245 | gnorm 2.44 | 2276s |
| step 3550 | loss 2.8749 | gnorm 2.23 | 2291s |
| step 3575 | loss 2.6986 | gnorm 2.35 | 2306s |
| step 3600 | loss 2.5832 | gnorm 2.27 | 2321s |
| step 3625 | loss 2.6425 | gnorm 2.26 | 2336s |
| step 3650 | loss 2.7245 | gnorm 2.27 | 2351s |
| step 3675 | loss 2.5597 | gnorm 2.29 | 2366s |
| step 3700 | loss 2.9487 | gnorm 2.20 | 2381s |
| step 3725 | loss 2.8018 | gnorm 2.26 | 2396s |
| step 3750 | loss 2.8344 | gnorm 2.32 | 2411s |
| val 2.7824 | ppl 16.16 | 8M tokens |
| step 3775 | loss 2.7982 | gnorm 2.31 | 2429s |
| step 3800 | loss 2.8159 | gnorm 2.21 | 2444s |
| step 3825 | loss 2.7541 | gnorm 2.27 | 2459s |
| step 3850 | loss 2.6004 | gnorm 2.32 | 2475s |
| step 3875 | loss 2.6492 | gnorm 2.30 | 2490s |
| step 3900 | loss 2.5851 | gnorm 2.19 | 2505s |
| step 3925 | loss 2.4567 | gnorm 2.39 | 2520s |
| step 3950 | loss 2.8217 | gnorm 2.25 | 2535s |
| step 3975 | loss 2.8198 | gnorm 2.31 | 2550s |
| step 4000 | loss 2.7746 | gnorm 2.45 | 2566s |
| val 2.7633 | ppl 15.85 | 8M tokens |
| step 4025 | loss 2.9212 | gnorm 2.43 | 2584s |
| step 4050 | loss 2.6847 | gnorm 2.25 | 2599s |
| step 4075 | loss 2.8598 | gnorm 2.36 | 2615s |
| step 4100 | loss 2.7371 | gnorm 2.29 | 2630s |
| step 4125 | loss 2.7010 | gnorm 2.60 | 2645s |
| step 4150 | loss 2.6612 | gnorm 2.29 | 2660s |
| step 4175 | loss 2.7117 | gnorm 2.47 | 2676s |
| step 4200 | loss 2.7607 | gnorm 2.30 | 2691s |
| step 4225 | loss 2.8989 | gnorm 2.57 | 2706s |
| step 4250 | loss 2.8335 | gnorm 2.48 | 2721s |
| val 2.7466 | ppl 15.59 | 9M tokens |
| step 4275 | loss 2.7723 | gnorm 2.65 | 2740s |
| step 4300 | loss 2.6089 | gnorm 2.32 | 2755s |
| step 4325 | loss 2.6483 | gnorm 2.29 | 2770s |
| step 4350 | loss 2.5808 | gnorm 2.36 | 2785s |
| step 4375 | loss 3.1040 | gnorm 2.95 | 2800s |
| step 4400 | loss 2.9378 | gnorm 2.48 | 2815s |
| step 4425 | loss 2.8229 | gnorm 2.37 | 2830s |
| step 4450 | loss 2.8420 | gnorm 2.31 | 2846s |
| step 4475 | loss 2.4412 | gnorm 2.31 | 2861s |
| step 4500 | loss 2.5466 | gnorm 2.40 | 2877s |
| val 2.7340 | ppl 15.39 | 9M tokens |
| step 4525 | loss 2.6072 | gnorm 2.36 | 2895s |
| step 4550 | loss 2.6989 | gnorm 2.46 | 2911s |
| step 4575 | loss 2.7203 | gnorm 2.41 | 2927s |
| step 4600 | loss 2.7147 | gnorm 2.43 | 2944s |
| step 4625 | loss 2.8740 | gnorm 2.34 | 2960s |
| step 4650 | loss 2.7108 | gnorm 2.37 | 2977s |
| step 4675 | loss 3.1500 | gnorm 2.43 | 2993s |
| step 4700 | loss 2.7334 | gnorm 2.59 | 3008s |
| step 4725 | loss 2.8229 | gnorm 2.41 | 3023s |
| step 4750 | loss 2.6176 | gnorm 2.33 | 3038s |
| val 2.7261 | ppl 15.27 | 10M tokens |
| step 4775 | loss 2.6204 | gnorm 2.28 | 3056s |
| step 4800 | loss 2.9064 | gnorm 2.38 | 3071s |
| step 4825 | loss 2.6115 | gnorm 2.50 | 3086s |
| step 4850 | loss 2.8141 | gnorm 2.48 | 3101s |
| step 4875 | loss 2.9632 | gnorm 2.51 | 3116s |
| step 4900 | loss 2.5242 | gnorm 2.53 | 3131s |
| step 4925 | loss 3.0410 | gnorm 2.56 | 3146s |
| step 4950 | loss 2.6725 | gnorm 2.47 | 3161s |
| step 4975 | loss 2.6360 | gnorm 2.42 | 3176s |
| step 5000 | loss 2.8666 | gnorm 2.64 | 3191s |
| val 2.7214 | ppl 15.20 | 10M tokens |
| step 5025 | loss 2.7423 | gnorm 2.26 | 3209s |
| step 5050 | loss 2.6553 | gnorm 2.43 | 3224s |
| step 5075 | loss 2.6794 | gnorm 2.35 | 3239s |
| step 5100 | loss 2.7391 | gnorm 2.38 | 3254s |
| step 5125 | loss 2.9630 | gnorm 2.35 | 3270s |
| step 5150 | loss 2.8359 | gnorm 2.27 | 3285s |
| step 5175 | loss 2.7692 | gnorm 2.51 | 3301s |
| step 5200 | loss 2.7218 | gnorm 2.24 | 3316s |
| step 5225 | loss 2.6277 | gnorm 2.44 | 3332s |
| step 5250 | loss 3.1225 | gnorm 2.51 | 3347s |
| val 2.7194 | ppl 15.17 | 11M tokens |
| step 5275 | loss 2.8204 | gnorm 2.35 | 3367s |
| step 5300 | loss 2.8327 | gnorm 2.34 | 3384s |
| step 5325 | loss 2.5840 | gnorm 2.23 | 3399s |
| step 5350 | loss 2.7542 | gnorm 2.36 | 3414s |
| step 5375 | loss 2.7365 | gnorm 2.32 | 3428s |
| step 5400 | loss 2.8519 | gnorm 2.33 | 3443s |
| step 5425 | loss 2.8636 | gnorm 2.54 | 3457s |
| step 5450 | loss 2.6134 | gnorm 2.36 | 3471s |
| step 5475 | loss 2.7233 | gnorm 2.65 | 3485s |
| step 5500 | loss 2.4590 | gnorm 2.47 | 3499s |
| val 2.7188 | ppl 15.16 | 11M tokens |
| saved native_transformer.pt |
|
|
