aramkris's picture
tinystories-colab: 5 files, 205 MB
f78fc59 verified
Raw History Blame Contribute Delete
21.2 kB
{ "model_type": "masked-diffusion", "params": 51140096, "model": { "vocab_size": 50304, "seq_len": 256, "d_model": 512, "n_layers": 8, "n_heads": 8, "d_ff": 2048, "dropout": 0.0, "diffusion_steps": 64, "mask_id": 50257, "tokenizer": "gpt2", "arch": "v1" }, "train": { "steps": 20000, "batch_size": 32, "grad_accum": 1, "lr": 0.0006, "min_lr": 6e-05, "warmup_steps": 1000, "weight_decay": 0.1, "grad_clip": 1.0, "eval_interval": 1000, "eval_batches": 40, "sample_interval": 5000, "seed": 42, "compile": true, "snapshot_every": 0 }, "datasets": [ "roneneldan/TinyStories" ], "device": "cuda", "dtype": "bfloat16", "gpu": "NVIDIA A100-SXM4-40GB", "seconds": 892, "updates": 20000, "tokens": 163840000, "tokens_per_sec": 232198, "best_bound": 2.578, "final_val": [ 20000, 3.272, 2.578 ], "history": { "train": [ [ 50, 9.3654, 12583 ], [ 100, 7.6773, 240852 ], [ 150, 6.2673, 240831 ], [ 200, 5.9523, 240909 ], [ 250, 6.0774, 240983 ], [ 300, 6.0543, 240827 ], [ 350, 5.9451, 240938 ], [ 400, 6.0008, 240853 ], [ 450, 6.0253, 240739 ], [ 500, 5.9665, 240781 ], [ 550, 6.0202, 240783 ], [ 600, 5.9691, 240775 ], [ 650, 5.9068, 240792 ], [ 700, 5.8754, 240813 ], [ 750, 5.8587, 240767 ], [ 800, 5.9111, 240675 ], [ 850, 5.9025, 240771 ], [ 900, 5.928, 240817 ], [ 950, 5.9477, 240793 ], [ 1000, 5.9166, 240794 ], [ 1050, 5.9181, 20898 ], [ 1100, 5.8359, 240897 ], [ 1150, 5.8851, 240851 ], [ 1200, 5.9865, 240784 ], [ 1250, 5.8334, 240759 ], [ 1300, 5.8594, 240704 ], [ 1350, 5.8411, 240729 ], [ 1400, 5.9324, 240637 ], [ 1450, 5.9112, 240697 ], [ 1500, 5.9153, 240722 ], [ 1550, 5.9317, 240694 ], [ 1600, 5.8622, 240704 ], [ 1650, 5.8957, 240802 ], [ 1700, 5.8938, 240814 ], [ 1750, 5.8267, 240781 ], [ 1800, 5.8535, 240539 ], [ 1850, 5.8473, 240709 ], [ 1900, 5.8751, 240696 ], [ 1950, 5.8409, 240759 ], [ 2000, 5.7728, 240799 ], [ 2050, 5.8877, 33388 ], [ 2100, 5.834, 240710 ], [ 2150, 5.8051, 240727 ], [ 2200, 5.9649, 240683 ], [ 2250, 5.8824, 240651 ], [ 2300, 5.7021, 240620 ], [ 2350, 5.9086, 240771 ], [ 2400, 5.8723, 240709 ], [ 2450, 5.718, 240701 ], [ 2500, 5.8124, 240693 ], [ 2550, 5.7558, 240592 ], [ 2600, 5.7514, 240755 ], [ 2650, 5.6855, 240750 ], [ 2700, 5.6781, 240687 ], [ 2750, 5.6791, 240689 ], [ 2800, 5.7621, 240658 ], [ 2850, 5.8346, 240670 ], [ 2900, 5.6288, 240659 ], [ 2950, 5.6348, 240721 ], [ 3000, 5.6906, 240651 ], [ 3050, 5.5769, 29041 ], [ 3100, 5.4977, 240788 ], [ 3150, 5.3325, 240757 ], [ 3200, 5.3667, 240704 ], [ 3250, 5.3754, 240642 ], [ 3300, 5.0938, 240606 ], [ 3350, 5.0143, 240717 ], [ 3400, 5.1491, 240681 ], [ 3450, 4.6034, 240699 ], [ 3500, 4.744, 240698 ], [ 3550, 4.63, 240639 ], [ 3600, 4.8174, 240573 ], [ 3650, 4.7472, 240672 ], [ 3700, 4.5971, 240690 ], [ 3750, 4.496, 240586 ], [ 3800, 4.2775, 240575 ], [ 3850, 4.5987, 240555 ], [ 3900, 4.4156, 240437 ], [ 3950, 4.1732, 240575 ], [ 4000, 4.3092, 240636 ], [ 4050, 4.2291, 86684 ], [ 4100, 4.2073, 240641 ], [ 4150, 4.3628, 240676 ], [ 4200, 4.2898, 240371 ], [ 4250, 4.2047, 240571 ], [ 4300, 4.123, 240602 ], [ 4350, 4.161, 240624 ], [ 4400, 4.1863, 240640 ], [ 4450, 4.4645, 240595 ], [ 4500, 4.0793, 240641 ], [ 4550, 4.2418, 240416 ], [ 4600, 4.1474, 240479 ], [ 4650, 4.3769, 240569 ], [ 4700, 3.9608, 240613 ], [ 4750, 4.2698, 240659 ], [ 4800, 4.402, 240523 ], [ 4850, 4.2029, 240593 ], [ 4900, 4.0464, 240587 ], [ 4950, 4.4047, 240587 ], [ 5000, 4.1646, 240536 ], [ 5050, 3.8636, 80982 ], [ 5100, 3.9524, 240593 ], [ 5150, 4.0385, 240577 ], [ 5200, 3.864, 240481 ], [ 5250, 3.9957, 240559 ], [ 5300, 4.1368, 240518 ], [ 5350, 3.9042, 240632 ], [ 5400, 4.3838, 240585 ], [ 5450, 4.1922, 240607 ], [ 5500, 3.8912, 240566 ], [ 5550, 4.2678, 240392 ], [ 5600, 3.8119, 240453 ], [ 5650, 3.9946, 240633 ], [ 5700, 4.1202, 240593 ], [ 5750, 3.7952, 240568 ], [ 5800, 3.8025, 240470 ], [ 5850, 3.8999, 240639 ], [ 5900, 4.3335, 240540 ], [ 5950, 4.0645, 240534 ], [ 6000, 4.0079, 240609 ], [ 6050, 3.8351, 86269 ], [ 6100, 4.1224, 240541 ], [ 6150, 3.9021, 240542 ], [ 6200, 3.7823, 240543 ], [ 6250, 3.9663, 240587 ], [ 6300, 3.5366, 240415 ], [ 6350, 3.8879, 240651 ], [ 6400, 4.1591, 240570 ], [ 6450, 3.8615, 240525 ], [ 6500, 4.0243, 240602 ], [ 6550, 3.9205, 240465 ], [ 6600, 4.0117, 240529 ], [ 6650, 3.6245, 240562 ], [ 6700, 3.9409, 240607 ], [ 6750, 4.0205, 240521 ], [ 6800, 3.92, 240537 ], [ 6850, 4.1234, 240402 ], [ 6900, 4.2936, 240417 ], [ 6950, 3.9846, 240522 ], [ 7000, 3.6249, 240547 ], [ 7050, 3.6288, 21156 ], [ 7100, 3.9811, 240614 ], [ 7150, 3.772, 240566 ], [ 7200, 3.9164, 240604 ], [ 7250, 4.34, 240529 ], [ 7300, 4.1949, 240527 ], [ 7350, 3.5575, 240625 ], [ 7400, 3.4182, 240555 ], [ 7450, 3.8584, 240639 ], [ 7500, 3.8529, 240622 ], [ 7550, 3.5358, 240594 ], [ 7600, 3.1945, 240641 ], [ 7650, 3.8072, 240629 ], [ 7700, 3.7462, 240628 ], [ 7750, 3.7555, 240634 ], [ 7800, 3.7594, 240452 ], [ 7850, 3.5872, 240622 ], [ 7900, 3.6351, 240562 ], [ 7950, 3.762, 240628 ], [ 8000, 3.664, 240569 ], [ 8050, 3.3608, 101376 ], [ 8100, 3.6921, 240496 ], [ 8150, 3.9227, 240530 ], [ 8200, 3.7933, 240539 ], [ 8250, 3.64, 240544 ], [ 8300, 3.7705, 240582 ], [ 8350, 3.983, 240628 ], [ 8400, 3.764, 240576 ], [ 8450, 3.8729, 240526 ], [ 8500, 3.8002, 240495 ], [ 8550, 3.6245, 240366 ], [ 8600, 3.8307, 240584 ], [ 8650, 3.5463, 240568 ], [ 8700, 3.8133, 240526 ], [ 8750, 3.7021, 240476 ], [ 8800, 3.9399, 240417 ], [ 8850, 3.8035, 240517 ], [ 8900, 4.1148, 240572 ], [ 8950, 3.7602, 240541 ], [ 9000, 3.3614, 240548 ], [ 9050, 3.3933, 33794 ], [ 9100, 4.1981, 240625 ], [ 9150, 3.4701, 240602 ], [ 9200, 3.4082, 240453 ], [ 9250, 3.5668, 240527 ], [ 9300, 3.2345, 240511 ], [ 9350, 2.9621, 240601 ], [ 9400, 3.342, 240665 ], [ 9450, 4.0445, 240617 ], [ 9500, 3.7439, 240510 ], [ 9550, 3.9178, 240512 ], [ 9600, 3.219, 240485 ], [ 9650, 3.7014, 240464 ], [ 9700, 3.561, 240614 ], [ 9750, 3.9113, 240583 ], [ 9800, 3.1792, 240495 ], [ 9850, 3.9671, 240585 ], [ 9900, 3.4089, 240466 ], [ 9950, 3.383, 240476 ], [ 10000, 3.7221, 240521 ], [ 10050, 3.3264, 83336 ], [ 10100, 3.3827, 240509 ], [ 10150, 3.5121, 240531 ], [ 10200, 3.3206, 240617 ], [ 10250, 3.4927, 240560 ], [ 10300, 3.6308, 240497 ], [ 10350, 3.3925, 240576 ], [ 10400, 3.9438, 240459 ], [ 10450, 3.7036, 240537 ], [ 10500, 3.4268, 240622 ], [ 10550, 3.7639, 240452 ], [ 10600, 3.2888, 240546 ], [ 10650, 3.4895, 240567 ], [ 10700, 3.7271, 240568 ], [ 10750, 3.2719, 240528 ], [ 10800, 3.3534, 240444 ], [ 10850, 3.4275, 240593 ], [ 10900, 4.0305, 240557 ], [ 10950, 3.6131, 240461 ], [ 11000, 3.5787, 240517 ], [ 11050, 3.4003, 85873 ], [ 11100, 3.7417, 240657 ], [ 11150, 3.4597, 240530 ], [ 11200, 3.2819, 240513 ], [ 11250, 3.6202, 240467 ], [ 11300, 3.1076, 240493 ], [ 11350, 3.5127, 240529 ], [ 11400, 3.7594, 240547 ], [ 11450, 3.4851, 240560 ], [ 11500, 3.6771, 240589 ], [ 11550, 3.498, 240458 ], [ 11600, 3.6514, 240540 ], [ 11650, 3.1594, 240563 ], [ 11700, 3.5242, 240559 ], [ 11750, 3.6953, 240493 ], [ 11800, 3.5489, 240512 ], [ 11850, 3.7794, 240505 ], [ 11900, 3.9901, 240385 ], [ 11950, 3.6348, 240442 ], [ 12000, 3.2631, 240556 ], [ 12050, 3.2353, 86160 ], [ 12100, 3.6327, 240600 ], [ 12150, 3.3913, 240565 ], [ 12200, 3.5767, 240549 ], [ 12250, 4.0666, 240553 ], [ 12300, 3.8866, 240508 ], [ 12350, 3.1807, 240634 ], [ 12400, 2.9919, 240549 ], [ 12450, 3.5492, 240631 ], [ 12500, 3.4941, 240585 ], [ 12550, 3.1709, 240474 ], [ 12600, 2.7805, 240449 ], [ 12650, 3.5046, 240619 ], [ 12700, 3.3684, 240535 ], [ 12750, 3.3808, 240582 ], [ 12800, 3.4562, 240458 ], [ 12850, 3.2485, 240488 ], [ 12900, 3.2963, 240411 ], [ 12950, 3.423, 240476 ], [ 13000, 3.356, 240602 ], [ 13050, 2.9646, 87522 ], [ 13100, 3.3928, 240517 ], [ 13150, 3.6437, 240461 ], [ 13200, 3.5363, 240444 ], [ 13250, 3.3328, 240432 ], [ 13300, 3.4656, 240470 ], [ 13350, 3.6955, 240443 ], [ 13400, 3.4545, 240547 ], [ 13450, 3.5515, 240482 ], [ 13500, 3.5083, 240400 ], [ 13550, 3.3598, 240414 ], [ 13600, 3.5747, 240515 ], [ 13650, 3.2373, 240518 ], [ 13700, 3.4993, 240546 ], [ 13750, 3.4221, 240469 ], [ 13800, 3.6739, 240462 ], [ 13850, 3.5259, 240394 ], [ 13900, 3.8367, 240455 ], [ 13950, 3.4751, 240514 ], [ 14000, 3.0708, 240508 ], [ 14050, 3.0935, 86881 ], [ 14100, 3.9898, 240510 ], [ 14150, 3.196, 240432 ], [ 14200, 3.0799, 240478 ], [ 14250, 3.2873, 240522 ], [ 14300, 2.9298, 240462 ], [ 14350, 2.6375, 240433 ], [ 14400, 3.048, 240487 ], [ 14450, 3.8086, 240429 ], [ 14500, 3.485, 240476 ], [ 14550, 3.6542, 240415 ], [ 14600, 2.9478, 240520 ], [ 14650, 3.4496, 240486 ], [ 14700, 3.3129, 240459 ], [ 14750, 3.6733, 240423 ], [ 14800, 2.8789, 240372 ], [ 14850, 3.7313, 240404 ], [ 14900, 3.1533, 240340 ], [ 14950, 3.1047, 240473 ], [ 15000, 3.4705, 240428 ], [ 15050, 3.0847, 25457 ], [ 15100, 3.1109, 240542 ], [ 15150, 3.2515, 240425 ], [ 15200, 3.0532, 240560 ], [ 15250, 3.241, 240653 ], [ 15300, 3.3811, 240475 ], [ 15350, 3.1495, 240581 ], [ 15400, 3.7248, 240633 ], [ 15450, 3.4557, 240571 ], [ 15500, 3.1844, 240524 ], [ 15550, 3.5391, 240377 ], [ 15600, 3.0183, 240462 ], [ 15650, 3.2386, 240564 ], [ 15700, 3.4943, 240513 ], [ 15750, 3.0256, 240569 ], [ 15800, 3.1075, 240522 ], [ 15850, 3.1645, 240422 ], [ 15900, 3.8695, 240383 ], [ 15950, 3.3968, 240520 ], [ 16000, 3.3563, 240382 ], [ 16050, 3.1605, 88188 ], [ 16100, 3.5128, 240617 ], [ 16150, 3.2292, 240480 ], [ 16200, 3.0376, 240500 ], [ 16250, 3.4051, 240636 ], [ 16300, 2.8595, 240528 ], [ 16350, 3.2708, 240454 ], [ 16400, 3.525, 240476 ], [ 16450, 3.2774, 240585 ], [ 16500, 3.4587, 240502 ], [ 16550, 3.2628, 240438 ], [ 16600, 3.4367, 240351 ], [ 16650, 2.9168, 240281 ], [ 16700, 3.3175, 240570 ], [ 16750, 3.5122, 240457 ], [ 16800, 3.338, 240149 ], [ 16850, 3.5787, 240321 ], [ 16900, 3.812, 240201 ], [ 16950, 3.4265, 240521 ], [ 17000, 3.0267, 240542 ], [ 17050, 3.0192, 87078 ], [ 17100, 3.4253, 240397 ], [ 17150, 3.1709, 240493 ], [ 17200, 3.3734, 240545 ], [ 17250, 3.8902, 240534 ], [ 17300, 3.7132, 240483 ], [ 17350, 2.9636, 240636 ], [ 17400, 2.7466, 240576 ], [ 17450, 3.3358, 240511 ], [ 17500, 3.3143, 240524 ], [ 17550, 2.9542, 240543 ], [ 17600, 2.558, 240493 ], [ 17650, 3.3127, 240505 ], [ 17700, 3.1562, 240535 ], [ 17750, 3.1615, 240468 ], [ 17800, 3.2703, 240423 ], [ 17850, 3.041, 240497 ], [ 17900, 3.1001, 240530 ], [ 17950, 3.192, 240548 ], [ 18000, 3.1583, 240499 ], [ 18050, 2.7559, 87096 ], [ 18100, 3.2183, 240424 ], [ 18150, 3.4624, 240496 ], [ 18200, 3.3584, 240576 ], [ 18250, 3.1317, 240553 ], [ 18300, 3.2774, 240547 ], [ 18350, 3.5103, 240572 ], [ 18400, 3.2602, 240446 ], [ 18450, 3.3711, 240394 ], [ 18500, 3.3291, 240529 ], [ 18550, 3.1873, 240450 ], [ 18600, 3.4217, 240573 ], [ 18650, 3.0286, 240547 ], [ 18700, 3.3213, 240440 ], [ 18750, 3.2607, 240438 ], [ 18800, 3.4965, 240434 ], [ 18850, 3.3608, 240498 ], [ 18900, 3.68, 240514 ], [ 18950, 3.2916, 240559 ], [ 19000, 2.9071, 240570 ], [ 19050, 2.9109, 20671 ], [ 19100, 3.8441, 240742 ], [ 19150, 3.02, 240723 ], [ 19200, 2.8922, 240667 ], [ 19250, 3.1147, 240500 ], [ 19300, 2.7329, 240631 ], [ 19350, 2.4512, 240606 ], [ 19400, 2.8811, 240602 ], [ 19450, 3.6681, 240681 ], [ 19500, 3.3401, 240703 ], [ 19550, 3.5185, 240632 ], [ 19600, 2.7894, 240545 ], [ 19650, 3.3131, 240649 ], [ 19700, 3.1683, 240566 ], [ 19750, 3.5193, 240636 ], [ 19800, 2.7262, 240649 ], [ 19850, 3.6013, 240690 ], [ 19900, 3.0088, 240673 ], [ 19950, 2.9481, 240463 ], [ 20000, 3.3314, 240505 ] ], "val": [ [ 1000, 5.9095, 5.8989 ], [ 2000, 5.8464, 5.8367 ], [ 3000, 5.6095, 5.5341 ], [ 4000, 4.4095, 3.9025 ], [ 5000, 4.097, 3.522 ], [ 6000, 3.9142, 3.3095 ], [ 7000, 3.7981, 3.1725 ], [ 8000, 3.7007, 3.0582 ], [ 9000, 3.6137, 2.9588 ], [ 10000, 3.5431, 2.8818 ], [ 11000, 3.4963, 2.8244 ], [ 12000, 3.4492, 2.7748 ], [ 13000, 3.4077, 2.7275 ], [ 14000, 3.3747, 2.6908 ], [ 15000, 3.3418, 2.6554 ], [ 16000, 3.3257, 2.6353 ], [ 17000, 3.3037, 2.6123 ], [ 18000, 3.2901, 2.5963 ], [ 19000, 3.2805, 2.5861 ], [ 20000, 3.272, 2.578 ] ] }, "samples": [ { "prompt": "Once upon a time", "text": "Once upon a time, there was little girl named Tim. He loved to play with his his toy car all day long. They would play together in the park every day and they were having lots of fun. \n\nOne day, Benny's friend Billy came over visit him. She saw another boy playing on a bench. Lily looked at her and said, \"Hi Max! Do you like like my car?\" His mom smiled and said, \"I want to share your car!\"\nTimmy was so happy that he could play with the red car. It was very nice, but it didn too know how to don't talk to others. But", "seconds": 0.295 }, { "prompt": "Lily wanted to", "text": "Lily wanted to see the swings and play in the park. She saw a big, of flowers. It was a nice slide. They had never seen before.\n\n\"Can we go on it?\" asked Lily.\n\n\"Yes, you can help me!\" she said. \"It will be friends.\".\"\n\nThey looked at each other and smiled. They felt happy for being lot of fun.\n\nBut then they heard a loud noise. The boy did not like to slide. He tripped over the rock He fell off the ground. Lily ran up from his mom and hugged him.\n\n\"That's okay,\" he", "seconds": 0.283 }, { "prompt": "The little dog was sad because", "text": "The little dog was sad because he couldn't find his his friends. The had a great idea. He worked very hard and finally decided to go! \n\nHe put all the way back on the slide, but it was too late. His mom saw him and said she was so happy. She hugged her mom and \"It looks like fun!\" \n\nSo they went down the slide again. they both went home. They had a wonderful day at the park.", "seconds": 0.281 } ], "finished_at": "2026-11-17 06:24" }
{
"model_type": "masked-diffusion",
"params": 51140096,
"model": {
"vocab_size": 50304,
"seq_len": 256,
"d_model": 512,
"n_layers": 8,
"n_heads": 8,
"d_ff": 2048,
"dropout": 0.0,
"diffusion_steps": 64,
"mask_id": 50257,
"tokenizer": "gpt2",
"arch": "v1"
},
"train": {
"steps": 20000,
"batch_size": 32,
"grad_accum": 1,
"lr": 0.0006,
"min_lr": 6e-05,
"warmup_steps": 1000,
"weight_decay": 0.1,
"grad_clip": 1.0,
"eval_interval": 1000,
"eval_batches": 40,
"sample_interval": 5000,
"seed": 42,
"compile": true,
"snapshot_every": 0
},
"datasets": [
"roneneldan/TinyStories"
],
"device": "cuda",
"dtype": "bfloat16",
"gpu": "NVIDIA A100-SXM4-40GB",
"seconds": 892,
"updates": 20000,
"tokens": 163840000,
"tokens_per_sec": 232198,
"best_bound": 2.578,
"final_val": [
20000,
3.272,
2.578
],
"history": {
"train": [
[
50,
9.3654,
12583
],
[
100,
7.6773,
240852
],
[
150,
6.2673,
240831
],
[
200,
5.9523,
240909
],
[
250,
6.0774,
240983
],
[
300,
6.0543,
240827
],
[
350,
5.9451,
240938
],
[
400,
6.0008,
240853
],
[
450,
6.0253,
240739
],
[
500,
5.9665,
240781
],