Hugging Face's logo

AbstractPhil
/
david-shared-space

Image Classification
TensorBoard
English
imagenet
multi-scale
feature-geometry
david
Eval Results (legacy)
Model card Files
xet
 Metrics Community
david-shared-space / weights /David-decoupled-deep_efficiency /20251013_004438 /best_model_acc61.80_metadata.json
AbstractPhil's picture
AbstractPhil
Update best_model_acc61.80_metadata.json - Run 20251013_004438
bfc769a verified
raw
history blame contribute delete
280 kB
{
 "epoch": 4,
 "optimizer_state_dict": {
 "state": {
 "0": {
 "step": "tensor(18770.)",
 "exp_avg": "tensor([[ 1.2475e-04, -8.8248e-05, 3.4245e-05, ..., -3.0798e-05,\n 2.5917e-05, -3.2490e-05],\n [ 6.7354e-05, 1.9629e-05, -2.5300e-05, ..., 2.5833e-05,\n 1.2038e-06, 1.9475e-05],\n [ 4.2822e-05, -5.7324e-07, 1.4546e-05, ..., 1.4081e-05,\n -8.3127e-06, 1.0111e-05],\n ...,\n [-4.8073e-06, 2.2384e-05, 2.3074e-05, ..., 6.5221e-06,\n -3.2563e-05, 7.8924e-06],\n [-3.2600e-05, -3.8011e-06, 5.7604e-05, ..., 2.7371e-05,\n -3.0006e-06, -2.9308e-05],\n [ 2.6444e-05, -4.5827e-05, -7.5257e-06, ..., 5.9208e-05,\n 1.1548e-05, 1.6020e-05]], device='cuda:0')",
 "exp_avg_sq": "tensor([[1.5463e-07, 7.3231e-08, 4.2427e-08, ..., 5.7490e-08, 2.3230e-08,\n 2.9127e-08],\n [5.4946e-08, 9.4572e-08, 4.1066e-08, ..., 3.1191e-08, 1.3575e-08,\n 1.2694e-08],\n [3.6099e-08, 5.8283e-08, 2.3126e-08, ..., 9.8196e-09, 1.1572e-08,\n 9.1587e-09],\n ...,\n [3.6223e-08, 6.9699e-08, 2.0543e-08, ..., 1.0908e-08, 1.1603e-08,\n 1.1360e-08],\n [7.7915e-09, 4.6478e-08, 1.3913e-08, ..., 2.1031e-08, 5.4136e-09,\n 6.1192e-09],\n [1.7553e-08, 7.2068e-08, 3.3667e-08, ..., 3.4888e-08, 8.7606e-09,\n 1.2293e-08]], device='cuda:0')"
 },
 "1": {
 "step": "tensor(18770.)",
 "exp_avg": "tensor([ 3.2724e-04, 9.6550e-04, 7.2119e-04, 8.4574e-04, -7.1493e-05,\n 9.0531e-04, -4.1847e-04, -6.4849e-04, -3.9629e-04, 8.6342e-04,\n 3.5101e-04, -1.3185e-03, 8.1707e-04, 2.6977e-03, -1.3589e-03,\n -5.4419e-04, -7.7863e-04, 2.6638e-03, 2.5722e-04, -7.9687e-04,\n -2.5210e-04, 1.1559e-03, 5.8767e-04, 2.6374e-03, 5.7066e-04,\n -6.3458e-04, -3.5647e-04, 3.1694e-04, -9.5988e-04, -9.9757e-04,\n 1.8936e-03, 5.4902e-04, -4.2497e-05, 4.8280e-04, -1.7229e-04,\n -1.2503e-03, -3.0880e-03, 1.8172e-04, 9.9548e-04, -6.7916e-04,\n -9.3907e-04, 3.9218e-04, 8.1726e-04, 1.6329e-03, -5.1816e-03,\n -2.6506e-04, -5.9047e-05, -4.4314e-04, 4.2196e-04, 1.0723e-03,\n 4.0831e-04, 7.9981e-05, 1.4974e-03, -1.6085e-03, -2.7369e-04,\n -4.7615e-04, -2.8010e-05, -1.3147e-03, 3.1600e-03, -1.1887e-03,\n 5.2511e-04, -2.9428e-03, -1.2371e-03, 1.3703e-03, -1.4190e-04,\n -2.5938e-04, 1.6246e-03, -9.7164e-04, -5.6166e-04, 7.9945e-04,\n -1.5039e-04, -1.0767e-03, -7.6465e-04, -1.1602e-03, 1.7678e-06,\n 1.1966e-03, 4.5028e-04, 6.6930e-04, -9.1531e-04, 1.1678e-04,\n 1.1724e-03, -1.2154e-03, 6.1155e-04, -2.9441e-03, 5.2610e-05,\n 1.9843e-04, -7.1190e-04, 2.7134e-04, -7.4588e-04, -1.2964e-03,\n -1.3837e-04, -2.3964e-04, -9.2301e-04, -1.9103e-04, -9.0324e-04,\n 8.2054e-04, -2.5957e-04, 7.3716e-04, 9.1443e-04, 6.9205e-04,\n -1.9951e-03, 2.5659e-04, -6.9689e-05, -2.4430e-04, -1.0903e-03,\n 2.4202e-04, -6.4741e-04, -1.4788e-03, -2.7925e-04, 4.3318e-04,\n 2.7873e-04, -1.2257e-03, 1.0225e-03, -6.8112e-04, -8.2426e-04,\n -3.0449e-03, 1.3022e-03, 1.2033e-03, 4.0923e-04, -1.6993e-03,\n -1.2639e-04, -2.1325e-04, 9.7664e-04, 7.0450e-04, 6.6926e-05,\n 1.4075e-03, 9.3209e-04, 7.0601e-04, 7.5382e-04, 1.4536e-03,\n -4.6674e-04, 5.6713e-04, -2.3815e-04, 4.6937e-04, 6.6900e-04,\n -2.0177e-03, 7.2646e-05, -6.5545e-04, 1.7533e-04, -1.7193e-04,\n 1.2463e-03, -5.5487e-04, -5.8309e-04, 5.3846e-05, -1.6839e-03,\n 4.5639e-04, 1.7469e-03, -6.1322e-04, 1.5200e-03, -2.4688e-03,\n 1.7313e-04, 9.2542e-04, -4.4293e-04, 7.9368e-04, 1.4636e-03,\n 1.4158e-03, 2.3105e-03, 1.3637e-03, -1.7384e-03, 9.6356e-04,\n 1.8071e-03, 1.3676e-04, -1.8301e-03, 1.3776e-05, 6.1462e-05,\n 6.9458e-04, 1.4289e-03, 3.1261e-04, 2.2417e-03, -4.3028e-04,\n -5.1467e-04, 5.9909e-04, 3.4296e-04, 2.2998e-04, 1.7941e-03,\n 7.2921e-04, -1.3838e-03, -6.2724e-04, 6.1316e-04, -1.8205e-03,\n 9.4783e-05, -7.8335e-04, -9.6619e-04, -1.4676e-04, 1.4322e-03,\n 7.8271e-05, -6.6329e-04, 4.9124e-04, -2.0587e-04, 5.6471e-04,\n -2.6997e-04, -4.0308e-04, 9.3750e-04, 1.0563e-03, 9.8428e-04,\n -4.8636e-04, 1.3207e-03, 1.7692e-03, -6.8437e-04, 3.5385e-04,\n -2.6110e-04, 4.0174e-04, -4.9252e-04, 6.1574e-04, 1.1169e-04,\n -5.3648e-04, 1.0965e-04, 1.2122e-03, 1.0847e-03, 3.4712e-04,\n -1.9944e-04, -6.4302e-04, 1.6283e-04, 7.6221e-04, -6.4201e-04,\n -5.2528e-04, -2.0601e-04, -1.0693e-03, 5.7191e-04, -5.3828e-04,\n -1.0179e-03, 1.0992e-03, 4.0311e-04, -8.0872e-05, 5.7817e-04,\n -2.3530e-04, -3.5260e-04, -1.8255e-03, -1.3900e-03, -1.1393e-03,\n 2.3000e-05, -2.8554e-04, 1.3612e-03, -5.2370e-04, 1.2861e-03,\n 8.9711e-04, -1.0072e-03, 1.9536e-03, -1.6058e-04, -1.5261e-04,\n -8.2386e-04, 5.0463e-04, 6.2336e-05, 3.5574e-04, 3.5982e-04,\n -9.2445e-04, 1.7574e-03, -9.4963e-04, 9.6953e-04, 2.0181e-03,\n -2.0730e-04, 1.4610e-03, -1.9174e-04, 9.3127e-04, -1.7718e-03,\n -6.5506e-04, 4.1169e-05, -1.0872e-03, 1.4111e-03, -8.9851e-04,\n -8.2003e-04, 3.5892e-04, -8.3984e-04, -3.6360e-04, 1.8619e-03,\n -1.1263e-03, -5.9708e-05, 1.9011e-03, -1.9790e-03, 5.5285e-04,\n 3.4602e-04, -5.4613e-04, 3.9772e-04, -7.3283e-04, -1.8317e-03,\n 6.2165e-04, -9.1811e-04, 7.9345e-04, 1.7719e-03, -6.2397e-04,\n -5.7612e-04, 9.6045e-05, -2.2437e-05, -1.3949e-03, 5.2531e-04,\n 1.1155e-04, 1.6607e-04, -1.1117e-03, -8.0046e-04, 1.1156e-03,\n -1.5496e-03, 4.1790e-05, 1.1150e-04, 1.0131e-04, -4.0196e-04,\n 8.5400e-05, 1.1629e-03, -9.2110e-04, 8.9489e-06, -9.0572e-05,\n 4.5139e-04, 3.0287e-04, 2.3032e-03, 1.5610e-03, 1.3566e-03,\n 6.6669e-04, -1.0223e-03, -1.9575e-03, -1.2566e-03, 9.7590e-04,\n -1.9224e-04, 1.3321e-03, 1.0968e-03, 8.7956e-04, 2.2564e-03,\n 1.6461e-04, -3.5246e-03, -3.8237e-04, 1.2356e-04, 1.0848e-04,\n -6.6976e-05, 8.1669e-04, 3.2880e-04, -7.4263e-04, -1.7132e-04,\n 7.7663e-04, 1.2149e-04, -2.2031e-03, -1.1285e-03, 1.4704e-04,\n 1.9755e-03, 9.1778e-04, -9.4226e-04, 9.3921e-04, -1.1027e-03,\n 9.8169e-05, 5.9785e-04, 8.4220e-04, -1.2268e-03, 1.2048e-03,\n 6.5783e-04, -1.1269e-03, 9.6678e-04, 9.2466e-04, 1.6951e-03,\n -6.4264e-04, -2.7731e-04, -2.3123e-03, 1.3396e-03, -1.0580e-03,\n -1.9172e-03, -4.3277e-04, 9.7867e-04, -1.3271e-03, -3.2489e-04,\n 6.4182e-05, 8.2065e-04, -3.2651e-03, -1.5122e-03, 5.5068e-05,\n -5.4978e-04, -7.5277e-04, 6.1291e-04, 4.3252e-04, 3.4022e-04,\n -1.0440e-03, -1.8261e-03, 1.3840e-03, -1.3162e-03, -2.1240e-04,\n 9.1522e-05, -8.1613e-04, 3.5252e-04, -6.7751e-04, 1.7136e-04,\n -2.9403e-03, 1.4228e-03, -1.0955e-03, -1.5857e-05, 2.0645e-03,\n 1.4238e-03, 4.3689e-04, -2.6405e-04, -1.4588e-04, -3.5663e-04,\n 3.4358e-04, 6.1477e-04, 6.4017e-04, -1.6860e-04, -2.0635e-03,\n -9.6126e-04, 8.4239e-04, -8.1784e-04, -2.6825e-04, 1.1634e-03,\n -1.6886e-03, -8.0316e-04, -5.9102e-04, 8.9293e-04, 3.9875e-04,\n 2.2537e-04, 3.9430e-04, -7.6473e-04, -8.5382e-04, -6.4160e-04,\n 2.1653e-04, 1.0535e-04, -1.1273e-03, -5.6633e-04, -1.8079e-05,\n 1.6792e-03, -3.1422e-04, 6.2231e-04, 2.9423e-04, -5.8637e-04,\n -5.7138e-04, -9.8490e-04, -2.7716e-04, -3.0944e-04, -1.2160e-03,\n 1.1204e-03, -1.3848e-03, 4.6321e-04, -5.2149e-04, -6.4512e-05,\n -2.1001e-04, -7.1796e-06, -3.8195e-04, 1.5389e-03, 1.4444e-03,\n 4.9063e-04, -4.4935e-04, 6.2075e-04, 4.3321e-05, 4.7167e-04,\n 3.1627e-04, -4.9126e-04, -1.6294e-04, -1.4634e-03, 1.7772e-03,\n -2.2796e-04, 1.1767e-03, -2.5141e-04, 6.8044e-04, 5.0925e-04,\n 3.5674e-04, 1.0370e-03, 6.9995e-04, -6.4161e-04, -4.8798e-04,\n 1.4986e-03, -7.0554e-04, -1.5603e-04, -1.8551e-04, 2.7407e-04,\n -1.0907e-03, 4.4921e-04, -9.2735e-04, 1.0055e-03, 2.3597e-04,\n 6.0945e-05, 2.1697e-05, -1.3317e-03, -2.0696e-05, 3.6842e-04,\n -7.5031e-04, 3.9840e-04, -3.7423e-04, -1.0549e-03, 4.0936e-04,\n 1.7108e-03, 3.0006e-04, -7.5346e-04, -2.3081e-03, 1.1892e-03,\n 7.1135e-04, -3.3822e-04, -1.5742e-03, 3.5667e-04, -1.3251e-04,\n -1.7204e-03, -1.1175e-03, -6.4916e-04, 3.7651e-05, 9.9433e-04,\n 1.6312e-04, -3.7931e-04, -6.4380e-05, -9.2121e-04, 1.9324e-04,\n 1.5857e-03, -2.2741e-03, -1.9970e-04, -1.4868e-03, 7.4812e-04,\n 2.8939e-04, 7.4753e-05, -1.1576e-04, -9.1946e-04, -1.6221e-04,\n 1.3556e-03, 1.5899e-04, -9.7367e-04, -1.8273e-04, 4.3730e-04,\n -9.2054e-04, -8.4454e-04, -9.7345e-04, -3.7767e-06, 2.7019e-04,\n -1.2539e-05, -5.1183e-06], device='cuda:0')",
 "exp_avg_sq": "tensor([3.6652e-05, 2.2382e-05, 1.2020e-05, 1.1845e-05, 1.5437e-05, 2.0584e-05,\n 1.4676e-05, 2.1301e-05, 1.1875e-05, 1.1656e-05, 1.7615e-05, 1.8299e-05,\n 1.2602e-05, 1.7928e-05, 1.3113e-05, 1.5750e-05, 1.0220e-05, 2.0011e-05,\n 7.3975e-06, 6.1028e-06, 9.0480e-06, 1.6054e-05, 8.9944e-06, 1.7045e-05,\n 1.5926e-05, 8.1849e-06, 2.9992e-05, 2.7082e-05, 1.1627e-05, 5.8314e-06,\n 1.5352e-05, 1.8349e-05, 1.6503e-05, 1.7440e-05, 2.7219e-05, 1.3336e-05,\n 2.0100e-05, 1.1691e-05, 1.7367e-05, 9.8678e-06, 2.4831e-05, 1.3270e-05,\n 1.1270e-05, 2.5679e-05, 1.8457e-05, 1.9834e-05, 1.3585e-05, 1.7814e-05,\n 2.2514e-05, 9.8880e-06, 9.9050e-06, 2.1653e-05, 1.8925e-05, 1.5104e-05,\n 7.6096e-06, 1.3599e-05, 2.1264e-05, 1.3911e-05, 4.0780e-05, 1.6366e-05,\n 2.2948e-05, 2.0489e-05, 3.0605e-05, 1.7514e-05, 9.1813e-06, 2.7593e-05,\n 1.5481e-05, 9.1793e-06, 1.4704e-05, 1.5460e-05, 1.5921e-05, 1.7362e-05,\n 1.0788e-05, 1.8238e-05, 9.7083e-06, 5.5550e-05, 1.9637e-05, 1.5641e-05,\n 1.7271e-05, 1.4769e-05, 1.2119e-05, 1.8292e-05, 1.9197e-05, 2.8177e-05,\n 1.5350e-05, 1.5579e-05, 1.8193e-05, 1.4329e-05, 1.1956e-05, 1.6335e-05,\n 1.3564e-05, 1.6998e-05, 1.4743e-05, 1.7101e-05, 1.4774e-05, 1.3959e-05,\n 9.7226e-06, 1.5966e-05, 8.0702e-06, 1.5484e-05, 2.9545e-05, 1.2473e-05,\n 3.4895e-06, 2.4652e-05, 2.3045e-05, 1.2984e-05, 2.9184e-05, 3.5591e-05,\n 1.1752e-05, 2.1826e-05, 1.0977e-05, 3.5394e-05, 2.0091e-05, 1.7920e-05,\n 1.2713e-05, 3.3006e-05, 1.6073e-05, 1.2139e-05, 2.1890e-05, 1.9086e-05,\n 7.9048e-06, 1.1890e-05, 3.3296e-06, 1.2120e-05, 2.1928e-05, 1.7880e-05,\n 1.2278e-05, 1.8626e-05, 1.8208e-05, 1.5245e-05, 1.3625e-05, 1.0346e-05,\n 1.1833e-05, 1.7278e-05, 1.5581e-05, 1.6119e-05, 1.5044e-05, 1.7718e-05,\n 1.3347e-05, 1.3848e-05, 1.2393e-05, 1.2778e-05, 1.3844e-05, 1.4411e-05,\n 1.8729e-05, 1.4351e-05, 2.0794e-05, 1.3073e-05, 2.6676e-05, 2.9968e-05,\n 1.4712e-05, 2.3646e-05, 3.2593e-05, 2.0315e-05, 1.8201e-05, 1.6386e-05,\n 2.5690e-05, 1.4106e-05, 2.3263e-05, 1.3642e-05, 1.4141e-05, 1.2829e-05,\n 2.9713e-05, 1.1924e-05, 1.4806e-05, 1.0254e-05, 2.3107e-05, 1.5137e-05,\n 1.6256e-05, 1.6233e-05, 2.0792e-05, 1.5230e-05, 1.2485e-05, 2.8795e-05,\n 2.3656e-05, 1.5082e-05, 1.8440e-05, 6.7588e-06, 1.1728e-05, 9.1253e-06,\n 1.7059e-05, 2.2971e-05, 1.9053e-05, 2.2173e-05, 2.3215e-05, 1.7937e-05,\n 2.7363e-05, 2.5983e-05, 8.3548e-06, 1.7383e-05, 1.0401e-05, 1.4975e-05,\n 1.3756e-05, 2.1488e-05, 2.0435e-05, 1.0574e-05, 1.0380e-05, 1.2434e-05,\n 1.9611e-05, 1.3330e-05, 1.8425e-05, 1.3477e-05, 1.2783e-05, 1.4294e-05,\n 1.5513e-05, 1.0745e-05, 1.0360e-05, 1.5933e-05, 2.6195e-05, 2.6913e-05,\n 2.1136e-05, 2.2301e-05, 2.8992e-05, 1.0909e-05, 2.3288e-05, 1.3715e-05,\n 1.1758e-05, 1.5509e-05, 1.3489e-05, 1.7178e-05, 2.2397e-05, 1.3507e-05,\n 2.1707e-05, 7.0654e-06, 2.5719e-05, 1.2404e-05, 1.7186e-05, 1.3086e-05,\n 1.4760e-05, 1.5990e-05, 1.9818e-05, 9.4018e-06, 1.7975e-05, 1.3357e-05,\n 1.5623e-05, 1.9181e-05, 7.8438e-06, 3.3929e-05, 1.3993e-05, 1.3961e-05,\n 1.4846e-05, 9.7617e-06, 1.3841e-05, 2.6084e-05, 5.6440e-06, 1.3155e-05,\n 1.1720e-05, 1.3905e-05, 2.7714e-05, 3.7034e-05, 2.2992e-05, 2.7427e-05,\n 7.2087e-06, 2.7652e-05, 1.8125e-05, 1.5855e-05, 1.6247e-05, 1.9295e-05,\n 8.5599e-06, 2.0077e-05, 1.2421e-05, 7.1984e-06, 1.6072e-05, 1.5172e-05,\n 3.8789e-05, 9.2748e-06, 1.8925e-05, 1.9061e-05, 1.1333e-05, 2.7598e-05,\n 1.0236e-05, 1.2347e-05, 1.6113e-05, 1.0241e-05, 9.2839e-06, 1.7228e-05,\n 8.5888e-06, 7.7675e-06, 2.3037e-05, 1.4367e-05, 7.0433e-06, 1.4449e-05,\n 1.3362e-05, 9.6306e-06, 1.1154e-05, 2.3605e-05, 7.6821e-06, 3.0813e-05,\n 1.1234e-05, 1.6161e-05, 1.2602e-05, 4.0477e-06, 1.0950e-05, 7.4150e-06,\n 1.4218e-05, 1.9313e-05, 1.7682e-05, 1.3775e-05, 2.2332e-05, 7.1565e-06,\n 2.0749e-05, 1.7928e-05, 3.1190e-05, 1.9790e-05, 2.3679e-05, 8.6864e-06,\n 3.5778e-05, 1.5878e-05, 1.3686e-05, 1.1882e-05, 1.0024e-05, 8.8602e-06,\n 2.0453e-05, 1.2723e-05, 2.2781e-05, 7.1401e-06, 1.1845e-05, 1.1301e-05,\n 2.2736e-05, 1.4055e-05, 1.7958e-05, 2.0006e-05, 1.5324e-05, 6.4574e-06,\n 1.0214e-05, 1.9189e-05, 1.1362e-05, 2.8132e-05, 9.5085e-06, 8.7212e-06,\n 1.3384e-05, 1.3110e-05, 1.6949e-05, 1.7454e-05, 1.4275e-05, 1.2136e-05,\n 1.9467e-05, 1.4876e-05, 3.5061e-05, 1.7572e-05, 1.7230e-05, 1.2647e-05,\n 2.3171e-05, 1.8038e-05, 1.2896e-05, 2.0206e-05, 1.0194e-05, 1.1080e-05,\n 2.2034e-05, 1.3382e-05, 1.6906e-05, 1.1227e-05, 1.7996e-05, 1.5037e-05,\n 1.1345e-05, 1.5209e-05, 4.8357e-05, 3.3065e-05, 8.9965e-06, 1.9854e-05,\n 2.3837e-05, 3.4260e-05, 2.0032e-05, 1.3230e-05, 2.2891e-05, 1.9318e-05,\n 1.6946e-05, 1.4879e-05, 7.5489e-06, 1.4568e-05, 8.8565e-06, 2.5653e-05,\n 1.0141e-05, 1.7999e-05, 1.8348e-05, 2.3419e-05, 6.0358e-06, 2.0895e-05,\n 1.5757e-05, 3.0850e-05, 1.4894e-05, 4.4184e-06, 1.0753e-05, 1.1537e-05,\n 4.3231e-05, 1.6914e-05, 1.0268e-05, 1.6784e-05, 1.2945e-05, 2.4521e-05,\n 2.3684e-05, 1.9363e-05, 1.6120e-05, 3.6591e-06, 1.4350e-05, 2.0746e-05,\n 2.2970e-05, 9.1096e-06, 2.6724e-05, 1.5220e-05, 1.7037e-05, 1.4794e-05,\n 2.6240e-05, 6.5338e-05, 2.2096e-05, 1.1798e-05, 7.4205e-06, 1.1910e-05,\n 5.1618e-06, 1.9299e-05, 2.1871e-05, 9.4997e-06, 2.1577e-05, 1.1887e-05,\n 1.5365e-05, 1.1021e-05, 8.1860e-06, 2.7265e-05, 3.6621e-05, 1.3906e-05,\n 1.1247e-05, 1.6697e-05, 1.6742e-05, 1.1880e-05, 1.4613e-05, 1.3541e-05,\n 8.4800e-06, 9.5274e-06, 1.0838e-05, 3.2176e-05, 1.7078e-05, 5.8262e-06,\n 2.0640e-05, 1.6431e-05, 1.2784e-05, 4.7111e-06, 1.7574e-05, 2.0300e-05,\n 2.0577e-05, 2.4336e-05, 1.4969e-05, 1.6778e-05, 9.8902e-06, 2.7155e-05,\n 7.1760e-06, 1.3723e-05, 1.7299e-05, 1.5594e-05, 1.5776e-05, 1.4397e-05,\n 1.3035e-05, 1.0849e-05, 1.2599e-05, 1.8785e-05, 7.6585e-06, 1.4751e-05,\n 9.8908e-06, 1.2239e-05, 1.5900e-05, 2.3978e-05, 2.5784e-05, 2.3887e-05,\n 1.9357e-05, 2.3390e-05, 8.2405e-06, 1.5867e-05, 1.3201e-05, 2.8494e-05,\n 1.9573e-05, 1.7275e-05, 1.5794e-05, 1.5679e-05, 3.2007e-05, 1.4984e-05,\n 1.1601e-05, 1.7054e-05, 2.5158e-05, 2.4607e-05, 1.5088e-05, 2.8827e-05,\n 1.9070e-05, 1.1517e-05, 1.2810e-05, 1.5062e-05, 1.8854e-05, 6.4400e-06,\n 1.1734e-05, 1.7022e-05, 1.5911e-05, 1.2964e-05, 3.6444e-05, 1.2039e-05,\n 7.8633e-06, 4.2234e-05, 1.5382e-05, 1.8746e-05, 2.3018e-05, 2.1398e-05,\n 1.2954e-05, 1.7581e-05, 1.2429e-05, 2.1231e-05, 1.6945e-05, 6.2536e-06,\n 1.3270e-05, 2.0110e-05, 3.1961e-05, 1.6218e-05, 1.1271e-05, 1.4051e-05,\n 8.2279e-06, 1.5168e-05], device='cuda:0')"
 },
 "2": {
 "step": "tensor(18770.)",
 "exp_avg": "tensor([[-2.7117e-05, 7.1326e-06, -1.3436e-05, ..., -1.3205e-06,\n 2.6239e-05, -1.4667e-05],\n [ 1.5352e-05, 2.5810e-05, -1.8285e-05, ..., 3.0376e-05,\n -2.1467e-05, 2.9033e-05],\n [-3.6795e-05, -7.4696e-05, 1.6477e-05, ..., -1.1278e-06,\n 3.4494e-05, -7.7093e-06],\n ...,\n [ 5.6855e-05, -9.4623e-06, -4.8620e-05, ..., 3.2631e-07,\n 5.8535e-05, -3.0723e-05],\n [-4.7838e-05, 1.6894e-05, 1.2468e-05, ..., -4.0220e-05,\n -2.5762e-05, 2.4387e-05],\n [-1.7163e-05, 1.1858e-05, -6.2163e-05, ..., 3.8668e-05,\n -9.0917e-06, 1.4622e-05]], device='cuda:0')",
 "exp_avg_sq": "tensor([[1.1530e-08, 3.0784e-09, 2.2770e-08, ..., 6.7084e-09, 6.5294e-09,\n 2.6308e-09],\n [1.7382e-08, 1.0477e-08, 8.5509e-09, ..., 1.1726e-08, 2.7736e-09,\n 9.5639e-09],\n [1.2122e-08, 2.5782e-08, 9.7254e-09, ..., 1.5026e-08, 3.3445e-09,\n 5.7137e-09],\n ...,\n [1.6990e-08, 9.9563e-09, 1.9609e-08, ..., 1.3553e-08, 6.3471e-09,\n 7.3710e-09],\n [1.8216e-08, 8.0162e-09, 8.0580e-09, ..., 2.8438e-08, 6.4148e-09,\n 6.0606e-09],\n [1.2055e-08, 7.5633e-09, 1.0372e-08, ..., 1.6103e-08, 5.4728e-09,\n 5.4833e-09]], device='cuda:0')"
 },
 "3": {
 "step": "tensor(18770.)",
 "exp_avg": "tensor([[ 1.4266e-06, -2.1885e-06, -2.5025e-06, ..., 1.3764e-06,\n -3.7134e-07, -5.3113e-07],\n [ 2.8543e-07, -1.6980e-06, -2.1767e-07, ..., 9.0480e-06,\n 1.5468e-06, -1.6950e-06],\n [-1.2929e-06, 5.7159e-06, 3.2085e-06, ..., -6.6143e-07,\n -3.8738e-07, 1.8927e-06],\n ...,\n [-2.2186e-06, 2.5340e-07, 3.9710e-06, ..., 6.4956e-07,\n 1.0617e-06, -3.0555e-06],\n [-3.4079e-06, 5.3047e-06, -2.0676e-06, ..., 2.6578e-07,\n 3.2414e-07, -1.6788e-06],\n [-1.8041e-06, -7.8921e-06, -1.7947e-06, ..., 2.7251e-06,\n 2.6077e-06, -4.4334e-07]], device='cuda:0')",
 "exp_avg_sq": "tensor([[1.7530e-10, 3.2546e-10, 1.2037e-10, ..., 5.2297e-11, 3.4594e-11,\n 5.4057e-11],\n [6.7218e-11, 2.7617e-10, 1.2059e-10, ..., 1.4952e-10, 3.4211e-11,\n 3.5038e-11],\n [3.3920e-11, 3.2457e-10, 8.7233e-11, ..., 2.1633e-11, 1.9753e-11,\n 2.3624e-11],\n ...,\n [7.5153e-11, 1.7259e-10, 1.8299e-10, ..., 9.4165e-11, 3.2541e-11,\n 3.9700e-11],\n [1.0947e-10, 2.6541e-10, 2.0181e-10, ..., 2.7300e-10, 5.7992e-11,\n 5.2114e-11],\n [7.0716e-11, 3.9753e-10, 9.9011e-11, ..., 4.8464e-11, 3.8947e-11,\n 4.2024e-11]], device='cuda:0')"
 },
 "4": {
 "step": "tensor(18770.)",
 "exp_avg": "tensor([ 3.1742e-05, 4.3764e-05, -2.1853e-05, ..., -3.1103e-05,\n -9.3194e-06, 8.6038e-05], device='cuda:0')",
 "exp_avg_sq": "tensor([6.6108e-08, 6.9011e-08, 2.7508e-08, ..., 5.6129e-08, 1.0285e-07,\n 6.0181e-08], device='cuda:0')"
 },
 "5": {
 "step": "tensor(18770.)",
 "exp_avg": "tensor([[-5.7288e-08, 4.1847e-08, 2.1826e-07, ..., -5.2141e-07,\n -1.2532e-06, 2.0150e-07],\n [-1.4668e-07, 6.5801e-07, -8.6539e-07, ..., 3.6658e-07,\n 1.7531e-06, 9.1764e-07],\n [-4.4507e-08, 1.7833e-07, -8.5421e-07, ..., 7.1757e-07,\n 2.1635e-07, -5.5878e-07],\n ...,\n [-9.0736e-08, 7.3605e-07, -9.6422e-07, ..., 1.0620e-06,\n -1.8536e-06, -1.6486e-07],\n [ 5.6037e-09, 2.2460e-08, -1.2498e-07, ..., -5.2301e-08,\n -7.5965e-08, -2.6813e-07],\n [-7.0668e-07, -1.2072e-08, 8.7106e-07, ..., -1.2187e-07,\n 1.7976e-07, -2.6553e-07]], device='cuda:0')",
 "exp_avg_sq": "tensor([[3.1214e-12, 7.6865e-12, 9.1341e-12, ..., 4.3005e-12, 1.6174e-11,\n 2.2095e-12],\n [7.8872e-12, 6.1090e-12, 1.9646e-11, ..., 1.0154e-11, 3.5262e-11,\n 1.3856e-11],\n [3.5319e-12, 4.4752e-12, 1.7495e-11, ..., 9.3366e-12, 2.3363e-11,\n 5.7428e-12],\n ...,\n [1.0291e-11, 7.9110e-12, 2.3765e-11, ..., 1.7422e-11, 2.6820e-11,\n 4.8245e-12],\n [1.0062e-11, 1.6821e-11, 2.1469e-11, ..., 9.7843e-12, 3.4114e-11,\n 3.7510e-12],\n [7.8247e-12, 3.9781e-12, 2.6650e-11, ..., 9.7855e-12, 3.5537e-11,\n 1.0755e-11]], device='cuda:0')"
 },
 "6": {
 "step": "tensor(18770.)",
 "exp_avg": "tensor([[ 2.5060e-07, 2.8529e-06, 1.4991e-06, ..., -1.5416e-06,\n -1.8915e-06, 2.8012e-07],\n [-7.0396e-08, -9.0883e-07, -1.4366e-06, ..., 1.5131e-06,\n 9.4621e-07, 1.1405e-06],\n [ 2.1608e-06, -5.0394e-06, 2.8799e-07, ..., 2.2988e-06,\n 1.1482e-06, 8.7939e-07],\n ...,\n [ 1.3024e-07, 1.3843e-06, 2.2766e-07, ..., -4.6271e-07,\n 1.0592e-07, -6.8413e-07],\n [ 1.4333e-06, 1.0525e-06, -6.1737e-07, ..., 7.8842e-08,\n -4.8054e-07, 2.8829e-07],\n [ 2.2067e-06, -2.3216e-06, 1.3553e-07, ..., -6.0301e-09,\n -1.5729e-06, -1.1900e-06]], device='cuda:0')",
 "exp_avg_sq": "tensor([[3.8340e-11, 1.8185e-10, 6.7563e-11, ..., 1.5447e-10, 2.3182e-11,\n 3.2926e-11],\n [5.3641e-11, 1.8247e-10, 7.0314e-11, ..., 8.2520e-11, 2.8662e-11,\n 4.8769e-11],\n [1.1675e-10, 2.7232e-10, 1.0883e-10, ..., 6.3229e-11, 9.8681e-11,\n 6.1542e-11],\n ...,\n [4.6717e-11, 1.0394e-10, 4.8007e-11, ..., 2.5479e-11, 1.5196e-11,\n 1.5488e-11],\n [3.9673e-11, 2.0038e-10, 5.1644e-11, ..., 1.9034e-11, 1.5607e-11,\n 1.6378e-11],\n [3.4000e-11, 1.5481e-10, 4.7604e-11, ..., 2.2572e-11, 1.9736e-11,\n 1.9092e-11]], device='cuda:0')"
 },
 "7": {
 "step": "tensor(18770.)",
 "exp_avg": "tensor([-4.4498e-05, 5.0143e-05, 7.7162e-05, ..., -1.2141e-06,\n 3.2148e-05, -3.9617e-05], device='cuda:0')",
 "exp_avg_sq": "tensor([4.7635e-08, 4.5959e-08, 6.9401e-08, ..., 1.9311e-08, 1.7964e-08,\n 3.0347e-08], device='cuda:0')"
 },
 "8": {
 "step": "tensor(18770.)",
 "exp_avg": "tensor([[ 5.2772e-08, 3.3764e-07, 3.8929e-07, ..., 1.6307e-07,\n -3.0901e-07, -1.7945e-07],\n [ 2.4581e-07, -8.4338e-07, 4.2604e-07, ..., -1.1598e-07,\n 7.4940e-07, 1.2947e-07],\n [ 1.1678e-10, -2.9179e-07, -1.7681e-08, ..., 1.9367e-07,\n 5.5313e-07, -3.9676e-07],\n ...,\n [ 3.9521e-07, -3.2558e-07, 4.1946e-07, ..., -1.5368e-07,\n -1.1743e-07, -1.1159e-07],\n [ 6.0265e-08, -6.6202e-07, -5.1759e-07, ..., -8.1554e-08,\n -4.1845e-07, -1.2426e-07],\n [ 1.2932e-07, 5.8153e-07, 3.7829e-07, ..., -2.2516e-07,\n -2.1018e-07, 1.8009e-07]], device='cuda:0')",
 "exp_avg_sq": "tensor([[1.2548e-12, 7.0629e-12, 1.5717e-12, ..., 1.0939e-12, 3.6564e-12,\n 9.2885e-13],\n [2.3716e-12, 2.0642e-11, 4.2158e-12, ..., 1.3148e-12, 4.1384e-12,\n 2.1374e-12],\n [2.9131e-12, 7.5080e-12, 8.1649e-12, ..., 1.4816e-12, 5.0112e-12,\n 1.4300e-12],\n ...,\n [3.1891e-12, 9.3246e-12, 4.3705e-12, ..., 3.7775e-12, 6.0891e-12,\n 2.9382e-12],\n [3.5263e-12, 8.0319e-12, 5.3877e-12, ..., 6.5886e-13, 5.2265e-12,\n 2.5696e-12],\n [2.4254e-12, 7.5540e-12, 4.7884e-12, ..., 1.5925e-12, 3.7092e-12,\n 3.3972e-12]], device='cuda:0')"
 },
 "9": {
 "step": "tensor(18770.)",
 "exp_avg": "tensor([[-9.8045e-07, 5.6409e-07, -7.8397e-08, ..., 5.0120e-07,\n 1.8539e-06, -5.1905e-07],\n [ 1.4263e-06, 1.2876e-06, 1.5577e-06, ..., 1.1674e-06,\n -3.7675e-07, -1.6034e-06],\n [ 5.7560e-07, 5.3610e-06, -4.1292e-07, ..., -2.8707e-06,\n 4.3741e-07, 2.3656e-06],\n ...,\n [ 6.1097e-07, 5.2372e-06, 1.0317e-06, ..., 1.7069e-07,\n 7.8203e-07, 4.1442e-07],\n [-6.2729e-07, -4.6529e-06, 1.6917e-06, ..., -1.3277e-06,\n 1.7017e-06, 4.8219e-07],\n [ 3.0791e-06, -1.5855e-06, -2.8126e-06, ..., -5.0293e-06,\n 1.2630e-06, 1.5485e-06]], device='cuda:0')",
 "exp_avg_sq": "tensor([[2.5823e-11, 2.8058e-11, 3.1300e-11, ..., 1.2390e-10, 1.6198e-11,\n 1.7027e-11],\n [4.5713e-11, 1.1654e-10, 2.4808e-11, ..., 8.5729e-11, 2.4476e-11,\n 2.3534e-11],\n [2.8849e-11, 1.7666e-10, 2.3565e-11, ..., 6.5841e-11, 1.3971e-11,\n 2.7496e-11],\n ...,\n [5.3470e-11, 1.1250e-10, 1.0096e-10, ..., 7.3800e-11, 1.6525e-11,\n 1.9874e-11],\n [7.6841e-11, 1.6486e-10, 5.2614e-11, ..., 1.5226e-10, 3.0100e-11,\n 3.2258e-11],\n [5.2496e-11, 5.4216e-11, 1.8450e-11, ..., 9.4637e-11, 2.3283e-11,\n 2.4701e-11]], device='cuda:0')"
 },
 "10": {
 "step": "tensor(18770.)",
 "exp_avg": "tensor([-2.9698e-05, 2.6058e-05, -1.2850e-06, ..., -8.7871e-06,\n -8.6780e-05, 1.1647e-05], device='cuda:0')",
 "exp_avg_sq": "tensor([3.9352e-08, 3.8418e-08, 2.8973e-08, ..., 3.1119e-08, 6.2135e-08,\n 3.8241e-08], device='cuda:0')"
 },
 "11": {
 "step": "tensor(18770.)",
 "exp_avg": "tensor([[-5.6058e-07, -9.9053e-08, -3.1411e-07, ..., 2.1242e-07,\n 1.9723e-07, -5.7792e-07],\n [-2.9696e-07, -3.6670e-08, 1.0531e-06, ..., 5.0759e-07,\n -1.0127e-07, 1.8106e-08],\n [-4.6461e-07, 3.3086e-07, -6.1314e-07, ..., -2.6650e-07,\n 7.0479e-07, 2.9863e-07],\n ...,\n [-2.5156e-07, -2.0352e-07, -2.9356e-07, ..., -1.9092e-07,\n -4.1734e-07, -3.7719e-07],\n [-1.7794e-09, -3.8551e-07, 2.9651e-07, ..., -1.3482e-07,\n -8.2661e-07, 1.3419e-06],\n [ 2.0013e-07, -3.6560e-07, -4.7935e-07, ..., 1.2399e-07,\n 7.0006e-07, -7.9462e-07]], device='cuda:0')",
 "exp_avg_sq": "tensor([[2.7022e-12, 8.0061e-13, 2.5992e-12, ..., 5.5481e-13, 3.9024e-12,\n 3.6316e-12],\n [1.9189e-12, 1.4043e-12, 3.6699e-12, ..., 1.1557e-12, 1.3099e-11,\n 7.9973e-12],\n [2.2969e-12, 2.4349e-12, 3.7880e-12, ..., 4.9839e-12, 8.3292e-12,\n 6.0249e-12],\n ...,\n [1.8745e-12, 2.6737e-12, 5.8208e-12, ..., 9.8743e-13, 1.0010e-11,\n 9.8618e-12],\n [3.6707e-12, 1.8414e-12, 3.0156e-12, ..., 4.0051e-12, 7.4434e-12,\n 9.2318e-12],\n [1.9182e-12, 4.7357e-12, 9.7345e-12, ..., 2.9228e-12, 1.1942e-11,\n 6.7218e-12]], device='cuda:0')"
 },
 "12": {
 "step": "tensor(18770.)",
 "exp_avg": "tensor([[-3.8046e-07, 5.3547e-07, -1.0945e-06, ..., 1.1870e-06,\n -7.0817e-07, -1.2356e-07],\n [-5.5852e-06, 3.2626e-06, 1.6435e-06, ..., -3.1160e-06,\n 1.3983e-06, -1.4129e-06],\n [ 2.5731e-08, -3.0001e-06, -1.4673e-06, ..., -6.7798e-07,\n 1.5433e-07, 4.4028e-07],\n ...,\n [ 7.5991e-07, 3.9363e-07, 6.0677e-07, ..., 9.1059e-07,\n -3.6386e-07, -2.7988e-08],\n [ 5.8013e-07, -3.2153e-06, 2.6805e-06, ..., -5.4799e-07,\n -1.0285e-06, -1.5781e-06],\n [-1.3506e-06, 2.5692e-06, 2.5860e-07, ..., -2.3614e-06,\n -1.6418e-07, -1.8356e-06]], device='cuda:0')",
 "exp_avg_sq": "tensor([[1.3665e-11, 2.4322e-11, 5.0393e-12, ..., 1.3054e-11, 6.2574e-12,\n 6.6014e-12],\n [3.1788e-11, 1.1576e-10, 2.8135e-11, ..., 4.8336e-11, 1.6383e-11,\n 1.4316e-11],\n [3.5234e-11, 1.6940e-10, 3.7419e-11, ..., 1.6441e-11, 1.1142e-11,\n 1.4647e-11],\n ...,\n [1.1318e-11, 1.2735e-10, 2.9449e-11, ..., 7.5785e-12, 5.6347e-12,\n 7.9066e-12],\n [4.6952e-11, 8.8722e-11, 1.0095e-10, ..., 3.9565e-11, 1.3571e-11,\n 2.9440e-11],\n [1.1156e-10, 1.5259e-10, 4.2226e-11, ..., 5.2572e-11, 2.7758e-11,\n 3.1159e-11]], device='cuda:0')"
 },
 "13": {
 "step": "tensor(18770.)",
 "exp_avg": "tensor([ 2.7487e-05, -8.4890e-05, 2.8998e-05, ..., 1.1210e-05,\n -5.5292e-07, -7.6093e-05], device='cuda:0')",
 "exp_avg_sq": "tensor([9.9523e-09, 2.3606e-08, 1.9859e-08, ..., 1.0101e-08, 2.8750e-08,\n 4.0338e-08], device='cuda:0')"
 },
 "14": {
 "step": "tensor(18770.)",
 "exp_avg": "tensor([[-1.4610e-08, 1.6840e-07, 1.6884e-07, ..., 4.8052e-07,\n 4.9161e-07, 1.0567e-07],\n [ 9.1085e-08, -8.0781e-08, -1.4629e-07, ..., -1.1267e-06,\n -4.5708e-07, 7.1371e-08],\n [-5.7663e-08, 1.0224e-07, -1.4200e-07, ..., -1.6262e-08,\n -1.8249e-07, 9.3265e-08],\n ...,\n [ 2.0388e-07, 1.9650e-07, -1.3702e-07, ..., 1.9744e-07,\n 8.0482e-07, 7.4573e-08],\n [ 2.0212e-07, 9.7165e-08, 3.8903e-08, ..., -5.5296e-07,\n -4.9012e-07, -1.0885e-07],\n [-2.7380e-07, 9.3235e-08, -2.9152e-07, ..., 5.0494e-07,\n -2.5569e-08, -1.2451e-07]], device='cuda:0')",
 "exp_avg_sq": "tensor([[5.4336e-13, 2.8776e-13, 4.3722e-13, ..., 5.9143e-13, 1.4288e-12,\n 6.6123e-13],\n [1.0099e-12, 6.1066e-13, 5.2825e-13, ..., 1.4346e-12, 2.8352e-12,\n 1.8183e-12],\n [1.0904e-12, 6.1624e-13, 8.7285e-13, ..., 1.6067e-12, 3.2462e-12,\n 1.2015e-12],\n ...,\n [1.0005e-12, 6.5441e-13, 7.4552e-13, ..., 1.0981e-12, 6.6275e-12,\n 1.2183e-12],\n [1.3173e-12, 4.2919e-13, 7.5407e-13, ..., 9.9566e-13, 2.7065e-12,\n 1.1677e-12],\n [1.1746e-12, 9.4784e-13, 1.7370e-12, ..., 1.2851e-12, 2.2124e-12,\n 2.2498e-12]], device='cuda:0')"
 },
 "15": {
 "step": "tensor(18770.)",
 "exp_avg": "tensor([[-1.2429e-06, -2.7146e-06, 2.3265e-06, ..., 1.1756e-06,\n -1.1814e-07, 2.5167e-07],\n [ 8.0127e-07, -2.2512e-06, 1.4184e-06, ..., 4.6858e-07,\n 2.5714e-07, -2.4942e-07],\n [-1.0229e-08, -4.8725e-07, 1.9901e-06, ..., -3.0888e-06,\n 1.9963e-07, -5.0301e-07],\n ...,\n [ 6.1404e-07, -2.3396e-07, 1.6069e-07, ..., 2.1369e-07,\n 1.0187e-07, 7.4590e-07],\n [ 5.5341e-07, -5.3150e-07, 6.7607e-07, ..., -1.6304e-06,\n 3.7019e-07, 2.2446e-07],\n [-2.0486e-06, -4.4795e-06, -2.9358e-06, ..., -4.6949e-06,\n -1.5186e-06, -1.1857e-06]], device='cuda:0')",
 "exp_avg_sq": "tensor([[1.0717e-11, 4.8227e-11, 2.9240e-11, ..., 1.1888e-11, 8.8254e-12,\n 9.1401e-12],\n [2.8910e-12, 5.3663e-11, 8.8684e-12, ..., 2.5556e-12, 1.5375e-12,\n 3.5529e-12],\n [3.3164e-11, 7.2006e-11, 4.2380e-11, ..., 7.5707e-11, 1.4994e-11,\n 3.2627e-11],\n ...,\n [4.3775e-11, 4.0806e-11, 8.5986e-12, ..., 9.5173e-12, 1.0200e-11,\n 7.5268e-12],\n [1.7334e-11, 3.2006e-11, 1.1848e-11, ..., 4.4585e-11, 1.2046e-11,\n 1.6301e-11],\n [2.7960e-11, 7.3469e-11, 1.7470e-11, ..., 5.9162e-11, 1.3605e-11,\n 2.2486e-11]], device='cuda:0')"
 },
 "16": {
 "step": "tensor(18770.)",
 "exp_avg": "tensor([-4.8573e-05, 1.5333e-05, -4.0909e-05, ..., 2.9608e-05,\n -2.5947e-05, -6.2804e-05], device='cuda:0')",
 "exp_avg_sq": "tensor([1.1724e-08, 3.3158e-09, 2.8156e-08, ..., 1.9569e-08, 2.1420e-08,\n 2.5522e-08], device='cuda:0')"
 },
 "17": {
 "step": "tensor(18770.)",
 "exp_avg": "tensor([[ 2.5973e-07, 1.3009e-08, -3.9073e-07, ..., -2.1492e-08,\n 1.8929e-07, -2.9568e-07],\n [-6.0787e-08, 3.3794e-08, 1.8787e-07, ..., -3.5845e-07,\n -4.0188e-08, -2.9507e-07],\n [-2.9000e-07, -7.1405e-09, 2.8017e-07, ..., 2.8011e-07,\n 1.8058e-07, 2.1797e-07],\n ...,\n [-1.2145e-07, -9.4210e-08, -2.8943e-07, ..., -2.5416e-07,\n 3.3040e-07, -2.0719e-07],\n [-4.5035e-08, -2.6203e-08, 7.9390e-07, ..., -3.0139e-07,\n 2.1511e-08, -2.3334e-07],\n [-3.2142e-07, -1.1642e-08, 3.7213e-07, ..., -3.3815e-07,\n 4.1193e-09, 1.1560e-07]], device='cuda:0')",
 "exp_avg_sq": "tensor([[2.6387e-13, 8.0272e-14, 3.2749e-12, ..., 5.8322e-13, 6.8508e-13,\n 9.0418e-13],\n [5.6940e-13, 1.0697e-13, 5.4247e-12, ..., 9.6362e-13, 6.5786e-13,\n 1.2449e-12],\n [4.2170e-13, 1.0527e-13, 4.8519e-12, ..., 9.1566e-13, 5.1054e-13,\n 7.6506e-13],\n ...,\n [3.5889e-13, 6.9938e-13, 1.6291e-12, ..., 2.6396e-12, 7.6814e-13,\n 1.0430e-12],\n [3.5958e-13, 1.2873e-13, 1.5563e-11, ..., 4.0956e-12, 7.3218e-13,\n 1.2992e-12],\n [5.1424e-13, 2.7397e-13, 3.7659e-12, ..., 2.3409e-12, 6.3247e-13,\n 8.8436e-13]], device='cuda:0')"
 },
 "18": {
 "step": "tensor(18770.)",
 "exp_avg": "tensor([[-2.3467e-06, -1.3482e-06, 2.0532e-07, ..., -4.5243e-07,\n 3.4254e-07, -1.4343e-06],\n [ 7.3334e-07, -2.0907e-09, -2.7858e-07, ..., -7.7580e-07,\n 3.6349e-07, 5.1859e-07],\n [ 7.9965e-07, -4.1570e-07, 4.5750e-07, ..., 1.8000e-06,\n -3.0573e-07, 5.5032e-07],\n ...,\n [-6.6968e-07, 1.4768e-06, 5.0105e-07, ..., 1.1866e-06,\n 3.4684e-07, 2.3503e-07],\n [-3.5187e-07, 1.8868e-06, 8.3324e-07, ..., 9.4897e-07,\n 8.4573e-07, -5.1147e-07],\n [ 3.3273e-07, -1.6733e-07, -2.2199e-07, ..., -2.3472e-06,\n -2.9515e-06, 2.3658e-06]], device='cuda:0')",
 "exp_avg_sq": "tensor([[5.3199e-11, 5.8581e-11, 2.9661e-11, ..., 1.5516e-11, 6.5304e-12,\n 2.0524e-11],\n [3.0154e-12, 4.6882e-12, 1.8081e-12, ..., 1.6975e-11, 2.0028e-12,\n 1.6791e-12],\n [3.6813e-11, 9.1670e-11, 2.2515e-11, ..., 7.9340e-11, 1.5318e-11,\n 2.4704e-11],\n ...,\n [7.0152e-11, 1.4174e-10, 4.9782e-11, ..., 2.1971e-11, 2.4352e-11,\n 1.5468e-11],\n [3.1845e-11, 1.2814e-10, 3.1809e-11, ..., 1.5565e-11, 1.3570e-11,\n 1.1131e-11],\n [7.7046e-11, 7.8013e-11, 3.9113e-11, ..., 3.8365e-11, 1.9236e-11,\n 1.7290e-11]], device='cuda:0')"
 },
 "19": {
 "step": "tensor(18770.)",
 "exp_avg": "tensor([-3.4225e-05, -7.4492e-06, 4.5972e-05, ..., 6.0461e-06,\n 5.1716e-06, -5.9582e-06], device='cuda:0')",
 "exp_avg_sq": "tensor([1.5933e-08, 3.9410e-09, 3.2882e-08, ..., 2.5824e-08, 2.4481e-08,\n 2.7148e-08], device='cuda:0')"
 },
 "20": {
 "step": "tensor(18770.)",
 "exp_avg": "tensor([[-1.9876e-08, -4.7424e-08, 9.7142e-08, ..., -4.5700e-07,\n -2.4048e-07, -1.1842e-08],\n [-6.7388e-09, -5.3154e-08, -4.5641e-07, ..., -1.7152e-07,\n -1.5785e-07, 6.5537e-08],\n [-1.7910e-07, 1.0823e-09, -8.1099e-08, ..., 1.1357e-07,\n -1.7498e-08, -1.9648e-07],\n ...,\n [ 1.2003e-07, 5.2718e-08, -3.1803e-07, ..., -1.1568e-07,\n 1.5072e-07, 4.4175e-07],\n [-1.4947e-07, 3.1584e-08, -3.8052e-07, ..., -1.0921e-07,\n -8.5453e-07, 2.5549e-07],\n [-2.8579e-09, -2.6537e-08, 1.8336e-07, ..., 2.2120e-07,\n 3.2577e-07, 9.1655e-07]], device='cuda:0')",
 "exp_avg_sq": "tensor([[5.1869e-13, 4.5245e-14, 1.5363e-12, ..., 2.0064e-12, 1.0721e-12,\n 7.3385e-13],\n [3.9868e-13, 1.1372e-13, 1.4831e-12, ..., 8.7460e-13, 1.5944e-12,\n 1.0084e-12],\n [5.7041e-13, 7.2257e-14, 9.5117e-13, ..., 1.8222e-12, 2.7471e-12,\n 1.2020e-12],\n ...,\n [5.8345e-13, 8.5990e-14, 1.7328e-12, ..., 1.4757e-12, 2.3918e-12,\n 1.3868e-12],\n [4.8605e-13, 7.6492e-14, 1.5821e-12, ..., 1.8371e-12, 2.1161e-12,\n 9.7700e-13],\n [4.9566e-13, 9.5283e-14, 1.4862e-12, ..., 9.5671e-13, 2.9568e-12,\n 2.1713e-12]], device='cuda:0')"
 },
 "21": {
 "step": "tensor(18770.)",
 "exp_avg": "tensor([[-3.0455e-06, 1.3595e-06, -1.0314e-06, ..., -1.1259e-08,\n -4.0614e-07, -8.8315e-08],\n [-2.1284e-06, 8.4948e-07, -6.1691e-07, ..., 2.4046e-07,\n -7.0542e-07, -2.7418e-07],\n [ 1.0047e-07, -2.7090e-07, 7.6741e-07, ..., -1.9722e-07,\n -6.6346e-07, 2.3107e-07],\n ...,\n [ 1.4200e-06, -3.3883e-07, 3.7504e-07, ..., 1.2727e-06,\n 5.1534e-07, 2.0338e-07],\n [-2.4420e-06, 1.5082e-06, 1.2304e-07, ..., -3.4048e-06,\n -1.0482e-06, -2.8356e-06],\n [-1.9501e-06, 3.2583e-07, 5.0894e-07, ..., -2.5582e-06,\n 9.8611e-08, -2.4865e-06]], device='cuda:0')",
 "exp_avg_sq": "tensor([[6.2367e-11, 3.7858e-11, 6.2543e-12, ..., 5.2480e-12, 6.2830e-12,\n 4.8628e-12],\n [1.0357e-10, 9.1133e-11, 1.5474e-11, ..., 5.3887e-11, 2.3592e-11,\n 2.2926e-11],\n [5.0384e-11, 9.0067e-11, 1.1035e-11, ..., 1.1645e-11, 5.3465e-11,\n 1.8425e-11],\n ...,\n [5.0556e-12, 6.0405e-12, 8.4736e-12, ..., 3.3276e-11, 1.1886e-11,\n 5.4443e-12],\n [8.8483e-11, 1.4319e-10, 6.0144e-11, ..., 8.9488e-11, 3.3759e-11,\n 5.6658e-11],\n [2.0733e-11, 2.3643e-11, 9.7478e-12, ..., 2.2711e-11, 1.0965e-11,\n 1.0666e-11]], device='cuda:0')"
 },
 "22": {
 "step": "tensor(18770.)",
 "exp_avg": "tensor([-2.7782e-05, -1.9407e-05, -4.6221e-06, ..., 1.8426e-05,\n -5.6938e-05, -6.6409e-05], device='cuda:0')",
 "exp_avg_sq": "tensor([8.2499e-09, 3.1067e-08, 2.5329e-08, ..., 1.0570e-08, 4.8851e-08,\n 1.4073e-08], device='cuda:0')"
 },
 "23": {
 "step": "tensor(18770.)",
 "exp_avg": "tensor([[-2.8572e-08, -7.3500e-08, 1.7660e-07, ..., 1.2014e-07,\n 3.1310e-08, -1.3504e-07],\n [ 7.8785e-08, -1.1122e-07, -5.7274e-08, ..., -3.3810e-07,\n -1.7857e-07, 2.5066e-07],\n [ 7.6067e-08, -5.8670e-08, 2.9662e-07, ..., -1.4302e-07,\n -1.9163e-08, -1.8818e-07],\n ...,\n [-9.8896e-09, 1.5777e-07, -1.0787e-07, ..., 2.1952e-09,\n 2.8745e-07, 1.0957e-08],\n [-1.5256e-07, 8.6956e-08, 1.1144e-07, ..., -1.8395e-08,\n -6.5054e-08, -6.0123e-09],\n [-1.3180e-07, -1.9209e-07, 7.9341e-08, ..., 4.5164e-07,\n 2.8018e-07, -2.5329e-07]], device='cuda:0')",
 "exp_avg_sq": "tensor([[9.1376e-14, 1.1353e-13, 1.9893e-13, ..., 2.0515e-13, 2.2430e-13,\n 1.4454e-13],\n [2.7119e-13, 2.7395e-13, 7.0952e-13, ..., 4.9596e-13, 5.5119e-13,\n 2.7193e-13],\n [1.3867e-13, 2.5638e-13, 7.6324e-13, ..., 3.6382e-13, 7.6630e-13,\n 3.0569e-13],\n ...,\n [1.5485e-13, 3.5359e-13, 6.8434e-13, ..., 3.4613e-13, 5.1267e-13,\n 3.1754e-13],\n [1.8738e-13, 5.1657e-13, 8.3274e-13, ..., 4.4671e-13, 7.2248e-13,\n 1.5047e-13],\n [2.4438e-13, 3.1956e-13, 4.8017e-13, ..., 8.9313e-13, 9.4199e-13,\n 1.4955e-13]], device='cuda:0')"
 },
 "24": {
 "step": "tensor(18770.)",
 "exp_avg": "tensor([[-6.0152e-07, 4.7335e-07, -4.3645e-07, ..., -2.1140e-07,\n -2.2704e-07, -4.3348e-07],\n [-5.7035e-07, 5.2636e-07, -1.7710e-07, ..., -9.1330e-07,\n 4.0551e-08, 8.7207e-08],\n [ 9.9842e-07, -7.7750e-07, -9.7964e-07, ..., -7.0653e-07,\n -1.8600e-07, 1.2530e-08],\n ...,\n [ 1.8388e-06, -4.6496e-07, -6.3926e-09, ..., 1.5679e-07,\n 9.2658e-08, -9.6524e-08],\n [ 5.2940e-07, 8.2293e-07, -1.2409e-06, ..., -9.6591e-07,\n -5.8645e-08, 8.7221e-08],\n [ 3.2981e-07, 1.8521e-07, 4.6953e-08, ..., 8.0237e-07,\n 1.0648e-06, 7.3919e-07]], device='cuda:0')",
 "exp_avg_sq": "tensor([[3.1774e-11, 2.8621e-11, 5.9522e-12, ..., 1.4658e-11, 9.7617e-12,\n 4.1521e-12],\n [6.1762e-12, 1.1316e-11, 4.9287e-12, ..., 3.6805e-11, 5.8284e-12,\n 9.9291e-12],\n [1.1097e-11, 2.4530e-11, 8.9494e-12, ..., 2.4048e-11, 6.1454e-12,\n 6.0618e-12],\n ...,\n [6.3987e-11, 3.4882e-11, 6.1074e-12, ..., 1.6408e-11, 1.1517e-11,\n 6.8585e-12],\n [9.7879e-12, 3.2428e-11, 3.5937e-11, ..., 1.5443e-11, 4.4663e-12,\n 5.9994e-12],\n [1.1869e-11, 2.6895e-11, 3.0800e-11, ..., 2.2439e-11, 4.0696e-12,\n 5.2204e-12]], device='cuda:0')"
 },
 "25": {
 "step": "tensor(18770.)",
 "exp_avg": "tensor([-2.2879e-05, -8.5621e-06, -1.0966e-05, ..., 7.8707e-06,\n -2.3814e-06, 2.7406e-05], device='cuda:0')",
 "exp_avg_sq": "tensor([1.0234e-08, 1.1773e-08, 8.9774e-09, ..., 1.4038e-08, 9.5747e-09,\n 8.1955e-09], device='cuda:0')"
 },
 "26": {
 "step": "tensor(18770.)",
 "exp_avg": "tensor([[ 9.1578e-08, 5.6776e-08, -1.2551e-07, ..., -1.6488e-07,\n 9.7023e-08, 9.9153e-08],\n [-2.0860e-08, 4.6499e-08, -1.7529e-07, ..., -1.5612e-07,\n -1.0974e-07, 2.4516e-07],\n [ 2.3571e-08, 1.6226e-07, 8.7782e-08, ..., 1.6039e-07,\n 4.5144e-07, -1.0189e-07],\n ...,\n [ 2.1020e-08, 2.0511e-07, 1.1542e-07, ..., -5.1509e-08,\n -8.3069e-08, 3.3476e-07],\n [ 1.3464e-07, 1.0658e-07, 1.6636e-07, ..., 4.0314e-07,\n -1.2301e-07, -1.9143e-08],\n [ 3.5329e-11, -2.7440e-09, -1.9639e-07, ..., -9.0350e-08,\n -7.4138e-08, 9.4324e-08]], device='cuda:0')",
 "exp_avg_sq": "tensor([[1.2718e-13, 1.7092e-13, 9.9785e-14, ..., 1.3494e-13, 2.2620e-13,\n 1.6140e-13],\n [1.7842e-13, 2.3807e-13, 1.9356e-13, ..., 2.4270e-13, 3.8066e-13,\n 1.4877e-13],\n [3.0382e-13, 2.3346e-13, 2.5929e-13, ..., 3.8648e-13, 4.7062e-13,\n 3.2531e-13],\n ...,\n [1.6772e-13, 1.9009e-13, 1.8452e-13, ..., 2.4082e-13, 3.3624e-13,\n 2.3773e-13],\n [1.5251e-13, 2.9922e-13, 1.8366e-13, ..., 6.1362e-13, 1.9679e-13,\n 2.4797e-13],\n [1.8970e-13, 1.9461e-13, 1.9765e-13, ..., 3.8122e-13, 4.9354e-13,\n 1.9908e-13]], device='cuda:0')"
 },
 "27": {
 "step": "tensor(18770.)",
 "exp_avg": "tensor([-5.6052e-45], device='cuda:0')",
 "exp_avg_sq": "tensor([5.4406e-14], device='cuda:0')"
 },
 "28": {
 "step": "tensor(18770.)",
 "exp_avg": "tensor([-5.6052e-45, 5.6052e-45, -5.6052e-45], device='cuda:0')",
 "exp_avg_sq": "tensor([7.3159e-16, 4.7397e-16, 1.6954e-16], device='cuda:0')"
 },
 "29": {
 "step": "tensor(18770.)",
 "exp_avg": "tensor([ 5.6052e-45, -5.6052e-45, 5.6052e-45, -5.6052e-45, -5.6052e-45,\n 5.6052e-45, 5.6052e-45, -5.6052e-45, -5.6052e-45], device='cuda:0')",
 "exp_avg_sq": "tensor([5.4837e-15, 7.2569e-16, 1.5098e-15, 6.3389e-16, 2.8795e-16, 7.1459e-16,\n 4.4033e-15, 6.5519e-15, 3.0341e-16], device='cuda:0')"
 },
 "31": {
 "step": "tensor(18770.)",
 "exp_avg": "tensor([[-5.6052e-45, 5.6052e-45, -5.6052e-45, ..., 5.6052e-45,\n 5.6052e-45, 5.6052e-45],\n [ 5.6052e-45, 5.6052e-45, 5.6052e-45, ..., -5.6052e-45,\n -5.6052e-45, -5.6052e-45],\n [-5.6052e-45, 5.6052e-45, -5.6052e-45, ..., -5.6052e-45,\n 5.6052e-45, -5.6052e-45],\n ...,\n [ 5.6052e-45, 5.6052e-45, -5.6052e-45, ..., 5.6052e-45,\n -5.6052e-45, 5.6052e-45],\n [ 5.6052e-45, -5.6052e-45, -5.6052e-45, ..., 5.6052e-45,\n 5.6052e-45, 5.6052e-45],\n [ 5.6052e-45, 5.6052e-45, -5.6052e-45, ..., 5.6052e-45,\n 5.6052e-45, 5.6052e-45]], device='cuda:0')",
 "exp_avg_sq": "tensor([[1.8425e-20, 2.1220e-21, 3.8940e-21, ..., 2.5786e-21, 6.1196e-22,\n 7.9702e-21],\n [5.1186e-22, 6.4150e-21, 2.3710e-22, ..., 5.3472e-23, 7.2824e-23,\n 3.3950e-22],\n [3.6977e-20, 4.8471e-21, 5.5590e-20, ..., 3.7992e-21, 5.2143e-21,\n 4.7379e-21],\n ...,\n [6.6713e-22, 5.0433e-22, 1.8359e-22, ..., 3.4211e-22, 6.1542e-23,\n 2.3344e-22],\n [3.6118e-22, 5.5625e-22, 1.1105e-21, ..., 4.2279e-22, 2.9352e-22,\n 3.9497e-22],\n [2.6374e-20, 4.5177e-21, 1.0982e-20, ..., 8.3332e-21, 2.3254e-21,\n 8.3522e-21]], device='cuda:0')"
 },
 "32": {
 "step": "tensor(18770.)",
 "exp_avg": "tensor([-5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45, -5.6052e-45,\n -5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45,\n -5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45,\n 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45,\n 5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45,\n 5.6052e-45, 5.6052e-45, -5.6052e-45, -5.6052e-45, -5.6052e-45,\n 5.6052e-45, -5.6052e-45, -5.6052e-45, -5.6052e-45, 5.6052e-45,\n -5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45,\n -5.6052e-45, -5.6052e-45, -5.6052e-45, 5.6052e-45, 5.6052e-45,\n 5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45, -5.6052e-45,\n -5.6052e-45, -5.6052e-45, 5.6052e-45, -5.6052e-45, -5.6052e-45,\n 5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45,\n 5.6052e-45, -5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45,\n 5.6052e-45, 5.6052e-45, -5.6052e-45, -5.6052e-45, 5.6052e-45,\n 5.6052e-45, -5.6052e-45, -5.6052e-45, -5.6052e-45, -5.6052e-45,\n 5.6052e-45, 5.6052e-45, -5.6052e-45, -5.6052e-45, 5.6052e-45,\n 5.6052e-45, -5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45,\n -5.6052e-45, -5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45,\n -5.6052e-45, -5.6052e-45, -5.6052e-45, -5.6052e-45, 5.6052e-45,\n 5.6052e-45, -5.6052e-45, 5.6052e-45, -5.6052e-45, -5.6052e-45,\n 5.6052e-45, -5.6052e-45, -5.6052e-45, -5.6052e-45, -5.6052e-45,\n -5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45,\n -5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45,\n -5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45,\n -5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45, 5.6052e-45,\n -5.6052e-45, 5.6052e-45, -5.6052e-45, -5.6052e-45, -5.6052e-45,\n -5.6052e-45, -5.6052e-45, -5.6052e-45, 5.6052e-45, -5.6052e-45,\n 5.6052e-45, -5.6052e-45, -5.6052e-45, 5.6052e-45, 5.6052e-45,\n 5.6052e-45, -5.6052e-45, -5.6052e-45, 5.6052e-45, 5.6052e-45,\n -5.6052e-45, -5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45,\n -5.6052e-45, -5.6052e-45, -5.6052e-45, 5.6052e-45, 5.6052e-45,\n 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45,\n 5.6052e-45, -5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45,\n 5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45,\n 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45,\n -5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45, 5.6052e-45,\n -5.6052e-45, -5.6052e-45, 5.6052e-45, -5.6052e-45, -5.6052e-45,\n -5.6052e-45, 5.6052e-45, -5.6052e-45, -5.6052e-45, -5.6052e-45,\n 5.6052e-45, 5.6052e-45, -5.6052e-45, -5.6052e-45, 5.6052e-45,\n 5.6052e-45, -5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45,\n -5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45, -5.6052e-45,\n 5.6052e-45, -5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45,\n 5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45, 5.6052e-45,\n -5.6052e-45, -5.6052e-45, -5.6052e-45, 5.6052e-45, 5.6052e-45,\n -5.6052e-45, -5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45,\n 5.6052e-45, -5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45,\n -5.6052e-45, -5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45,\n -5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45,\n 5.6052e-45, -5.6052e-45, -5.6052e-45, 5.6052e-45, 5.6052e-45,\n 5.6052e-45, -5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45,\n 5.6052e-45, -5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45,\n 5.6052e-45], device='cuda:0')",
 "exp_avg_sq": "tensor([2.4504e-17, 5.5326e-19, 4.7516e-17, 6.3465e-18, 8.7319e-19, 2.3060e-17,\n 2.8687e-19, 1.0939e-16, 4.4901e-19, 4.5589e-17, 4.5220e-17, 4.8920e-18,\n 2.9057e-17, 6.6564e-19, 3.0352e-18, 5.5814e-17, 1.6868e-18, 9.1166e-18,\n 6.2269e-17, 2.0692e-17, 6.0919e-17, 9.4906e-19, 7.8458e-18, 1.7179e-17,\n 6.1160e-19, 1.0422e-16, 4.0096e-19, 1.8495e-17, 5.0024e-17, 1.2101e-18,\n 7.2072e-18, 2.0558e-19, 3.3016e-17, 1.0066e-17, 7.6768e-18, 2.2449e-17,\n 7.4569e-19, 4.0306e-18, 2.3571e-18, 1.6011e-17, 1.1773e-18, 8.5420e-18,\n 1.0961e-17, 3.3842e-17, 1.2479e-17, 2.7418e-18, 1.6955e-17, 2.8496e-19,\n 9.5383e-18, 8.8560e-17, 1.8653e-17, 1.2148e-17, 9.6372e-18, 2.9807e-19,\n 2.8303e-17, 4.2226e-19, 4.2760e-19, 7.7268e-17, 1.5393e-16, 6.3861e-19,\n 3.0697e-18, 7.3479e-18, 4.9527e-17, 1.5937e-18, 7.6123e-19, 6.0178e-19,\n 5.3902e-19, 1.6768e-17, 4.9876e-18, 9.3278e-18, 1.0113e-18, 5.9490e-19,\n 1.8766e-16, 1.7449e-17, 5.6176e-19, 1.5633e-17, 7.4111e-18, 2.2286e-18,\n 7.6056e-19, 1.7878e-18, 1.2246e-17, 2.0515e-18, 1.1297e-17, 2.4023e-18,\n 1.4762e-18, 1.2278e-18, 3.1767e-18, 5.2534e-18, 1.8201e-17, 4.2347e-17,\n 1.2555e-17, 5.0347e-17, 1.0081e-18, 7.6440e-17, 1.1295e-19, 9.5848e-18,\n 7.2256e-17, 7.6492e-19, 1.1392e-17, 1.6777e-17, 4.5076e-18, 1.9753e-17,\n 8.0285e-18, 1.5221e-17, 3.7033e-18, 2.9968e-17, 1.4789e-18, 5.3055e-18,\n 1.6095e-17, 1.5891e-18, 2.0989e-17, 2.5518e-17, 9.0831e-18, 2.7587e-17,\n 1.2918e-16, 7.1936e-19, 1.2151e-18, 6.1646e-18, 3.8696e-18, 1.1162e-18,\n 2.4083e-17, 1.8442e-17, 8.2951e-18, 1.4063e-17, 3.0067e-17, 1.9859e-18,\n 1.1686e-17, 7.8328e-18, 3.7302e-17, 6.1633e-18, 1.4550e-18, 3.2244e-17,\n 3.0778e-19, 1.4364e-18, 5.2775e-18, 4.3017e-17, 5.9065e-18, 1.1132e-16,\n 4.3955e-18, 2.3490e-18, 2.5699e-18, 1.2197e-17, 4.5183e-18, 5.8591e-19,\n 1.9556e-18, 8.9275e-17, 2.0485e-18, 4.2931e-17, 2.7020e-17, 4.3358e-18,\n 1.4571e-16, 7.5993e-19, 2.0663e-17, 2.4353e-17, 5.1836e-18, 5.1970e-19,\n 8.7927e-20, 5.1980e-17, 2.0361e-16, 1.5852e-18, 4.3301e-17, 1.3685e-17,\n 5.0792e-18, 8.7919e-18, 2.5424e-17, 5.7493e-17, 1.7926e-16, 4.2409e-17,\n 1.2862e-16, 4.6552e-17, 6.2746e-17, 1.3807e-17, 2.6425e-17, 1.4460e-17,\n 7.0124e-18, 2.8339e-17, 7.9435e-18, 1.0340e-16, 1.6106e-17, 2.3578e-17,\n 2.3629e-17, 3.4818e-18, 3.6370e-18, 6.0100e-17, 2.8437e-17, 2.4658e-17,\n 3.9786e-17, 5.2056e-19, 1.3240e-18, 3.2256e-17, 1.2098e-17, 2.5204e-17,\n 1.0052e-16, 5.1715e-18, 2.2265e-18, 1.7094e-19, 1.5008e-18, 1.0034e-18,\n 5.5649e-17, 1.3285e-16, 2.1399e-17, 2.3088e-19, 4.6442e-19, 2.6702e-19,\n 6.3860e-19, 9.2392e-19, 1.9978e-17, 4.8556e-19, 2.9137e-17, 5.3386e-17,\n 1.1702e-17, 1.7465e-16, 2.5778e-17, 7.1535e-18, 1.3590e-17, 2.2283e-17,\n 2.1863e-18, 2.0552e-18, 3.6158e-18, 4.0909e-17, 6.9869e-17, 8.6856e-19,\n 1.6479e-18, 7.2590e-20, 2.1360e-16, 1.0843e-16, 1.3672e-17, 2.9866e-18,\n 3.4032e-17, 4.3402e-18, 7.6359e-18, 5.6596e-19, 3.8148e-17, 1.1343e-18,\n 3.2516e-18, 1.4922e-18, 1.8526e-17, 1.5227e-17, 1.2029e-17, 1.2095e-18,\n 1.9326e-17, 4.2144e-17, 8.2464e-17, 1.1937e-17, 1.9933e-18, 3.0776e-17,\n 3.2355e-17, 2.1316e-17, 1.0283e-17, 1.8422e-18, 7.3749e-18, 6.5908e-18,\n 6.2854e-17, 9.9691e-19, 4.9841e-19, 4.9726e-17], device='cuda:0')"
 },
 "33": {
 "step": "tensor(18770.)",
 "exp_avg": "tensor([ 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45,\n -5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45,\n -5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45,\n 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45,\n 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45,\n 5.6052e-45, -5.6052e-45, -5.6052e-45, -5.6052e-45, -5.6052e-45,\n 5.6052e-45, -5.6052e-45, -5.6052e-45, -5.6052e-45, 5.6052e-45,\n -5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45,\n 5.6052e-45, -5.6052e-45, -5.6052e-45, 5.6052e-45, 5.6052e-45,\n 5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45, -5.6052e-45,\n -5.6052e-45, -5.6052e-45, 5.6052e-45, -5.6052e-45, -5.6052e-45,\n -5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45,\n 5.6052e-45, -5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45,\n 5.6052e-45, -5.6052e-45, -5.6052e-45, 5.6052e-45, 5.6052e-45,\n -5.6052e-45, -5.6052e-45, -5.6052e-45, 5.6052e-45, 5.6052e-45,\n 5.6052e-45, 5.6052e-45, -5.6052e-45, -5.6052e-45, 5.6052e-45,\n 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45,\n 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45,\n -5.6052e-45, -5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45,\n 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45,\n 5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45, 5.6052e-45,\n 5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45,\n 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45,\n 5.6052e-45, -5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45,\n -5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45,\n -5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45,\n -5.6052e-45, -5.6052e-45, -5.6052e-45, 5.6052e-45, 5.6052e-45,\n 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45,\n 5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45, 5.6052e-45,\n -5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45,\n -5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45, 5.6052e-45,\n 5.6052e-45, -5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45,\n 5.6052e-45, -5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45,\n 5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45,\n 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45,\n -5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45,\n -5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45,\n 5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45, -5.6052e-45,\n 5.6052e-45, 5.6052e-45, -5.6052e-45, -5.6052e-45, 5.6052e-45,\n -5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45,\n 5.6052e-45, -5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45,\n 5.6052e-45, -5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45,\n 5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45,\n -5.6052e-45, -5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45,\n -5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45,\n 5.6052e-45, -5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45,\n -5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45,\n -5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45,\n 5.6052e-45, -5.6052e-45, -5.6052e-45, 5.6052e-45, 5.6052e-45,\n 5.6052e-45, -5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45,\n 5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45,\n 5.6052e-45], device='cuda:0')",
 "exp_avg_sq": "tensor([5.0824e-20, 9.1062e-22, 6.1050e-20, 4.4230e-21, 1.2253e-21, 2.1508e-20,\n 6.8885e-22, 3.0395e-19, 1.5183e-22, 5.5022e-20, 1.5236e-19, 2.1876e-21,\n 6.9637e-20, 6.0946e-22, 3.5619e-22, 4.2197e-19, 4.0306e-21, 4.6037e-20,\n 9.9015e-19, 5.6939e-20, 6.6791e-20, 1.9562e-21, 1.9028e-21, 4.6848e-20,\n 2.8597e-22, 5.2051e-19, 9.5902e-22, 2.1624e-20, 5.0857e-20, 1.5258e-20,\n 4.7113e-21, 5.5309e-22, 3.6627e-20, 3.1447e-21, 1.4196e-20, 1.6723e-20,\n 1.0311e-21, 1.9318e-21, 5.7705e-22, 4.4707e-20, 3.2550e-21, 6.5912e-21,\n 3.4896e-20, 8.6645e-20, 2.9065e-20, 1.6351e-20, 1.3176e-20, 6.9314e-22,\n 2.2195e-20, 6.6915e-19, 8.2428e-21, 1.5006e-21, 1.0184e-20, 1.2603e-21,\n 2.0984e-20, 4.3897e-22, 9.0654e-22, 1.7957e-19, 8.7023e-20, 2.3474e-22,\n 3.1440e-20, 3.6632e-21, 5.6357e-20, 1.4402e-21, 6.8402e-22, 1.6513e-22,\n 1.5143e-21, 8.7662e-21, 7.2073e-22, 1.4280e-20, 1.1661e-21, 9.0134e-22,\n 2.7271e-19, 4.9704e-20, 8.8603e-22, 1.8541e-20, 1.0313e-20, 1.5859e-21,\n 1.5191e-21, 1.3866e-20, 1.8703e-19, 3.4196e-22, 2.7875e-20, 9.8636e-22,\n 9.2872e-22, 1.0377e-21, 3.3219e-21, 9.1305e-21, 8.2063e-20, 6.4815e-20,\n 1.4236e-20, 3.3297e-19, 1.6150e-21, 1.2403e-19, 6.6860e-22, 3.9553e-20,\n 2.1193e-19, 1.1291e-21, 1.1592e-20, 9.0233e-20, 5.2520e-21, 2.3324e-20,\n 2.3870e-20, 6.2367e-20, 4.5192e-22, 1.4375e-20, 5.3339e-21, 5.4791e-21,\n 3.1620e-21, 7.5464e-21, 8.2050e-20, 1.2251e-19, 2.0673e-20, 4.2326e-19,\n 4.0938e-19, 2.4240e-22, 2.9865e-21, 3.7219e-21, 2.0374e-20, 3.2456e-22,\n 3.2785e-20, 8.2324e-21, 1.7805e-20, 4.6896e-21, 6.7081e-20, 2.5487e-20,\n 1.7365e-20, 5.4784e-21, 1.1352e-19, 5.7803e-22, 5.3506e-22, 4.3443e-20,\n 2.6989e-21, 4.2890e-21, 1.1125e-21, 5.8216e-20, 1.3135e-20, 3.6709e-19,\n 4.4891e-21, 7.5693e-22, 1.8176e-21, 8.5962e-21, 1.8223e-21, 4.3055e-22,\n 5.2157e-22, 7.9551e-20, 8.9148e-22, 9.5444e-20, 1.1821e-20, 2.2421e-21,\n 1.6357e-19, 1.2864e-21, 1.2212e-20, 1.7606e-20, 2.1155e-20, 4.3408e-22,\n 1.8202e-22, 1.4749e-19, 2.7547e-18, 1.9553e-21, 5.1527e-19, 2.4033e-20,\n 4.0567e-21, 9.0255e-21, 6.1419e-20, 6.9755e-20, 2.1344e-19, 2.1080e-20,\n 1.3575e-19, 1.0779e-19, 2.1741e-19, 3.0359e-20, 4.6810e-20, 1.1803e-19,\n 4.7222e-21, 9.9588e-21, 1.0962e-20, 9.2140e-20, 1.0416e-20, 2.3849e-19,\n 1.8268e-20, 5.3458e-21, 7.8534e-21, 5.6725e-20, 2.0131e-19, 1.2068e-20,\n 2.1487e-19, 4.8361e-22, 4.1810e-21, 3.0606e-20, 6.5600e-21, 1.4491e-19,\n 1.8060e-19, 1.5906e-20, 1.1828e-21, 4.7738e-22, 1.2157e-21, 1.6508e-21,\n 1.3546e-19, 1.6428e-18, 1.2572e-20, 6.7107e-22, 1.5226e-21, 3.1595e-23,\n 1.9116e-21, 1.0057e-21, 9.6174e-21, 7.8620e-22, 2.4846e-20, 1.3346e-19,\n 8.6309e-20, 2.6745e-19, 7.8229e-20, 2.4244e-21, 2.6911e-20, 1.7701e-20,\n 1.2116e-21, 2.1158e-21, 1.2896e-21, 1.4608e-19, 1.3965e-19, 1.9316e-21,\n 6.3288e-21, 1.3614e-21, 3.7737e-19, 1.4372e-19, 8.5785e-21, 1.0921e-21,\n 6.9496e-20, 4.7084e-21, 1.7891e-20, 2.5551e-22, 1.4130e-19, 3.0426e-21,\n 8.7117e-22, 6.0248e-22, 5.8637e-20, 6.8994e-21, 7.5870e-21, 1.8724e-21,\n 1.6745e-20, 5.9836e-20, 1.1661e-19, 9.2313e-21, 2.4828e-21, 6.8564e-20,\n 2.0432e-19, 4.0831e-20, 2.1864e-20, 1.2427e-21, 1.5865e-20, 9.1637e-22,\n 3.6570e-19, 1.7937e-21, 7.2915e-22, 2.8984e-19], device='cuda:0')"
 },
 "34": {
 "step": "tensor(18770.)",
 "exp_avg": "tensor([ 5.6052e-45, -5.6052e-45, -5.6052e-45, 5.6052e-45, -5.6052e-45,\n -5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45,\n -5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45,\n 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45,\n 5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45,\n 5.6052e-45, 5.6052e-45, -5.6052e-45, -5.6052e-45, -5.6052e-45,\n 5.6052e-45, 5.6052e-45, -5.6052e-45, -5.6052e-45, 5.6052e-45,\n -5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45,\n -5.6052e-45, -5.6052e-45, -5.6052e-45, 5.6052e-45, 5.6052e-45,\n 5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45, -5.6052e-45,\n -5.6052e-45, -5.6052e-45, 5.6052e-45, -5.6052e-45, -5.6052e-45,\n 5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45,\n 5.6052e-45, -5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45,\n 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45,\n 5.6052e-45, -5.6052e-45, -5.6052e-45, 5.6052e-45, -5.6052e-45,\n 5.6052e-45, 5.6052e-45, -5.6052e-45, -5.6052e-45, 5.6052e-45,\n 5.6052e-45, -5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45,\n 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45,\n -5.6052e-45, -5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45,\n 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45,\n 5.6052e-45, -5.6052e-45, -5.6052e-45, 5.6052e-45, -5.6052e-45,\n -5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45,\n 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45,\n 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45,\n -5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45, 5.6052e-45,\n -5.6052e-45, 5.6052e-45, -5.6052e-45, -5.6052e-45, -5.6052e-45,\n -5.6052e-45, -5.6052e-45, -5.6052e-45, 5.6052e-45, 5.6052e-45,\n 5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45, 5.6052e-45,\n 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45,\n -5.6052e-45, -5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45,\n -5.6052e-45, -5.6052e-45, -5.6052e-45, 5.6052e-45, 5.6052e-45,\n 5.6052e-45, -5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45,\n 5.6052e-45, -5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45,\n 5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45,\n 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45,\n -5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45, 5.6052e-45,\n -5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45, -5.6052e-45,\n 5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45, -5.6052e-45,\n 5.6052e-45, 5.6052e-45, -5.6052e-45, -5.6052e-45, 5.6052e-45,\n 5.6052e-45, -5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45,\n -5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45, -5.6052e-45,\n 5.6052e-45, -5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45,\n 5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45, 5.6052e-45,\n -5.6052e-45, -5.6052e-45, -5.6052e-45, 5.6052e-45, 5.6052e-45,\n -5.6052e-45, -5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45,\n 5.6052e-45, -5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45,\n -5.6052e-45, -5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45,\n -5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45,\n 5.6052e-45, -5.6052e-45, -5.6052e-45, 5.6052e-45, 5.6052e-45,\n 5.6052e-45, -5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45,\n 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45,\n 5.6052e-45], device='cuda:0')",
 "exp_avg_sq": "tensor([4.0649e-20, 3.4500e-21, 7.1845e-20, 7.8943e-21, 3.9515e-21, 6.2692e-20,\n 7.7965e-21, 3.3742e-19, 5.6990e-21, 8.8171e-20, 8.9591e-20, 7.8395e-21,\n 1.3285e-19, 1.1245e-21, 4.0666e-21, 2.2212e-19, 1.3788e-20, 5.6395e-20,\n 2.3499e-19, 7.8535e-20, 2.0095e-19, 1.3874e-20, 9.4743e-21, 3.2350e-20,\n 1.7891e-22, 3.5440e-19, 4.4760e-22, 1.3898e-20, 5.3726e-20, 8.7741e-21,\n 3.3687e-20, 1.9520e-22, 4.7918e-20, 1.0089e-20, 3.8357e-20, 5.6780e-21,\n 3.7191e-21, 1.5097e-20, 1.1681e-20, 5.9701e-20, 7.8375e-21, 9.9464e-21,\n 3.1523e-20, 1.5002e-19, 4.5569e-20, 2.3275e-20, 5.0718e-20, 2.3201e-21,\n 2.5976e-20, 2.5532e-19, 1.7859e-20, 5.4393e-21, 2.4245e-20, 4.0332e-21,\n 3.2906e-20, 3.5226e-22, 1.4114e-21, 2.6775e-19, 2.0126e-19, 9.9710e-22,\n 9.9819e-21, 1.6346e-20, 1.6996e-19, 7.0981e-21, 3.3700e-21, 5.7581e-21,\n 3.2535e-22, 2.2558e-20, 6.2547e-21, 3.8186e-20, 5.9664e-21, 1.2812e-21,\n 2.1962e-19, 3.5535e-20, 1.1322e-21, 5.3013e-20, 2.3055e-20, 5.4058e-21,\n 1.1265e-20, 1.6747e-20, 9.6209e-20, 1.7952e-20, 2.6124e-20, 2.9502e-21,\n 2.9332e-22, 8.3401e-21, 2.4634e-21, 3.0767e-20, 9.4618e-20, 1.1380e-19,\n 1.5555e-20, 1.1364e-19, 4.9635e-21, 9.4452e-20, 1.0021e-22, 5.8294e-20,\n 8.3349e-20, 3.3242e-21, 1.3274e-20, 4.0754e-20, 1.6184e-20, 2.8405e-20,\n 2.2053e-20, 5.6055e-20, 3.4364e-21, 2.0861e-20, 1.1365e-20, 1.9108e-20,\n 9.8011e-21, 3.3897e-21, 6.9024e-20, 1.0857e-19, 4.1397e-20, 1.4946e-19,\n 1.4237e-19, 1.2670e-21, 5.0042e-21, 1.4469e-20, 2.8455e-20, 2.5347e-21,\n 4.6749e-20, 3.7239e-20, 2.9476e-20, 4.1266e-20, 1.0415e-19, 1.4074e-20,\n 3.0203e-20, 1.5888e-20, 5.4733e-20, 1.4144e-20, 3.2761e-21, 4.8234e-20,\n 4.9475e-21, 5.7040e-21, 1.7623e-20, 7.1218e-20, 1.8287e-20, 1.2723e-19,\n 2.0515e-20, 3.0165e-21, 6.3857e-21, 1.8167e-20, 4.1264e-21, 4.0760e-21,\n 3.7161e-21, 1.0045e-19, 6.3468e-21, 8.7391e-20, 3.4022e-20, 6.6769e-21,\n 1.5176e-19, 1.2904e-21, 1.2208e-20, 9.2924e-20, 4.3938e-20, 1.6741e-21,\n 8.3765e-23, 2.0403e-19, 6.8378e-19, 1.7869e-20, 2.1747e-19, 2.4499e-20,\n 2.1884e-20, 1.0466e-20, 4.7422e-20, 1.6397e-19, 4.8859e-19, 1.4019e-19,\n 1.3670e-19, 7.1282e-20, 1.9485e-19, 5.7700e-20, 1.1693e-19, 8.7913e-20,\n 1.3248e-20, 2.6856e-20, 1.0867e-20, 8.7860e-20, 6.2066e-20, 1.4482e-19,\n 1.5294e-20, 1.7059e-20, 1.9397e-20, 6.1911e-20, 1.2171e-19, 1.9836e-20,\n 1.9718e-19, 4.5290e-21, 1.7531e-21, 1.0674e-20, 2.3986e-20, 1.2124e-19,\n 1.1295e-19, 1.6815e-20, 9.6923e-21, 1.2739e-21, 2.2194e-21, 3.9760e-21,\n 1.6670e-19, 4.8752e-19, 5.1158e-21, 3.1568e-22, 6.3396e-21, 5.7382e-22,\n 6.7272e-21, 2.4480e-21, 2.2252e-20, 4.5464e-21, 3.2950e-20, 1.4522e-19,\n 2.2343e-20, 4.4079e-19, 8.7809e-20, 9.7673e-21, 7.2114e-20, 4.2638e-20,\n 2.4411e-20, 1.8033e-20, 1.9312e-20, 1.6419e-19, 1.7878e-19, 4.0618e-21,\n 5.5831e-21, 3.0935e-22, 5.6732e-19, 3.2371e-19, 1.4454e-20, 1.0910e-20,\n 8.2654e-20, 9.7937e-21, 2.9780e-20, 1.1566e-21, 1.5556e-19, 2.3547e-21,\n 2.7935e-21, 6.3385e-21, 8.4796e-20, 4.3206e-20, 3.6607e-20, 8.4826e-21,\n 8.5661e-20, 5.1093e-20, 7.4030e-20, 2.7166e-20, 2.8326e-21, 1.1002e-19,\n 5.1670e-20, 9.1034e-20, 4.9389e-20, 3.3808e-21, 2.7551e-20, 9.7512e-21,\n 2.6569e-19, 4.9856e-22, 2.9189e-21, 2.2766e-19], device='cuda:0')"
 },
 "35": {
 "step": "tensor(18770.)",
 "exp_avg": "tensor([[ 5.6052e-45, 5.6052e-45, -5.6052e-45, ..., 5.6052e-45,\n 5.6052e-45, 5.6052e-45],\n [ 5.6052e-45, 5.6052e-45, 5.6052e-45, ..., 5.6052e-45,\n 5.6052e-45, 5.6052e-45],\n [ 5.6052e-45, 5.6052e-45, -5.6052e-45, ..., 5.6052e-45,\n 5.6052e-45, 5.6052e-45],\n ...,\n [-5.6052e-45, -5.6052e-45, 5.6052e-45, ..., -5.6052e-45,\n 5.6052e-45, 5.6052e-45],\n [-5.6052e-45, 5.6052e-45, -5.6052e-45, ..., -5.6052e-45,\n -5.6052e-45, -5.6052e-45],\n [ 5.6052e-45, 5.6052e-45, -5.6052e-45, ..., 5.6052e-45,\n 5.6052e-45, 5.6052e-45]], device='cuda:0')",
 "exp_avg_sq": "tensor([[7.5002e-22, 3.2088e-21, 2.4055e-22, ..., 2.2077e-21, 4.4193e-22,\n 3.6984e-22],\n [1.0831e-21, 4.5053e-21, 7.8908e-22, ..., 1.2446e-21, 8.1896e-22,\n 3.2713e-21],\n [1.5448e-20, 3.7325e-21, 3.0472e-20, ..., 3.4352e-21, 2.2575e-21,\n 3.0067e-21],\n ...,\n [4.6740e-21, 2.9400e-22, 8.2633e-22, ..., 1.3222e-20, 5.1288e-22,\n 1.6807e-21],\n [2.9049e-20, 3.2242e-21, 1.4297e-20, ..., 9.3255e-21, 1.4760e-21,\n 5.5846e-21],\n [1.2672e-21, 1.6801e-21, 1.9705e-21, ..., 5.3523e-22, 1.8365e-22,\n 5.5699e-22]], device='cuda:0')"
 },
 "36": {
 "step": "tensor(18770.)",
 "exp_avg": "tensor([ 5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45,\n -5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45, -5.6052e-45,\n 5.6052e-45, -5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45,\n 5.6052e-45, -5.6052e-45, -5.6052e-45, 5.6052e-45, 5.6052e-45,\n 5.6052e-45, -5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45,\n 5.6052e-45, -5.6052e-45, -5.6052e-45, -5.6052e-45, 5.6052e-45,\n 5.6052e-45, 5.6052e-45, -5.6052e-45, -5.6052e-45, 5.6052e-45,\n -5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45,\n 5.6052e-45, -5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45,\n -5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45, -5.6052e-45,\n -5.6052e-45, -5.6052e-45, 5.6052e-45, -5.6052e-45, -5.6052e-45,\n -5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45,\n -5.6052e-45, -5.6052e-45, 5.6052e-45, -5.6052e-45, -5.6052e-45,\n 5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45,\n 5.6052e-45, -5.6052e-45, -5.6052e-45, -5.6052e-45, -5.6052e-45,\n -5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45,\n -5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45,\n 5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45, -5.6052e-45,\n 5.6052e-45, -5.6052e-45, 5.6052e-45, -5.6052e-45, -5.6052e-45,\n 5.6052e-45, -5.6052e-45, -5.6052e-45, 5.6052e-45, -5.6052e-45,\n 5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45, -5.6052e-45,\n -5.6052e-45, 5.6052e-45, -5.6052e-45, -5.6052e-45, -5.6052e-45,\n -5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45, -5.6052e-45,\n -5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45,\n -5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45,\n 5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45, -5.6052e-45,\n -5.6052e-45, -5.6052e-45, -5.6052e-45, -5.6052e-45, -5.6052e-45,\n 5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45, 5.6052e-45,\n 5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45, 5.6052e-45,\n -5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45, 5.6052e-45,\n -5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45, 5.6052e-45,\n -5.6052e-45, -5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45,\n 5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45,\n 5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45,\n 5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45, -5.6052e-45,\n -5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45,\n 5.6052e-45, -5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45,\n -5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45, -5.6052e-45,\n 5.6052e-45, -5.6052e-45, -5.6052e-45, 5.6052e-45, 5.6052e-45,\n 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45,\n -5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45,\n 5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45, -5.6052e-45,\n 5.6052e-45, -5.6052e-45, -5.6052e-45, 5.6052e-45, 5.6052e-45,\n -5.6052e-45, -5.6052e-45, -5.6052e-45, 5.6052e-45, 5.6052e-45,\n 5.6052e-45, -5.6052e-45, -5.6052e-45, 5.6052e-45, 5.6052e-45,\n 5.6052e-45, -5.6052e-45, -5.6052e-45, -5.6052e-45, 5.6052e-45,\n -5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45,\n 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45,\n 5.6052e-45, -5.6052e-45, -5.6052e-45, 5.6052e-45, -5.6052e-45,\n 5.6052e-45, -5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45,\n 5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45, -5.6052e-45,\n 5.6052e-45], device='cuda:0')",
 "exp_avg_sq": "tensor([1.4792e-18, 1.4248e-18, 1.5029e-17, 2.5682e-17, 2.9205e-17, 2.4405e-17,\n 7.4945e-19, 5.5275e-17, 2.7103e-19, 1.9394e-18, 1.6785e-18, 5.5078e-18,\n 1.1586e-17, 2.1634e-18, 3.0586e-17, 1.9395e-17, 1.3129e-19, 2.8026e-18,\n 5.3025e-17, 4.0657e-17, 1.2486e-17, 3.6096e-19, 1.3036e-17, 1.0430e-17,\n 3.3956e-19, 6.7730e-17, 1.1697e-17, 3.5959e-17, 5.0774e-18, 1.4833e-18,\n 3.7168e-18, 9.3729e-19, 8.9090e-17, 4.1771e-18, 1.3190e-18, 9.1171e-18,\n 6.3101e-18, 2.8167e-18, 7.3772e-17, 2.8693e-18, 5.9200e-18, 2.6513e-17,\n 5.8366e-19, 7.4267e-17, 2.8557e-18, 3.5419e-18, 3.1403e-17, 1.8749e-19,\n 5.1262e-18, 5.4677e-17, 1.6965e-17, 9.9983e-18, 4.5234e-18, 1.3357e-17,\n 6.5150e-17, 1.4731e-18, 3.5940e-19, 2.4561e-17, 2.9356e-16, 1.0208e-17,\n 1.4734e-17, 1.1701e-17, 2.5044e-17, 3.7267e-18, 1.2461e-17, 3.9663e-19,\n 3.6378e-17, 3.6424e-17, 2.5103e-17, 8.5331e-18, 4.8767e-18, 1.5479e-17,\n 5.3981e-17, 9.8555e-18, 1.8119e-18, 1.9121e-19, 4.6720e-18, 4.0895e-18,\n 2.4493e-19, 4.2617e-18, 9.5211e-18, 6.6592e-18, 2.2702e-18, 1.0086e-18,\n 1.9968e-18, 2.8728e-17, 2.7633e-18, 8.5565e-19, 6.3043e-19, 4.1252e-17,\n 5.7959e-18, 4.4066e-17, 2.2592e-18, 2.0807e-17, 7.6788e-17, 6.3781e-18,\n 1.4154e-17, 4.5844e-19, 3.1245e-18, 1.7125e-17, 1.2690e-18, 2.6048e-18,\n 1.6687e-18, 9.7097e-19, 8.9036e-18, 4.4122e-17, 4.9376e-18, 8.6075e-19,\n 1.6839e-18, 1.0717e-17, 2.0335e-17, 7.0312e-18, 2.7361e-19, 1.4855e-19,\n 9.5428e-18, 4.5071e-18, 3.4664e-18, 8.4655e-18, 7.1481e-18, 1.0170e-17,\n 6.4484e-17, 5.1666e-17, 3.0027e-18, 1.3419e-18, 1.0042e-18, 5.3296e-17,\n 8.7414e-18, 4.8686e-19, 2.0031e-17, 7.2814e-17, 1.4060e-17, 2.8659e-17,\n 1.3150e-18, 5.5146e-18, 2.0997e-17, 2.5040e-18, 1.9758e-17, 5.0369e-17,\n 4.9408e-18, 1.8002e-19, 3.2066e-17, 1.4802e-19, 6.4140e-18, 1.0054e-17,\n 8.0779e-19, 3.6628e-17, 3.7529e-17, 1.5538e-18, 1.7816e-17, 2.7901e-17,\n 1.0442e-16, 1.4752e-18, 2.4425e-17, 2.2721e-17, 4.9976e-17, 8.3926e-18,\n 3.9233e-18, 6.2988e-18, 1.0659e-17, 1.5221e-19, 4.9348e-17, 3.0637e-18,\n 4.4039e-19, 6.3396e-18, 3.5947e-19, 2.6529e-17, 1.4721e-17, 1.8952e-17,\n 2.7082e-17, 6.5929e-20, 6.9900e-17, 6.1078e-19, 1.2977e-17, 1.8282e-17,\n 2.4152e-17, 1.2705e-17, 5.4027e-18, 2.3937e-19, 9.2129e-17, 9.4562e-20,\n 4.3554e-18, 6.5107e-18, 1.1022e-17, 1.4177e-17, 4.7225e-18, 1.6644e-18,\n 1.5403e-17, 6.2584e-18, 7.1816e-18, 4.0683e-17, 6.2067e-18, 2.1446e-17,\n 5.3477e-17, 2.5077e-17, 1.6352e-17, 2.1845e-17, 9.5530e-20, 3.2427e-17,\n 1.2620e-17, 3.6893e-17, 2.8564e-18, 2.2778e-18, 1.8358e-18, 1.7657e-18,\n 1.5979e-17, 2.3143e-17, 3.1391e-18, 3.4175e-18, 5.4592e-19, 1.7777e-18,\n 4.0890e-19, 2.5090e-18, 2.6581e-17, 1.1232e-17, 1.3559e-17, 1.0649e-17,\n 1.8302e-17, 9.8486e-18, 3.3688e-18, 2.3105e-19, 3.0543e-17, 1.3947e-18,\n 1.4871e-18, 1.2747e-18, 8.6241e-20, 2.7206e-17, 5.3934e-19, 4.4930e-17,\n 1.4297e-17, 8.0878e-18, 1.0497e-17, 3.0141e-18, 2.8355e-17, 1.7866e-18,\n 5.1488e-19, 9.0243e-19, 2.3283e-17, 7.0493e-18, 9.2148e-18, 7.1515e-18,\n 1.6070e-17, 1.6733e-17, 3.4199e-17, 1.4721e-17, 1.0263e-17, 6.4665e-17,\n 4.3664e-18, 8.6907e-18, 3.7951e-19, 6.7393e-20, 6.1360e-18, 1.3005e-17,\n 1.5195e-17, 8.9923e-18, 3.3904e-17, 1.9255e-18], device='cuda:0')"
 },
 "37": {
 "step": "tensor(18770.)",
 "exp_avg": "tensor([ 5.6052e-45, -5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45,\n 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45,\n -5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45,\n 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45,\n -5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45, -5.6052e-45,\n 5.6052e-45, -5.6052e-45, -5.6052e-45, 5.6052e-45, 5.6052e-45,\n 5.6052e-45, -5.6052e-45, -5.6052e-45, -5.6052e-45, -5.6052e-45,\n -5.6052e-45, -5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45,\n -5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45, -5.6052e-45,\n 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45,\n -5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45,\n -5.6052e-45, -5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45,\n 5.6052e-45, -5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45,\n 5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45,\n 5.6052e-45, -5.6052e-45, -5.6052e-45, -5.6052e-45, 5.6052e-45,\n 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45,\n -5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45,\n 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45,\n 5.6052e-45, -5.6052e-45, 5.6052e-45, -5.6052e-45, -5.6052e-45,\n 5.6052e-45, -5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45,\n 5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45, -5.6052e-45,\n -5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45,\n -5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45,\n 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45,\n -5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45, -5.6052e-45,\n 5.6052e-45, 5.6052e-45, -5.6052e-45, -5.6052e-45, -5.6052e-45,\n -5.6052e-45, -5.6052e-45, 5.6052e-45, -5.6052e-45, -5.6052e-45,\n 5.6052e-45, 5.6052e-45, -5.6052e-45, -5.6052e-45, 5.6052e-45,\n 5.6052e-45, -5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45,\n -5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45, 5.6052e-45,\n -5.6052e-45, -5.6052e-45, -5.6052e-45, 5.6052e-45, 5.6052e-45,\n 5.6052e-45, -5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45,\n 5.6052e-45, 5.6052e-45, -5.6052e-45, -5.6052e-45, -5.6052e-45,\n 5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45, 5.6052e-45,\n 5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45,\n -5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45, 5.6052e-45,\n 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45,\n -5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45,\n -5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45, 5.6052e-45,\n 5.6052e-45, -5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45,\n -5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45,\n 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45,\n -5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45, 5.6052e-45,\n -5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45,\n 5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45, 5.6052e-45,\n 5.6052e-45, 5.6052e-45, -5.6052e-45, -5.6052e-45, -5.6052e-45,\n -5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45, -5.6052e-45,\n 5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45, 5.6052e-45,\n 5.6052e-45, -5.6052e-45, -5.6052e-45, 5.6052e-45, 5.6052e-45,\n 5.6052e-45, -5.6052e-45, -5.6052e-45, 5.6052e-45, 5.6052e-45,\n -5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45, -5.6052e-45,\n 5.6052e-45], device='cuda:0')",
 "exp_avg_sq": "tensor([7.2306e-22, 2.4501e-21, 1.8673e-20, 4.5356e-20, 1.0944e-19, 5.1416e-20,\n 1.9498e-21, 4.7855e-20, 2.4821e-21, 9.7350e-22, 1.3846e-21, 1.6591e-21,\n 4.2451e-20, 2.3741e-21, 4.3079e-20, 5.5986e-20, 8.3863e-22, 1.9450e-20,\n 3.6004e-19, 2.4022e-19, 1.2513e-20, 1.3006e-21, 3.3241e-20, 6.1936e-20,\n 1.6325e-22, 2.0288e-19, 1.2284e-20, 1.1360e-19, 2.0905e-21, 8.7174e-22,\n 7.2227e-21, 3.7460e-22, 2.1503e-19, 2.1983e-20, 8.1548e-21, 2.4761e-21,\n 3.0006e-21, 6.6543e-22, 8.9953e-19, 1.0832e-20, 5.4625e-21, 1.6943e-19,\n 7.3641e-22, 2.2638e-19, 1.3689e-21, 4.6048e-21, 4.0687e-20, 4.9177e-22,\n 4.9603e-21, 3.0076e-19, 9.6334e-21, 2.0218e-22, 1.3212e-20, 3.0840e-20,\n 2.3681e-19, 3.0966e-21, 2.8719e-21, 1.4454e-20, 8.6073e-19, 1.5674e-20,\n 5.9647e-20, 7.1354e-21, 4.8498e-20, 8.3228e-21, 1.0048e-20, 4.8728e-22,\n 2.8024e-19, 5.7337e-19, 4.6443e-20, 5.8732e-20, 2.3615e-21, 1.3607e-20,\n 1.8561e-19, 1.1301e-20, 1.5094e-21, 7.6622e-22, 4.1388e-21, 3.7667e-21,\n 7.3009e-22, 6.4048e-20, 3.1253e-20, 1.2278e-20, 8.5615e-22, 3.8677e-22,\n 7.4913e-22, 1.4969e-19, 1.3277e-21, 7.1439e-22, 2.5503e-21, 1.6077e-19,\n 4.3218e-20, 3.3839e-19, 8.6848e-22, 2.6877e-19, 1.3271e-19, 2.7370e-21,\n 8.1525e-21, 7.7959e-22, 1.0893e-20, 6.1158e-20, 9.0203e-21, 1.6420e-21,\n 1.6293e-20, 2.3370e-22, 9.8080e-21, 5.9894e-20, 1.6245e-20, 2.7433e-21,\n 2.2961e-21, 5.6864e-21, 6.7439e-20, 3.9509e-20, 1.9968e-22, 7.1216e-22,\n 1.2016e-20, 1.0895e-20, 4.5849e-21, 9.7027e-21, 2.6405e-21, 1.2456e-20,\n 1.1939e-19, 1.5070e-19, 2.6934e-21, 6.9173e-21, 1.0196e-21, 2.3558e-19,\n 1.2114e-20, 4.9152e-21, 1.1578e-19, 2.8266e-19, 3.0329e-20, 6.9058e-20,\n 1.2714e-21, 1.1161e-19, 6.6615e-20, 1.1099e-21, 1.0052e-19, 1.4179e-19,\n 1.7540e-20, 1.0701e-21, 5.0980e-20, 6.9335e-22, 3.4458e-21, 1.4944e-20,\n 1.2400e-21, 5.0119e-20, 1.4039e-19, 7.0355e-21, 2.0592e-19, 9.6212e-20,\n 1.3539e-19, 1.4777e-21, 6.0017e-20, 3.0959e-20, 4.6285e-19, 2.4019e-21,\n 7.4105e-21, 5.3096e-21, 1.0051e-20, 5.9506e-22, 1.3778e-19, 5.2655e-22,\n 1.1491e-21, 2.2051e-19, 1.0648e-21, 3.6798e-20, 2.9676e-21, 1.5708e-20,\n 3.8368e-20, 5.7113e-22, 1.7781e-19, 5.8468e-21, 2.9252e-20, 9.0824e-20,\n 3.1830e-20, 4.5887e-21, 6.1831e-21, 3.9312e-22, 4.6994e-19, 1.2571e-21,\n 1.1282e-20, 7.5716e-20, 2.0269e-20, 2.9780e-20, 2.5069e-20, 2.0338e-22,\n 1.2304e-19, 1.8586e-20, 1.7928e-20, 4.3915e-20, 3.1128e-21, 8.9452e-20,\n 8.1800e-20, 5.0465e-20, 2.2888e-20, 1.6798e-19, 2.1118e-21, 4.4211e-20,\n 1.1220e-20, 7.5030e-20, 2.4059e-21, 1.0551e-20, 4.5831e-22, 1.9644e-20,\n 1.3611e-20, 2.7460e-20, 7.7810e-22, 8.7452e-21, 3.2316e-22, 4.3886e-22,\n 8.1146e-23, 9.1671e-21, 2.6562e-20, 5.6200e-21, 3.6671e-20, 2.2306e-20,\n 2.1807e-20, 4.5623e-20, 1.5528e-21, 8.5638e-22, 1.3709e-20, 2.1813e-21,\n 1.4323e-21, 8.1954e-22, 6.5648e-21, 2.2317e-20, 2.7566e-22, 1.3474e-19,\n 2.8662e-20, 1.1262e-20, 3.6617e-20, 2.5385e-21, 1.7073e-19, 7.8517e-21,\n 1.3191e-21, 6.9788e-22, 1.8076e-19, 2.7664e-21, 2.0481e-20, 1.1030e-19,\n 4.2264e-20, 5.1073e-20, 1.0096e-19, 4.4769e-20, 1.1223e-20, 2.8708e-19,\n 8.8035e-21, 6.7541e-21, 1.7222e-22, 2.4622e-22, 4.0518e-21, 1.6230e-20,\n 3.8582e-20, 6.0813e-21, 8.1334e-20, 5.1954e-21], device='cuda:0')"
 },
 "38": {
 "step": "tensor(18770.)",
 "exp_avg": "tensor([ 5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45,\n -5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45, -5.6052e-45,\n 5.6052e-45, -5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45,\n 5.6052e-45, -5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45,\n 5.6052e-45, -5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45,\n 5.6052e-45, -5.6052e-45, -5.6052e-45, -5.6052e-45, 5.6052e-45,\n 5.6052e-45, 5.6052e-45, -5.6052e-45, -5.6052e-45, 5.6052e-45,\n -5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45,\n 5.6052e-45, -5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45,\n -5.6052e-45, 5.6052e-45, -5.6052e-45, -5.6052e-45, -5.6052e-45,\n -5.6052e-45, -5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45,\n -5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45, -5.6052e-45,\n 5.6052e-45, -5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45,\n 5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45,\n 5.6052e-45, -5.6052e-45, -5.6052e-45, -5.6052e-45, -5.6052e-45,\n -5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45,\n -5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45,\n 5.6052e-45, 5.6052e-45, -5.6052e-45, -5.6052e-45, -5.6052e-45,\n 5.6052e-45, -5.6052e-45, 5.6052e-45, -5.6052e-45, -5.6052e-45,\n 5.6052e-45, -5.6052e-45, -5.6052e-45, 5.6052e-45, -5.6052e-45,\n 5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45, -5.6052e-45,\n -5.6052e-45, 5.6052e-45, -5.6052e-45, -5.6052e-45, 5.6052e-45,\n 5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45, -5.6052e-45,\n -5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45,\n -5.6052e-45, -5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45,\n 5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45, -5.6052e-45,\n -5.6052e-45, -5.6052e-45, -5.6052e-45, -5.6052e-45, -5.6052e-45,\n 5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45, -5.6052e-45,\n 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45,\n -5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45, 5.6052e-45,\n -5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45, 5.6052e-45,\n 5.6052e-45, -5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45,\n 5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45,\n 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45,\n 5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45, -5.6052e-45,\n -5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45,\n 5.6052e-45, -5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45,\n -5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45,\n 5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45, 5.6052e-45,\n 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45,\n -5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45,\n 5.6052e-45, 5.6052e-45, -5.6052e-45, -5.6052e-45, -5.6052e-45,\n 5.6052e-45, -5.6052e-45, -5.6052e-45, 5.6052e-45, 5.6052e-45,\n -5.6052e-45, -5.6052e-45, -5.6052e-45, 5.6052e-45, -5.6052e-45,\n 5.6052e-45, -5.6052e-45, -5.6052e-45, -5.6052e-45, -5.6052e-45,\n 5.6052e-45, -5.6052e-45, -5.6052e-45, -5.6052e-45, 5.6052e-45,\n -5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45, -5.6052e-45,\n -5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45,\n 5.6052e-45, -5.6052e-45, -5.6052e-45, 5.6052e-45, -5.6052e-45,\n 5.6052e-45, -5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45,\n 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45,\n 5.6052e-45], device='cuda:0')",
 "exp_avg_sq": "tensor([2.4860e-21, 4.0901e-21, 3.9614e-20, 3.2640e-20, 1.1029e-19, 8.0547e-20,\n 4.5672e-21, 1.5536e-19, 8.0100e-22, 7.0848e-21, 3.2397e-21, 1.1920e-20,\n 5.6474e-20, 1.6798e-20, 5.3356e-20, 8.2770e-20, 1.3849e-22, 1.9115e-20,\n 2.0738e-19, 1.2995e-19, 4.5563e-20, 5.6946e-21, 3.0846e-20, 4.9076e-20,\n 5.8152e-22, 2.3806e-19, 5.5991e-20, 9.8356e-20, 6.2252e-21, 2.5456e-21,\n 1.3938e-20, 7.5160e-23, 1.1970e-19, 3.3019e-20, 3.0348e-21, 5.2017e-21,\n 1.1783e-20, 1.1249e-20, 3.1052e-19, 8.1573e-21, 1.7791e-20, 6.9627e-20,\n 1.2078e-21, 2.5697e-19, 1.2152e-20, 1.8700e-20, 8.4831e-20, 2.6342e-22,\n 1.0825e-20, 1.7454e-19, 2.4722e-20, 1.1681e-20, 1.8022e-20, 1.0731e-19,\n 1.2940e-19, 1.4875e-20, 1.0054e-21, 8.6040e-20, 4.3601e-19, 4.0812e-20,\n 2.2199e-20, 2.2898e-20, 9.6042e-20, 3.1435e-20, 1.4614e-20, 4.5344e-22,\n 1.2060e-19, 2.5969e-19, 4.8331e-20, 4.0550e-20, 1.3547e-20, 2.9380e-20,\n 9.1884e-20, 1.9482e-20, 9.0409e-21, 2.5127e-21, 1.4059e-20, 6.9862e-21,\n 7.6952e-22, 3.3929e-20, 2.1599e-20, 2.2003e-20, 3.3792e-21, 1.8400e-21,\n 1.6507e-21, 1.2064e-19, 5.0061e-21, 3.5327e-21, 1.1321e-20, 1.1692e-19,\n 3.9461e-20, 1.3638e-19, 5.6106e-21, 1.3217e-19, 1.0187e-19, 2.6660e-20,\n 2.5742e-20, 4.1704e-21, 8.2189e-21, 6.0564e-20, 1.0125e-20, 3.9172e-21,\n 2.4849e-20, 4.5436e-21, 2.5645e-20, 5.0489e-20, 3.2307e-20, 1.6675e-20,\n 4.9437e-21, 1.0728e-20, 6.8972e-20, 3.1118e-20, 1.2396e-21, 2.4211e-21,\n 3.5768e-20, 2.5538e-20, 1.3165e-20, 3.2247e-20, 3.3634e-20, 2.9757e-20,\n 8.1436e-20, 1.3707e-19, 1.0233e-20, 3.2031e-21, 1.7102e-21, 1.7202e-19,\n 2.2478e-20, 2.2833e-21, 1.0854e-19, 6.1899e-20, 9.5836e-20, 5.9674e-20,\n 1.8924e-20, 4.8012e-20, 8.5838e-20, 7.6395e-21, 1.0367e-19, 9.2669e-20,\n 3.1691e-20, 2.3723e-22, 1.1095e-19, 2.8638e-22, 1.1167e-20, 3.3172e-20,\n 1.3193e-21, 5.5850e-20, 6.8367e-20, 2.1505e-20, 6.5965e-20, 8.8436e-20,\n 1.2647e-19, 1.7719e-21, 2.5333e-20, 7.6782e-20, 2.1109e-19, 8.1963e-21,\n 4.1865e-21, 2.7819e-20, 3.6646e-20, 2.3526e-22, 1.5021e-19, 4.9378e-21,\n 1.5689e-21, 8.8013e-20, 9.8282e-22, 9.3674e-20, 4.0364e-20, 6.2020e-20,\n 3.9710e-20, 8.1069e-23, 2.0491e-19, 9.5755e-21, 4.9815e-20, 9.1999e-20,\n 5.5076e-20, 1.3601e-20, 1.6120e-20, 5.3539e-22, 3.0209e-19, 3.8875e-22,\n 7.3012e-21, 4.8825e-20, 4.5437e-20, 2.7465e-20, 3.2878e-20, 1.3523e-21,\n 7.9439e-20, 4.0297e-20, 1.0112e-20, 2.9376e-20, 1.0590e-20, 5.6396e-20,\n 5.2060e-20, 4.8965e-20, 6.3354e-20, 6.2896e-20, 6.1300e-22, 7.5887e-20,\n 4.2555e-20, 1.2438e-19, 3.1090e-21, 7.5469e-21, 9.4397e-21, 1.1298e-20,\n 3.8819e-20, 6.6706e-20, 4.8330e-21, 2.6233e-20, 1.2203e-21, 1.2634e-20,\n 6.5282e-22, 3.3054e-20, 9.0718e-20, 1.9047e-20, 7.0177e-20, 2.7020e-20,\n 5.9236e-20, 9.6519e-20, 1.4122e-20, 1.4135e-21, 7.0599e-20, 1.3520e-20,\n 2.9603e-20, 6.4838e-21, 6.8546e-22, 8.9319e-20, 4.4878e-21, 9.4180e-20,\n 4.2665e-20, 3.6496e-20, 3.2052e-20, 7.2222e-21, 1.4669e-19, 8.2200e-21,\n 7.9757e-21, 2.3867e-22, 1.0500e-19, 1.8839e-20, 4.1106e-20, 3.8046e-20,\n 7.2064e-20, 3.7492e-20, 6.3639e-20, 5.2221e-20, 2.4580e-20, 2.2432e-19,\n 2.3745e-20, 3.2107e-20, 5.4454e-23, 8.3572e-22, 1.5066e-20, 2.1650e-20,\n 6.6299e-20, 1.2688e-20, 9.1610e-20, 1.1262e-20], device='cuda:0')"
 },
 "39": {
 "step": "tensor(18770.)",
 "exp_avg": "tensor([[ 5.6052e-45, -5.6052e-45, -5.6052e-45, ..., 5.6052e-45,\n -5.6052e-45, -5.6052e-45],\n [ 5.6052e-45, -5.6052e-45, -5.6052e-45, ..., 5.6052e-45,\n -5.6052e-45, 5.6052e-45],\n [ 5.6052e-45, 5.6052e-45, -5.6052e-45, ..., 5.6052e-45,\n 5.6052e-45, 5.6052e-45],\n ...,\n [-5.6052e-45, -5.6052e-45, 5.6052e-45, ..., -5.6052e-45,\n -5.6052e-45, -5.6052e-45],\n [ 5.6052e-45, 5.6052e-45, 5.6052e-45, ..., 5.6052e-45,\n 5.6052e-45, 5.6052e-45],\n [ 5.6052e-45, 5.6052e-45, -5.6052e-45, ..., 5.6052e-45,\n 5.6052e-45, 5.6052e-45]], device='cuda:0')",
 "exp_avg_sq": "tensor([[8.3201e-22, 2.3555e-21, 4.3219e-21, ..., 1.4691e-21, 3.4638e-22,\n 2.0136e-21],\n [9.9977e-21, 1.3055e-20, 2.5298e-21, ..., 5.8966e-21, 1.3808e-21,\n 8.1939e-21],\n [3.1372e-20, 1.0647e-20, 2.2168e-20, ..., 4.7589e-21, 3.0019e-21,\n 5.4443e-21],\n ...,\n [7.6541e-20, 6.0782e-21, 1.0966e-20, ..., 1.0981e-20, 4.0144e-21,\n 1.1925e-20],\n [2.8158e-21, 5.9960e-21, 1.1269e-20, ..., 1.4447e-21, 4.3782e-22,\n 1.0028e-21],\n [1.3836e-21, 6.5444e-22, 4.3164e-21, ..., 7.1013e-22, 8.7366e-22,\n 3.8733e-22]], device='cuda:0')"
 },
 "40": {
 "step": "tensor(18770.)",
 "exp_avg": "tensor([ 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45,\n -5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45,\n -5.6052e-45, 5.6052e-45, -5.6052e-45, -5.6052e-45, -5.6052e-45,\n 5.6052e-45, -5.6052e-45, -5.6052e-45, 5.6052e-45, 5.6052e-45,\n 5.6052e-45, 5.6052e-45, -5.6052e-45, -5.6052e-45, -5.6052e-45,\n 5.6052e-45, 5.6052e-45, -5.6052e-45, -5.6052e-45, 5.6052e-45,\n 5.6052e-45, -5.6052e-45, -5.6052e-45, -5.6052e-45, -5.6052e-45,\n -5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45, -5.6052e-45,\n 5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45, 5.6052e-45,\n 5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45, -5.6052e-45,\n 5.6052e-45, -5.6052e-45, 5.6052e-45, -5.6052e-45, -5.6052e-45,\n -5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45,\n -5.6052e-45, -5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45,\n -5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45,\n 5.6052e-45, 5.6052e-45, -5.6052e-45, -5.6052e-45, 5.6052e-45,\n 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45,\n -5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45, -5.6052e-45,\n 5.6052e-45, -5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45,\n 5.6052e-45, -5.6052e-45, -5.6052e-45, -5.6052e-45, -5.6052e-45,\n 5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45,\n -5.6052e-45, -5.6052e-45, -5.6052e-45, -5.6052e-45, -5.6052e-45,\n -5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45, -5.6052e-45,\n -5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45,\n -5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45,\n -5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45,\n 5.6052e-45, 5.6052e-45, -5.6052e-45, -5.6052e-45, -5.6052e-45,\n 5.6052e-45, -5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45,\n 5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45, 5.6052e-45,\n 5.6052e-45, -5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45,\n -5.6052e-45, 5.6052e-45, -5.6052e-45, -5.6052e-45, -5.6052e-45,\n -5.6052e-45, -5.6052e-45, -5.6052e-45, 5.6052e-45, 5.6052e-45,\n -5.6052e-45, -5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45,\n 5.6052e-45, -5.6052e-45, 5.6052e-45, -5.6052e-45, -5.6052e-45,\n 5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45,\n 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45,\n 5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45, 5.6052e-45,\n -5.6052e-45, -5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45,\n -5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45, -5.6052e-45,\n -5.6052e-45, -5.6052e-45, -5.6052e-45, 5.6052e-45, 5.6052e-45,\n -5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45,\n -5.6052e-45, -5.6052e-45, -5.6052e-45, -5.6052e-45, 5.6052e-45,\n -5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45, 5.6052e-45,\n 5.6052e-45, -5.6052e-45, -5.6052e-45, 5.6052e-45, 5.6052e-45,\n -5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45, 5.6052e-45,\n -5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45,\n 5.6052e-45, -5.6052e-45, -5.6052e-45, -5.6052e-45, 5.6052e-45,\n -5.6052e-45, -5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45,\n -5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45,\n 5.6052e-45, 5.6052e-45, -5.6052e-45, -5.6052e-45, -5.6052e-45,\n 5.6052e-45, -5.6052e-45, -5.6052e-45, 5.6052e-45, 5.6052e-45,\n 5.6052e-45, -5.6052e-45, 5.6052e-45, -5.6052e-45, -5.6052e-45,\n 5.6052e-45], device='cuda:0')",
 "exp_avg_sq": "tensor([6.4436e-18, 9.1057e-18, 1.6452e-17, 1.9487e-17, 1.1280e-17, 1.7460e-17,\n 2.2794e-18, 2.6680e-18, 7.5770e-17, 4.2982e-18, 1.1681e-17, 2.1725e-17,\n 8.7060e-20, 4.9519e-17, 1.0221e-18, 9.6015e-18, 1.4899e-17, 3.0526e-18,\n 1.4653e-16, 3.6666e-17, 1.3050e-16, 3.2218e-18, 3.2557e-18, 2.1096e-17,\n 4.1775e-18, 1.3853e-17, 5.8128e-17, 4.9831e-17, 2.8245e-18, 2.1073e-17,\n 1.3007e-17, 2.0656e-17, 2.5808e-16, 2.0228e-18, 6.0644e-17, 1.5827e-17,\n 1.7044e-17, 5.1232e-18, 2.2544e-16, 1.4062e-17, 5.6197e-19, 3.1373e-17,\n 2.8074e-19, 6.7614e-18, 2.8361e-17, 9.2505e-19, 3.1378e-17, 1.6850e-17,\n 1.8516e-18, 5.1608e-17, 1.5129e-18, 6.8811e-17, 2.4545e-17, 1.9501e-19,\n 1.0639e-17, 2.3417e-17, 2.2363e-17, 4.3940e-17, 2.7747e-16, 1.9464e-17,\n 5.1938e-17, 1.7165e-17, 7.2266e-17, 7.5294e-18, 4.3452e-18, 1.6141e-18,\n 4.3353e-17, 2.0422e-18, 3.3800e-19, 5.7627e-17, 7.6800e-19, 2.4692e-18,\n 2.0298e-16, 2.6407e-17, 6.8202e-18, 5.7838e-18, 2.0556e-17, 2.3128e-17,\n 3.1560e-17, 1.6517e-18, 1.9175e-17, 2.6987e-18, 1.6565e-17, 3.3042e-17,\n 8.5190e-18, 8.2840e-18, 1.4874e-17, 1.1970e-17, 9.4650e-18, 2.3937e-17,\n 6.2739e-18, 3.9272e-18, 9.5031e-19, 6.1736e-17, 2.0073e-17, 1.7441e-17,\n 1.4186e-18, 4.7981e-18, 4.1343e-19, 7.1695e-19, 1.7194e-18, 4.8288e-19,\n 1.1048e-18, 1.3840e-17, 2.3045e-17, 2.8707e-17, 3.2137e-17, 1.3785e-18,\n 1.0690e-17, 1.0317e-17, 4.3864e-17, 2.1021e-17, 1.6876e-17, 3.0665e-17,\n 7.7018e-17, 1.0900e-17, 2.9028e-17, 2.2962e-17, 1.6774e-17, 8.5830e-19,\n 5.4045e-17, 1.4868e-17, 2.1288e-17, 7.8723e-19, 2.3296e-18, 2.5202e-17,\n 1.2669e-17, 6.8456e-18, 5.4288e-17, 1.0152e-16, 1.0405e-17, 2.8714e-17,\n 1.3842e-17, 5.9183e-17, 4.3195e-19, 4.1305e-18, 5.2364e-18, 7.5547e-17,\n 4.6524e-18, 6.4718e-19, 1.6108e-16, 1.1894e-17, 2.4614e-18, 4.2526e-18,\n 1.7890e-18, 7.2385e-17, 4.6093e-17, 4.4558e-17, 3.6422e-17, 1.1693e-17,\n 1.8781e-17, 1.6459e-17, 1.1367e-18, 5.4517e-17, 3.0045e-17, 1.5740e-17,\n 1.1905e-19, 4.8671e-19, 1.8562e-16, 9.3184e-19, 5.7173e-17, 9.2423e-18,\n 1.5783e-17, 2.3667e-18, 2.3278e-17, 1.0882e-16, 1.9456e-16, 4.5494e-17,\n 1.1926e-17, 1.9339e-18, 9.7053e-17, 2.5377e-17, 1.3399e-18, 5.2638e-18,\n 6.4733e-19, 1.5602e-18, 8.6873e-18, 3.8954e-17, 1.3745e-16, 5.6286e-18,\n 1.7406e-17, 1.1495e-17, 1.0765e-18, 4.0679e-18, 2.6679e-17, 3.4003e-17,\n 3.0732e-18, 4.3551e-18, 2.3865e-17, 2.9290e-17, 2.0257e-18, 4.6215e-18,\n 9.2025e-17, 1.5552e-17, 1.9528e-18, 2.1606e-18, 1.2064e-17, 4.5345e-17,\n 5.0756e-17, 9.6736e-17, 3.7921e-17, 1.4029e-18, 1.2797e-18, 4.0868e-17,\n 3.5324e-17, 1.4748e-17, 6.3047e-20, 7.4384e-19, 3.4536e-18, 9.6739e-18,\n 1.5378e-18, 5.3162e-19, 3.0700e-17, 7.6352e-17, 6.7553e-19, 3.0903e-17,\n 1.5483e-18, 1.1778e-18, 3.5369e-17, 2.5227e-17, 1.0307e-18, 2.5135e-17,\n 2.0454e-19, 4.4116e-17, 1.4772e-16, 5.2616e-18, 4.9838e-18, 2.4154e-17,\n 3.7759e-17, 4.3228e-17, 3.3001e-17, 3.1698e-18, 9.8069e-18, 1.1432e-17,\n 6.0612e-18, 2.7171e-18, 7.6929e-17, 4.3432e-17, 2.7569e-17, 2.1340e-18,\n 1.5528e-18, 2.0690e-17, 2.1523e-17, 5.7748e-19, 5.3623e-18, 1.1114e-17,\n 6.7087e-17, 1.3943e-18, 2.0707e-17, 1.8191e-17, 7.4337e-18, 2.3005e-18,\n 4.0076e-19, 7.3366e-17, 6.7759e-18, 2.2384e-18], device='cuda:0')"
 },
 "41": {
 "step": "tensor(18770.)",
 "exp_avg": "tensor([ 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45,\n -5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45,\n -5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45,\n 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45,\n 5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45, 5.6052e-45,\n 5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45, 5.6052e-45,\n 5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45, 5.6052e-45,\n 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45,\n 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45,\n 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45,\n -5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45,\n -5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45,\n -5.6052e-45, -5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45,\n 5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45,\n 5.6052e-45, -5.6052e-45, -5.6052e-45, -5.6052e-45, 5.6052e-45,\n 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45,\n -5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45,\n 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45,\n 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45,\n 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45,\n 5.6052e-45, -5.6052e-45, -5.6052e-45, -5.6052e-45, 5.6052e-45,\n -5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45,\n 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45,\n 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45,\n -5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45,\n 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45,\n 5.6052e-45, -5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45,\n 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45,\n 5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45, -5.6052e-45,\n 5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45, -5.6052e-45,\n 5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45, 5.6052e-45,\n 5.6052e-45, -5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45,\n 5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45,\n 5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45,\n 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45,\n -5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45, 5.6052e-45,\n 5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45,\n 5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45, -5.6052e-45,\n 5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45, 5.6052e-45,\n -5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45,\n -5.6052e-45, -5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45,\n 5.6052e-45, -5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45,\n -5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45,\n 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45,\n -5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45,\n 5.6052e-45, 5.6052e-45, -5.6052e-45, -5.6052e-45, 5.6052e-45,\n 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45,\n -5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45,\n 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45,\n 5.6052e-45, -5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45,\n 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45,\n 5.6052e-45], device='cuda:0')",
 "exp_avg_sq": "tensor([5.0097e-21, 6.0178e-21, 1.6738e-20, 1.9421e-20, 1.6098e-20, 8.1828e-20,\n 1.3349e-20, 2.8808e-21, 2.3871e-19, 2.5842e-21, 2.3619e-21, 2.7256e-20,\n 3.6959e-22, 2.2968e-20, 2.3216e-21, 2.2631e-20, 1.0727e-19, 2.4611e-21,\n 1.4683e-18, 3.4605e-19, 1.6057e-18, 1.4257e-20, 4.9824e-21, 5.9260e-19,\n 7.8921e-21, 8.0961e-21, 3.5213e-19, 1.2576e-20, 8.1025e-22, 1.2286e-19,\n 6.3029e-21, 8.5687e-21, 4.8496e-19, 1.2691e-20, 2.7643e-19, 4.4846e-20,\n 4.9827e-21, 1.0458e-21, 1.0822e-18, 1.7910e-20, 1.8298e-21, 2.9173e-19,\n 3.2089e-22, 4.1621e-21, 1.2792e-19, 3.2766e-22, 6.1759e-20, 1.4942e-20,\n 2.3896e-21, 7.2895e-20, 9.0414e-22, 1.1786e-19, 3.3972e-20, 5.0769e-21,\n 3.6269e-21, 6.6344e-21, 1.0318e-19, 3.8114e-20, 3.1116e-19, 3.8258e-20,\n 2.4611e-20, 2.8570e-21, 2.3707e-19, 1.7553e-20, 1.6884e-21, 3.5163e-21,\n 8.6508e-19, 1.7451e-20, 1.2080e-21, 3.3045e-19, 1.9869e-21, 1.1095e-21,\n 2.6664e-19, 1.2279e-20, 3.4497e-21, 9.9039e-22, 2.7364e-20, 2.6169e-20,\n 2.1212e-20, 1.0342e-21, 5.8141e-21, 2.6215e-21, 1.8835e-19, 3.0168e-20,\n 3.2587e-21, 2.1258e-21, 3.9707e-21, 1.3340e-20, 1.3390e-20, 5.0620e-20,\n 9.3587e-21, 3.0165e-20, 5.6871e-22, 5.8826e-20, 2.8751e-21, 5.5669e-20,\n 3.0575e-21, 4.8200e-20, 3.0374e-22, 1.0318e-21, 5.7350e-22, 2.8891e-21,\n 3.1076e-22, 1.1196e-20, 3.9434e-20, 6.4668e-21, 1.5349e-19, 2.5984e-21,\n 1.3326e-20, 1.2666e-20, 4.3629e-19, 1.8769e-19, 7.3507e-20, 5.1194e-20,\n 1.8228e-19, 1.0876e-19, 4.9898e-19, 1.4084e-20, 8.0329e-21, 1.7135e-21,\n 3.8129e-20, 3.3801e-21, 3.9019e-20, 5.5857e-21, 1.3823e-21, 3.2034e-20,\n 7.0242e-21, 2.9764e-21, 1.1146e-19, 1.9263e-19, 1.2985e-20, 6.6058e-22,\n 7.7715e-21, 6.2276e-19, 2.0720e-21, 1.5591e-21, 3.1909e-21, 5.6530e-19,\n 1.8837e-21, 2.6212e-22, 8.4326e-19, 1.0958e-20, 1.2672e-21, 2.6941e-21,\n 8.8867e-21, 9.6505e-20, 3.2763e-20, 1.5676e-19, 2.3456e-21, 2.1107e-22,\n 1.6661e-20, 5.1166e-20, 9.4672e-22, 3.5858e-19, 2.6167e-19, 2.0270e-20,\n 7.8599e-22, 9.1581e-22, 4.4310e-19, 5.4646e-22, 5.2108e-19, 6.5238e-22,\n 8.7984e-21, 1.3404e-21, 1.1772e-20, 1.8977e-19, 1.6940e-19, 1.3759e-20,\n 7.1574e-21, 8.3826e-22, 1.1756e-19, 1.3846e-19, 1.5457e-21, 7.6409e-20,\n 1.2777e-21, 1.8683e-21, 4.5561e-21, 3.9491e-21, 2.1580e-18, 3.0967e-20,\n 3.0343e-20, 3.9250e-21, 4.2079e-22, 8.7968e-23, 2.2230e-20, 6.7046e-20,\n 6.9985e-21, 2.7940e-21, 1.9558e-20, 8.4368e-21, 2.5981e-21, 8.6727e-21,\n 1.7001e-19, 8.6823e-21, 2.4224e-21, 6.8715e-22, 2.6641e-20, 5.7674e-20,\n 1.1490e-19, 1.0108e-18, 1.8472e-20, 5.6825e-22, 1.4558e-21, 4.1101e-21,\n 1.7136e-20, 1.9820e-20, 3.4951e-22, 1.2139e-21, 6.2806e-21, 3.1061e-21,\n 3.6528e-22, 3.2961e-21, 7.4093e-20, 7.1437e-20, 7.9848e-22, 2.4085e-20,\n 1.9744e-21, 4.8066e-21, 4.4507e-20, 2.8024e-20, 2.2711e-21, 4.2256e-20,\n 6.3851e-22, 4.0727e-20, 5.0891e-19, 4.5749e-22, 1.1069e-20, 9.1964e-21,\n 2.3655e-20, 4.2743e-19, 2.4940e-19, 3.0180e-21, 1.0133e-19, 4.4546e-21,\n 1.7946e-19, 4.3336e-22, 5.3781e-19, 1.4351e-20, 3.2746e-20, 7.0814e-20,\n 8.5361e-21, 6.4618e-20, 1.9090e-20, 8.1265e-22, 1.1678e-20, 4.5072e-21,\n 6.0377e-20, 2.2188e-21, 3.6306e-20, 5.7152e-20, 1.4372e-21, 3.3348e-21,\n 1.4753e-21, 5.1444e-20, 6.1100e-21, 2.1610e-21], device='cuda:0')"
 },
 "42": {
 "step": "tensor(18770.)",
 "exp_avg": "tensor([ 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45,\n -5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45,\n -5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45, -5.6052e-45,\n 5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45, 5.6052e-45,\n 5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45, 5.6052e-45,\n 5.6052e-45, 5.6052e-45, -5.6052e-45, -5.6052e-45, 5.6052e-45,\n 5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45, 5.6052e-45,\n 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45,\n 5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45, 5.6052e-45,\n 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45,\n 5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45,\n -5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45,\n -5.6052e-45, -5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45,\n 5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45,\n 5.6052e-45, 5.6052e-45, -5.6052e-45, -5.6052e-45, 5.6052e-45,\n 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45,\n -5.6052e-45, -5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45,\n 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45,\n 5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45, -5.6052e-45,\n 5.6052e-45, -5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45,\n -5.6052e-45, -5.6052e-45, -5.6052e-45, -5.6052e-45, 5.6052e-45,\n -5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45,\n 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45,\n 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45,\n -5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45,\n 5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45, -5.6052e-45,\n 5.6052e-45, -5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45,\n 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45,\n 5.6052e-45, -5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45,\n -5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45,\n -5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45, 5.6052e-45,\n 5.6052e-45, -5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45,\n 5.6052e-45, -5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45,\n 5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45, -5.6052e-45,\n 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45,\n 5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45, 5.6052e-45,\n 5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45,\n 5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45, -5.6052e-45,\n 5.6052e-45, -5.6052e-45, -5.6052e-45, 5.6052e-45, -5.6052e-45,\n -5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45,\n -5.6052e-45, -5.6052e-45, -5.6052e-45, -5.6052e-45, 5.6052e-45,\n 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45,\n 5.6052e-45, -5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45,\n 5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45, 5.6052e-45,\n -5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45,\n 5.6052e-45, 5.6052e-45, -5.6052e-45, -5.6052e-45, 5.6052e-45,\n 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45,\n -5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45,\n -5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45, -5.6052e-45,\n 5.6052e-45, -5.6052e-45, -5.6052e-45, 5.6052e-45, 5.6052e-45,\n 5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45,\n 5.6052e-45], device='cuda:0')",
 "exp_avg_sq": "tensor([3.3059e-20, 2.9286e-20, 3.8082e-20, 5.1728e-20, 5.0036e-20, 5.4994e-20,\n 1.2747e-20, 4.1490e-21, 2.6043e-19, 5.6728e-21, 9.9940e-21, 9.5216e-20,\n 6.0279e-22, 2.6479e-20, 2.4262e-21, 5.2111e-20, 8.2889e-20, 5.8714e-21,\n 5.7742e-19, 2.3678e-19, 5.5132e-19, 2.8848e-20, 1.1733e-20, 1.7458e-19,\n 1.6074e-20, 5.3007e-20, 2.1098e-19, 2.4584e-20, 3.0329e-21, 1.2072e-19,\n 4.4591e-20, 1.9940e-20, 1.7351e-19, 1.9185e-20, 1.6955e-19, 2.8023e-20,\n 2.8813e-20, 1.2813e-20, 6.8556e-19, 4.9289e-20, 8.1347e-22, 1.8790e-19,\n 3.4894e-21, 3.0810e-20, 9.4478e-20, 2.1154e-21, 4.3866e-20, 2.9837e-20,\n 7.3711e-21, 1.5157e-19, 1.0365e-21, 8.0076e-20, 9.5658e-20, 6.3716e-21,\n 7.2538e-21, 1.1583e-20, 1.3105e-19, 1.4917e-19, 3.2020e-19, 6.4856e-20,\n 2.7323e-20, 5.1638e-21, 3.0074e-19, 3.9768e-20, 2.7908e-21, 1.1189e-20,\n 3.0246e-19, 1.8715e-20, 3.1782e-23, 2.5677e-19, 5.2914e-21, 1.1885e-21,\n 1.4893e-19, 2.6937e-20, 9.5286e-21, 2.6472e-20, 8.9984e-20, 3.2452e-20,\n 8.4547e-20, 1.0574e-20, 4.0388e-21, 4.5263e-21, 1.2232e-19, 3.7081e-20,\n 3.6512e-21, 2.3829e-20, 2.0984e-21, 3.5246e-20, 3.7043e-20, 7.8303e-20,\n 4.2022e-20, 2.4446e-20, 7.7895e-22, 1.0061e-19, 1.3463e-20, 9.8019e-20,\n 1.4767e-22, 2.3592e-20, 1.3159e-21, 3.6605e-22, 4.9208e-21, 3.7994e-21,\n 5.8814e-22, 1.1494e-20, 3.0129e-20, 1.0552e-20, 1.9911e-19, 2.2092e-21,\n 6.2193e-21, 7.3276e-21, 1.7067e-19, 1.4217e-19, 1.1745e-19, 1.4399e-19,\n 7.9261e-20, 9.1177e-20, 2.3719e-19, 9.0463e-20, 6.5753e-20, 7.3326e-22,\n 5.3998e-20, 5.0363e-20, 5.4651e-20, 1.1936e-21, 1.3313e-20, 6.5876e-20,\n 1.9230e-20, 1.9743e-20, 1.2087e-19, 8.3371e-20, 3.6236e-20, 3.8854e-20,\n 3.8288e-20, 2.4014e-19, 3.4649e-22, 1.1502e-20, 4.5037e-20, 2.1246e-19,\n 2.6927e-20, 1.0159e-20, 5.8674e-19, 2.7811e-20, 5.0616e-22, 1.2933e-20,\n 1.2660e-20, 9.7898e-20, 7.0564e-20, 6.6828e-20, 3.0407e-20, 8.9359e-21,\n 2.1858e-20, 4.9939e-20, 2.3080e-21, 2.6955e-19, 2.0985e-19, 3.0228e-20,\n 1.5853e-21, 8.5744e-22, 5.1287e-19, 3.3814e-21, 2.6468e-19, 4.7145e-21,\n 2.4834e-20, 1.3989e-21, 2.5456e-20, 3.4661e-19, 5.0981e-19, 1.1438e-19,\n 1.1142e-20, 3.1900e-21, 2.8586e-19, 1.6102e-19, 7.7179e-21, 7.8604e-20,\n 3.7596e-21, 1.0375e-21, 1.1541e-20, 2.0748e-20, 5.6831e-19, 3.1507e-20,\n 3.4739e-20, 7.7320e-21, 6.0353e-21, 1.8641e-21, 1.2643e-19, 8.0939e-20,\n 2.6715e-20, 1.5864e-20, 2.6607e-20, 2.4583e-20, 1.1132e-20, 1.5734e-20,\n 8.1995e-20, 2.1002e-20, 3.5158e-21, 7.8343e-21, 6.5673e-20, 9.6848e-20,\n 1.3069e-19, 3.9933e-19, 2.4843e-20, 1.1248e-21, 3.9974e-21, 1.6924e-21,\n 5.8145e-20, 3.3458e-20, 1.6975e-22, 1.1685e-20, 1.2144e-20, 1.6647e-20,\n 1.6633e-21, 4.5251e-21, 9.7056e-20, 9.5353e-20, 7.3269e-21, 3.6813e-20,\n 2.0450e-21, 1.7939e-20, 1.1849e-19, 9.7905e-20, 1.7458e-21, 1.1281e-19,\n 7.6050e-22, 1.6063e-19, 3.9842e-19, 1.5483e-20, 1.6693e-20, 1.3925e-20,\n 5.5533e-20, 2.5163e-19, 1.1629e-19, 7.3330e-21, 1.0204e-19, 3.7158e-20,\n 7.6260e-20, 3.1648e-21, 3.9337e-19, 1.2068e-19, 9.1201e-20, 3.4392e-20,\n 4.9741e-21, 9.1766e-20, 3.5286e-20, 1.8515e-21, 1.0620e-20, 2.6272e-20,\n 3.4573e-20, 2.1898e-21, 7.8509e-20, 5.2794e-20, 1.1458e-20, 3.2199e-21,\n 1.0467e-21, 8.3326e-20, 1.5717e-20, 1.6496e-20], device='cuda:0')"
 },
 "43": {
 "step": "tensor(18770.)",
 "exp_avg": "tensor([[-5.6052e-45, -5.6052e-45, -5.6052e-45, ..., -5.6052e-45,\n -5.6052e-45, -5.6052e-45],\n [ 5.6052e-45, -5.6052e-45, -5.6052e-45, ..., 5.6052e-45,\n -5.6052e-45, 5.6052e-45],\n [ 5.6052e-45, -5.6052e-45, -5.6052e-45, ..., 5.6052e-45,\n 5.6052e-45, -5.6052e-45],\n ...,\n [-5.6052e-45, -5.6052e-45, 5.6052e-45, ..., -5.6052e-45,\n -5.6052e-45, -5.6052e-45],\n [-5.6052e-45, -5.6052e-45, 5.6052e-45, ..., 5.6052e-45,\n 5.6052e-45, -5.6052e-45],\n [ 5.6052e-45, 5.6052e-45, -5.6052e-45, ..., 5.6052e-45,\n 5.6052e-45, 5.6052e-45]], device='cuda:0')",
 "exp_avg_sq": "tensor([[2.8306e-21, 1.0161e-21, 3.3142e-22, ..., 7.5415e-21, 5.3917e-22,\n 3.1452e-21],\n [2.9060e-21, 1.0023e-20, 2.0905e-21, ..., 4.4959e-22, 1.2097e-21,\n 9.9599e-21],\n [4.0659e-20, 8.3997e-21, 3.6440e-20, ..., 4.2808e-21, 3.5236e-21,\n 2.7197e-22],\n ...,\n [2.0875e-20, 2.2612e-21, 9.4004e-21, ..., 1.6436e-20, 1.2395e-21,\n 3.0665e-21],\n [1.8572e-20, 2.6181e-20, 4.5072e-20, ..., 5.0111e-21, 5.4687e-22,\n 4.4904e-21],\n [3.7689e-21, 1.3347e-21, 4.5389e-21, ..., 1.0818e-21, 4.5561e-22,\n 1.4588e-21]], device='cuda:0')"
 },
 "44": {
 "step": "tensor(18770.)",
 "exp_avg": "tensor([ 5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45, 5.6052e-45,\n -5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45,\n -5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45,\n 5.6052e-45, -5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45,\n 5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45,\n 5.6052e-45, -5.6052e-45, -5.6052e-45, 5.6052e-45, 5.6052e-45,\n 5.6052e-45, -5.6052e-45, -5.6052e-45, -5.6052e-45, 5.6052e-45,\n -5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45,\n 5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45, 5.6052e-45,\n 5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45, -5.6052e-45,\n -5.6052e-45, 5.6052e-45, -5.6052e-45, -5.6052e-45, -5.6052e-45,\n -5.6052e-45, 5.6052e-45, -5.6052e-45, -5.6052e-45, 5.6052e-45,\n -5.6052e-45, -5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45,\n 5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45, -5.6052e-45,\n 5.6052e-45, -5.6052e-45, -5.6052e-45, -5.6052e-45, 5.6052e-45,\n 5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45, -5.6052e-45,\n 5.6052e-45, 5.6052e-45, -5.6052e-45, -5.6052e-45, -5.6052e-45,\n 5.6052e-45, -5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45,\n -5.6052e-45, -5.6052e-45, -5.6052e-45, -5.6052e-45, -5.6052e-45,\n 5.6052e-45, -5.6052e-45, -5.6052e-45, -5.6052e-45, -5.6052e-45,\n 5.6052e-45, 5.6052e-45, -5.6052e-45, -5.6052e-45, -5.6052e-45,\n -5.6052e-45, 5.6052e-45, -5.6052e-45, -5.6052e-45, -5.6052e-45,\n 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45,\n 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45,\n -5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45,\n 5.6052e-45, -5.6052e-45, -5.6052e-45, 5.6052e-45, -5.6052e-45,\n 5.6052e-45, -5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45,\n 5.6052e-45, -5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45,\n 5.6052e-45, -5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45,\n -5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45,\n -5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45, 5.6052e-45,\n -5.6052e-45, -5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45,\n -5.6052e-45, -5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45,\n 5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45,\n 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45,\n -5.6052e-45, 5.6052e-45, -5.6052e-45, -5.6052e-45, 5.6052e-45,\n 5.6052e-45, -5.6052e-45, 5.6052e-45, -5.6052e-45, -5.6052e-45,\n 5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45, -5.6052e-45,\n 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45,\n 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45,\n -5.6052e-45, -5.6052e-45, -5.6052e-45, -5.6052e-45, 5.6052e-45,\n 5.6052e-45, 5.6052e-45, -5.6052e-45, -5.6052e-45, 5.6052e-45,\n 5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45, 5.6052e-45,\n 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45,\n 5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45, 5.6052e-45,\n 5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45, 5.6052e-45,\n 5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45, -5.6052e-45,\n -5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45,\n 5.6052e-45, -5.6052e-45, -5.6052e-45, 5.6052e-45, 5.6052e-45,\n 5.6052e-45, -5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45,\n 5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45, -5.6052e-45,\n 5.6052e-45], device='cuda:0')",
 "exp_avg_sq": "tensor([8.3813e-18, 4.9832e-18, 2.4527e-17, 2.4282e-17, 1.8994e-18, 1.4168e-17,\n 2.7105e-18, 4.5122e-18, 9.6332e-18, 3.6025e-18, 3.4551e-17, 7.8381e-17,\n 4.6347e-17, 1.5949e-19, 1.1117e-18, 1.1049e-17, 9.5922e-18, 5.9541e-18,\n 5.1455e-17, 9.0252e-19, 9.0280e-17, 4.8430e-18, 2.4955e-17, 1.7065e-17,\n 4.6257e-18, 3.0844e-17, 2.1820e-18, 1.5329e-16, 2.9632e-17, 3.7917e-18,\n 6.1853e-17, 2.1827e-17, 1.3720e-17, 1.1895e-18, 2.6793e-17, 1.7309e-17,\n 1.2109e-18, 1.0189e-17, 4.6330e-17, 1.5062e-17, 7.7148e-18, 5.4235e-18,\n 6.2948e-18, 2.5265e-17, 3.1209e-18, 1.5856e-18, 7.6973e-18, 1.2052e-18,\n 4.6200e-17, 7.3564e-17, 5.4276e-17, 4.1060e-19, 1.5287e-17, 2.5051e-17,\n 3.8156e-17, 1.8377e-17, 4.5381e-17, 6.1986e-19, 7.1718e-18, 2.1172e-18,\n 1.7824e-17, 3.9184e-17, 4.9219e-19, 2.0373e-17, 6.0420e-19, 3.7883e-18,\n 2.8619e-17, 3.3174e-17, 1.3657e-17, 2.0141e-18, 7.0850e-18, 2.5068e-19,\n 1.2475e-16, 6.4382e-17, 3.3243e-19, 3.6917e-18, 3.5756e-18, 1.4728e-17,\n 1.4457e-18, 1.2602e-17, 1.6844e-19, 7.8910e-18, 2.6912e-18, 1.8138e-17,\n 1.0079e-17, 2.0079e-17, 1.3958e-17, 2.5304e-17, 3.7100e-17, 3.6155e-17,\n 1.3678e-18, 5.0308e-18, 1.9529e-17, 2.1730e-17, 3.1184e-17, 7.8256e-18,\n 1.2539e-17, 4.9261e-18, 9.3596e-18, 3.1392e-18, 5.8067e-18, 2.4261e-18,\n 1.5139e-18, 1.2062e-17, 5.6203e-18, 1.1065e-17, 7.1740e-18, 1.1071e-17,\n 1.3061e-17, 5.1791e-18, 1.9466e-17, 3.8465e-18, 2.0504e-17, 8.9570e-18,\n 1.7586e-18, 7.5620e-18, 8.3722e-18, 5.0346e-17, 1.6235e-17, 1.3859e-18,\n 1.8709e-17, 8.7995e-17, 1.8418e-17, 1.6525e-16, 1.8425e-17, 6.5320e-17,\n 1.4431e-18, 5.4309e-17, 4.6503e-18, 8.1604e-18, 2.8894e-18, 5.4174e-17,\n 8.6021e-18, 1.9245e-17, 2.7670e-19, 1.5032e-18, 2.9339e-17, 4.9175e-18,\n 2.2657e-17, 1.4038e-17, 1.2948e-17, 3.5671e-17, 1.8064e-18, 3.9278e-17,\n 5.4644e-18, 4.0399e-17, 2.4051e-17, 3.9818e-17, 8.9242e-18, 1.8156e-17,\n 7.8499e-17, 1.2536e-17, 1.4350e-17, 6.6441e-17, 2.4584e-19, 2.2987e-17,\n 3.6366e-18, 8.7436e-18, 5.5471e-18, 1.6540e-17, 5.9028e-17, 2.0818e-17,\n 3.1545e-17, 1.2149e-18, 1.0978e-17, 4.1927e-18, 7.1856e-17, 1.2638e-18,\n 6.2457e-17, 6.2961e-18, 3.0296e-17, 1.6488e-17, 1.1104e-17, 1.0964e-18,\n 1.8583e-17, 1.6095e-16, 1.4497e-18, 1.8691e-17, 5.2486e-17, 1.4592e-19,\n 6.4119e-19, 2.5237e-17, 8.0190e-19, 3.8737e-17, 4.7606e-17, 4.9437e-19,\n 2.4834e-18, 6.8946e-18, 5.3266e-17, 3.2705e-17, 2.5330e-18, 1.1126e-17,\n 9.6920e-19, 2.7962e-18, 1.3995e-17, 4.1904e-18, 3.4338e-19, 2.5916e-17,\n 8.8429e-18, 7.5341e-17, 2.9149e-18, 3.4610e-17, 1.1280e-18, 1.8958e-17,\n 2.6854e-17, 5.5821e-17, 5.9471e-19, 7.1469e-18, 3.9489e-18, 6.1589e-18,\n 4.6666e-19, 1.3891e-17, 1.2042e-17, 5.3675e-17, 4.2028e-17, 9.7984e-18,\n 3.8593e-18, 2.1730e-18, 6.0924e-18, 1.2661e-17, 1.2097e-18, 4.8188e-17,\n 1.6022e-18, 1.4782e-17, 3.0191e-17, 1.0116e-17, 1.5825e-18, 4.5126e-17,\n 1.5430e-18, 2.4290e-17, 4.3847e-19, 2.0916e-18, 4.6512e-19, 4.1969e-18,\n 7.0392e-18, 1.1550e-18, 3.3395e-17, 3.4198e-18, 1.4545e-18, 2.1753e-17,\n 9.6926e-18, 7.6413e-18, 4.0266e-18, 3.8946e-18, 4.8090e-18, 7.0494e-17,\n 3.8173e-17, 3.7309e-19, 1.2620e-17, 1.7077e-19, 8.8683e-18, 7.5585e-18,\n 4.9408e-17, 4.1904e-17, 2.3901e-17, 2.4600e-18], device='cuda:0')"
 },
 "45": {
 "step": "tensor(18770.)",
 "exp_avg": "tensor([ 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45,\n -5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45,\n -5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45,\n 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45,\n 5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45,\n 5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45, 5.6052e-45,\n 5.6052e-45, -5.6052e-45, -5.6052e-45, -5.6052e-45, 5.6052e-45,\n -5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45,\n 5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45, 5.6052e-45,\n 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45,\n -5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45,\n -5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45,\n -5.6052e-45, -5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45,\n 5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45,\n 5.6052e-45, -5.6052e-45, -5.6052e-45, -5.6052e-45, 5.6052e-45,\n 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45,\n -5.6052e-45, 5.6052e-45, -5.6052e-45, -5.6052e-45, -5.6052e-45,\n 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45,\n 5.6052e-45, -5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45,\n 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45,\n 5.6052e-45, -5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45,\n -5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45, -5.6052e-45,\n 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45,\n 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45,\n -5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45,\n 5.6052e-45, -5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45,\n 5.6052e-45, -5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45,\n 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45,\n 5.6052e-45, -5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45,\n 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45,\n 5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45, 5.6052e-45,\n -5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45,\n 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45,\n 5.6052e-45, 5.6052e-45, -5.6052e-45, -5.6052e-45, 5.6052e-45,\n 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45,\n -5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45,\n -5.6052e-45, -5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45,\n -5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45, -5.6052e-45,\n 5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45, 5.6052e-45,\n 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45,\n 5.6052e-45, -5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45,\n 5.6052e-45, -5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45,\n 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45,\n 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45,\n 5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45, 5.6052e-45,\n 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45,\n 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45,\n -5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45,\n 5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45, 5.6052e-45,\n 5.6052e-45, -5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45,\n 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45,\n -5.6052e-45], device='cuda:0')",
 "exp_avg_sq": "tensor([1.5322e-20, 8.7343e-21, 3.7207e-20, 6.4883e-20, 2.7174e-21, 9.1163e-21,\n 1.1208e-20, 2.7634e-21, 1.5370e-21, 1.2732e-21, 1.6867e-20, 2.1195e-19,\n 1.9583e-19, 1.4307e-22, 9.3620e-24, 3.5202e-20, 3.7352e-20, 2.6349e-20,\n 2.0092e-19, 2.2310e-21, 3.0242e-19, 4.1760e-21, 2.5813e-20, 1.2745e-19,\n 1.9547e-20, 5.9837e-20, 2.0092e-21, 2.8131e-19, 3.5047e-20, 3.4787e-21,\n 1.5210e-19, 9.4176e-21, 3.0244e-21, 8.7823e-22, 5.9594e-20, 9.8386e-21,\n 1.4654e-21, 8.9856e-21, 2.0694e-19, 5.2238e-20, 1.2887e-21, 2.3876e-20,\n 5.4513e-21, 3.7032e-20, 1.3691e-21, 4.8210e-21, 3.2276e-21, 1.7518e-21,\n 1.3325e-19, 2.6252e-19, 9.6106e-20, 4.9804e-22, 4.9003e-20, 3.4052e-20,\n 3.4843e-20, 3.4029e-20, 6.4193e-19, 1.2530e-21, 1.7397e-20, 7.6056e-22,\n 2.0261e-20, 9.9471e-20, 3.8586e-21, 7.4807e-20, 5.4655e-23, 3.2053e-22,\n 6.4344e-20, 3.0978e-19, 2.5453e-21, 4.9403e-21, 6.4634e-21, 9.9685e-22,\n 1.8818e-19, 1.6010e-19, 2.7678e-22, 3.2383e-22, 2.0257e-21, 1.0579e-20,\n 3.0899e-21, 7.9430e-20, 8.9845e-22, 1.0134e-20, 8.4031e-21, 5.1302e-21,\n 5.1593e-21, 1.5280e-20, 1.4416e-20, 7.2059e-20, 8.7931e-20, 4.1829e-20,\n 6.2489e-21, 4.9201e-21, 1.7724e-19, 3.0995e-20, 4.6477e-20, 1.5422e-20,\n 8.4585e-21, 7.2979e-21, 1.5854e-20, 4.0526e-22, 2.7219e-20, 2.5208e-21,\n 5.6346e-22, 1.4535e-20, 1.4948e-21, 8.8184e-21, 1.3368e-20, 4.3607e-21,\n 1.1931e-20, 5.5535e-21, 1.1019e-19, 4.4424e-21, 7.4482e-20, 5.2475e-20,\n 2.5162e-22, 6.5007e-20, 5.1291e-20, 4.7868e-20, 1.0022e-20, 7.8526e-22,\n 1.7573e-20, 2.1391e-19, 2.5941e-20, 6.5177e-19, 2.1908e-20, 2.9533e-19,\n 8.9782e-21, 9.7465e-20, 1.7365e-20, 4.7813e-21, 4.2921e-22, 1.4407e-19,\n 3.0920e-21, 9.3523e-20, 1.5708e-21, 5.1633e-22, 1.3137e-19, 1.5288e-20,\n 3.4576e-20, 7.9734e-21, 3.6760e-20, 9.7808e-20, 2.3059e-21, 8.2100e-20,\n 6.8393e-21, 2.2803e-20, 1.7205e-20, 2.6855e-19, 3.3302e-21, 4.6538e-20,\n 1.1778e-19, 1.4401e-20, 1.5654e-20, 2.1766e-19, 3.2134e-21, 1.7621e-20,\n 1.2149e-21, 3.6722e-20, 4.9067e-21, 3.5068e-21, 2.7559e-19, 2.5146e-20,\n 7.0505e-20, 4.1691e-22, 6.6271e-20, 4.9127e-21, 6.7571e-20, 5.6421e-21,\n 9.6616e-20, 1.0082e-21, 1.7000e-20, 4.8714e-21, 8.2061e-21, 7.1877e-22,\n 1.5273e-20, 2.0143e-19, 1.4515e-21, 1.5051e-20, 4.5599e-19, 4.0142e-22,\n 2.6130e-22, 1.8109e-19, 2.6802e-21, 4.1994e-20, 7.6984e-19, 5.1522e-22,\n 4.1605e-21, 1.9452e-20, 5.0472e-20, 2.5861e-20, 1.6460e-21, 6.7243e-20,\n 1.0986e-21, 3.8408e-22, 5.7221e-21, 5.3529e-21, 2.0960e-21, 1.8936e-20,\n 2.6684e-20, 5.8394e-19, 4.5841e-21, 3.9263e-20, 1.2942e-21, 1.9183e-20,\n 2.2822e-20, 1.2598e-19, 7.3734e-22, 1.9301e-20, 4.3591e-21, 1.0263e-21,\n 1.6389e-21, 1.0228e-20, 2.1770e-20, 6.7726e-20, 8.3741e-20, 1.5574e-20,\n 3.4049e-21, 2.2363e-21, 4.7327e-21, 1.1533e-20, 1.0139e-21, 1.2083e-19,\n 1.3059e-20, 1.6498e-20, 2.5219e-20, 2.8290e-21, 1.4144e-21, 3.1474e-20,\n 5.8903e-21, 4.9958e-20, 1.8907e-21, 4.5447e-22, 8.6268e-22, 1.6373e-21,\n 1.1598e-20, 2.3328e-22, 2.0139e-19, 5.5923e-22, 3.6470e-22, 2.9462e-20,\n 6.9442e-21, 2.5650e-21, 3.6736e-21, 2.8963e-20, 3.3022e-22, 2.1980e-19,\n 6.0682e-20, 1.1409e-21, 2.4079e-20, 1.3721e-21, 3.3965e-21, 7.3991e-21,\n 5.5484e-20, 3.3941e-20, 3.8363e-20, 9.0123e-22], device='cuda:0')"
 },
 "46": {
 "step": "tensor(18770.)",
 "exp_avg": "tensor([ 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45,\n -5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45,\n -5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45,\n 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45,\n 5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45,\n 5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45, 5.6052e-45,\n 5.6052e-45, -5.6052e-45, -5.6052e-45, 5.6052e-45, 5.6052e-45,\n -5.6052e-45, -5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45,\n 5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45, 5.6052e-45,\n 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45,\n -5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45,\n -5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45, 5.6052e-45,\n -5.6052e-45, -5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45,\n 5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45,\n 5.6052e-45, 5.6052e-45, -5.6052e-45, -5.6052e-45, 5.6052e-45,\n 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45,\n 5.6052e-45, 5.6052e-45, -5.6052e-45, -5.6052e-45, -5.6052e-45,\n 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45,\n 5.6052e-45, -5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45,\n 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45,\n 5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45, -5.6052e-45,\n -5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45, -5.6052e-45,\n 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45,\n 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45,\n -5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45,\n 5.6052e-45, -5.6052e-45, -5.6052e-45, 5.6052e-45, -5.6052e-45,\n 5.6052e-45, -5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45,\n 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45,\n 5.6052e-45, -5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45,\n -5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45,\n -5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45, -5.6052e-45,\n -5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45,\n 5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45,\n 5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45,\n 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45,\n -5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45, 5.6052e-45,\n 5.6052e-45, -5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45,\n 5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45, -5.6052e-45,\n 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45,\n 5.6052e-45, -5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45,\n 5.6052e-45, -5.6052e-45, -5.6052e-45, -5.6052e-45, 5.6052e-45,\n 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45,\n -5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45,\n 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45,\n 5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45, 5.6052e-45,\n 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45,\n 5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45, 5.6052e-45,\n -5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45,\n 5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45, 5.6052e-45,\n 5.6052e-45, -5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45,\n 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45,\n 5.6052e-45], device='cuda:0')",
 "exp_avg_sq": "tensor([2.4828e-20, 1.5180e-20, 3.2417e-20, 4.6754e-20, 7.7241e-21, 3.5878e-20,\n 8.2288e-21, 7.1303e-21, 3.4305e-20, 7.5615e-21, 4.9113e-20, 1.8441e-19,\n 1.9485e-19, 3.4743e-22, 1.7527e-21, 5.0872e-20, 3.4027e-20, 3.4603e-20,\n 1.5574e-19, 7.9411e-21, 2.4088e-19, 1.5777e-20, 4.1024e-20, 4.3308e-20,\n 3.6976e-20, 1.0785e-19, 8.0803e-21, 2.0844e-19, 5.2001e-20, 5.7175e-21,\n 2.1524e-19, 9.2583e-21, 9.3235e-21, 1.9213e-21, 8.3144e-20, 1.0298e-20,\n 2.3000e-21, 4.0074e-20, 1.2283e-19, 6.1473e-20, 1.7027e-20, 1.9272e-20,\n 1.0434e-20, 9.9783e-20, 1.1580e-20, 1.4468e-20, 1.5778e-20, 4.6640e-21,\n 5.2909e-20, 1.9212e-19, 6.5116e-20, 1.5382e-21, 3.2739e-20, 3.7318e-20,\n 6.1006e-20, 2.3533e-20, 2.2487e-19, 4.7583e-21, 1.1873e-20, 7.3177e-21,\n 2.7222e-20, 4.3062e-20, 6.9182e-22, 5.6503e-20, 1.9394e-22, 1.7756e-20,\n 1.1151e-19, 1.0149e-19, 2.2797e-20, 6.8628e-21, 1.8325e-20, 3.4311e-22,\n 1.9710e-19, 1.1453e-19, 1.0762e-21, 8.8839e-21, 1.6570e-20, 2.6661e-20,\n 2.3426e-21, 3.3051e-20, 8.9145e-23, 1.7484e-20, 1.3638e-20, 2.6791e-20,\n 1.1960e-20, 5.1133e-20, 1.4004e-20, 9.1814e-20, 1.0792e-19, 8.1497e-20,\n 3.3728e-21, 6.8220e-21, 4.9791e-20, 3.9616e-20, 4.4418e-20, 3.4278e-20,\n 1.2959e-20, 1.3454e-20, 9.3615e-21, 4.0254e-21, 3.5643e-20, 1.0560e-21,\n 2.5795e-21, 3.4418e-20, 4.3448e-21, 6.2870e-21, 4.1281e-20, 2.2627e-20,\n 1.0673e-20, 7.0666e-21, 8.8608e-20, 1.6652e-20, 8.5845e-20, 4.8539e-20,\n 3.5528e-21, 4.9790e-20, 5.3388e-20, 8.1828e-20, 5.0901e-20, 2.3888e-21,\n 2.2838e-20, 2.1770e-19, 4.7171e-20, 4.1891e-19, 4.3626e-20, 1.7239e-19,\n 2.1461e-20, 1.1322e-19, 1.6744e-20, 2.1866e-20, 7.5237e-21, 8.0419e-20,\n 1.4617e-20, 4.8254e-20, 2.7008e-22, 7.9252e-21, 5.8833e-20, 1.6775e-20,\n 8.7186e-20, 1.7608e-20, 3.8167e-20, 5.0716e-20, 1.9026e-21, 8.6065e-20,\n 1.8950e-20, 4.8142e-20, 4.4053e-20, 1.1062e-19, 1.0416e-20, 5.1890e-20,\n 9.4829e-20, 2.3675e-20, 1.5112e-20, 2.2155e-19, 1.2278e-21, 2.2831e-20,\n 4.1861e-21, 4.7041e-20, 1.8697e-20, 3.4829e-20, 1.3615e-19, 2.6542e-20,\n 6.2144e-20, 1.7362e-21, 5.7241e-20, 1.0956e-20, 1.8470e-19, 4.2329e-21,\n 3.6862e-20, 8.3501e-21, 7.5248e-20, 4.8601e-20, 3.4083e-20, 6.3725e-21,\n 3.1103e-20, 1.4224e-19, 7.4398e-21, 1.8901e-20, 1.5812e-19, 2.1349e-21,\n 1.5975e-21, 4.3048e-20, 6.5938e-21, 4.0352e-20, 1.6681e-19, 8.7621e-22,\n 1.4086e-20, 3.7407e-20, 8.3534e-20, 2.2527e-20, 3.4736e-21, 6.0888e-20,\n 4.1948e-22, 4.0551e-21, 3.7360e-20, 7.8822e-21, 1.2240e-21, 5.8386e-20,\n 2.3318e-20, 2.0967e-19, 3.7270e-21, 2.7898e-20, 9.1563e-21, 2.0958e-20,\n 5.5653e-20, 1.6785e-19, 4.9115e-22, 2.7312e-20, 6.2237e-21, 1.4839e-20,\n 8.9524e-22, 3.0941e-20, 3.8692e-20, 8.1148e-20, 1.5125e-19, 1.6009e-20,\n 2.3020e-20, 3.4729e-21, 1.7768e-20, 4.3355e-20, 7.1239e-21, 1.4372e-19,\n 1.4878e-20, 3.1445e-20, 7.3949e-20, 2.7965e-20, 3.6239e-21, 6.9645e-20,\n 6.2805e-21, 8.7518e-20, 2.6183e-21, 4.8262e-21, 2.2505e-21, 9.7581e-21,\n 1.5883e-20, 3.0276e-21, 1.3383e-19, 1.0176e-20, 7.7519e-21, 3.9952e-20,\n 4.2174e-20, 8.1595e-21, 3.6510e-21, 1.7779e-20, 6.9969e-21, 2.1936e-19,\n 5.6589e-20, 1.0039e-21, 4.5176e-20, 4.2911e-22, 1.5601e-20, 1.6306e-20,\n 1.6851e-19, 5.1782e-20, 2.3181e-20, 1.2896e-20], device='cuda:0')"
 },
 "47": {
 "step": "tensor(18770.)",
 "exp_avg": "tensor([[ 5.6052e-45, 5.6052e-45, 5.6052e-45, ..., 5.6052e-45,\n 5.6052e-45, 5.6052e-45],\n [ 5.6052e-45, -5.6052e-45, -5.6052e-45, ..., 5.6052e-45,\n -5.6052e-45, 5.6052e-45],\n [ 5.6052e-45, 5.6052e-45, -5.6052e-45, ..., 5.6052e-45,\n 5.6052e-45, 5.6052e-45],\n ...,\n [-5.6052e-45, -5.6052e-45, 5.6052e-45, ..., 5.6052e-45,\n 5.6052e-45, -5.6052e-45],\n [-5.6052e-45, -5.6052e-45, 5.6052e-45, ..., 5.6052e-45,\n -5.6052e-45, 5.6052e-45],\n [ 5.6052e-45, 5.6052e-45, -5.6052e-45, ..., -5.6052e-45,\n 5.6052e-45, -5.6052e-45]], device='cuda:0')",
 "exp_avg_sq": "tensor([[6.1359e-21, 5.4114e-21, 3.3513e-21, ..., 3.5764e-21, 1.3648e-22,\n 7.0411e-22],\n [5.6877e-22, 1.7979e-21, 8.9313e-22, ..., 6.4565e-22, 7.6304e-23,\n 3.7074e-22],\n [1.8346e-20, 4.4528e-21, 4.9139e-20, ..., 9.8520e-21, 1.7475e-21,\n 9.3816e-21],\n ...,\n [3.0575e-20, 1.3830e-20, 1.2071e-20, ..., 5.3318e-21, 2.3469e-22,\n 5.3177e-21],\n [1.0842e-19, 1.7543e-20, 5.7103e-20, ..., 1.5577e-20, 4.7631e-21,\n 1.4661e-20],\n [2.8095e-21, 3.5581e-21, 9.9857e-22, ..., 4.7685e-21, 6.9195e-22,\n 3.2847e-22]], device='cuda:0')"
 },
 "48": {
 "step": "tensor(18770.)",
 "exp_avg": "tensor([-5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45,\n -5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45,\n 5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45, 5.6052e-45,\n 5.6052e-45, -5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45,\n 5.6052e-45, 5.6052e-45, -5.6052e-45, -5.6052e-45, -5.6052e-45,\n 5.6052e-45, -5.6052e-45, -5.6052e-45, -5.6052e-45, 5.6052e-45,\n 5.6052e-45, -5.6052e-45, -5.6052e-45, -5.6052e-45, 5.6052e-45,\n 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45,\n 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45,\n 5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45,\n -5.6052e-45, -5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45,\n -5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45,\n 5.6052e-45, -5.6052e-45, 5.6052e-45, -5.6052e-45, -5.6052e-45,\n -5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45,\n -5.6052e-45, 5.6052e-45, -5.6052e-45, -5.6052e-45, -5.6052e-45,\n 5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45, 5.6052e-45,\n -5.6052e-45, -5.6052e-45, -5.6052e-45, -5.6052e-45, 5.6052e-45,\n 5.6052e-45, -5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45,\n 5.6052e-45, -5.6052e-45, -5.6052e-45, -5.6052e-45, 5.6052e-45,\n -5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45,\n 5.6052e-45, 5.6052e-45, -5.6052e-45, -5.6052e-45, 5.6052e-45,\n -5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45, -5.6052e-45,\n -5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45,\n 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45,\n -5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45,\n 5.6052e-45, -5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45,\n 5.6052e-45, -5.6052e-45, 5.6052e-45, -5.6052e-45, -5.6052e-45,\n 5.6052e-45, -5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45,\n -5.6052e-45, -5.6052e-45, -5.6052e-45, 5.6052e-45, 5.6052e-45,\n -5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45,\n -5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45, 5.6052e-45,\n -5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45,\n 5.6052e-45, -5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45,\n 5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45,\n 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45,\n -5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45, -5.6052e-45,\n -5.6052e-45, -5.6052e-45, -5.6052e-45, -5.6052e-45, 5.6052e-45,\n 5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45, -5.6052e-45,\n 5.6052e-45, -5.6052e-45, -5.6052e-45, -5.6052e-45, 5.6052e-45,\n 5.6052e-45, -5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45,\n -5.6052e-45, -5.6052e-45, -5.6052e-45, -5.6052e-45, 5.6052e-45,\n 5.6052e-45, -5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45,\n 5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45, 5.6052e-45,\n -5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45,\n 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45,\n 5.6052e-45, -5.6052e-45, -5.6052e-45, -5.6052e-45, 5.6052e-45,\n 5.6052e-45, -5.6052e-45, -5.6052e-45, 5.6052e-45, -5.6052e-45,\n -5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45,\n 5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45, -5.6052e-45,\n 5.6052e-45, -5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45,\n 5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45, -5.6052e-45,\n 5.6052e-45], device='cuda:0')",
 "exp_avg_sq": "tensor([9.3748e-18, 2.7841e-19, 4.3347e-17, 4.3203e-17, 8.1691e-19, 2.3056e-17,\n 1.4628e-18, 6.5889e-17, 1.3865e-17, 3.2704e-17, 6.6735e-17, 2.8626e-17,\n 8.0906e-18, 2.1978e-18, 3.9018e-18, 1.4083e-17, 6.4531e-18, 4.2340e-17,\n 5.3208e-19, 1.5918e-16, 1.7686e-16, 6.8782e-19, 2.1017e-17, 6.3421e-17,\n 1.3628e-17, 3.3388e-18, 1.9349e-17, 7.9224e-17, 6.9562e-17, 6.9251e-18,\n 4.8086e-17, 2.6744e-18, 6.5698e-18, 6.7576e-19, 1.5851e-16, 2.6238e-17,\n 1.5147e-17, 4.4657e-19, 1.0086e-16, 4.6667e-18, 2.0990e-17, 3.2195e-18,\n 4.7918e-19, 3.0127e-18, 4.8768e-17, 3.7118e-18, 7.1521e-18, 3.1702e-18,\n 1.6142e-17, 4.7924e-18, 1.2795e-16, 7.0015e-17, 2.2522e-17, 1.0576e-18,\n 5.9849e-17, 5.0925e-17, 5.5099e-18, 8.8870e-19, 6.0910e-17, 4.0351e-18,\n 1.7357e-17, 1.5457e-17, 1.1604e-16, 8.3476e-18, 5.9152e-17, 4.6436e-18,\n 4.9744e-19, 3.9812e-17, 4.6918e-17, 1.0769e-18, 1.1409e-18, 1.0479e-17,\n 8.6633e-17, 9.6595e-17, 2.1787e-19, 6.1528e-19, 7.6397e-19, 1.9036e-18,\n 1.6849e-18, 2.5042e-17, 2.2525e-17, 2.2913e-17, 3.7952e-17, 7.1124e-17,\n 5.5247e-19, 7.2437e-18, 4.6145e-18, 1.7028e-18, 5.9355e-17, 1.8275e-17,\n 4.1621e-18, 1.8979e-17, 2.9083e-17, 1.6164e-16, 2.2942e-18, 2.6407e-18,\n 8.3857e-18, 5.2940e-19, 2.0539e-17, 1.9945e-17, 3.7709e-19, 5.7871e-19,\n 1.1833e-16, 1.9714e-17, 1.6070e-17, 6.3876e-18, 7.1574e-17, 3.1011e-19,\n 4.3463e-17, 2.1071e-17, 5.4990e-18, 4.0970e-17, 4.6304e-18, 1.5333e-19,\n 1.3857e-16, 1.8130e-17, 8.9615e-19, 5.2641e-18, 4.7981e-17, 1.0145e-17,\n 1.0758e-16, 1.3853e-16, 1.4400e-17, 1.5816e-16, 1.6355e-18, 7.8922e-19,\n 8.9891e-19, 7.2444e-18, 7.6617e-19, 3.2945e-16, 7.0219e-18, 2.5472e-17,\n 1.3301e-18, 9.2048e-17, 1.7808e-16, 2.3298e-19, 1.8174e-17, 1.1271e-16,\n 1.0073e-16, 1.0383e-17, 1.0999e-18, 2.0569e-17, 1.9320e-18, 3.6877e-18,\n 3.9702e-19, 1.9225e-17, 2.1339e-17, 2.0669e-19, 9.0780e-19, 1.9938e-19,\n 7.5857e-17, 1.7192e-16, 2.6422e-18, 1.9107e-17, 1.8433e-17, 3.6626e-17,\n 1.7770e-17, 5.4722e-17, 1.6312e-16, 6.5906e-19, 4.5074e-17, 6.2525e-17,\n 1.9227e-18, 1.0055e-17, 3.3812e-17, 4.2756e-16, 7.4899e-18, 1.5355e-18,\n 1.1423e-17, 2.6110e-17, 3.8401e-18, 6.2309e-17, 1.7721e-19, 1.2838e-17,\n 1.5556e-19, 1.7415e-16, 4.9829e-18, 3.8061e-17, 8.2740e-17, 9.5137e-18,\n 3.8862e-18, 2.5292e-17, 8.7141e-19, 3.5697e-18, 5.1086e-17, 4.2672e-19,\n 1.1136e-16, 2.9061e-18, 3.9127e-17, 5.0271e-18, 5.9824e-17, 3.6015e-19,\n 5.0625e-18, 2.7482e-17, 5.6907e-18, 1.3955e-17, 1.1207e-18, 8.2285e-17,\n 1.2639e-16, 1.2564e-16, 4.0649e-18, 9.8618e-17, 8.9122e-19, 8.2934e-18,\n 3.6584e-18, 1.4877e-18, 6.3074e-18, 1.9256e-17, 2.1129e-17, 5.2507e-19,\n 2.0415e-18, 1.7291e-18, 9.0179e-17, 6.0649e-19, 2.8924e-17, 3.4244e-17,\n 2.3471e-17, 2.7459e-17, 6.2302e-17, 6.3648e-18, 6.8757e-19, 2.6588e-17,\n 1.0295e-17, 7.1211e-17, 2.7493e-17, 8.1791e-18, 1.5520e-17, 6.0540e-17,\n 7.2152e-17, 4.9726e-17, 3.9037e-17, 1.2452e-18, 5.7198e-18, 5.4157e-17,\n 8.1552e-18, 4.0572e-17, 1.8261e-17, 2.0485e-17, 3.2290e-17, 3.0448e-17,\n 1.1685e-17, 1.4025e-17, 9.0468e-17, 8.8256e-18, 1.4233e-18, 1.2747e-18,\n 6.2674e-17, 8.7877e-19, 5.5143e-17, 8.3856e-20, 4.8778e-18, 5.8101e-17,\n 1.3744e-16, 5.2180e-17, 1.1467e-16, 3.2583e-18], device='cuda:0')"
 },
 "49": {
 "step": "tensor(18770.)",
 "exp_avg": "tensor([ 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45,\n -5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45,\n 5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45,\n 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45,\n 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45,\n 5.6052e-45, -5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45,\n 5.6052e-45, 5.6052e-45, -5.6052e-45, -5.6052e-45, 5.6052e-45,\n 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45,\n 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45,\n 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45,\n -5.6052e-45, -5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45,\n -5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45,\n 5.6052e-45, -5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45,\n 5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45,\n 5.6052e-45, 5.6052e-45, -5.6052e-45, -5.6052e-45, 5.6052e-45,\n 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45,\n -5.6052e-45, -5.6052e-45, -5.6052e-45, -5.6052e-45, -5.6052e-45,\n 5.6052e-45, -5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45,\n 5.6052e-45, -5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45,\n -5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45, 5.6052e-45,\n 5.6052e-45, -5.6052e-45, -5.6052e-45, -5.6052e-45, 5.6052e-45,\n 5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45, -5.6052e-45,\n 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45,\n 5.6052e-45, -5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45,\n -5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45,\n -5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45,\n 5.6052e-45, -5.6052e-45, -5.6052e-45, 5.6052e-45, -5.6052e-45,\n 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45,\n 5.6052e-45, -5.6052e-45, -5.6052e-45, 5.6052e-45, 5.6052e-45,\n -5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45,\n 5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45, 5.6052e-45,\n 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45,\n 5.6052e-45, -5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45,\n 5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45,\n 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45,\n -5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45, 5.6052e-45,\n -5.6052e-45, -5.6052e-45, -5.6052e-45, -5.6052e-45, 5.6052e-45,\n -5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45, -5.6052e-45,\n 5.6052e-45, -5.6052e-45, -5.6052e-45, 5.6052e-45, 5.6052e-45,\n 5.6052e-45, -5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45,\n -5.6052e-45, -5.6052e-45, -5.6052e-45, 5.6052e-45, -5.6052e-45,\n 5.6052e-45, -5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45,\n 5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45, 5.6052e-45,\n -5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45,\n 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45,\n 5.6052e-45, -5.6052e-45, -5.6052e-45, -5.6052e-45, 5.6052e-45,\n 5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45, -5.6052e-45,\n -5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45,\n 5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45, -5.6052e-45,\n -5.6052e-45, -5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45,\n 5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45,\n 5.6052e-45], device='cuda:0')",
 "exp_avg_sq": "tensor([3.0152e-21, 8.2942e-22, 2.8985e-20, 8.1994e-20, 9.6796e-22, 5.0488e-20,\n 3.7025e-21, 3.3007e-20, 1.6038e-20, 2.0561e-20, 9.0395e-20, 1.1445e-20,\n 1.8969e-19, 6.8755e-21, 3.1798e-21, 2.4297e-20, 3.0949e-21, 2.2896e-19,\n 4.2620e-21, 2.7734e-19, 9.5559e-19, 1.3013e-21, 1.7290e-20, 8.7009e-19,\n 1.3230e-20, 3.5213e-22, 5.7032e-21, 3.2000e-19, 5.1190e-20, 3.2793e-21,\n 1.6217e-19, 4.3191e-21, 2.7309e-20, 9.5572e-21, 8.5724e-19, 1.4254e-19,\n 9.7929e-21, 1.9944e-22, 5.5874e-19, 1.0720e-20, 2.5805e-20, 1.0301e-20,\n 3.4013e-22, 7.3413e-22, 9.4449e-20, 4.7704e-21, 2.1747e-21, 8.2273e-22,\n 1.6463e-19, 2.8867e-21, 3.1247e-19, 9.7394e-20, 1.5714e-20, 2.9718e-22,\n 3.8663e-20, 7.4973e-20, 5.5079e-21, 3.5274e-21, 9.4675e-21, 2.0727e-21,\n 1.5409e-20, 1.0103e-20, 1.3013e-18, 1.4104e-20, 1.6483e-19, 1.3592e-20,\n 3.8931e-21, 1.6699e-20, 2.1034e-20, 1.1804e-21, 4.0018e-21, 3.1528e-21,\n 2.9160e-20, 6.8479e-20, 1.9085e-21, 5.1076e-22, 8.9416e-22, 8.9828e-22,\n 1.2167e-21, 3.0547e-20, 3.4500e-20, 1.2247e-20, 4.4020e-20, 4.7306e-20,\n 2.3886e-21, 4.3714e-21, 2.1805e-21, 3.2883e-21, 1.7960e-19, 5.2042e-20,\n 1.2034e-20, 1.9097e-20, 5.1847e-20, 3.2073e-19, 7.5904e-23, 3.8614e-21,\n 2.5962e-21, 3.7306e-22, 1.3739e-19, 1.2174e-19, 1.2306e-21, 9.2035e-22,\n 2.6048e-19, 2.0905e-20, 3.0931e-20, 3.9032e-21, 2.0510e-19, 2.2289e-21,\n 3.2119e-20, 6.2495e-21, 9.9712e-22, 3.7932e-19, 1.3964e-20, 5.0787e-22,\n 6.3138e-19, 1.1501e-19, 1.8294e-21, 1.5479e-21, 1.1493e-19, 4.2268e-21,\n 1.2953e-19, 6.3874e-19, 1.0417e-20, 2.3081e-19, 8.1327e-21, 2.7242e-21,\n 1.0463e-21, 1.0061e-21, 1.7127e-20, 1.5619e-18, 7.3845e-21, 1.5677e-20,\n 1.3323e-21, 3.9227e-19, 1.6331e-19, 8.1665e-22, 2.8173e-19, 7.5686e-19,\n 1.6537e-19, 2.8287e-21, 1.1136e-20, 6.8669e-21, 1.7998e-21, 4.6642e-21,\n 1.2695e-21, 5.2797e-21, 1.1844e-20, 8.0760e-22, 1.3449e-21, 1.7554e-21,\n 4.3275e-20, 1.6051e-18, 1.1477e-21, 3.7260e-20, 1.2856e-20, 2.5518e-20,\n 4.4583e-20, 1.7357e-19, 4.2358e-19, 1.2014e-21, 1.6675e-20, 8.1380e-20,\n 3.9376e-21, 4.4100e-21, 2.7744e-19, 1.4102e-18, 5.5218e-21, 3.3015e-21,\n 7.5797e-21, 3.3011e-20, 9.4191e-22, 5.5500e-20, 6.9888e-22, 3.5244e-20,\n 1.0297e-21, 3.2486e-19, 3.4878e-21, 7.6593e-21, 2.6185e-19, 1.8395e-20,\n 1.9116e-21, 1.0536e-19, 5.6349e-22, 1.2606e-21, 5.7132e-19, 3.9659e-22,\n 1.2763e-18, 1.6531e-21, 7.9813e-20, 2.6733e-21, 2.8569e-20, 5.7050e-22,\n 4.2529e-21, 1.1198e-20, 1.9377e-21, 7.9086e-21, 5.5849e-22, 1.3671e-19,\n 5.5227e-19, 2.9231e-19, 9.6558e-22, 8.6914e-20, 8.8934e-22, 1.0052e-21,\n 9.0931e-23, 8.9926e-22, 6.6460e-21, 6.6889e-20, 2.5766e-20, 2.2493e-21,\n 2.8580e-21, 6.4559e-21, 2.5297e-19, 1.3894e-21, 7.8160e-20, 4.1103e-20,\n 2.9323e-20, 1.5722e-19, 4.3376e-19, 5.9735e-21, 2.0056e-21, 7.5103e-20,\n 1.3154e-20, 1.4737e-19, 1.3171e-20, 3.0562e-22, 8.1995e-21, 5.8365e-20,\n 1.2873e-19, 2.5820e-19, 1.3973e-19, 3.2773e-22, 3.5770e-21, 6.2279e-20,\n 3.1848e-21, 3.3348e-20, 6.1336e-20, 8.4765e-21, 1.5719e-19, 1.3128e-19,\n 1.2782e-20, 1.5091e-20, 1.0848e-19, 2.7339e-21, 7.6673e-22, 2.0227e-21,\n 7.7014e-20, 1.2910e-21, 1.7388e-19, 2.4404e-21, 7.1172e-21, 1.5949e-19,\n 1.1985e-19, 9.0406e-20, 1.0587e-18, 4.7440e-21], device='cuda:0')"
 },
 "50": {
 "step": "tensor(18770.)",
 "exp_avg": "tensor([ 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45,\n -5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45,\n 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45,\n 5.6052e-45, -5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45,\n 5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45, -5.6052e-45,\n 5.6052e-45, -5.6052e-45, -5.6052e-45, -5.6052e-45, 5.6052e-45,\n 5.6052e-45, 5.6052e-45, -5.6052e-45, -5.6052e-45, 5.6052e-45,\n 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45,\n 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45,\n 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45,\n -5.6052e-45, -5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45,\n -5.6052e-45, 5.6052e-45, -5.6052e-45, -5.6052e-45, 5.6052e-45,\n 5.6052e-45, -5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45,\n 5.6052e-45, -5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45,\n -5.6052e-45, 5.6052e-45, -5.6052e-45, -5.6052e-45, -5.6052e-45,\n 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45,\n -5.6052e-45, -5.6052e-45, -5.6052e-45, -5.6052e-45, -5.6052e-45,\n 5.6052e-45, -5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45,\n 5.6052e-45, -5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45,\n -5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45,\n 5.6052e-45, 5.6052e-45, -5.6052e-45, -5.6052e-45, 5.6052e-45,\n 5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45, -5.6052e-45,\n 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45,\n 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45,\n -5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45,\n 5.6052e-45, -5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45,\n 5.6052e-45, -5.6052e-45, 5.6052e-45, -5.6052e-45, -5.6052e-45,\n -5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45,\n -5.6052e-45, -5.6052e-45, -5.6052e-45, 5.6052e-45, -5.6052e-45,\n -5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45, 5.6052e-45,\n 5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45, 5.6052e-45,\n -5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45,\n 5.6052e-45, -5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45,\n 5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45,\n 5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45, 5.6052e-45,\n -5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45, 5.6052e-45,\n -5.6052e-45, 5.6052e-45, -5.6052e-45, -5.6052e-45, 5.6052e-45,\n 5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45,\n 5.6052e-45, -5.6052e-45, -5.6052e-45, -5.6052e-45, 5.6052e-45,\n 5.6052e-45, -5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45,\n -5.6052e-45, -5.6052e-45, -5.6052e-45, 5.6052e-45, 5.6052e-45,\n 5.6052e-45, -5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45,\n 5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45, 5.6052e-45,\n -5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45,\n 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45,\n 5.6052e-45, -5.6052e-45, -5.6052e-45, -5.6052e-45, 5.6052e-45,\n 5.6052e-45, -5.6052e-45, -5.6052e-45, 5.6052e-45, -5.6052e-45,\n -5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45,\n 5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45, -5.6052e-45,\n 5.6052e-45, -5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45,\n 5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45,\n 5.6052e-45], device='cuda:0')",
 "exp_avg_sq": "tensor([2.0123e-20, 4.4328e-22, 1.4627e-19, 6.8675e-20, 2.9771e-21, 3.4987e-20,\n 1.3263e-20, 1.5907e-19, 6.2523e-20, 7.7468e-20, 1.2459e-19, 8.0886e-20,\n 6.2192e-20, 2.5536e-21, 6.4243e-21, 4.0265e-20, 1.9856e-20, 1.7034e-19,\n 1.3959e-21, 5.4579e-19, 5.7219e-19, 1.2349e-21, 3.3626e-20, 1.9178e-19,\n 2.4137e-20, 1.3675e-20, 1.4583e-20, 1.6661e-19, 9.8436e-20, 3.8357e-20,\n 2.0852e-19, 1.6961e-21, 2.0351e-20, 1.5219e-20, 5.9449e-19, 8.8191e-20,\n 4.0518e-20, 2.9654e-21, 3.0134e-19, 1.6897e-20, 6.8905e-20, 2.3075e-20,\n 2.1533e-21, 1.5328e-20, 1.9421e-19, 1.6342e-20, 1.7635e-20, 8.3830e-21,\n 8.9219e-20, 1.9368e-20, 1.8774e-19, 5.7355e-20, 9.6499e-20, 8.3666e-21,\n 6.2414e-20, 8.5822e-20, 1.8522e-20, 2.0555e-21, 7.6213e-20, 1.1394e-20,\n 6.4646e-20, 3.8802e-20, 5.0420e-19, 2.6500e-20, 4.4018e-20, 1.2623e-20,\n 9.7473e-22, 7.6247e-20, 7.2827e-20, 1.2113e-21, 1.1628e-20, 1.3284e-20,\n 1.4460e-19, 2.0145e-19, 1.1576e-21, 3.4398e-21, 4.2203e-21, 2.5097e-21,\n 4.3354e-21, 1.0352e-19, 2.4718e-20, 2.6959e-20, 4.7337e-20, 8.6600e-20,\n 5.8914e-22, 2.2737e-20, 3.6340e-21, 7.0684e-21, 2.2552e-19, 7.6395e-20,\n 3.0118e-20, 3.5491e-20, 8.1890e-20, 1.6957e-19, 5.0783e-21, 9.8899e-21,\n 2.6333e-20, 3.1011e-22, 7.6141e-20, 1.0265e-19, 1.2738e-21, 1.4534e-21,\n 1.4587e-19, 4.6861e-20, 3.4552e-20, 1.5239e-20, 3.0013e-19, 8.4919e-22,\n 2.6082e-20, 3.6428e-20, 1.0826e-20, 2.1565e-19, 2.8723e-20, 4.1191e-22,\n 2.4944e-19, 1.1976e-19, 1.0537e-21, 1.3994e-20, 1.6226e-19, 1.4944e-20,\n 2.2712e-19, 3.2246e-19, 2.7558e-20, 4.3998e-19, 3.8513e-21, 3.3190e-21,\n 6.3412e-21, 2.4347e-20, 1.5977e-21, 6.5178e-19, 2.6058e-20, 2.9692e-20,\n 6.5292e-21, 1.7881e-19, 4.1458e-19, 7.6551e-22, 1.1170e-19, 3.6281e-19,\n 3.5465e-19, 1.8375e-20, 7.3785e-21, 3.6228e-20, 3.8830e-21, 5.7221e-21,\n 1.1474e-21, 1.8345e-20, 3.1109e-20, 6.1346e-22, 1.1790e-21, 4.6300e-22,\n 1.0669e-19, 6.0979e-19, 2.6554e-21, 8.3053e-20, 4.1378e-20, 2.1344e-20,\n 6.3275e-20, 2.4353e-19, 4.1748e-19, 2.1013e-21, 1.7151e-19, 5.5288e-20,\n 5.6830e-21, 1.3419e-20, 1.3757e-19, 1.2537e-18, 1.5626e-20, 5.1079e-21,\n 1.3254e-20, 4.8294e-20, 1.5329e-20, 1.8376e-19, 3.7707e-22, 6.5275e-20,\n 4.8297e-22, 1.7030e-19, 1.1180e-20, 3.7836e-20, 3.0361e-19, 3.4876e-20,\n 4.3816e-21, 6.3139e-20, 1.2760e-21, 5.8250e-21, 2.2812e-19, 6.6217e-22,\n 4.7725e-19, 1.3407e-20, 6.5583e-20, 1.2406e-20, 1.8616e-19, 3.8974e-21,\n 1.4174e-20, 3.2676e-20, 8.6206e-21, 4.1969e-20, 3.4730e-21, 1.7379e-19,\n 3.3159e-19, 3.1170e-19, 1.4773e-21, 1.4364e-19, 7.4332e-21, 1.4481e-20,\n 5.1479e-21, 7.9166e-21, 2.4929e-20, 9.3214e-20, 4.4841e-20, 1.1993e-21,\n 1.4432e-20, 5.7459e-21, 2.5245e-19, 1.2341e-21, 1.1040e-19, 6.5369e-20,\n 4.7480e-20, 1.1886e-19, 1.9654e-19, 2.1101e-20, 2.1507e-21, 9.6303e-20,\n 1.9118e-20, 2.6778e-19, 6.8027e-20, 2.6008e-20, 1.8486e-20, 6.3204e-20,\n 1.6500e-19, 1.8878e-19, 1.3276e-19, 1.8501e-21, 1.4341e-20, 1.7047e-19,\n 2.0586e-21, 6.6454e-20, 8.1072e-20, 4.9495e-20, 1.3864e-19, 4.3396e-20,\n 5.1190e-20, 5.4036e-20, 1.1468e-19, 3.2790e-20, 7.3243e-21, 5.4651e-21,\n 1.1535e-19, 1.3658e-21, 1.6611e-19, 1.0752e-21, 1.1394e-20, 1.4168e-19,\n 4.3380e-19, 6.8732e-20, 2.7448e-19, 1.1613e-20], device='cuda:0')"
 },
 "51": {
 "step": "tensor(18770.)",
 "exp_avg": "tensor([[ 5.6052e-45, -5.6052e-45, 5.6052e-45, ..., 5.6052e-45,\n 5.6052e-45, 5.6052e-45],\n [ 5.6052e-45, 5.6052e-45, 5.6052e-45, ..., 5.6052e-45,\n 5.6052e-45, 5.6052e-45],\n [ 5.6052e-45, 5.6052e-45, -5.6052e-45, ..., 5.6052e-45,\n 5.6052e-45, 5.6052e-45],\n ...,\n [-5.6052e-45, 5.6052e-45, -5.6052e-45, ..., 5.6052e-45,\n -5.6052e-45, 5.6052e-45],\n [ 5.6052e-45, 5.6052e-45, -5.6052e-45, ..., 5.6052e-45,\n 5.6052e-45, 5.6052e-45],\n [ 5.6052e-45, 5.6052e-45, -5.6052e-45, ..., 5.6052e-45,\n 5.6052e-45, 5.6052e-45]], device='cuda:0')",
 "exp_avg_sq": "tensor([[5.2162e-20, 6.3662e-21, 2.4277e-20, ..., 2.5847e-20, 1.7239e-21,\n 1.9756e-20],\n [8.9552e-21, 1.9439e-21, 4.0771e-22, ..., 1.0962e-20, 3.5012e-21,\n 4.8006e-21],\n [8.0928e-20, 2.5099e-20, 1.5473e-19, ..., 2.3161e-20, 2.4919e-21,\n 1.6471e-20],\n ...,\n [1.3591e-21, 3.9302e-21, 5.8346e-22, ..., 1.4519e-21, 7.9527e-22,\n 4.9511e-22],\n [8.3544e-20, 6.0892e-20, 8.8604e-20, ..., 1.2351e-19, 4.7325e-21,\n 7.5413e-20],\n [1.7066e-20, 2.9906e-20, 2.7134e-20, ..., 5.5840e-21, 1.5436e-21,\n 1.1953e-20]], device='cuda:0')"
 },
 "52": {
 "step": "tensor(18770.)",
 "exp_avg": "tensor([ 5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45,\n -5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45,\n 5.6052e-45, -5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45,\n 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45,\n -5.6052e-45, -5.6052e-45, -5.6052e-45, 5.6052e-45, -5.6052e-45,\n 5.6052e-45, -5.6052e-45, -5.6052e-45, -5.6052e-45, -5.6052e-45,\n 5.6052e-45, -5.6052e-45, -5.6052e-45, 5.6052e-45, 5.6052e-45,\n -5.6052e-45, -5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45,\n 5.6052e-45, -5.6052e-45, -5.6052e-45, 5.6052e-45, 5.6052e-45,\n -5.6052e-45, -5.6052e-45, 5.6052e-45, -5.6052e-45, -5.6052e-45,\n -5.6052e-45, -5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45,\n -5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45,\n 5.6052e-45, -5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45,\n -5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45,\n 5.6052e-45, 5.6052e-45, -5.6052e-45, -5.6052e-45, -5.6052e-45,\n -5.6052e-45, 5.6052e-45, -5.6052e-45, -5.6052e-45, 5.6052e-45,\n 5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45, -5.6052e-45,\n 5.6052e-45, -5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45,\n -5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45, -5.6052e-45,\n 5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45, -5.6052e-45,\n 5.6052e-45, -5.6052e-45, -5.6052e-45, 5.6052e-45, 5.6052e-45,\n 5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45, -5.6052e-45,\n -5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45,\n 5.6052e-45, -5.6052e-45, 5.6052e-45, -5.6052e-45, -5.6052e-45,\n -5.6052e-45, -5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45,\n 5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45, -5.6052e-45,\n -5.6052e-45, -5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45,\n -5.6052e-45, -5.6052e-45, 5.6052e-45, -5.6052e-45, -5.6052e-45,\n 5.6052e-45, -5.6052e-45, -5.6052e-45, -5.6052e-45, 5.6052e-45,\n 5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45, -5.6052e-45,\n -5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45,\n 5.6052e-45, -5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45,\n 5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45, -5.6052e-45,\n 5.6052e-45, 5.6052e-45, -5.6052e-45, -5.6052e-45, -5.6052e-45,\n 5.6052e-45, -5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45,\n -5.6052e-45, -5.6052e-45, -5.6052e-45, 5.6052e-45, 5.6052e-45,\n -5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45,\n 5.6052e-45, -5.6052e-45, 5.6052e-45, -5.6052e-45, -5.6052e-45,\n 5.6052e-45, -5.6052e-45, -5.6052e-45, 5.6052e-45, -5.6052e-45,\n 5.6052e-45, -5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45,\n -5.6052e-45, -5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45,\n 5.6052e-45, -5.6052e-45, -5.6052e-45, 5.6052e-45, -5.6052e-45,\n -5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45,\n -5.6052e-45, -5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45,\n -5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45, 5.6052e-45,\n -5.6052e-45, -5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45,\n -5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45,\n 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45,\n 5.6052e-45, -5.6052e-45, -5.6052e-45, 5.6052e-45, -5.6052e-45,\n 5.6052e-45, -5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45,\n 5.6052e-45, -5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45,\n 5.6052e-45], device='cuda:0')",
 "exp_avg_sq": "tensor([7.0748e-17, 1.5217e-17, 1.3836e-16, 9.7247e-19, 4.2789e-17, 1.8098e-16,\n 2.7553e-18, 9.7510e-18, 4.7735e-18, 1.4168e-19, 2.4623e-17, 1.4549e-17,\n 1.1108e-16, 9.2615e-18, 1.6463e-17, 4.8100e-17, 7.5086e-19, 3.2001e-17,\n 1.4658e-16, 2.0638e-16, 6.1123e-19, 3.4088e-18, 5.5024e-18, 1.1038e-16,\n 1.0964e-16, 1.1857e-16, 3.0981e-17, 1.6604e-16, 2.4434e-17, 1.5039e-16,\n 3.2862e-16, 5.0701e-17, 3.0315e-16, 7.1389e-19, 3.9267e-17, 4.5220e-18,\n 6.5514e-18, 2.5907e-17, 1.1100e-16, 2.1210e-17, 1.0635e-17, 5.1219e-17,\n 8.9123e-18, 6.0173e-17, 1.0091e-16, 9.1653e-18, 3.5662e-17, 2.2592e-17,\n 5.8238e-17, 1.6515e-16, 2.2522e-17, 2.5760e-16, 3.7904e-19, 1.8322e-18,\n 1.8108e-16, 7.0719e-19, 2.8785e-18, 3.4533e-17, 6.7337e-17, 4.3986e-18,\n 6.0317e-19, 3.7620e-18, 1.2563e-16, 1.0269e-18, 8.7570e-17, 5.0444e-18,\n 1.2020e-17, 8.4115e-17, 1.2659e-16, 4.9056e-19, 1.5316e-18, 3.8922e-18,\n 2.6058e-16, 1.6883e-17, 6.9120e-18, 5.0122e-18, 8.9986e-18, 2.5579e-18,\n 4.8376e-17, 1.7458e-17, 1.1212e-18, 2.0222e-18, 2.2481e-17, 8.4866e-19,\n 4.2255e-18, 5.0886e-17, 3.4896e-17, 4.5069e-18, 2.2362e-17, 7.5454e-18,\n 9.9121e-17, 5.7344e-19, 3.1114e-18, 1.7853e-19, 1.1153e-16, 1.9921e-18,\n 1.5147e-17, 9.5549e-17, 4.9216e-17, 1.1242e-17, 9.7785e-18, 4.3860e-18,\n 2.8030e-17, 6.0266e-17, 1.7882e-17, 2.4172e-18, 2.1496e-17, 3.4987e-17,\n 1.2893e-16, 6.0672e-18, 1.5234e-17, 5.0452e-19, 4.7067e-18, 2.9794e-16,\n 3.6086e-16, 1.3532e-17, 2.5258e-18, 5.7641e-18, 1.0012e-18, 5.8259e-18,\n 2.1497e-17, 6.5550e-18, 2.2833e-17, 1.5899e-17, 3.0766e-17, 1.1846e-16,\n 2.7819e-18, 2.5997e-17, 1.0546e-16, 1.8994e-16, 1.0433e-16, 3.1111e-17,\n 1.6410e-17, 2.6241e-17, 1.9862e-18, 5.0925e-17, 5.1930e-18, 4.1167e-17,\n 4.4972e-19, 5.7588e-19, 1.7184e-16, 3.7453e-17, 2.1674e-17, 3.9055e-17,\n 1.4754e-17, 2.5843e-18, 4.8976e-18, 4.8196e-18, 5.1300e-18, 6.1510e-18,\n 1.6567e-17, 9.5371e-19, 2.1540e-17, 2.6664e-19, 7.9376e-17, 6.1713e-18,\n 8.2844e-19, 1.0692e-17, 3.2899e-16, 6.0786e-17, 1.4234e-16, 1.2461e-16,\n 6.2473e-17, 5.2489e-19, 2.9120e-18, 1.1567e-16, 3.6340e-17, 9.1123e-18,\n 3.9667e-16, 7.0743e-18, 1.1986e-16, 3.0782e-16, 2.1086e-16, 3.4702e-17,\n 2.1243e-16, 1.8083e-16, 9.7514e-17, 3.5911e-17, 4.0390e-17, 1.6732e-18,\n 2.4416e-17, 1.1323e-17, 8.4423e-18, 5.3565e-18, 2.5272e-17, 5.1776e-18,\n 3.3059e-18, 7.0541e-19, 1.1696e-16, 3.5882e-18, 4.5209e-17, 5.8956e-19,\n 2.0388e-16, 2.5924e-17, 3.5650e-18, 2.0237e-16, 2.1558e-17, 6.0692e-18,\n 7.6638e-17, 6.1932e-17, 7.6022e-17, 7.3736e-17, 6.0858e-18, 1.4103e-18,\n 2.2805e-18, 4.1350e-17, 3.9457e-17, 4.0536e-19, 1.5642e-17, 1.3409e-17,\n 5.7462e-16, 1.7219e-16, 2.0885e-17, 6.7650e-18, 2.8962e-17, 3.8585e-17,\n 2.6430e-17, 1.0531e-17, 1.4487e-16, 1.0176e-16, 3.5285e-17, 1.9287e-18,\n 4.2117e-18, 4.5968e-17, 2.2062e-16, 3.0343e-18, 6.8551e-18, 8.3173e-17,\n 9.2792e-17, 9.1682e-17, 7.0396e-17, 2.2493e-17, 9.4851e-18, 6.1106e-17,\n 5.6892e-17, 8.0576e-19, 1.8173e-16, 4.0474e-18, 4.1732e-17, 1.3553e-16,\n 4.8392e-17, 3.2750e-17, 1.3258e-16, 1.2106e-17, 1.7275e-17, 4.3954e-18,\n 1.2657e-16, 1.3590e-16, 3.5061e-17, 1.3465e-17, 1.7042e-17, 1.5440e-16,\n 5.3313e-17, 1.7327e-18, 3.6205e-16, 3.3428e-17], device='cuda:0')"
 },
 "53": {
 "step": "tensor(18770.)",
 "exp_avg": "tensor([ 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45,\n -5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45,\n 5.6052e-45, -5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45,\n 5.6052e-45, -5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45,\n 5.6052e-45, -5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45,\n 5.6052e-45, -5.6052e-45, -5.6052e-45, -5.6052e-45, -5.6052e-45,\n 5.6052e-45, 5.6052e-45, -5.6052e-45, -5.6052e-45, 5.6052e-45,\n 5.6052e-45, -5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45,\n -5.6052e-45, -5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45,\n 5.6052e-45, -5.6052e-45, 5.6052e-45, -5.6052e-45, -5.6052e-45,\n -5.6052e-45, -5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45,\n -5.6052e-45, 5.6052e-45, -5.6052e-45, -5.6052e-45, 5.6052e-45,\n 5.6052e-45, -5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45,\n 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45,\n 5.6052e-45, 5.6052e-45, -5.6052e-45, -5.6052e-45, -5.6052e-45,\n -5.6052e-45, -5.6052e-45, -5.6052e-45, -5.6052e-45, 5.6052e-45,\n 5.6052e-45, -5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45,\n 5.6052e-45, -5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45,\n -5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45,\n 5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45, -5.6052e-45,\n 5.6052e-45, -5.6052e-45, -5.6052e-45, 5.6052e-45, 5.6052e-45,\n -5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45,\n -5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45,\n 5.6052e-45, 5.6052e-45, -5.6052e-45, -5.6052e-45, 5.6052e-45,\n -5.6052e-45, -5.6052e-45, -5.6052e-45, 5.6052e-45, 5.6052e-45,\n 5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45, -5.6052e-45,\n -5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45,\n -5.6052e-45, -5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45,\n 5.6052e-45, -5.6052e-45, -5.6052e-45, -5.6052e-45, 5.6052e-45,\n -5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45,\n -5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45,\n 5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45, 5.6052e-45,\n 5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45,\n 5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45,\n 5.6052e-45, -5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45,\n -5.6052e-45, -5.6052e-45, -5.6052e-45, 5.6052e-45, 5.6052e-45,\n -5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45,\n 5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45,\n 5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45, 5.6052e-45,\n 5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45, 5.6052e-45,\n 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45,\n 5.6052e-45, -5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45,\n 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45,\n 5.6052e-45, -5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45,\n -5.6052e-45, -5.6052e-45, -5.6052e-45, 5.6052e-45, 5.6052e-45,\n 5.6052e-45, -5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45,\n -5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45, 5.6052e-45,\n 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45,\n 5.6052e-45, 5.6052e-45, -5.6052e-45, -5.6052e-45, 5.6052e-45,\n -5.6052e-45, -5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45,\n 5.6052e-45, -5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45,\n -5.6052e-45], device='cuda:0')",
 "exp_avg_sq": "tensor([2.6456e-19, 1.8064e-20, 4.9715e-19, 3.0503e-21, 3.2376e-20, 2.6888e-19,\n 3.7369e-21, 5.8629e-21, 3.3417e-21, 1.3539e-21, 1.3288e-20, 5.3076e-21,\n 2.7128e-19, 7.1236e-22, 1.1609e-20, 6.8081e-20, 3.0347e-21, 2.3081e-19,\n 6.5000e-19, 1.2155e-18, 4.9320e-21, 8.9195e-21, 3.4675e-21, 3.3860e-19,\n 1.9503e-19, 2.1650e-19, 8.0379e-20, 7.1528e-19, 6.8338e-21, 1.1450e-19,\n 4.1434e-18, 1.5443e-19, 1.2871e-18, 3.7111e-21, 1.5508e-19, 9.7620e-22,\n 8.7450e-21, 5.7775e-21, 2.1288e-19, 6.7809e-21, 1.1163e-20, 1.3515e-19,\n 1.2804e-20, 2.1230e-20, 4.3063e-20, 2.4964e-21, 1.9882e-19, 1.3850e-20,\n 2.0238e-19, 2.9393e-19, 7.1647e-21, 6.4027e-19, 2.7044e-21, 1.3987e-21,\n 3.6218e-19, 2.2327e-21, 1.6462e-21, 8.8662e-21, 4.1457e-20, 2.6627e-21,\n 1.4765e-22, 7.0489e-21, 5.2978e-19, 3.3839e-22, 9.0131e-20, 4.7387e-21,\n 4.5339e-20, 7.7872e-19, 1.4978e-19, 3.5463e-21, 1.4473e-21, 9.5989e-22,\n 1.1139e-18, 8.5672e-20, 5.0103e-20, 1.4743e-19, 6.0396e-21, 1.6938e-20,\n 6.1161e-20, 2.9921e-19, 1.0816e-20, 8.6640e-21, 1.1499e-20, 4.4527e-21,\n 2.5602e-21, 2.4143e-19, 9.3042e-20, 4.1637e-21, 1.0795e-20, 1.7732e-19,\n 2.1643e-19, 2.4054e-21, 3.8204e-21, 1.4333e-21, 1.5338e-19, 4.1863e-21,\n 3.7718e-20, 2.5132e-19, 4.4287e-20, 7.2267e-22, 1.1750e-20, 5.4309e-20,\n 1.4747e-19, 2.1463e-20, 2.3858e-20, 3.8493e-21, 8.4449e-20, 9.2205e-20,\n 2.0019e-19, 3.3018e-21, 1.6115e-20, 9.3848e-22, 5.5670e-21, 1.0199e-18,\n 5.5553e-18, 1.1942e-20, 4.6596e-21, 2.4093e-21, 3.1718e-21, 5.8261e-21,\n 6.2376e-20, 1.1491e-21, 1.7513e-20, 1.1682e-20, 9.2219e-21, 3.7836e-19,\n 3.7649e-21, 1.5776e-20, 1.5963e-19, 2.8988e-19, 3.9294e-20, 4.3399e-20,\n 3.7904e-20, 1.8457e-19, 9.6377e-22, 1.3410e-19, 2.9023e-20, 2.3985e-19,\n 4.3403e-21, 4.0862e-21, 4.0137e-19, 1.1910e-19, 1.2234e-20, 7.1224e-19,\n 5.2379e-21, 8.5679e-21, 6.3707e-21, 2.3233e-21, 9.5927e-21, 2.0893e-21,\n 1.8608e-19, 1.8856e-21, 1.1953e-20, 4.9205e-21, 2.2473e-19, 3.4500e-21,\n 3.2922e-21, 2.7441e-21, 2.1495e-18, 2.1185e-20, 5.4360e-19, 4.7476e-19,\n 4.4652e-20, 7.4861e-22, 4.0340e-21, 1.3666e-19, 1.2433e-20, 1.2792e-20,\n 5.7069e-19, 5.8381e-21, 3.2665e-19, 1.1138e-18, 5.7070e-19, 2.4285e-19,\n 3.5352e-19, 3.2488e-19, 1.5285e-19, 1.9345e-20, 8.2818e-20, 2.1561e-21,\n 1.0047e-19, 4.3468e-21, 1.8817e-20, 1.6307e-20, 2.0869e-19, 2.3631e-20,\n 3.6403e-21, 3.8317e-22, 3.7245e-19, 1.5941e-20, 6.0291e-20, 3.8842e-21,\n 5.9117e-19, 2.2282e-20, 4.8480e-21, 1.2362e-18, 7.3859e-21, 4.5733e-21,\n 2.4040e-19, 1.0443e-19, 2.6885e-19, 4.9063e-20, 7.4615e-21, 1.4859e-21,\n 1.8003e-21, 1.2726e-19, 2.0955e-19, 1.6154e-21, 2.9935e-21, 4.7149e-21,\n 1.8668e-18, 3.0438e-19, 5.5054e-20, 5.4082e-21, 5.6637e-20, 3.4234e-20,\n 6.9972e-20, 4.5317e-20, 1.1665e-19, 2.2222e-19, 2.8009e-21, 1.3543e-21,\n 2.5516e-21, 2.6495e-20, 7.0536e-19, 4.4394e-21, 1.1513e-19, 5.7988e-20,\n 6.0444e-19, 1.4235e-19, 2.0625e-19, 5.6346e-20, 5.8352e-21, 1.2053e-19,\n 1.6698e-19, 1.7351e-21, 6.2188e-19, 2.2040e-21, 1.3307e-19, 2.0147e-19,\n 3.8056e-20, 1.5475e-19, 2.8870e-19, 1.6984e-20, 4.1757e-21, 5.6666e-21,\n 8.5491e-19, 2.6522e-19, 3.9626e-20, 1.4714e-20, 3.7244e-21, 5.7160e-19,\n 4.9397e-19, 1.2868e-21, 3.4544e-18, 1.4665e-20], device='cuda:0')"
 },
 "54": {
 "step": "tensor(18770.)",
 "exp_avg": "tensor([ 5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45,\n -5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45, -5.6052e-45,\n 5.6052e-45, -5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45,\n 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45,\n -5.6052e-45, -5.6052e-45, -5.6052e-45, 5.6052e-45, -5.6052e-45,\n 5.6052e-45, -5.6052e-45, -5.6052e-45, -5.6052e-45, -5.6052e-45,\n 5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45, 5.6052e-45,\n -5.6052e-45, -5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45,\n 5.6052e-45, -5.6052e-45, -5.6052e-45, 5.6052e-45, 5.6052e-45,\n -5.6052e-45, -5.6052e-45, 5.6052e-45, -5.6052e-45, -5.6052e-45,\n -5.6052e-45, -5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45,\n -5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45,\n 5.6052e-45, -5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45,\n -5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45,\n 5.6052e-45, -5.6052e-45, -5.6052e-45, -5.6052e-45, -5.6052e-45,\n -5.6052e-45, 5.6052e-45, -5.6052e-45, -5.6052e-45, 5.6052e-45,\n 5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45,\n 5.6052e-45, -5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45,\n -5.6052e-45, -5.6052e-45, 5.6052e-45, -5.6052e-45, -5.6052e-45,\n 5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45, -5.6052e-45,\n -5.6052e-45, -5.6052e-45, -5.6052e-45, 5.6052e-45, 5.6052e-45,\n 5.6052e-45, 5.6052e-45, -5.6052e-45, -5.6052e-45, -5.6052e-45,\n -5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45,\n 5.6052e-45, -5.6052e-45, 5.6052e-45, -5.6052e-45, -5.6052e-45,\n -5.6052e-45, -5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45,\n -5.6052e-45, -5.6052e-45, 5.6052e-45, -5.6052e-45, -5.6052e-45,\n -5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45, -5.6052e-45,\n -5.6052e-45, -5.6052e-45, 5.6052e-45, -5.6052e-45, -5.6052e-45,\n 5.6052e-45, -5.6052e-45, -5.6052e-45, -5.6052e-45, 5.6052e-45,\n 5.6052e-45, -5.6052e-45, 5.6052e-45, -5.6052e-45, -5.6052e-45,\n -5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45,\n 5.6052e-45, -5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45,\n 5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45, -5.6052e-45,\n 5.6052e-45, 5.6052e-45, -5.6052e-45, -5.6052e-45, -5.6052e-45,\n 5.6052e-45, -5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45,\n -5.6052e-45, -5.6052e-45, -5.6052e-45, 5.6052e-45, 5.6052e-45,\n -5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45,\n 5.6052e-45, -5.6052e-45, 5.6052e-45, -5.6052e-45, -5.6052e-45,\n 5.6052e-45, -5.6052e-45, -5.6052e-45, 5.6052e-45, -5.6052e-45,\n 5.6052e-45, -5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45,\n -5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45,\n 5.6052e-45, -5.6052e-45, -5.6052e-45, 5.6052e-45, -5.6052e-45,\n 5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45,\n -5.6052e-45, -5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45,\n -5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45, 5.6052e-45,\n -5.6052e-45, -5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45,\n -5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45,\n 5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45, 5.6052e-45,\n 5.6052e-45, -5.6052e-45, -5.6052e-45, 5.6052e-45, -5.6052e-45,\n 5.6052e-45, -5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45,\n 5.6052e-45, -5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45,\n 5.6052e-45], device='cuda:0')",
 "exp_avg_sq": "tensor([1.7054e-19, 6.0470e-20, 4.2920e-19, 9.6717e-21, 1.1998e-19, 3.6957e-19,\n 1.1621e-20, 5.7459e-20, 3.2221e-20, 6.1929e-21, 3.2208e-20, 2.3032e-20,\n 4.4885e-19, 8.7921e-21, 4.0957e-20, 1.5101e-19, 8.7533e-21, 1.8102e-19,\n 4.8999e-19, 7.8091e-19, 6.2285e-21, 6.2383e-20, 2.0557e-20, 4.3731e-19,\n 1.2620e-19, 4.0243e-19, 4.8717e-20, 2.9156e-19, 6.3928e-20, 3.1143e-19,\n 1.2945e-18, 6.1794e-20, 5.0505e-19, 1.4121e-21, 1.9410e-19, 1.4519e-20,\n 1.3294e-19, 1.1952e-19, 3.4018e-19, 1.9268e-20, 4.7933e-20, 5.3506e-20,\n 2.4475e-20, 2.4785e-19, 3.9268e-19, 2.0282e-20, 1.8138e-19, 6.3294e-20,\n 1.0590e-19, 3.4023e-19, 1.2469e-20, 4.6599e-19, 2.6238e-21, 1.3402e-21,\n 4.1011e-19, 1.3220e-20, 1.9027e-20, 1.3107e-19, 1.1792e-19, 1.6295e-20,\n 5.9691e-21, 1.2397e-20, 4.4601e-19, 6.9077e-21, 3.4477e-19, 4.4931e-20,\n 6.3686e-20, 3.8081e-19, 2.4550e-19, 1.1600e-21, 1.0730e-20, 7.0309e-21,\n 3.3849e-19, 3.6955e-20, 1.0283e-19, 1.2120e-19, 2.0370e-20, 6.4972e-20,\n 1.5632e-19, 1.4920e-19, 5.5932e-21, 1.5537e-20, 1.1180e-19, 1.7067e-20,\n 4.4308e-21, 2.1430e-19, 7.0610e-20, 1.4638e-20, 1.0493e-19, 1.0854e-19,\n 1.9906e-19, 1.7155e-21, 1.5213e-20, 5.3287e-22, 1.8450e-19, 2.7710e-20,\n 7.3796e-20, 3.9410e-19, 4.9405e-20, 1.3042e-20, 5.9085e-20, 9.3700e-20,\n 9.1391e-20, 1.9529e-19, 4.1165e-20, 1.2658e-21, 1.1422e-19, 1.1545e-19,\n 1.9915e-19, 9.0655e-21, 4.6401e-20, 1.3737e-22, 3.2953e-20, 1.1011e-18,\n 1.2331e-18, 3.6073e-20, 2.2948e-20, 3.3226e-20, 1.2119e-20, 3.3994e-20,\n 8.7973e-20, 1.2084e-20, 6.4407e-20, 4.4761e-20, 1.0283e-19, 2.7933e-19,\n 1.3491e-20, 6.5195e-20, 1.7066e-19, 3.0065e-19, 1.9041e-19, 5.3593e-20,\n 5.4285e-20, 1.4878e-19, 5.7803e-21, 1.1333e-19, 8.5587e-20, 1.5153e-19,\n 5.5796e-21, 6.3875e-21, 5.6518e-19, 1.3517e-19, 8.7690e-20, 6.5903e-19,\n 4.5378e-20, 3.0409e-21, 1.6829e-20, 1.8008e-20, 8.7754e-21, 2.0858e-20,\n 9.7599e-20, 1.3835e-20, 9.5371e-20, 1.3695e-21, 2.8697e-19, 1.1348e-20,\n 1.5589e-20, 4.7588e-20, 1.0631e-18, 1.3445e-19, 4.6451e-19, 3.6886e-19,\n 1.9558e-19, 1.6527e-20, 9.8576e-21, 3.9793e-19, 1.0934e-19, 1.4835e-19,\n 7.9636e-19, 1.3850e-20, 3.6463e-19, 6.9251e-19, 7.3465e-19, 2.5672e-19,\n 6.8437e-19, 2.1496e-19, 1.1800e-19, 4.9230e-20, 1.6818e-19, 5.0228e-21,\n 5.2973e-20, 4.3089e-20, 4.0152e-20, 1.9496e-20, 1.5133e-19, 1.5445e-20,\n 3.4284e-20, 1.2481e-20, 1.2802e-19, 3.5292e-20, 1.0259e-19, 6.2374e-21,\n 2.2890e-19, 2.9548e-20, 5.6389e-20, 7.8328e-19, 2.8893e-20, 1.4301e-20,\n 2.1509e-19, 2.1996e-19, 2.6447e-19, 1.4801e-19, 3.2766e-20, 1.9864e-21,\n 2.3388e-20, 1.7221e-19, 2.4289e-19, 9.8505e-21, 2.0802e-20, 4.6668e-20,\n 1.5154e-18, 4.7118e-19, 7.7582e-20, 1.5328e-20, 1.1091e-19, 4.5361e-20,\n 1.2903e-19, 4.9589e-20, 5.0146e-19, 3.3749e-19, 6.6701e-20, 1.2133e-20,\n 8.4991e-20, 1.8996e-19, 5.9882e-19, 1.8044e-20, 1.2043e-19, 2.9882e-19,\n 2.7442e-19, 3.2108e-19, 9.1980e-20, 5.3481e-20, 5.1938e-20, 2.3485e-19,\n 9.4103e-20, 4.5442e-22, 7.1138e-19, 7.2495e-21, 1.5991e-19, 5.1328e-19,\n 1.8976e-19, 1.4267e-19, 1.5609e-19, 4.7472e-20, 2.9584e-20, 1.4083e-20,\n 2.7437e-19, 4.4817e-19, 9.7587e-20, 4.9487e-20, 4.1559e-20, 2.2306e-19,\n 3.0075e-19, 9.3791e-22, 1.3514e-18, 1.5943e-19], device='cuda:0')"
 },
 "55": {
 "step": "tensor(18770.)",
 "exp_avg": "tensor([[-5.6052e-45, 5.6052e-45, 5.6052e-45, ..., 5.6052e-45,\n 5.6052e-45, 5.6052e-45],\n [-5.6052e-45, -5.6052e-45, -5.6052e-45, ..., 5.6052e-45,\n -5.6052e-45, 5.6052e-45],\n [-5.6052e-45, 5.6052e-45, -5.6052e-45, ..., 5.6052e-45,\n -5.6052e-45, 5.6052e-45],\n ...,\n [-5.6052e-45, -5.6052e-45, 5.6052e-45, ..., -5.6052e-45,\n -5.6052e-45, -5.6052e-45],\n [-5.6052e-45, -5.6052e-45, 5.6052e-45, ..., -5.6052e-45,\n -5.6052e-45, -5.6052e-45],\n [ 5.6052e-45, 5.6052e-45, -5.6052e-45, ..., 5.6052e-45,\n 5.6052e-45, 5.6052e-45]], device='cuda:0')",
 "exp_avg_sq": "tensor([[2.3588e-20, 1.4237e-20, 2.4794e-20, ..., 1.0962e-20, 9.2195e-22,\n 7.5319e-21],\n [3.3052e-21, 1.0523e-20, 1.1546e-21, ..., 2.8970e-21, 1.1040e-21,\n 7.4936e-22],\n [3.4761e-22, 6.9136e-22, 2.7527e-21, ..., 6.2819e-22, 2.9236e-22,\n 4.5630e-22],\n ...,\n [1.9376e-20, 1.2620e-20, 1.6895e-20, ..., 3.0002e-22, 1.3560e-21,\n 1.7029e-21],\n [4.3939e-20, 7.8188e-21, 3.0262e-20, ..., 1.3963e-20, 2.6745e-21,\n 1.6887e-20],\n [5.4711e-20, 3.5705e-20, 3.2846e-20, ..., 7.2761e-20, 1.0916e-21,\n 2.2475e-20]], device='cuda:0')"
 },
 "56": {
 "step": "tensor(18770.)",
 "exp_avg": "tensor([ 5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45,\n -5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45,\n 5.6052e-45, -5.6052e-45, -5.6052e-45, -5.6052e-45, 5.6052e-45,\n 5.6052e-45, -5.6052e-45, -5.6052e-45, 5.6052e-45, 5.6052e-45,\n 5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45,\n 5.6052e-45, -5.6052e-45, -5.6052e-45, -5.6052e-45, 5.6052e-45,\n -5.6052e-45, -5.6052e-45, -5.6052e-45, -5.6052e-45, 5.6052e-45,\n -5.6052e-45, -5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45,\n 5.6052e-45, -5.6052e-45, -5.6052e-45, 5.6052e-45, 5.6052e-45,\n -5.6052e-45, -5.6052e-45, 5.6052e-45, -5.6052e-45, -5.6052e-45,\n -5.6052e-45, -5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45,\n -5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45,\n -5.6052e-45, -5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45,\n -5.6052e-45, -5.6052e-45, -5.6052e-45, -5.6052e-45, 5.6052e-45,\n 5.6052e-45, 5.6052e-45, -5.6052e-45, -5.6052e-45, 5.6052e-45,\n 5.6052e-45, -5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45,\n 5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45, -5.6052e-45,\n 5.6052e-45, -5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45,\n 5.6052e-45, -5.6052e-45, -5.6052e-45, -5.6052e-45, -5.6052e-45,\n 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45,\n -5.6052e-45, 5.6052e-45, -5.6052e-45, -5.6052e-45, 5.6052e-45,\n -5.6052e-45, -5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45,\n -5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45, -5.6052e-45,\n 5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45,\n 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45,\n 5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45, -5.6052e-45,\n -5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45,\n 5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45, 5.6052e-45,\n 5.6052e-45, -5.6052e-45, -5.6052e-45, 5.6052e-45, 5.6052e-45,\n -5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45, -5.6052e-45,\n -5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45, 5.6052e-45,\n -5.6052e-45, -5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45,\n 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45,\n 5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45,\n 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45,\n -5.6052e-45, -5.6052e-45, -5.6052e-45, 5.6052e-45, 5.6052e-45,\n -5.6052e-45, -5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45,\n 5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45, -5.6052e-45,\n 5.6052e-45, -5.6052e-45, -5.6052e-45, 5.6052e-45, -5.6052e-45,\n 5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45, 5.6052e-45,\n -5.6052e-45, -5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45,\n 5.6052e-45, -5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45,\n -5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45, -5.6052e-45,\n -5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45, 5.6052e-45,\n -5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45,\n 5.6052e-45, -5.6052e-45, -5.6052e-45, -5.6052e-45, 5.6052e-45,\n 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45,\n -5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45,\n 5.6052e-45, -5.6052e-45, -5.6052e-45, 5.6052e-45, 5.6052e-45,\n 5.6052e-45, -5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45,\n 5.6052e-45, -5.6052e-45, 5.6052e-45, -5.6052e-45, -5.6052e-45,\n 5.6052e-45], device='cuda:0')",
 "exp_avg_sq": "tensor([4.3300e-17, 5.1471e-18, 6.3220e-20, 7.2483e-18, 5.5413e-17, 6.1036e-17,\n 1.8178e-18, 7.7751e-18, 5.1069e-17, 2.8532e-17, 2.2659e-18, 6.0784e-17,\n 1.6498e-17, 2.7193e-17, 2.4451e-17, 1.0572e-17, 8.2793e-19, 1.2287e-17,\n 2.9676e-17, 7.7030e-17, 9.3277e-19, 8.0526e-18, 6.8893e-18, 7.3197e-17,\n 8.9299e-19, 7.4691e-17, 4.6318e-17, 2.4847e-17, 4.7410e-17, 3.1834e-18,\n 3.7954e-17, 4.3070e-17, 4.7018e-17, 3.1726e-18, 6.5102e-17, 9.0134e-18,\n 7.8465e-19, 2.1902e-18, 5.1528e-18, 4.3569e-17, 2.6190e-17, 5.4098e-18,\n 3.2976e-19, 3.4856e-17, 2.2685e-19, 3.5244e-19, 4.0562e-18, 6.6700e-17,\n 2.7689e-17, 3.8477e-17, 3.9289e-17, 4.2940e-17, 2.7291e-18, 4.7658e-17,\n 2.1943e-17, 1.5417e-17, 4.2279e-18, 7.1130e-19, 2.5590e-16, 1.9930e-17,\n 7.8813e-18, 3.5712e-17, 4.6192e-17, 6.0765e-18, 1.8764e-18, 6.3901e-18,\n 3.3338e-17, 4.0158e-17, 6.0953e-17, 2.8301e-18, 9.4696e-19, 4.0119e-18,\n 2.5074e-16, 2.4372e-17, 7.5277e-18, 1.5821e-17, 3.4569e-19, 3.6932e-17,\n 1.2531e-17, 1.5777e-17, 1.3197e-17, 1.8808e-17, 8.4833e-19, 9.6524e-18,\n 1.5298e-17, 2.3315e-17, 1.5438e-17, 2.6307e-18, 1.9424e-17, 2.4425e-17,\n 1.2837e-18, 1.3950e-18, 1.7227e-17, 8.6121e-17, 3.2268e-18, 6.7143e-18,\n 7.0518e-17, 6.6037e-19, 2.0252e-18, 3.0095e-17, 5.5003e-18, 4.2845e-18,\n 1.6766e-17, 1.1705e-17, 3.0387e-18, 7.8844e-17, 1.5205e-19, 8.8233e-17,\n 2.5337e-19, 2.3137e-17, 3.0378e-17, 1.1282e-17, 5.6571e-19, 3.0356e-17,\n 1.9282e-16, 5.9651e-19, 1.4080e-17, 4.8358e-17, 5.5075e-20, 2.3440e-18,\n 1.1860e-18, 4.3347e-18, 1.1659e-18, 1.1243e-16, 8.1512e-17, 1.4844e-18,\n 1.1934e-17, 3.6371e-18, 4.6896e-17, 1.2959e-17, 4.4711e-17, 5.9343e-18,\n 1.1014e-17, 9.7465e-17, 1.8044e-16, 4.3989e-19, 8.3345e-18, 3.4475e-17,\n 4.2705e-17, 1.1058e-19, 1.0899e-16, 6.4203e-18, 8.8201e-17, 2.5456e-17,\n 8.1338e-19, 5.7487e-19, 1.9117e-17, 2.8665e-17, 5.7268e-18, 4.1475e-18,\n 3.5552e-18, 2.6698e-17, 1.0406e-17, 8.3616e-17, 1.3680e-17, 4.0305e-17,\n 5.3527e-18, 2.0130e-17, 1.5762e-16, 1.1623e-18, 1.3971e-16, 2.5697e-18,\n 1.4042e-17, 4.9943e-18, 1.1010e-17, 2.3777e-16, 1.3867e-17, 4.7067e-17,\n 3.1713e-17, 5.3544e-19, 6.5876e-17, 3.4637e-17, 1.9221e-17, 2.8649e-18,\n 1.3900e-17, 4.0524e-17, 2.7771e-18, 4.6819e-17, 7.8905e-17, 1.6332e-17,\n 5.8631e-17, 1.3638e-18, 6.4526e-18, 7.0897e-19, 7.1675e-17, 1.8224e-18,\n 4.0963e-18, 1.0542e-17, 3.6213e-18, 5.1731e-18, 1.8347e-18, 3.0408e-18,\n 6.0013e-17, 2.0663e-18, 6.4374e-19, 1.7998e-18, 2.7127e-19, 1.5431e-17,\n 4.8980e-18, 3.3494e-19, 3.2312e-17, 3.6440e-18, 8.4006e-18, 1.6119e-17,\n 3.1585e-17, 8.3213e-18, 2.8793e-17, 1.3597e-18, 1.2481e-18, 1.3196e-18,\n 3.1898e-17, 2.7148e-16, 2.2682e-18, 1.5897e-17, 1.4124e-19, 5.9635e-17,\n 1.4856e-17, 6.3437e-19, 5.5102e-17, 5.4053e-17, 7.7089e-19, 1.5087e-17,\n 8.6942e-18, 6.8894e-17, 8.2071e-17, 7.5423e-17, 1.4689e-17, 9.9580e-19,\n 7.0707e-17, 4.2218e-17, 7.7874e-19, 1.3380e-17, 5.4488e-17, 1.4678e-17,\n 7.0121e-18, 1.5105e-17, 5.0695e-18, 2.3210e-18, 6.4855e-19, 3.6333e-19,\n 1.1986e-19, 2.1589e-17, 7.7848e-18, 3.9699e-17, 2.6552e-17, 5.3177e-19,\n 2.4642e-17, 2.8573e-18, 1.2957e-18, 1.4984e-19, 1.2162e-17, 2.0853e-17,\n 1.2496e-17, 3.6375e-17, 4.4442e-17, 1.7268e-16], device='cuda:0')"
 },
 "57": {
 "step": "tensor(18770.)",
 "exp_avg": "tensor([ 5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45,\n -5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45,\n 5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45,\n 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45,\n 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45,\n 5.6052e-45, -5.6052e-45, -5.6052e-45, -5.6052e-45, 5.6052e-45,\n 5.6052e-45, -5.6052e-45, -5.6052e-45, 5.6052e-45, 5.6052e-45,\n 5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45, -5.6052e-45,\n 5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45, 5.6052e-45,\n -5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45,\n 5.6052e-45, -5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45,\n -5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45,\n -5.6052e-45, -5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45,\n 5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45,\n 5.6052e-45, 5.6052e-45, -5.6052e-45, -5.6052e-45, 5.6052e-45,\n 5.6052e-45, -5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45,\n 5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45, -5.6052e-45,\n 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45,\n 5.6052e-45, -5.6052e-45, 5.6052e-45, -5.6052e-45, -5.6052e-45,\n 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45,\n 5.6052e-45, -5.6052e-45, -5.6052e-45, 5.6052e-45, 5.6052e-45,\n -5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45,\n 5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45, -5.6052e-45,\n 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45,\n -5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45,\n 5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45, -5.6052e-45,\n -5.6052e-45, -5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45,\n -5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45,\n 5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45, 5.6052e-45,\n -5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45, -5.6052e-45,\n 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45,\n 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45,\n 5.6052e-45, -5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45,\n 5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45,\n 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45,\n 5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45, 5.6052e-45,\n -5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45,\n 5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45, -5.6052e-45,\n 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45,\n 5.6052e-45, 5.6052e-45, -5.6052e-45, -5.6052e-45, 5.6052e-45,\n 5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45,\n 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45,\n -5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45,\n -5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45,\n -5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45,\n 5.6052e-45, -5.6052e-45, -5.6052e-45, 5.6052e-45, 5.6052e-45,\n 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45,\n 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45,\n 5.6052e-45, -5.6052e-45, -5.6052e-45, 5.6052e-45, 5.6052e-45,\n 5.6052e-45, -5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45,\n 5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45, -5.6052e-45,\n 5.6052e-45], device='cuda:0')",
 "exp_avg_sq": "tensor([1.5300e-19, 3.2459e-21, 2.4887e-21, 4.7895e-21, 6.6478e-20, 6.3084e-20,\n 2.9071e-21, 1.0822e-20, 6.5403e-20, 2.0793e-20, 9.9179e-22, 5.1277e-20,\n 1.3361e-19, 2.2094e-20, 1.8647e-20, 2.1026e-20, 1.0013e-21, 2.7199e-20,\n 2.1898e-20, 2.0600e-19, 3.3457e-21, 1.6802e-20, 1.2203e-21, 6.2962e-19,\n 1.2390e-21, 1.8229e-19, 1.4191e-20, 9.7280e-21, 1.7019e-20, 9.6535e-22,\n 4.1807e-19, 2.2173e-20, 3.5070e-20, 6.9934e-22, 4.1422e-19, 3.5879e-20,\n 1.4353e-21, 5.1162e-22, 1.9569e-21, 5.1523e-20, 4.8397e-20, 2.0098e-20,\n 7.2429e-21, 9.7257e-20, 2.1270e-21, 5.0220e-21, 6.1699e-21, 1.9931e-19,\n 3.1774e-20, 2.4553e-19, 2.1542e-20, 3.6260e-20, 1.7922e-21, 8.8113e-20,\n 2.4540e-20, 1.2606e-20, 1.0856e-21, 3.4519e-21, 2.0900e-19, 3.6624e-20,\n 2.7162e-21, 3.9709e-20, 1.9132e-19, 1.5674e-21, 4.9440e-21, 1.6948e-19,\n 2.1493e-19, 4.1958e-20, 1.5904e-20, 1.0627e-20, 1.6076e-21, 4.6037e-22,\n 8.6684e-19, 6.8431e-21, 5.5572e-21, 2.5528e-20, 1.3303e-21, 7.3999e-20,\n 1.5053e-20, 9.2465e-20, 7.4998e-20, 5.0090e-20, 3.4468e-22, 8.5159e-22,\n 2.8276e-21, 3.0839e-20, 5.6163e-21, 2.2412e-22, 2.8229e-20, 9.5476e-20,\n 2.8642e-21, 2.2282e-21, 4.2800e-20, 1.1368e-19, 4.4544e-21, 1.4454e-20,\n 3.8977e-19, 5.6264e-23, 5.0278e-22, 9.4120e-21, 2.6405e-21, 9.7712e-22,\n 6.7600e-21, 2.2887e-20, 1.4674e-21, 8.0464e-20, 2.1743e-22, 2.9531e-19,\n 3.7191e-22, 4.0204e-20, 6.5311e-20, 1.9444e-20, 8.2009e-22, 3.7827e-19,\n 6.5352e-19, 1.2027e-21, 8.9899e-20, 4.2876e-20, 1.0961e-21, 1.1553e-21,\n 1.5172e-21, 1.4991e-21, 2.7437e-21, 8.5355e-20, 1.5976e-19, 2.2426e-21,\n 6.9740e-21, 4.4202e-21, 1.5219e-19, 3.4720e-20, 3.4596e-20, 1.3638e-21,\n 1.7195e-20, 5.3024e-19, 3.1728e-19, 1.2156e-21, 2.4672e-21, 1.1599e-19,\n 1.2812e-19, 9.4512e-22, 6.0608e-19, 1.4288e-20, 1.3552e-19, 2.9220e-20,\n 5.5700e-22, 1.0815e-21, 6.7287e-21, 2.7837e-20, 4.2859e-21, 1.8970e-21,\n 7.9258e-22, 4.9364e-20, 8.5432e-20, 1.1116e-18, 2.2010e-20, 1.3857e-19,\n 1.1696e-21, 5.2628e-20, 2.1623e-19, 4.5472e-21, 4.5558e-19, 2.0959e-21,\n 1.0800e-20, 2.9779e-20, 5.3400e-20, 7.3242e-19, 3.6606e-21, 3.5852e-20,\n 6.5082e-21, 1.3962e-21, 9.9553e-20, 1.0367e-19, 2.7585e-19, 2.2218e-21,\n 1.4528e-20, 2.1476e-20, 7.7434e-21, 2.4033e-20, 5.1810e-19, 8.0829e-21,\n 2.9260e-20, 3.9212e-21, 1.4206e-20, 5.8743e-22, 5.7348e-19, 1.3500e-21,\n 6.6416e-21, 1.6350e-20, 1.0146e-21, 1.1366e-20, 2.3876e-21, 2.8286e-21,\n 5.2125e-20, 1.7775e-21, 1.0305e-21, 1.7485e-21, 3.6633e-22, 6.8163e-21,\n 3.1982e-21, 1.5723e-21, 6.3060e-20, 7.4783e-22, 6.5819e-21, 4.0276e-21,\n 3.5888e-20, 3.1794e-20, 4.3118e-20, 1.1167e-21, 5.2249e-21, 2.1184e-21,\n 2.2807e-20, 8.9354e-19, 1.8635e-21, 8.0703e-21, 1.2541e-21, 6.0745e-20,\n 1.4687e-20, 4.9561e-22, 4.6149e-19, 1.5457e-19, 2.6096e-21, 3.0128e-20,\n 1.6530e-20, 5.4704e-19, 1.1481e-19, 7.1625e-20, 4.2588e-20, 1.4514e-21,\n 2.8899e-19, 1.7963e-19, 6.4268e-21, 2.5683e-20, 3.4755e-19, 3.6197e-20,\n 9.8506e-21, 4.9481e-21, 8.7515e-21, 4.2081e-21, 4.4026e-22, 8.3259e-22,\n 4.6206e-21, 8.1606e-20, 7.8580e-22, 6.5838e-20, 5.0028e-20, 1.5311e-21,\n 1.1625e-20, 1.5090e-21, 1.1797e-21, 1.3849e-21, 2.0388e-20, 1.8679e-20,\n 2.3775e-20, 8.4351e-21, 2.8332e-19, 7.6530e-19], device='cuda:0')"
 },
 "58": {
 "step": "tensor(18770.)",
 "exp_avg": "tensor([ 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45,\n -5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45,\n 5.6052e-45, -5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45,\n 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45,\n 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45,\n 5.6052e-45, -5.6052e-45, -5.6052e-45, -5.6052e-45, 5.6052e-45,\n 5.6052e-45, -5.6052e-45, -5.6052e-45, 5.6052e-45, 5.6052e-45,\n 5.6052e-45, -5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45,\n 5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45, -5.6052e-45,\n -5.6052e-45, -5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45,\n -5.6052e-45, -5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45,\n -5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45,\n -5.6052e-45, -5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45,\n 5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45,\n 5.6052e-45, 5.6052e-45, -5.6052e-45, -5.6052e-45, 5.6052e-45,\n 5.6052e-45, -5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45,\n 5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45, -5.6052e-45,\n 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45,\n 5.6052e-45, -5.6052e-45, 5.6052e-45, -5.6052e-45, -5.6052e-45,\n 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45,\n 5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45, 5.6052e-45,\n -5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45,\n 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45,\n 5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45,\n 5.6052e-45, -5.6052e-45, -5.6052e-45, 5.6052e-45, 5.6052e-45,\n 5.6052e-45, 5.6052e-45, -5.6052e-45, -5.6052e-45, -5.6052e-45,\n -5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45,\n 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45,\n 5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45, 5.6052e-45,\n -5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45, -5.6052e-45,\n 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45,\n 5.6052e-45, -5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45,\n 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45,\n 5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45, -5.6052e-45,\n 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45,\n -5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45, 5.6052e-45,\n -5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45,\n 5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45, -5.6052e-45,\n -5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45, -5.6052e-45,\n 5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45, -5.6052e-45,\n -5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45,\n 5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45, 5.6052e-45,\n -5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45, -5.6052e-45,\n -5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45,\n -5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45,\n 5.6052e-45, -5.6052e-45, -5.6052e-45, -5.6052e-45, 5.6052e-45,\n 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45,\n -5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45,\n 5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45, 5.6052e-45,\n 5.6052e-45, -5.6052e-45, 5.6052e-45, -5.6052e-45, -5.6052e-45,\n 5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45, -5.6052e-45,\n 5.6052e-45], device='cuda:0')",
 "exp_avg_sq": "tensor([1.3998e-19, 1.6087e-20, 4.9431e-22, 2.5270e-20, 1.4329e-19, 1.0212e-19,\n 1.4317e-20, 1.3521e-20, 1.4715e-19, 8.1448e-20, 3.0870e-21, 8.5748e-20,\n 6.7052e-20, 3.0083e-20, 4.0371e-20, 4.7682e-20, 4.4911e-21, 3.6947e-20,\n 1.0420e-19, 3.1241e-19, 1.1616e-21, 4.3955e-20, 8.3454e-21, 2.0356e-19,\n 1.5306e-21, 2.7278e-19, 5.0434e-20, 2.5193e-20, 5.9988e-20, 1.1840e-20,\n 1.7642e-19, 7.5232e-21, 6.8538e-20, 4.9967e-21, 2.6613e-19, 2.7304e-20,\n 5.2231e-21, 1.0904e-20, 2.0221e-20, 3.1478e-20, 7.8427e-20, 3.9374e-20,\n 7.0058e-21, 1.6383e-19, 8.1396e-22, 1.4408e-20, 2.7510e-20, 1.8000e-19,\n 4.6827e-20, 1.3002e-19, 4.6355e-20, 3.1090e-20, 1.4602e-20, 1.4159e-19,\n 1.8607e-20, 2.4251e-20, 1.8383e-20, 1.2287e-21, 3.2853e-19, 4.1391e-20,\n 8.1008e-21, 4.5081e-20, 2.0375e-19, 9.9299e-21, 1.1017e-20, 5.5304e-20,\n 1.1909e-19, 5.8052e-20, 9.4974e-20, 2.1020e-20, 2.2454e-21, 5.1370e-21,\n 2.0023e-19, 3.3111e-20, 1.0984e-20, 4.8158e-20, 6.8352e-21, 8.4590e-20,\n 1.7389e-20, 1.1265e-19, 8.6611e-20, 6.4867e-20, 1.5725e-21, 7.2334e-21,\n 1.3978e-20, 5.8471e-20, 1.2239e-20, 7.6901e-21, 1.0907e-19, 9.5548e-20,\n 1.1886e-20, 5.6749e-21, 5.2950e-20, 8.5308e-20, 6.3310e-21, 4.4260e-20,\n 2.6744e-19, 2.0005e-21, 2.3173e-22, 1.6001e-20, 9.5128e-21, 6.9563e-21,\n 9.6702e-21, 4.0189e-20, 9.1998e-21, 5.6687e-20, 1.6481e-21, 2.4572e-19,\n 4.8323e-22, 2.8842e-20, 7.5113e-20, 6.3456e-20, 5.1467e-21, 1.4150e-19,\n 2.4793e-19, 6.1823e-21, 1.0108e-19, 1.0473e-19, 7.9572e-22, 7.2069e-21,\n 7.2003e-22, 9.3413e-21, 3.9063e-21, 3.0097e-19, 2.1509e-19, 2.0930e-21,\n 2.6388e-20, 1.1510e-20, 6.9940e-20, 3.0830e-20, 3.4844e-20, 4.4036e-22,\n 4.2866e-20, 3.2119e-19, 3.7084e-19, 3.1286e-22, 3.3957e-20, 1.1660e-19,\n 2.0907e-19, 7.5210e-22, 4.3779e-19, 3.1150e-20, 1.1882e-19, 6.0280e-20,\n 1.1290e-21, 2.3866e-21, 2.6903e-20, 6.8908e-20, 8.9073e-21, 8.8771e-21,\n 6.2246e-21, 1.0837e-19, 5.6827e-20, 3.6200e-19, 4.4736e-20, 9.1116e-20,\n 5.9363e-21, 1.0201e-19, 4.3793e-19, 4.1144e-21, 4.7459e-19, 1.0859e-21,\n 5.1823e-20, 2.3922e-20, 2.6555e-20, 7.3408e-19, 4.0762e-20, 1.3964e-19,\n 3.3917e-20, 1.3575e-21, 1.7921e-19, 1.4237e-19, 1.2116e-19, 1.9094e-20,\n 6.4157e-20, 5.7123e-20, 1.5325e-20, 4.4511e-20, 2.9154e-19, 5.1627e-20,\n 5.7497e-20, 9.0850e-21, 3.6920e-20, 1.7278e-22, 3.5334e-19, 2.6789e-21,\n 3.0803e-20, 4.0721e-20, 3.4803e-21, 1.7271e-20, 3.2016e-21, 9.8667e-21,\n 5.8121e-20, 9.9056e-21, 2.1758e-21, 7.4923e-21, 1.1533e-21, 2.4947e-20,\n 8.0502e-22, 7.3241e-22, 3.3408e-20, 4.8890e-21, 1.7886e-20, 9.2094e-21,\n 6.3859e-20, 5.3180e-20, 5.0342e-20, 3.7228e-22, 7.7175e-21, 4.9607e-21,\n 2.6921e-20, 7.3592e-19, 8.2973e-21, 2.3867e-20, 2.2560e-21, 6.4618e-20,\n 3.7767e-20, 5.6131e-21, 2.5964e-19, 2.0252e-19, 1.4791e-21, 7.2150e-20,\n 3.2908e-20, 3.6451e-19, 1.8197e-19, 2.1798e-19, 2.3399e-20, 4.4891e-21,\n 1.5207e-19, 1.9853e-19, 9.0599e-21, 4.4783e-20, 2.0601e-19, 7.6518e-20,\n 7.0403e-21, 1.9338e-20, 4.0470e-20, 4.6411e-21, 1.9928e-21, 1.9635e-22,\n 4.8345e-22, 4.9059e-20, 8.9378e-21, 1.1997e-19, 3.9758e-20, 1.4758e-21,\n 3.3413e-20, 1.2953e-20, 6.0925e-21, 1.3752e-22, 3.4515e-20, 2.9836e-20,\n 5.0153e-20, 5.4701e-20, 1.6888e-19, 6.5403e-19], device='cuda:0')"
 },
 "59": {
 "step": "tensor(18770.)",
 "exp_avg": "tensor([[-5.6052e-45, -5.6052e-45, -5.6052e-45, ..., -5.6052e-45,\n -5.6052e-45, -5.6052e-45],\n [ 5.6052e-45, -5.6052e-45, -5.6052e-45, ..., 5.6052e-45,\n 5.6052e-45, 5.6052e-45],\n [ 5.6052e-45, -5.6052e-45, 5.6052e-45, ..., 5.6052e-45,\n 5.6052e-45, 5.6052e-45],\n ...,\n [-5.6052e-45, 5.6052e-45, 5.6052e-45, ..., -5.6052e-45,\n -5.6052e-45, -5.6052e-45],\n [ 5.6052e-45, -5.6052e-45, -5.6052e-45, ..., 5.6052e-45,\n 5.6052e-45, 5.6052e-45],\n [ 5.6052e-45, -5.6052e-45, -5.6052e-45, ..., 5.6052e-45,\n 5.6052e-45, 5.6052e-45]], device='cuda:0')",
 "exp_avg_sq": "tensor([[3.6531e-20, 2.1903e-20, 1.7566e-20, ..., 2.4673e-20, 2.1953e-21,\n 2.0269e-20],\n [8.1538e-22, 1.1980e-21, 3.0405e-21, ..., 1.1247e-21, 1.8795e-22,\n 9.7107e-22],\n [2.1310e-21, 5.8396e-22, 1.4721e-21, ..., 1.3512e-21, 5.5518e-22,\n 9.6562e-22],\n ...,\n [9.3134e-20, 2.0340e-20, 5.1851e-20, ..., 3.7380e-20, 5.5188e-21,\n 1.9004e-20],\n [4.4459e-21, 7.4218e-22, 1.0764e-20, ..., 3.0745e-21, 3.1874e-21,\n 3.6866e-21],\n [3.0073e-21, 6.7763e-21, 3.5139e-22, ..., 2.3357e-21, 1.9778e-21,\n 1.3001e-21]], device='cuda:0')"
 },
 "60": {
 "step": "tensor(18770.)",
 "exp_avg": "tensor([-5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45,\n -5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45, -5.6052e-45,\n -5.6052e-45, 5.6052e-45, -5.6052e-45, -5.6052e-45, 5.6052e-45,\n 5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45, 5.6052e-45,\n 5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45, -5.6052e-45,\n -5.6052e-45, -5.6052e-45, -5.6052e-45, -5.6052e-45, 5.6052e-45,\n -5.6052e-45, -5.6052e-45, -5.6052e-45, -5.6052e-45, -5.6052e-45,\n -5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45, 5.6052e-45,\n 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45,\n 5.6052e-45, 5.6052e-45, -5.6052e-45, -5.6052e-45, -5.6052e-45,\n -5.6052e-45, -5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45,\n 5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45, -5.6052e-45,\n -5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45,\n 5.6052e-45, -5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45,\n 5.6052e-45, 5.6052e-45, -5.6052e-45, -5.6052e-45, -5.6052e-45,\n 5.6052e-45, 5.6052e-45, -5.6052e-45, -5.6052e-45, 5.6052e-45,\n 5.6052e-45, -5.6052e-45, -5.6052e-45, -5.6052e-45, 5.6052e-45,\n -5.6052e-45, -5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45,\n -5.6052e-45, -5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45,\n 5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45, -5.6052e-45,\n 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45,\n 5.6052e-45, -5.6052e-45, -5.6052e-45, -5.6052e-45, -5.6052e-45,\n -5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45, -5.6052e-45,\n 5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45, 5.6052e-45,\n -5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45,\n 5.6052e-45, 5.6052e-45, -5.6052e-45, -5.6052e-45, -5.6052e-45,\n -5.6052e-45, -5.6052e-45, -5.6052e-45, -5.6052e-45, -5.6052e-45,\n 5.6052e-45, -5.6052e-45, -5.6052e-45, 5.6052e-45, 5.6052e-45,\n 5.6052e-45, -5.6052e-45, -5.6052e-45, 5.6052e-45, -5.6052e-45,\n -5.6052e-45, -5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45,\n 5.6052e-45, 5.6052e-45, -5.6052e-45, -5.6052e-45, 5.6052e-45,\n -5.6052e-45, -5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45,\n 5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45, -5.6052e-45,\n 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45,\n 5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45,\n 5.6052e-45, -5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45,\n -5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45,\n 5.6052e-45, -5.6052e-45, -5.6052e-45, 5.6052e-45, -5.6052e-45,\n 5.6052e-45, -5.6052e-45, -5.6052e-45, -5.6052e-45, 5.6052e-45,\n 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45,\n 5.6052e-45, -5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45,\n 5.6052e-45, -5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45,\n 5.6052e-45, -5.6052e-45, -5.6052e-45, -5.6052e-45, 5.6052e-45,\n 5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45, 5.6052e-45,\n -5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45,\n 5.6052e-45, 5.6052e-45, -5.6052e-45, -5.6052e-45, -5.6052e-45,\n -5.6052e-45, -5.6052e-45, -5.6052e-45, 5.6052e-45, 5.6052e-45,\n -5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45, -5.6052e-45,\n 5.6052e-45, -5.6052e-45, 5.6052e-45, -5.6052e-45, -5.6052e-45,\n 5.6052e-45, -5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45,\n 5.6052e-45, 5.6052e-45, -5.6052e-45, -5.6052e-45, 5.6052e-45,\n 5.6052e-45], device='cuda:0')",
 "exp_avg_sq": "tensor([8.6106e-17, 1.1032e-18, 1.6386e-18, 9.5258e-19, 5.8472e-18, 6.1470e-16,\n 6.6716e-18, 1.3909e-17, 4.1192e-17, 1.3842e-17, 2.8521e-17, 8.5789e-17,\n 1.3098e-17, 1.0152e-17, 5.3306e-17, 7.6622e-17, 1.2289e-17, 8.9665e-17,\n 6.2042e-16, 8.0083e-18, 1.7108e-16, 1.1041e-18, 1.1201e-17, 2.2586e-16,\n 1.6205e-16, 2.0303e-17, 2.1383e-16, 2.3002e-16, 3.4219e-17, 2.5935e-17,\n 2.9387e-17, 2.1089e-17, 3.8648e-17, 1.3786e-17, 3.2142e-18, 2.9468e-17,\n 3.3653e-17, 3.4421e-18, 1.3009e-16, 5.3078e-17, 2.0992e-18, 1.2607e-16,\n 1.9490e-17, 6.3032e-18, 2.9933e-18, 3.9238e-17, 7.7352e-17, 1.8965e-16,\n 9.3738e-17, 2.2173e-16, 5.1895e-17, 4.1200e-17, 8.2976e-16, 7.0371e-18,\n 2.1683e-16, 3.1485e-17, 1.2288e-16, 1.3979e-16, 4.1003e-16, 2.3552e-17,\n 1.3653e-15, 2.0584e-17, 5.3326e-17, 6.0869e-17, 7.7016e-19, 8.7782e-18,\n 5.1592e-17, 1.0338e-16, 8.8927e-16, 4.4269e-18, 2.5758e-17, 1.1799e-18,\n 8.4959e-16, 4.4030e-16, 1.3648e-17, 1.0878e-17, 1.0923e-16, 2.4876e-17,\n 6.6789e-17, 8.9740e-18, 5.4114e-17, 1.4489e-18, 5.8912e-18, 1.8893e-17,\n 4.9589e-17, 1.9583e-18, 7.5528e-18, 7.8305e-17, 9.6912e-19, 1.7369e-18,\n 1.6402e-17, 6.4541e-17, 1.8836e-17, 3.3276e-17, 5.5846e-18, 1.4523e-18,\n 1.2175e-16, 9.9246e-17, 6.0143e-17, 8.9187e-17, 1.1866e-18, 2.5913e-17,\n 3.1921e-17, 5.5556e-17, 5.5645e-18, 4.2009e-17, 6.7866e-18, 5.6736e-18,\n 5.2861e-18, 3.0397e-18, 1.9939e-16, 4.5579e-18, 7.2354e-18, 3.4299e-17,\n 1.2613e-16, 5.6262e-17, 6.2694e-17, 7.9255e-17, 8.2218e-17, 4.8181e-17,\n 5.2223e-18, 3.0555e-17, 4.3320e-18, 4.7562e-17, 1.3045e-17, 2.7895e-17,\n 2.0900e-18, 6.1863e-18, 1.5186e-16, 6.8231e-17, 3.0708e-17, 7.6185e-17,\n 2.4192e-17, 2.2125e-16, 3.9592e-17, 2.6181e-19, 3.6227e-18, 3.1434e-16,\n 1.9367e-16, 7.7299e-18, 8.5686e-17, 5.1367e-17, 1.8565e-18, 1.1181e-17,\n 3.3938e-18, 5.1455e-17, 1.0705e-17, 5.4706e-17, 1.1588e-16, 8.8539e-18,\n 2.8392e-17, 6.8014e-18, 2.2815e-17, 1.7233e-18, 8.4236e-18, 9.5341e-17,\n 3.8809e-16, 8.3282e-18, 5.2082e-16, 1.5883e-17, 3.8441e-17, 1.9257e-18,\n 3.6192e-17, 8.5162e-19, 2.1340e-17, 3.2519e-17, 1.6336e-16, 3.4078e-17,\n 5.2220e-18, 3.3035e-17, 2.9010e-17, 7.5165e-17, 7.5157e-19, 1.7391e-18,\n 2.8518e-16, 3.1303e-17, 2.6404e-18, 6.6352e-19, 6.5558e-17, 1.0297e-18,\n 1.4036e-17, 1.7028e-18, 7.7851e-17, 2.1482e-17, 1.7830e-16, 1.5453e-16,\n 9.2012e-18, 7.6332e-18, 1.1464e-18, 2.0279e-16, 4.2424e-18, 7.6581e-17,\n 1.1045e-16, 9.6995e-19, 2.7073e-17, 9.5618e-18, 3.6451e-18, 9.9190e-17,\n 1.5669e-16, 1.1407e-16, 9.0843e-17, 5.6533e-16, 5.4727e-18, 1.8646e-16,\n 9.0689e-18, 7.4238e-17, 1.3539e-16, 7.5153e-18, 9.5562e-18, 3.1331e-17,\n 2.7503e-16, 5.6029e-18, 4.6449e-18, 6.8819e-17, 3.5523e-17, 3.4697e-17,\n 3.0703e-18, 3.1873e-18, 1.7406e-16, 1.3419e-17, 7.1888e-17, 1.9861e-17,\n 2.1930e-18, 2.7859e-16, 5.7591e-18, 5.9688e-17, 3.0235e-17, 1.0560e-17,\n 9.6582e-18, 1.1752e-17, 6.9306e-17, 3.7080e-17, 1.8028e-17, 9.5673e-18,\n 2.1664e-17, 3.7494e-17, 6.9938e-17, 2.6408e-18, 3.3232e-18, 1.7904e-18,\n 4.7231e-17, 3.5017e-17, 1.6343e-18, 2.7608e-17, 2.7424e-17, 2.7926e-17,\n 4.4854e-17, 1.1023e-17, 7.3614e-17, 8.2691e-17, 1.0432e-18, 1.7013e-17,\n 1.5384e-18, 1.6629e-16, 1.5288e-17, 2.3962e-18], device='cuda:0')"
 },
 "61": {
 "step": "tensor(18770.)",
 "exp_avg": "tensor([ 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45,\n -5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45,\n -5.6052e-45, 5.6052e-45, -5.6052e-45, -5.6052e-45, 5.6052e-45,\n 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45,\n 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45,\n 5.6052e-45, 5.6052e-45, -5.6052e-45, -5.6052e-45, -5.6052e-45,\n 5.6052e-45, -5.6052e-45, -5.6052e-45, -5.6052e-45, 5.6052e-45,\n -5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45,\n 5.6052e-45, 5.6052e-45, -5.6052e-45, -5.6052e-45, 5.6052e-45,\n 5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45, -5.6052e-45,\n -5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45,\n 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45,\n 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45,\n 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45,\n -5.6052e-45, -5.6052e-45, -5.6052e-45, 5.6052e-45, 5.6052e-45,\n 5.6052e-45, 5.6052e-45, -5.6052e-45, -5.6052e-45, 5.6052e-45,\n 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45,\n 5.6052e-45, -5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45,\n -5.6052e-45, -5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45,\n 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45,\n -5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45,\n 5.6052e-45, -5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45,\n -5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45, -5.6052e-45,\n 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45,\n 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45,\n -5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45, -5.6052e-45,\n -5.6052e-45, -5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45,\n 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45,\n 5.6052e-45, 5.6052e-45, -5.6052e-45, -5.6052e-45, 5.6052e-45,\n 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45,\n 5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45,\n 5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45, -5.6052e-45,\n 5.6052e-45, -5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45,\n 5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45,\n -5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45,\n 5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45, 5.6052e-45,\n 5.6052e-45, 5.6052e-45, -5.6052e-45, -5.6052e-45, 5.6052e-45,\n 5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45,\n 5.6052e-45, -5.6052e-45, -5.6052e-45, 5.6052e-45, 5.6052e-45,\n 5.6052e-45, -5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45,\n 5.6052e-45, 5.6052e-45, -5.6052e-45, -5.6052e-45, -5.6052e-45,\n 5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45,\n 5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45,\n 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45,\n -5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45,\n -5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45, -5.6052e-45,\n -5.6052e-45, -5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45,\n -5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45, 5.6052e-45,\n 5.6052e-45, 5.6052e-45, -5.6052e-45, -5.6052e-45, 5.6052e-45,\n 5.6052e-45, -5.6052e-45, -5.6052e-45, 5.6052e-45, 5.6052e-45,\n -5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45,\n -5.6052e-45], device='cuda:0')",
 "exp_avg_sq": "tensor([6.7914e-19, 2.9416e-21, 1.1292e-20, 2.1344e-21, 2.8685e-21, 2.4127e-18,\n 1.5007e-20, 1.2853e-20, 2.1647e-20, 8.8091e-21, 3.3497e-20, 5.6879e-19,\n 3.8844e-21, 6.3520e-21, 5.0310e-20, 2.1654e-19, 1.1113e-20, 2.4140e-19,\n 6.0603e-18, 1.2026e-20, 1.2683e-18, 1.4608e-21, 2.4976e-20, 3.5718e-18,\n 1.7255e-19, 1.3042e-20, 3.1212e-18, 3.0879e-19, 2.3270e-20, 7.3887e-20,\n 3.7873e-20, 6.4060e-19, 6.6433e-21, 4.1464e-20, 2.0035e-21, 3.6649e-19,\n 4.6327e-20, 5.3384e-21, 1.8200e-18, 1.1734e-18, 3.5545e-21, 6.8400e-19,\n 2.2332e-20, 8.8621e-21, 4.9311e-21, 1.1319e-19, 3.5113e-20, 1.5420e-19,\n 1.3419e-19, 4.5269e-19, 5.1646e-22, 4.8114e-19, 4.7630e-18, 1.9357e-21,\n 1.0676e-18, 2.2568e-19, 3.1448e-19, 1.2612e-19, 1.0833e-18, 8.4150e-21,\n 6.4676e-18, 3.0339e-20, 3.0478e-20, 6.0173e-19, 5.5797e-21, 8.6922e-21,\n 7.7953e-20, 1.5973e-18, 2.1258e-18, 3.4386e-21, 2.1246e-20, 1.4033e-20,\n 2.6913e-18, 1.8139e-18, 1.7990e-20, 8.2031e-21, 1.3469e-19, 1.7156e-19,\n 6.0521e-20, 2.0349e-21, 1.7800e-19, 4.2096e-21, 8.8530e-21, 1.9765e-20,\n 6.0040e-19, 1.6932e-20, 3.8453e-20, 1.8743e-19, 4.7078e-21, 1.1526e-21,\n 2.6939e-19, 1.1525e-18, 8.6689e-20, 7.1343e-20, 3.9566e-21, 3.2125e-21,\n 2.9374e-19, 2.5050e-19, 1.0788e-19, 1.3532e-20, 8.7934e-22, 1.4991e-20,\n 2.8623e-19, 1.0874e-18, 2.2610e-20, 1.8075e-19, 1.4869e-19, 1.2875e-20,\n 1.7002e-21, 1.0494e-21, 1.8710e-19, 5.4552e-21, 5.6707e-20, 1.0472e-18,\n 7.8345e-19, 8.8129e-19, 3.4479e-19, 1.7616e-19, 8.1831e-20, 1.2137e-20,\n 1.3641e-21, 8.8848e-21, 4.3030e-21, 8.5727e-21, 1.3883e-19, 2.3437e-20,\n 1.3276e-21, 5.7303e-21, 5.4752e-19, 2.8717e-20, 1.2105e-19, 4.2260e-20,\n 1.3716e-20, 5.4956e-18, 3.1114e-19, 3.2969e-21, 9.0866e-21, 4.9681e-18,\n 6.0265e-19, 6.2584e-21, 5.8613e-20, 2.3516e-20, 2.5614e-21, 8.3778e-21,\n 1.8091e-21, 5.5013e-21, 1.0055e-20, 2.3398e-19, 1.9976e-18, 6.8250e-21,\n 2.7093e-20, 1.8562e-20, 2.2478e-19, 3.8682e-21, 3.1793e-20, 1.0032e-18,\n 8.7306e-19, 1.3616e-20, 8.5421e-19, 5.2655e-21, 8.3490e-21, 4.1825e-21,\n 2.4147e-20, 4.9438e-22, 1.7062e-21, 2.6031e-20, 2.0238e-19, 1.3830e-20,\n 1.9364e-21, 3.2877e-20, 2.4825e-20, 2.1698e-19, 1.7615e-21, 3.6628e-21,\n 3.9934e-19, 1.2967e-20, 2.5019e-21, 3.4603e-21, 2.3587e-19, 1.4331e-21,\n 4.3162e-20, 2.7497e-21, 2.7491e-20, 1.0081e-19, 1.8640e-19, 3.9534e-19,\n 2.2609e-19, 4.3135e-21, 5.8380e-22, 2.9464e-19, 9.1943e-20, 2.7038e-20,\n 1.0279e-18, 1.0502e-20, 8.7957e-20, 1.4696e-19, 9.3697e-21, 1.8623e-19,\n 7.0349e-19, 1.2708e-19, 1.5434e-18, 1.8738e-18, 6.3755e-21, 1.9478e-19,\n 2.1891e-21, 5.1618e-19, 7.2351e-20, 7.8241e-21, 6.1082e-22, 1.0814e-19,\n 1.2459e-18, 7.0078e-21, 3.7022e-21, 4.3219e-20, 3.9114e-20, 7.3481e-20,\n 4.6816e-21, 1.1660e-20, 2.5251e-18, 4.1139e-21, 4.2032e-20, 2.7867e-20,\n 2.6224e-21, 1.7101e-18, 3.0470e-20, 3.4704e-20, 4.1744e-20, 8.2848e-20,\n 1.3232e-21, 6.1713e-20, 3.3538e-19, 1.1153e-18, 8.7423e-19, 8.8768e-21,\n 1.7818e-19, 2.3918e-20, 4.6937e-19, 2.1555e-20, 3.8900e-21, 2.7136e-20,\n 4.8165e-20, 1.4158e-19, 4.0954e-21, 5.9954e-20, 3.3155e-22, 4.0144e-20,\n 2.2323e-20, 2.0706e-20, 5.6150e-19, 8.0715e-20, 6.2449e-21, 8.8974e-21,\n 3.2902e-21, 8.0107e-19, 6.9101e-21, 9.4365e-21], device='cuda:0')"
 },
 "62": {
 "step": "tensor(18770.)",
 "exp_avg": "tensor([ 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45,\n -5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45, -5.6052e-45,\n -5.6052e-45, 5.6052e-45, -5.6052e-45, -5.6052e-45, 5.6052e-45,\n 5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45, 5.6052e-45,\n 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45,\n -5.6052e-45, 5.6052e-45, -5.6052e-45, -5.6052e-45, 5.6052e-45,\n 5.6052e-45, -5.6052e-45, -5.6052e-45, -5.6052e-45, 5.6052e-45,\n -5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45, 5.6052e-45,\n 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45,\n 5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45, -5.6052e-45,\n -5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45,\n 5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45, -5.6052e-45,\n -5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45,\n 5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45,\n 5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45, -5.6052e-45,\n -5.6052e-45, 5.6052e-45, -5.6052e-45, -5.6052e-45, 5.6052e-45,\n 5.6052e-45, -5.6052e-45, -5.6052e-45, 5.6052e-45, 5.6052e-45,\n 5.6052e-45, -5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45,\n -5.6052e-45, -5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45,\n 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45,\n 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45,\n 5.6052e-45, -5.6052e-45, -5.6052e-45, 5.6052e-45, -5.6052e-45,\n -5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45, -5.6052e-45,\n 5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45, 5.6052e-45,\n 5.6052e-45, -5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45,\n 5.6052e-45, 5.6052e-45, -5.6052e-45, -5.6052e-45, -5.6052e-45,\n -5.6052e-45, -5.6052e-45, -5.6052e-45, -5.6052e-45, -5.6052e-45,\n -5.6052e-45, -5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45,\n 5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45, -5.6052e-45,\n -5.6052e-45, -5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45,\n 5.6052e-45, 5.6052e-45, -5.6052e-45, -5.6052e-45, 5.6052e-45,\n 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45,\n 5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45, 5.6052e-45,\n 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45,\n 5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45,\n 5.6052e-45, -5.6052e-45, -5.6052e-45, 5.6052e-45, 5.6052e-45,\n 5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45,\n 5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45, -5.6052e-45,\n 5.6052e-45, -5.6052e-45, -5.6052e-45, -5.6052e-45, 5.6052e-45,\n 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45,\n 5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45,\n 5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45,\n 5.6052e-45, -5.6052e-45, -5.6052e-45, -5.6052e-45, 5.6052e-45,\n 5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45, 5.6052e-45,\n -5.6052e-45, -5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45,\n 5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45, -5.6052e-45,\n -5.6052e-45, -5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45,\n -5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45,\n 5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45,\n 5.6052e-45, -5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45,\n 5.6052e-45, 5.6052e-45, -5.6052e-45, -5.6052e-45, 5.6052e-45,\n 5.6052e-45], device='cuda:0')",
 "exp_avg_sq": "tensor([4.6757e-19, 1.9618e-20, 1.1980e-21, 6.3111e-22, 4.0024e-20, 9.8578e-19,\n 7.3737e-20, 8.9102e-20, 2.6095e-19, 9.8859e-20, 2.0260e-19, 3.4315e-19,\n 5.4832e-20, 2.3987e-20, 1.0676e-19, 3.0463e-19, 7.6707e-20, 1.6324e-19,\n 2.8565e-18, 6.2630e-20, 5.8799e-19, 4.7375e-21, 6.0876e-20, 1.3598e-18,\n 2.7032e-19, 1.5313e-19, 8.9211e-19, 2.3839e-19, 1.4380e-19, 1.6335e-19,\n 4.1790e-20, 2.0287e-19, 1.4315e-20, 2.2338e-19, 1.1524e-20, 1.3125e-19,\n 1.4072e-19, 5.9076e-20, 6.9994e-19, 4.3615e-19, 2.0564e-20, 7.7891e-19,\n 1.2791e-19, 5.7535e-20, 1.0559e-20, 2.6077e-19, 3.7211e-19, 2.5700e-19,\n 5.5410e-20, 4.5644e-19, 5.5112e-20, 2.7767e-19, 3.5668e-18, 4.2618e-20,\n 1.0401e-18, 2.7151e-19, 6.9460e-19, 6.2666e-19, 1.0895e-18, 9.1016e-20,\n 2.5755e-18, 5.5389e-20, 2.3404e-19, 4.3417e-19, 2.2622e-21, 8.7908e-20,\n 3.5299e-20, 6.0297e-19, 1.7691e-18, 5.9955e-20, 8.6525e-20, 4.2221e-21,\n 1.1328e-18, 9.1654e-19, 6.0591e-20, 1.0180e-19, 4.7602e-19, 3.0312e-19,\n 1.1416e-19, 9.3331e-20, 3.8944e-19, 2.8924e-20, 9.1737e-20, 1.6563e-20,\n 1.7710e-19, 2.7312e-20, 8.3136e-20, 4.0709e-19, 2.8205e-20, 5.8997e-21,\n 2.4668e-19, 2.9770e-19, 1.2036e-19, 1.0749e-19, 3.6286e-20, 1.5115e-20,\n 6.1525e-19, 5.7457e-19, 8.9821e-20, 1.1639e-19, 5.4001e-21, 4.4320e-20,\n 1.8972e-19, 5.8022e-19, 5.2541e-20, 1.0675e-19, 1.8086e-19, 4.6063e-20,\n 1.9413e-20, 5.5641e-20, 2.4315e-19, 4.5102e-20, 8.5380e-20, 3.7831e-19,\n 1.7925e-19, 5.5633e-19, 4.4707e-19, 2.7198e-19, 3.4070e-19, 2.6299e-19,\n 4.7166e-20, 1.9549e-19, 2.4634e-20, 1.5491e-19, 2.0218e-19, 6.7423e-20,\n 9.4191e-21, 9.0248e-20, 1.3448e-19, 6.9448e-20, 1.1609e-19, 1.7504e-19,\n 1.2884e-19, 1.4063e-18, 1.8042e-19, 1.4487e-21, 5.7167e-20, 1.5248e-18,\n 9.9722e-19, 6.5806e-20, 3.9439e-19, 6.0314e-20, 3.8800e-22, 3.7793e-20,\n 1.8995e-20, 5.7726e-20, 3.1370e-20, 2.0706e-19, 5.1981e-19, 3.4259e-20,\n 1.2784e-19, 1.0381e-19, 9.6494e-20, 3.3601e-20, 8.5016e-20, 5.6274e-19,\n 7.2907e-19, 4.0081e-20, 1.9186e-18, 4.2002e-20, 2.0874e-19, 4.6057e-22,\n 9.5835e-20, 7.8062e-22, 5.1244e-21, 1.1754e-19, 5.1033e-19, 1.1290e-19,\n 6.6247e-22, 8.1303e-20, 1.0519e-19, 4.0766e-19, 1.7618e-20, 1.7397e-20,\n 1.2272e-18, 1.4908e-19, 4.5948e-21, 2.3462e-21, 2.9057e-19, 5.9037e-21,\n 4.4662e-20, 1.3622e-20, 4.1723e-19, 2.9781e-19, 8.3767e-19, 7.4929e-19,\n 1.4951e-19, 9.5662e-20, 3.8723e-22, 1.2447e-19, 3.5094e-20, 1.5955e-19,\n 4.0573e-19, 2.0640e-20, 1.7656e-19, 6.7069e-20, 2.7352e-20, 2.7437e-19,\n 4.1082e-19, 4.4180e-19, 5.9246e-19, 7.3397e-19, 2.6405e-20, 1.6695e-19,\n 5.8319e-20, 5.2888e-19, 1.3180e-19, 1.9986e-20, 1.6235e-19, 2.4503e-19,\n 8.6726e-19, 6.2649e-20, 3.0181e-20, 2.7155e-19, 2.3497e-19, 5.7941e-20,\n 3.7285e-21, 4.5770e-20, 1.0912e-18, 5.5328e-20, 1.5039e-19, 1.2061e-19,\n 4.6696e-20, 1.4225e-18, 2.6353e-20, 1.7182e-19, 2.3700e-19, 7.2223e-20,\n 1.0570e-20, 1.9305e-19, 1.4504e-19, 4.7584e-19, 3.7512e-19, 2.2596e-20,\n 2.3758e-19, 3.4164e-19, 4.7714e-19, 2.8042e-20, 6.3256e-20, 6.7329e-20,\n 2.3706e-19, 2.8784e-19, 1.1100e-21, 8.5772e-20, 1.6374e-20, 7.9135e-20,\n 3.2977e-20, 4.4063e-20, 3.9745e-19, 3.4904e-19, 9.2304e-21, 1.0436e-19,\n 1.8837e-20, 6.7357e-19, 1.1615e-19, 1.6298e-20], device='cuda:0')"
 },
 "63": {
 "step": "tensor(18770.)",
 "exp_avg": "tensor([[-5.6052e-45, -5.6052e-45, -5.6052e-45, ..., 5.6052e-45,\n -5.6052e-45, -5.6052e-45],\n [-5.6052e-45, 5.6052e-45, -5.6052e-45, ..., -5.6052e-45,\n -5.6052e-45, 5.6052e-45],\n [-5.6052e-45, 5.6052e-45, -5.6052e-45, ..., -5.6052e-45,\n -5.6052e-45, 5.6052e-45],\n ...,\n [ 5.6052e-45, -5.6052e-45, 5.6052e-45, ..., 5.6052e-45,\n 5.6052e-45, 5.6052e-45],\n [ 5.6052e-45, -5.6052e-45, 5.6052e-45, ..., 5.6052e-45,\n 5.6052e-45, 5.6052e-45],\n [-5.6052e-45, 5.6052e-45, -5.6052e-45, ..., -5.6052e-45,\n 5.6052e-45, 5.6052e-45]], device='cuda:0')",
 "exp_avg_sq": "tensor([[1.3664e-20, 4.9596e-22, 3.1473e-21, ..., 4.7537e-21, 4.3890e-20,\n 8.3328e-22],\n [4.2425e-21, 8.0149e-22, 9.4600e-21, ..., 1.7315e-20, 2.1054e-20,\n 2.8639e-21],\n [1.5256e-21, 7.3985e-23, 4.3444e-21, ..., 7.0665e-21, 9.2200e-21,\n 4.1397e-21],\n ...,\n [4.2029e-18, 1.0218e-19, 3.1824e-18, ..., 2.2172e-18, 1.1524e-17,\n 2.5094e-18],\n [2.1728e-18, 1.5258e-19, 2.5942e-18, ..., 1.7321e-18, 7.3538e-18,\n 3.9803e-19],\n [1.4664e-18, 8.7364e-20, 8.8499e-19, ..., 1.1888e-18, 4.1742e-18,\n 1.5625e-19]], device='cuda:0')"
 },
 "64": {
 "step": "tensor(18770.)",
 "exp_avg": "tensor([-5.6052e-45, -5.6052e-45, -5.6052e-45, 5.6052e-45, -5.6052e-45,\n -5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45, -5.6052e-45,\n -5.6052e-45, 5.6052e-45, -5.6052e-45, -5.6052e-45, 5.6052e-45,\n 5.6052e-45, -5.6052e-45, 5.6052e-45, -5.6052e-45, -5.6052e-45,\n -5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45, -5.6052e-45,\n -5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45,\n -5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45,\n -5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45, -5.6052e-45,\n -5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45, 5.6052e-45,\n 5.6052e-45, -5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45,\n -5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45,\n -5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45,\n -5.6052e-45, -5.6052e-45, -5.6052e-45, 5.6052e-45, -5.6052e-45,\n -5.6052e-45, -5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45,\n 5.6052e-45, 5.6052e-45, -5.6052e-45, -5.6052e-45, 5.6052e-45,\n -5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45, 5.6052e-45,\n 5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45, -5.6052e-45,\n 5.6052e-45, -5.6052e-45, -5.6052e-45, -5.6052e-45, 5.6052e-45,\n -5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45, 5.6052e-45,\n -5.6052e-45, -5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45,\n -5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45,\n 5.6052e-45, -5.6052e-45, -5.6052e-45, 5.6052e-45, 5.6052e-45,\n -5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45, -5.6052e-45,\n 5.6052e-45, -5.6052e-45, -5.6052e-45, -5.6052e-45, 5.6052e-45,\n 5.6052e-45, -5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45,\n 5.6052e-45, -5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45,\n 5.6052e-45, -5.6052e-45, -5.6052e-45, -5.6052e-45, 5.6052e-45,\n -5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45, -5.6052e-45,\n -5.6052e-45, -5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45,\n 5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45, -5.6052e-45,\n -5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45,\n 5.6052e-45, -5.6052e-45, -5.6052e-45, 5.6052e-45, 5.6052e-45,\n 5.6052e-45, -5.6052e-45, -5.6052e-45, -5.6052e-45, 5.6052e-45,\n -5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45,\n -5.6052e-45, -5.6052e-45, -5.6052e-45, -5.6052e-45, 5.6052e-45,\n -5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45,\n -5.6052e-45, -5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45,\n -5.6052e-45, -5.6052e-45, -5.6052e-45, 5.6052e-45, -5.6052e-45,\n 5.6052e-45, -5.6052e-45, -5.6052e-45, -5.6052e-45, -5.6052e-45,\n 5.6052e-45, -5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45,\n -5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45, -5.6052e-45,\n 5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45, -5.6052e-45,\n -5.6052e-45, -5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45,\n 5.6052e-45, -5.6052e-45, -5.6052e-45, -5.6052e-45, 5.6052e-45,\n -5.6052e-45, -5.6052e-45, 5.6052e-45, -5.6052e-45, -5.6052e-45,\n -5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45,\n 5.6052e-45, -5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45,\n -5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45, -5.6052e-45,\n -5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45, 5.6052e-45,\n -5.6052e-45, -5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45,\n 5.6052e-45, -5.6052e-45, 5.6052e-45, -5.6052e-45, -5.6052e-45,\n -5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45,\n -5.6052e-45, -5.6052e-45, 5.6052e-45, -5.6052e-45, -5.6052e-45,\n -5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45, 5.6052e-45,\n -5.6052e-45, -5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45,\n 5.6052e-45, -5.6052e-45, -5.6052e-45, 5.6052e-45, -5.6052e-45,\n 5.6052e-45, -5.6052e-45, -5.6052e-45, 5.6052e-45, 5.6052e-45,\n -5.6052e-45, -5.6052e-45, -5.6052e-45, -5.6052e-45, 5.6052e-45,\n -5.6052e-45, -5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45,\n -5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45,\n -5.6052e-45, -5.6052e-45, -5.6052e-45, 5.6052e-45, -5.6052e-45,\n 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45,\n 5.6052e-45, -5.6052e-45, 5.6052e-45, -5.6052e-45, -5.6052e-45,\n 5.6052e-45, -5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45,\n -5.6052e-45, -5.6052e-45, -5.6052e-45, -5.6052e-45, -5.6052e-45,\n -5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45,\n -5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45,\n 5.6052e-45, 5.6052e-45, -5.6052e-45, -5.6052e-45, 5.6052e-45,\n 5.6052e-45, -5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45,\n -5.6052e-45, -5.6052e-45, -5.6052e-45, 5.6052e-45, 5.6052e-45,\n -5.6052e-45, -5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45,\n 5.6052e-45, -5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45,\n -5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45, -5.6052e-45,\n -5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45, -5.6052e-45,\n -5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45,\n 5.6052e-45, -5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45,\n 5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45, 5.6052e-45,\n -5.6052e-45, -5.6052e-45, -5.6052e-45, 5.6052e-45, 5.6052e-45,\n -5.6052e-45, -5.6052e-45, -5.6052e-45, -5.6052e-45, 5.6052e-45,\n 5.6052e-45, -5.6052e-45, -5.6052e-45, -5.6052e-45, 5.6052e-45,\n -5.6052e-45, -5.6052e-45, -5.6052e-45, -5.6052e-45, -5.6052e-45,\n 5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45,\n 5.6052e-45, -5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45,\n -5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45, -5.6052e-45,\n -5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45,\n -5.6052e-45, -5.6052e-45, -5.6052e-45, 5.6052e-45, 5.6052e-45,\n -5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45, -5.6052e-45,\n -5.6052e-45, -5.6052e-45, -5.6052e-45, -5.6052e-45, 5.6052e-45,\n -5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45, -5.6052e-45,\n 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45,\n -5.6052e-45, -5.6052e-45, -5.6052e-45, -5.6052e-45, 5.6052e-45,\n -5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45, 5.6052e-45,\n 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45,\n -5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45, 5.6052e-45,\n 5.6052e-45, -5.6052e-45, -5.6052e-45, -5.6052e-45, 5.6052e-45,\n 5.6052e-45, -5.6052e-45, -5.6052e-45, 5.6052e-45, -5.6052e-45,\n -5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45, -5.6052e-45,\n -5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45, -5.6052e-45,\n 5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45, 5.6052e-45,\n 5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45, -5.6052e-45,\n -5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45,\n 5.6052e-45, -5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45,\n -5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45,\n 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45,\n -5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45,\n 5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45,\n -5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45,\n -5.6052e-45, -5.6052e-45, -5.6052e-45, 5.6052e-45, 5.6052e-45,\n 5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45, 5.6052e-45,\n -5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45, -5.6052e-45,\n 5.6052e-45, 5.6052e-45, -5.6052e-45, -5.6052e-45, -5.6052e-45,\n 5.6052e-45, -5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45,\n 5.6052e-45, -5.6052e-45, -5.6052e-45, 5.6052e-45, -5.6052e-45,\n 5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45, 5.6052e-45,\n 5.6052e-45, -5.6052e-45, -5.6052e-45, 5.6052e-45, 5.6052e-45,\n -5.6052e-45, -5.6052e-45, -5.6052e-45, -5.6052e-45, 5.6052e-45,\n 5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45, -5.6052e-45,\n 5.6052e-45, -5.6052e-45, -5.6052e-45, -5.6052e-45, -5.6052e-45,\n -5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45,\n -5.6052e-45, -5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45,\n -5.6052e-45, -5.6052e-45, 5.6052e-45, -5.6052e-45, -5.6052e-45,\n 5.6052e-45, -5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45,\n 5.6052e-45, -5.6052e-45, -5.6052e-45, -5.6052e-45, 5.6052e-45,\n 5.6052e-45, -5.6052e-45, -5.6052e-45, 5.6052e-45, -5.6052e-45,\n -5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45,\n -5.6052e-45, -5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45,\n 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45,\n 5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45,\n 5.6052e-45, 5.6052e-45, -5.6052e-45, -5.6052e-45, 5.6052e-45,\n 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45,\n -5.6052e-45, -5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45,\n 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45,\n -5.6052e-45, -5.6052e-45, -5.6052e-45, -5.6052e-45, 5.6052e-45,\n 5.6052e-45, -5.6052e-45, -5.6052e-45, 5.6052e-45, -5.6052e-45,\n -5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45, -5.6052e-45,\n -5.6052e-45, -5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45,\n -5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45, 5.6052e-45,\n 5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45,\n -5.6052e-45, -5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45,\n -5.6052e-45, -5.6052e-45, -5.6052e-45, -5.6052e-45, 5.6052e-45,\n -5.6052e-45, -5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45,\n 5.6052e-45, -5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45,\n -5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45, 5.6052e-45,\n 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45,\n 5.6052e-45, -5.6052e-45, -5.6052e-45, -5.6052e-45, -5.6052e-45,\n -5.6052e-45, -5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45,\n -5.6052e-45, -5.6052e-45, -5.6052e-45, -5.6052e-45, 5.6052e-45,\n 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45,\n -5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45, -5.6052e-45,\n -5.6052e-45, 5.6052e-45, -5.6052e-45, -5.6052e-45, -5.6052e-45,\n 5.6052e-45, -5.6052e-45, -5.6052e-45, -5.6052e-45, 5.6052e-45,\n 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45,\n -5.6052e-45, -5.6052e-45, -5.6052e-45, -5.6052e-45, 5.6052e-45,\n 5.6052e-45, 5.6052e-45, 5.6052e-45], device='cuda:0')",
 "exp_avg_sq": "tensor([2.5012e-20, 3.7804e-20, 1.7877e-20, 5.6742e-20, 2.0777e-20, 1.2296e-20,\n 5.4733e-20, 2.8618e-20, 2.1206e-20, 5.0659e-20, 2.4395e-20, 7.2623e-21,\n 7.5582e-20, 5.1294e-20, 2.0140e-20, 1.0143e-19, 9.9688e-20, 1.2213e-19,\n 1.8229e-20, 2.7827e-20, 2.4918e-20, 5.8171e-20, 4.3146e-20, 1.8355e-20,\n 1.9420e-20, 3.9781e-20, 1.0470e-20, 4.5081e-20, 4.5852e-20, 1.1242e-19,\n 4.0022e-20, 1.8506e-20, 8.3441e-20, 1.6716e-20, 1.1977e-20, 1.3440e-20,\n 3.8252e-20, 7.5736e-20, 2.0492e-19, 1.1219e-19, 9.5508e-21, 1.9687e-20,\n 4.0770e-20, 1.0636e-19, 1.6993e-20, 2.1581e-20, 3.4911e-20, 5.8492e-21,\n 2.3383e-20, 1.3156e-20, 2.1587e-20, 8.4918e-20, 2.1765e-21, 1.2052e-19,\n 2.4371e-20, 4.8409e-21, 2.9129e-20, 3.1680e-20, 6.0861e-20, 3.0227e-20,\n 3.1713e-20, 3.4185e-20, 2.4505e-21, 4.1736e-20, 2.3623e-20, 1.5065e-20,\n 4.1481e-20, 2.7872e-20, 2.1210e-20, 2.1946e-20, 1.2646e-20, 2.9702e-20,\n 2.0572e-21, 7.3751e-20, 3.7759e-20, 1.0479e-19, 1.5798e-20, 2.3008e-20,\n 4.4299e-20, 2.5766e-20, 5.9211e-21, 4.4565e-21, 2.4333e-20, 2.0006e-20,\n 3.3185e-20, 2.6580e-20, 1.5785e-20, 1.3213e-20, 1.0059e-19, 7.4249e-20,\n 1.5026e-20, 5.4031e-21, 6.5121e-20, 1.1256e-19, 1.8145e-20, 1.5889e-20,\n 2.5234e-20, 1.9219e-20, 3.5971e-20, 7.4084e-20, 1.2441e-20, 3.1303e-20,\n 1.5276e-21, 3.7610e-20, 1.6529e-20, 2.5594e-20, 2.0214e-20, 1.2300e-20,\n 1.6551e-20, 8.2078e-21, 3.8414e-20, 1.3304e-20, 1.0070e-20, 2.2959e-20,\n 9.8640e-21, 1.1613e-20, 2.3141e-20, 1.9750e-21, 1.0196e-20, 5.9365e-20,\n 4.2441e-20, 3.8087e-21, 3.9633e-20, 1.4894e-20, 2.2624e-20, 2.9960e-20,\n 2.2039e-20, 7.3554e-20, 3.8436e-20, 8.2602e-21, 2.7205e-20, 9.3457e-21,\n 5.8048e-20, 3.6279e-20, 1.0812e-20, 4.2599e-20, 3.3905e-20, 3.7382e-20,\n 2.7239e-20, 1.3010e-20, 6.0260e-21, 2.7539e-20, 1.3925e-20, 2.7290e-20,\n 2.9672e-21, 2.5837e-20, 8.2254e-21, 1.5818e-20, 9.1909e-21, 1.5734e-20,\n 1.7724e-20, 2.5412e-20, 1.2668e-20, 1.2306e-20, 1.4104e-21, 2.4886e-20,\n 2.2463e-20, 1.0483e-20, 4.0078e-20, 2.1579e-20, 2.7932e-20, 1.9178e-20,\n 4.2680e-20, 1.3379e-20, 5.1011e-21, 8.3169e-21, 3.1794e-20, 3.8061e-20,\n 1.8858e-20, 8.9506e-21, 1.9063e-20, 4.2610e-20, 3.1654e-20, 2.6732e-20,\n 1.4161e-20, 6.5848e-21, 1.5055e-20, 4.1102e-20, 2.0887e-20, 1.0290e-20,\n 1.9529e-20, 2.3807e-20, 2.3317e-20, 4.7787e-21, 1.2555e-20, 8.5066e-21,\n 9.9198e-21, 4.4324e-20, 8.5464e-21, 1.2563e-20, 1.9461e-20, 1.4052e-20,\n 3.3080e-20, 5.4338e-22, 5.6630e-21, 1.1861e-20, 8.9252e-20, 1.3177e-20,\n 8.5217e-21, 5.0293e-21, 1.7354e-19, 1.3163e-20, 4.7522e-21, 1.9218e-20,\n 7.0678e-20, 6.2084e-20, 8.5457e-20, 1.5157e-20, 2.3078e-20, 3.9481e-20,\n 1.0956e-20, 1.8726e-20, 8.3906e-20, 1.1569e-20, 2.6141e-20, 7.8489e-20,\n 1.4950e-20, 1.1533e-21, 2.2765e-20, 1.7565e-21, 1.2066e-20, 5.6616e-20,\n 4.1386e-20, 1.2516e-19, 1.2264e-20, 3.1354e-21, 1.9706e-20, 2.5503e-20,\n 1.0241e-20, 1.4524e-20, 1.3413e-21, 2.9588e-20, 1.2595e-20, 2.2455e-20,\n 1.1156e-20, 1.8326e-20, 1.1525e-20, 2.1220e-20, 5.9420e-20, 3.6320e-20,\n 2.0126e-20, 5.0275e-20, 6.1520e-21, 1.6522e-20, 2.3141e-20, 7.0732e-20,\n 4.5806e-20, 2.0583e-21, 2.1759e-20, 6.9083e-21, 3.3470e-20, 1.4307e-20,\n 1.6188e-20, 1.6562e-20, 2.8673e-20, 1.5123e-20, 3.9063e-28, 1.6267e-28,\n 3.8464e-28, 1.7607e-28, 4.2824e-28, 3.8960e-29, 7.3817e-29, 3.1576e-28,\n 1.9155e-28, 5.0120e-29, 2.9032e-28, 1.9808e-28, 3.5592e-28, 3.4157e-28,\n 3.7376e-28, 1.6437e-28, 4.0274e-28, 3.8538e-28, 3.0167e-28, 2.6688e-28,\n 2.1153e-28, 7.9622e-29, 7.4051e-29, 4.4470e-29, 6.8966e-29, 1.7704e-28,\n 9.4705e-29, 3.6637e-28, 2.5862e-28, 3.3779e-29, 1.6840e-28, 2.1959e-29,\n 1.1330e-28, 9.1041e-29, 1.2081e-28, 5.4726e-29, 1.4422e-28, 2.2018e-28,\n 5.7990e-28, 1.6960e-28, 4.3743e-29, 7.6366e-29, 1.7160e-28, 4.9326e-29,\n 5.1249e-29, 1.1547e-28, 1.1095e-28, 1.4789e-29, 9.8904e-29, 7.7874e-29,\n 1.1126e-28, 1.0071e-28, 1.7511e-28, 2.3171e-29, 1.2909e-29, 6.8507e-29,\n 9.0918e-30, 2.5629e-29, 1.3507e-28, 1.1797e-28, 8.1079e-29, 1.0868e-29,\n 5.4495e-29, 1.2910e-28, 1.1430e-28, 1.0532e-28, 4.1178e-29, 1.6104e-28,\n 1.9746e-28, 1.2081e-28, 3.8809e-28, 4.4388e-28, 1.8810e-28, 1.5348e-29,\n 4.0232e-28, 1.4131e-28, 9.3629e-28, 1.9580e-28, 4.3321e-28, 1.1058e-28,\n 2.1302e-28, 2.9354e-28, 1.1379e-28, 2.3677e-28, 2.3978e-28, 1.0934e-28,\n 2.3080e-28, 7.8039e-29, 2.0175e-28, 9.8738e-29, 2.7447e-28, 4.6898e-29,\n 5.1743e-29, 3.6002e-28, 3.0914e-28, 1.7321e-28, 4.9475e-29, 1.9857e-28,\n 5.9263e-29, 2.3460e-28, 1.1474e-28, 1.1961e-28, 1.3483e-28, 2.4354e-28,\n 2.1651e-28, 8.5917e-29, 2.1926e-28, 1.2802e-28, 8.0814e-29, 3.1997e-28,\n 1.4017e-28, 2.5400e-28, 2.4805e-28, 8.7595e-29, 2.7545e-29, 1.8377e-28,\n 2.4926e-28, 8.4359e-29, 6.3494e-29, 8.7018e-29, 1.8316e-29, 1.6453e-28,\n 2.4873e-28, 2.5902e-28, 6.9406e-29, 1.1339e-28, 1.8249e-28, 3.6216e-29,\n 5.7701e-28, 2.7706e-28, 2.5499e-28, 4.7652e-29, 2.4711e-28, 1.8525e-28,\n 3.5008e-28, 3.6698e-29, 2.6356e-28, 1.3004e-28, 9.3149e-30, 1.4321e-28,\n 1.3588e-28, 3.8186e-29, 1.0510e-28, 1.1900e-28, 1.3290e-28, 2.1077e-28,\n 3.9728e-28, 3.3301e-29, 7.4249e-29, 2.4992e-29, 1.2493e-28, 6.8132e-29,\n 2.2974e-28, 7.5431e-29, 3.0599e-28, 1.8855e-28, 1.2823e-28, 1.5974e-28,\n 4.3609e-29, 1.8886e-28, 2.1170e-29, 1.0999e-28, 1.6798e-28, 2.0732e-28,\n 8.8226e-29, 1.5461e-28, 1.3613e-28, 1.7807e-28, 3.4684e-28, 3.3230e-28,\n 1.7276e-28, 4.0582e-28, 1.5246e-28, 6.5873e-29, 1.5864e-28, 6.3423e-29,\n 2.8186e-29, 1.7694e-28, 1.2704e-28, 1.4837e-28, 1.5600e-28, 8.6555e-29,\n 2.9266e-28, 4.0843e-29, 2.4218e-28, 1.8713e-28, 1.1407e-28, 9.1337e-29,\n 1.0230e-28, 1.7958e-28, 1.1684e-28, 2.4617e-28, 8.6187e-29, 1.3065e-29,\n 1.3552e-28, 2.5640e-28, 4.6997e-28, 4.1078e-29, 2.4018e-28, 1.5362e-28,\n 4.8908e-29, 1.8051e-28, 3.5883e-28, 3.6732e-28, 1.2210e-28, 8.4597e-29,\n 9.2713e-29, 5.6437e-29, 2.7254e-28, 6.3557e-29, 4.5666e-29, 4.2471e-28,\n 2.1933e-28, 5.9574e-29, 6.2561e-29, 9.9203e-29, 1.4792e-29, 7.5175e-29,\n 8.4902e-29, 5.1487e-28, 1.9597e-28, 3.3939e-29, 7.3863e-29, 9.8466e-29,\n 2.1108e-28, 2.6695e-28, 4.0574e-28, 5.4967e-29, 3.6888e-28, 1.1554e-28,\n 8.1411e-29, 9.8382e-28, 4.9505e-28, 1.8373e-28, 4.8374e-28, 4.0957e-28,\n 1.3152e-28, 5.5622e-28, 2.2701e-28, 1.5775e-28, 8.7604e-28, 1.5654e-28,\n 1.2516e-28, 3.5163e-28, 3.6747e-28, 3.4166e-28, 1.1324e-27, 2.2890e-28,\n 3.5592e-28, 5.2035e-29, 1.1155e-27, 1.8992e-28, 2.4384e-28, 4.7684e-28,\n 9.2056e-29, 2.8876e-28, 5.6207e-17, 1.2421e-17, 9.6236e-18, 2.9110e-17,\n 3.2612e-17, 3.1531e-18, 1.9873e-17, 3.2903e-18, 3.5972e-17, 7.4134e-18,\n 2.0196e-17, 1.0298e-17, 2.2584e-17, 8.0290e-18, 1.6225e-17, 8.0579e-18,\n 2.2080e-18, 3.9037e-18, 2.6252e-18, 4.6229e-18, 5.5553e-17, 9.1968e-18,\n 2.2804e-17, 1.7741e-17, 2.2292e-18, 2.2431e-18, 1.2991e-17, 1.2854e-17,\n 2.9334e-18, 3.9018e-17, 1.9918e-17, 9.5876e-18, 2.0278e-18, 8.3522e-17,\n 1.4300e-17, 4.2630e-17, 3.1151e-18, 5.2772e-18, 2.0205e-17, 1.3703e-17,\n 6.6367e-18, 3.0721e-17, 9.4436e-18, 2.8724e-17, 5.9212e-18, 2.0039e-18,\n 8.8224e-18, 1.8696e-18, 1.7138e-17, 3.7773e-17, 7.3077e-18, 2.4527e-17,\n 1.4216e-17, 1.1700e-17, 1.6978e-17, 1.4808e-17, 2.6867e-18, 7.8343e-18,\n 2.8848e-18, 2.3866e-17, 3.9475e-17, 7.2316e-18, 1.3098e-17, 5.9112e-17,\n 8.1824e-18, 1.6375e-18, 1.2759e-17, 1.1176e-17, 8.0741e-18, 1.5486e-17,\n 1.8483e-17, 8.8080e-18, 3.5955e-17, 1.8612e-17, 9.3453e-18, 3.7453e-17,\n 3.5679e-17, 1.8276e-17, 1.0642e-17, 2.2200e-17, 7.8844e-18, 1.0264e-17,\n 1.0102e-17, 3.6254e-17, 1.8104e-17, 3.4840e-17, 2.2118e-18, 5.8408e-18,\n 7.6914e-18, 8.5357e-18, 5.8737e-18, 1.3230e-17, 2.0323e-17, 5.9933e-18,\n 4.3165e-18, 5.4356e-17, 3.8825e-18, 3.0439e-18, 3.1528e-17, 1.9959e-17,\n 1.4473e-17, 1.5946e-18, 2.4720e-17, 3.0008e-17, 1.2250e-17, 1.2859e-17,\n 8.5370e-18, 1.0076e-17, 7.7711e-18, 1.3445e-17, 7.0779e-18, 2.4385e-17,\n 9.3760e-18, 1.2250e-17, 1.1739e-17, 3.2211e-17, 2.6200e-18, 4.3685e-17,\n 1.4731e-17, 1.2404e-17, 3.9174e-17, 8.0279e-18, 2.7754e-17, 1.3773e-17,\n 1.1759e-17, 3.8249e-18, 3.1802e-17, 6.6110e-18, 4.7927e-18, 5.2923e-18,\n 2.3465e-17, 3.1010e-17, 1.0316e-17, 7.9092e-18, 2.4090e-17, 2.3937e-18,\n 2.5523e-18, 7.1668e-18, 2.7740e-17, 1.2152e-17, 7.8947e-18, 7.1929e-18,\n 1.9791e-17, 5.4493e-18, 1.8155e-17, 1.1654e-18, 1.2200e-17, 3.5457e-17,\n 7.5092e-18, 1.0263e-17, 2.7466e-17, 1.7329e-17, 1.1289e-17, 3.2086e-17,\n 2.9875e-17, 3.2376e-17, 2.9469e-18, 1.2174e-17, 5.5513e-18, 8.6754e-18,\n 9.3981e-18, 1.2906e-17, 2.0409e-17, 3.5755e-17, 6.1970e-18, 5.1627e-18,\n 2.5466e-18, 4.3904e-18, 1.8700e-17, 4.4584e-18, 3.9278e-17, 2.3376e-17,\n 7.1107e-18, 1.3659e-17, 6.3554e-18, 1.1552e-17, 9.6895e-18, 3.6739e-18,\n 2.6137e-17, 1.1169e-17, 1.8185e-17, 1.3318e-17, 7.6550e-18, 1.2576e-17,\n 1.5982e-17, 8.3993e-18, 6.8155e-18, 2.2387e-17, 4.3757e-19, 1.1217e-17,\n 2.0685e-17, 3.7613e-17, 6.7724e-18, 9.7006e-18, 3.2634e-17, 2.3446e-17,\n 1.9227e-18, 4.2628e-18, 2.2371e-18, 2.4559e-18, 1.4119e-17, 2.8531e-17,\n 1.1790e-18, 2.8771e-17, 1.2781e-17, 4.4378e-17, 3.9932e-17, 1.9846e-17,\n 2.8191e-17, 6.5405e-18, 9.5355e-18, 1.8853e-18, 1.7388e-17, 4.8619e-18,\n 4.3427e-18, 4.9907e-17, 1.8875e-17, 1.3056e-17, 4.3685e-17, 8.7334e-18,\n 9.0845e-18, 7.5823e-18, 1.6674e-17, 2.7809e-17, 2.2814e-17, 2.9159e-17,\n 1.9052e-17, 2.2138e-17, 9.3182e-19, 1.1919e-17, 9.6136e-18, 7.2922e-18,\n 2.2082e-18, 1.5091e-17, 1.8636e-17, 1.6956e-17, 6.2583e-18, 1.7404e-17,\n 1.8461e-17, 1.3524e-17, 1.1213e-17, 9.8086e-19, 1.4747e-17, 4.8863e-18,\n 1.4654e-17, 2.0780e-17, 2.2276e-17, 1.1659e-17, 3.2544e-18, 9.4654e-18,\n 1.5717e-17, 2.7641e-17, 1.5224e-17, 3.3058e-17, 1.3498e-17, 6.4595e-18],\n device='cuda:0')"
 },
 "65": {
 "step": "tensor(18770.)",
 "exp_avg": "tensor([[-5.6052e-45, -5.6052e-45, 5.6052e-45, ..., 5.6052e-45,\n -5.6052e-45, -5.6052e-45],\n [-5.6052e-45, 5.6052e-45, -5.6052e-45, ..., -5.6052e-45,\n -5.6052e-45, 5.6052e-45],\n [ 5.6052e-45, -5.6052e-45, 5.6052e-45, ..., 5.6052e-45,\n 5.6052e-45, -5.6052e-45],\n ...,\n [ 5.6052e-45, -5.6052e-45, 5.6052e-45, ..., -5.6052e-45,\n -5.6052e-45, -5.6052e-45],\n [ 5.6052e-45, -5.6052e-45, 5.6052e-45, ..., 5.6052e-45,\n 5.6052e-45, -5.6052e-45],\n [ 5.6052e-45, 5.6052e-45, -5.6052e-45, ..., -5.6052e-45,\n -5.6052e-45, 5.6052e-45]], device='cuda:0')",
 "exp_avg_sq": "tensor([[5.5471e-19, 4.7399e-18, 5.4008e-18, ..., 5.3688e-18, 2.0963e-18,\n 4.2860e-18],\n [7.6704e-19, 1.4472e-17, 1.8118e-17, ..., 2.4839e-17, 5.1033e-18,\n 1.1970e-17],\n [1.8026e-19, 4.8590e-18, 4.9809e-18, ..., 3.1929e-18, 9.5218e-19,\n 4.7058e-18],\n ...,\n [1.3488e-18, 2.3100e-17, 2.1326e-17, ..., 1.1294e-17, 5.1381e-18,\n 2.5522e-17],\n [4.0212e-18, 5.0202e-17, 4.8735e-17, ..., 3.4247e-17, 1.3237e-17,\n 5.1802e-17],\n [1.0307e-19, 3.3014e-18, 4.0963e-18, ..., 5.1892e-18, 1.0407e-18,\n 3.1181e-18]], device='cuda:0')"
 },
 "66": {
 "step": "tensor(18770.)",
 "exp_avg": "tensor([ 5.6052e-45, -5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45,\n 5.6052e-45, 5.6052e-45, -5.6052e-45, -5.6052e-45, -5.6052e-45,\n 5.6052e-45, -5.6052e-45, -5.6052e-45, 5.6052e-45, -5.6052e-45,\n 5.6052e-45, 5.6052e-45, -5.6052e-45, -5.6052e-45, 5.6052e-45,\n -5.6052e-45, -5.6052e-45, 5.6052e-45, -5.6052e-45, -5.6052e-45,\n -5.6052e-45, -5.6052e-45, 5.6052e-45, -5.6052e-45, -5.6052e-45,\n -5.6052e-45, -5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45,\n 5.6052e-45, 5.6052e-45, -5.6052e-45, -5.6052e-45, 5.6052e-45,\n 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45,\n -5.6052e-45, 5.6052e-45, -5.6052e-45, -5.6052e-45, -5.6052e-45,\n -5.6052e-45, -5.6052e-45, 5.6052e-45, -5.6052e-45, -5.6052e-45,\n 5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45, -5.6052e-45,\n 5.6052e-45, 5.6052e-45, -5.6052e-45, -5.6052e-45, -5.6052e-45,\n 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45,\n -5.6052e-45, -5.6052e-45, -5.6052e-45, -5.6052e-45, 5.6052e-45,\n 5.6052e-45, -5.6052e-45, -5.6052e-45, 5.6052e-45, 5.6052e-45,\n 5.6052e-45, 5.6052e-45, -5.6052e-45, -5.6052e-45, -5.6052e-45,\n 5.6052e-45, -5.6052e-45, -5.6052e-45, -5.6052e-45, 5.6052e-45,\n 5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45, -5.6052e-45,\n -5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45,\n -5.6052e-45, -5.6052e-45, -5.6052e-45, 5.6052e-45, 5.6052e-45,\n -5.6052e-45, -5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45,\n -5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45, -5.6052e-45,\n 5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45,\n 5.6052e-45, -5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45,\n -5.6052e-45, -5.6052e-45, -5.6052e-45, -5.6052e-45, -5.6052e-45,\n 5.6052e-45, -5.6052e-45, 5.6052e-45, -5.6052e-45, -5.6052e-45,\n -5.6052e-45, -5.6052e-45, -5.6052e-45, -5.6052e-45, 5.6052e-45,\n -5.6052e-45, -5.6052e-45, -5.6052e-45, 5.6052e-45, -5.6052e-45,\n 5.6052e-45, -5.6052e-45, -5.6052e-45, 5.6052e-45, 5.6052e-45,\n -5.6052e-45, -5.6052e-45, -5.6052e-45, -5.6052e-45, -5.6052e-45,\n 5.6052e-45, -5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45,\n 5.6052e-45, -5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45,\n -5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45,\n -5.6052e-45, -5.6052e-45, -5.6052e-45, -5.6052e-45, 5.6052e-45,\n -5.6052e-45, 5.6052e-45, -5.6052e-45, -5.6052e-45, -5.6052e-45,\n -5.6052e-45, -5.6052e-45, -5.6052e-45, -5.6052e-45, -5.6052e-45,\n -5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45, 5.6052e-45,\n -5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45,\n 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45,\n 5.6052e-45, -5.6052e-45, -5.6052e-45, -5.6052e-45, -5.6052e-45,\n -5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45,\n -5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45, -5.6052e-45,\n 5.6052e-45, -5.6052e-45, -5.6052e-45, -5.6052e-45, 5.6052e-45,\n -5.6052e-45, -5.6052e-45, -5.6052e-45, 5.6052e-45, 5.6052e-45,\n 5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45, -5.6052e-45,\n 5.6052e-45, -5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45,\n -5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45,\n 5.6052e-45, -5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45,\n 5.6052e-45, -5.6052e-45, 5.6052e-45, 5.6052e-45, 5.6052e-45,\n -5.6052e-45, 5.6052e-45, 5.6052e-45, -5.6052e-45, 5.6052e-45,\n -5.6052e-45], device='cuda:0')",
 "exp_avg_sq": "tensor([6.2650e-18, 1.9258e-17, 4.3591e-18, 2.5377e-17, 4.6531e-17, 9.2489e-18,\n 2.4325e-18, 1.0689e-17, 2.1240e-18, 1.0205e-17, 2.6411e-17, 2.0165e-17,\n 3.4549e-18, 2.1815e-17, 1.1677e-17, 7.4037e-18, 3.4045e-17, 6.8051e-18,\n 2.1673e-17, 3.1685e-17, 1.3702e-17, 1.1836e-17, 1.7348e-17, 4.4858e-17,\n 1.0091e-18, 8.2750e-18, 2.2365e-17, 4.5273e-17, 3.0662e-17, 2.0926e-17,\n 2.5193e-17, 1.3050e-17, 2.2965e-17, 8.7086e-18, 2.7457e-17, 3.4097e-17,\n 2.5308e-17, 3.4970e-17, 1.9176e-17, 2.3739e-17, 1.0489e-16, 1.9035e-17,\n 2.1656e-17, 8.1974e-18, 2.5065e-17, 4.9176e-17, 9.9151e-18, 2.2374e-17,\n 9.6735e-18, 4.1360e-17, 8.0827e-17, 2.2968e-17, 2.4490e-17, 9.8149e-18,\n 2.4871e-17, 5.5976e-17, 3.1208e-17, 1.2848e-17, 1.8282e-17, 1.9859e-17,\n 2.4739e-17, 8.3372e-18, 3.3339e-17, 1.9843e-17, 7.3441e-18, 2.8905e-17,\n 3.8750e-18, 2.1035e-17, 1.6556e-17, 5.4934e-18, 4.8936e-18, 1.3347e-17,\n 3.2716e-17, 5.3018e-18, 8.8474e-18, 1.0874e-17, 2.1656e-17, 8.6195e-18,\n 3.6040e-17, 1.1465e-17, 2.2972e-17, 6.3100e-18, 2.3911e-17, 1.8453e-17,\n 7.0863e-18, 6.7017e-18, 3.6533e-17, 1.5815e-17, 1.9558e-17, 8.1689e-17,\n 2.8303e-17, 1.6563e-17, 1.1496e-17, 2.5917e-17, 5.5648e-17, 6.6157e-17,\n 3.2273e-18, 8.5956e-18, 3.8098e-17, 1.6022e-17, 1.1547e-17, 2.7707e-17,\n 2.0611e-17, 4.7782e-17, 1.3712e-17, 2.2702e-17, 1.3217e-17, 1.3455e-17,\n 1.1393e-17, 5.0702e-18, 1.4989e-17, 1.6607e-17, 1.7622e-17, 7.9995e-18,\n 9.6699e-18, 4.8813e-17, 3.1571e-18, 4.4506e-18, 1.6420e-17, 5.1026e-18,\n 4.1597e-17, 6.5091e-18, 4.0327e-17, 1.8336e-17, 2.1122e-17, 1.7746e-17,\n 1.4702e-17, 1.4456e-17, 3.9454e-18, 9.8001e-18, 9.0474e-18, 1.6266e-17,\n 1.5721e-17, 2.1500e-17, 7.8009e-18, 1.4586e-17, 7.1365e-19, 2.7519e-17,\n 1.8133e-17, 2.2900e-17, 7.1240e-18, 1.2653e-17, 2.5933e-17, 5.3333e-17,\n 1.6776e-17, 2.2863e-18, 3.0757e-18, 1.9736e-17, 1.7325e-17, 2.1488e-17,\n 2.8607e-17, 3.1548e-17, 4.4558e-17, 3.2436e-17, 1.3991e-17, 3.2528e-17,\n 2.2917e-17, 6.0826e-18, 1.6996e-17, 2.1659e-17, 5.5179e-17, 3.2732e-18,\n 3.2605e-17, 1.1174e-17, 2.0856e-17, 3.2447e-17, 8.3758e-18, 4.5296e-17,\n 4.1499e-17, 2.6572e-17, 2.1131e-17, 8.3698e-18, 1.0822e-17, 1.1356e-16,\n 2.4171e-17, 1.3826e-16, 9.0255e-18, 2.6861e-17, 1.2296e-17, 1.1933e-17,\n 9.7589e-18, 1.4677e-17, 1.6835e-17, 2.1769e-17, 1.7621e-17, 1.6466e-17,\n 1.2141e-17, 3.5088e-17, 1.0086e-17, 9.0344e-17, 2.2627e-17, 2.2109e-17,\n 3.4995e-17, 1.7292e-17, 7.7570e-17, 1.1311e-17, 9.0573e-18, 2.2934e-17,\n 6.9508e-18, 2.2056e-17, 2.0249e-18, 1.6930e-17, 1.9686e-17, 1.1790e-17,\n 1.8417e-17, 1.7452e-17, 2.4866e-17, 2.5243e-17, 1.9055e-17, 4.5804e-17,\n 2.8866e-17, 2.9545e-17, 5.8467e-18, 4.5789e-17, 8.0283e-17, 5.3871e-18,\n 1.2460e-17, 2.8725e-17, 6.4179e-17, 6.7155e-17, 1.6334e-17, 5.0599e-18,\n 1.6582e-17, 1.3400e-17, 3.2849e-17, 1.8129e-17, 8.5053e-18, 1.5965e-17,\n 1.6636e-17, 4.9164e-17, 2.0937e-17, 2.8363e-17, 3.9045e-17, 9.6194e-17,\n 3.5822e-17, 1.2380e-17, 1.2456e-17, 1.8041e-17, 3.0058e-17, 1.2891e-17,\n 3.1587e-17, 1.5567e-17, 6.1802e-17, 1.7185e-17, 7.5590e-17, 1.4481e-17,\n 2.3166e-18, 3.1350e-17, 2.7741e-17, 2.2081e-17, 5.3889e-18, 2.1602e-17,\n 4.1632e-17, 2.1646e-17, 4.0869e-17, 7.8717e-18], device='cuda:0')"
 },
 "67": {
 "step": "tensor(18770.)",
 "exp_avg": "tensor([[ 5.6052e-45, -5.6052e-45, 5.6052e-45, ..., -5.6052e-45,\n 5.6052e-45, -5.6052e-45],\n [-5.6052e-45, -5.6052e-45, -5.6052e-45, ..., -5.6052e-45,\n 5.6052e-45, -5.6052e-45],\n [-5.6052e-45, 5.6052e-45, -5.6052e-45, ..., 5.6052e-45,\n -5.6052e-45, 5.6052e-45],\n ...,\n [ 5.6052e-45, -5.6052e-45, 5.6052e-45, ..., 5.6052e-45,\n -5.6052e-45, 5.6052e-45],\n [ 5.6052e-45, 5.6052e-45, 5.6052e-45, ..., 5.6052e-45,\n -5.6052e-45, 5.6052e-45],\n [-5.6052e-45, 5.6052e-45, 5.6052e-45, ..., -5.6052e-45,\n 5.6052e-45, -5.6052e-45]], device='cuda:0')",
 "exp_avg_sq": "tensor([[7.0325e-18, 1.1120e-15, 1.1760e-16, ..., 1.4931e-15, 5.3975e-18,\n 1.5179e-16],\n [7.0895e-17, 4.3728e-17, 3.4071e-17, ..., 2.9578e-16, 7.0893e-17,\n 5.1562e-17],\n [2.3360e-18, 1.1655e-15, 1.1169e-16, ..., 1.5643e-15, 3.3059e-18,\n 1.5999e-16],\n ...,\n [4.5437e-17, 1.3331e-17, 1.4542e-17, ..., 3.9501e-16, 9.6521e-18,\n 1.8405e-16],\n [3.5258e-16, 1.9687e-16, 1.7139e-16, ..., 1.6179e-15, 3.4582e-16,\n 3.9137e-16],\n [4.4345e-19, 5.1387e-19, 1.2443e-18, ..., 4.3880e-18, 3.8701e-19,\n 1.2781e-18]], device='cuda:0')"
 },
 "68": {
 "step": "tensor(18770.)",
 "exp_avg": "tensor([ 5.6052e-45, -5.6052e-45, 5.6052e-45, -5.6052e-45, -5.6052e-45,\n 5.6052e-45, 5.6052e-45, -5.6052e-45, -5.6052e-45], device='cuda:0')",
 "exp_avg_sq": "tensor([6.1382e-16, 8.2417e-17, 1.7250e-16, 7.1991e-17, 3.2211e-17, 8.0332e-17,\n 4.9510e-16, 7.4098e-16, 3.3748e-17], device='cuda:0')"
 },
 "69": {
 "step": "tensor(18770.)",
 "exp_avg": "tensor([[ 5.6052e-45, -5.6052e-45, 5.6052e-45, ..., -5.6052e-45,\n 5.6052e-45, -5.6052e-45],\n [-5.6052e-45, -5.6052e-45, -5.6052e-45, ..., -5.6052e-45,\n 5.6052e-45, -5.6052e-45],\n [-5.6052e-45, 5.6052e-45, -5.6052e-45, ..., 5.6052e-45,\n -5.6052e-45, 5.6052e-45],\n ...,\n [ 5.6052e-45, -5.6052e-45, 5.6052e-45, ..., 5.6052e-45,\n -5.6052e-45, 5.6052e-45],\n [ 5.6052e-45, 5.6052e-45, 5.6052e-45, ..., 5.6052e-45,\n -5.6052e-45, 5.6052e-45],\n [-5.6052e-45, 5.6052e-45, 5.6052e-45, ..., -5.6052e-45,\n 5.6052e-45, -5.6052e-45]], device='cuda:0')",
 "exp_avg_sq": "tensor([[6.7258e-18, 1.0497e-15, 1.1150e-16, ..., 1.4091e-15, 5.2295e-18,\n 1.4324e-16],\n [6.8327e-17, 4.2135e-17, 3.2848e-17, ..., 2.8510e-16, 6.8331e-17,\n 4.9708e-17],\n [2.2149e-18, 1.0998e-15, 1.0544e-16, ..., 1.4761e-15, 3.1212e-18,\n 1.5096e-16],\n ...,\n [4.4553e-17, 1.3077e-17, 1.4279e-17, ..., 3.8720e-16, 9.4616e-18,\n 1.8041e-16],\n [3.4044e-16, 1.9001e-16, 1.6534e-16, ..., 1.5646e-15, 3.3352e-16,\n 3.7988e-16],\n [4.3576e-19, 5.1226e-19, 1.2431e-18, ..., 4.3133e-18, 3.8036e-19,\n 1.2534e-18]], device='cuda:0')"
 },
 "70": {
 "step": "tensor(18770.)",
 "exp_avg": "tensor([ 5.6052e-45, -5.6052e-45, 5.6052e-45, -5.6052e-45, -5.6052e-45,\n 5.6052e-45, 5.6052e-45, -5.6052e-45, -5.6052e-45], device='cuda:0')",
 "exp_avg_sq": "tensor([6.0530e-16, 7.9802e-17, 1.6356e-16, 6.9706e-17, 3.1889e-17, 7.8934e-17,\n 4.8636e-16, 7.2180e-16, 3.3695e-17], device='cuda:0')"
 },
 "71": {
 "step": "tensor(18770.)",
 "exp_avg": "tensor([[ 5.6052e-45, -5.6052e-45, 5.6052e-45, ..., -5.6052e-45,\n 5.6052e-45, -5.6052e-45],\n [-5.6052e-45, -5.6052e-45, -5.6052e-45, ..., -5.6052e-45,\n 5.6052e-45, -5.6052e-45],\n [-5.6052e-45, 5.6052e-45, -5.6052e-45, ..., 5.6052e-45,\n -5.6052e-45, 5.6052e-45],\n ...,\n [ 5.6052e-45, -5.6052e-45, 5.6052e-45, ..., 5.6052e-45,\n -5.6052e-45, 5.6052e-45],\n [ 5.6052e-45, 5.6052e-45, 5.6052e-45, ..., 5.6052e-45,\n -5.6052e-45, 5.6052e-45],\n [-5.6052e-45, 5.6052e-45, 5.6052e-45, ..., -5.6052e-45,\n 5.6052e-45, -5.6052e-45]], device='cuda:0')",
 "exp_avg_sq": "tensor([[6.8748e-18, 1.0754e-15, 1.1403e-16, ..., 1.4438e-15, 5.3002e-18,\n 1.4677e-16],\n [6.8203e-17, 4.2087e-17, 3.2791e-17, ..., 2.8463e-16, 6.8208e-17,\n 4.9625e-17],\n [2.2651e-18, 1.1272e-15, 1.0804e-16, ..., 1.5128e-15, 3.1974e-18,\n 1.5472e-16],\n ...,\n [4.4553e-17, 1.3077e-17, 1.4279e-17, ..., 3.8720e-16, 9.4616e-18,\n 1.8041e-16],\n [3.3994e-16, 1.8971e-16, 1.6508e-16, ..., 1.5625e-15, 3.3299e-16,\n 3.7952e-16],\n [4.3574e-19, 5.1225e-19, 1.2431e-18, ..., 4.3132e-18, 3.8033e-19,\n 1.2534e-18]], device='cuda:0')"
 },
 "72": {
 "step": "tensor(18770.)",
 "exp_avg": "tensor([ 5.6052e-45, -5.6052e-45, 5.6052e-45, -5.6052e-45, -5.6052e-45,\n 5.6052e-45, 5.6052e-45, -5.6052e-45, -5.6052e-45], device='cuda:0')",
 "exp_avg_sq": "tensor([6.0885e-16, 7.9693e-17, 1.6728e-16, 6.9613e-17, 3.1885e-17, 7.8935e-17,\n 4.8636e-16, 7.2129e-16, 3.3695e-17], device='cuda:0')"
 }
 },
 "param_groups": [
 {
 "lr": 0.005000500000000001,
 "name": "scale_128",
 "betas": [
 0.9,
 0.999
 ],
 "eps": 1e-08,
 "weight_decay": 1e-05,
 "amsgrad": false,
 "maximize": false,
 "foreach": null,
 "capturable": false,
 "differentiable": false,
 "fused": null,
 "decoupled_weight_decay": true,
 "initial_lr": 0.01,
 "params": [
 0,
 1,
 2
 ]
 },
 {
 "lr": 0.005000500000000001,
 "name": "scale_256",
 "betas": [
 0.9,
 0.999
 ],
 "eps": 1e-08,
 "weight_decay": 1e-05,
 "amsgrad": false,
 "maximize": false,
 "foreach": null,
 "capturable": false,
 "differentiable": false,
 "fused": null,
 "decoupled_weight_decay": true,
 "initial_lr": 0.01,
 "params": [
 3,
 4,
 5
 ]
 },
 {
 "lr": 0.005000500000000001,
 "name": "scale_384",
 "betas": [
 0.9,
 0.999
 ],
 "eps": 1e-08,
 "weight_decay": 1e-05,
 "amsgrad": false,
 "maximize": false,
 "foreach": null,
 "capturable": false,
 "differentiable": false,
 "fused": null,
 "decoupled_weight_decay": true,
 "initial_lr": 0.01,
 "params": [
 6,
 7,
 8
 ]
 },
 {
 "lr": 0.005000500000000001,
 "name": "scale_448",
 "betas": [
 0.9,
 0.999
 ],
 "eps": 1e-08,
 "weight_decay": 1e-05,
 "amsgrad": false,
 "maximize": false,
 "foreach": null,
 "capturable": false,
 "differentiable": false,
 "fused": null,
 "decoupled_weight_decay": true,
 "initial_lr": 0.01,
 "params": [
 9,
 10,
 11
 ]
 },
 {
 "lr": 0.005000500000000001,
 "name": "scale_512",
 "betas": [
 0.9,
 0.999
 ],
 "eps": 1e-08,
 "weight_decay": 1e-05,
 "amsgrad": false,
 "maximize": false,
 "foreach": null,
 "capturable": false,
 "differentiable": false,
 "fused": null,
 "decoupled_weight_decay": true,
 "initial_lr": 0.01,
 "params": [
 12,
 13,
 14
 ]
 },
 {
 "lr": 0.005000500000000001,
 "name": "scale_576",
 "betas": [
 0.9,
 0.999
 ],
 "eps": 1e-08,
 "weight_decay": 1e-05,
 "amsgrad": false,
 "maximize": false,
 "foreach": null,
 "capturable": false,
 "differentiable": false,
 "fused": null,
 "decoupled_weight_decay": true,
 "initial_lr": 0.01,
 "params": [
 15,
 16,
 17
 ]
 },
 {
 "lr": 0.005000500000000001,
 "name": "scale_640",
 "betas": [
 0.9,
 0.999
 ],
 "eps": 1e-08,
 "weight_decay": 1e-05,
 "amsgrad": false,
 "maximize": false,
 "foreach": null,
 "capturable": false,
 "differentiable": false,
 "fused": null,
 "decoupled_weight_decay": true,
 "initial_lr": 0.01,
 "params": [
 18,
 19,
 20
 ]
 },
 {
 "lr": 0.005000500000000001,
 "name": "scale_768",
 "betas": [
 0.9,
 0.999
 ],
 "eps": 1e-08,
 "weight_decay": 1e-05,
 "amsgrad": false,
 "maximize": false,
 "foreach": null,
 "capturable": false,
 "differentiable": false,
 "fused": null,
 "decoupled_weight_decay": true,
 "initial_lr": 0.01,
 "params": [
 21,
 22,
 23
 ]
 },
 {
 "lr": 0.005000500000000001,
 "name": "scale_896",
 "betas": [
 0.9,
 0.999
 ],
 "eps": 1e-08,
 "weight_decay": 1e-05,
 "amsgrad": false,
 "maximize": false,
 "foreach": null,
 "capturable": false,
 "differentiable": false,
 "fused": null,
 "decoupled_weight_decay": true,
 "initial_lr": 0.01,
 "params": [
 24,
 25,
 26
 ]
 },
 {
 "lr": 0.0025005,
 "name": "fusion",
 "betas": [
 0.9,
 0.999
 ],
 "eps": 1e-08,
 "weight_decay": 1e-05,
 "amsgrad": false,
 "maximize": false,
 "foreach": null,
 "capturable": false,
 "differentiable": false,
 "fused": null,
 "decoupled_weight_decay": true,
 "initial_lr": 0.005,
 "params": [
 27,
 28,
 29,
 30,
 31,
 32,
 33,
 34,
 35,
 36,
 37,
 38,
 39,
 40,
 41,
 42,
 43,
 44,
 45,
 46,
 47,
 48,
 49,
 50,
 51,
 52,
 53,
 54,
 55,
 56,
 57,
 58,
 59,
 60,
 61,
 62,
 63,
 64,
 65,
 66,
 67,
 68,
 69,
 70,
 71,
 72
 ]
 }
 ]
 },
 "scheduler_state_dict": {
 "T_0": 10,
 "T_i": 10,
 "T_mult": 2,
 "eta_min": 1e-06,
 "T_cur": 5,
 "base_lrs": [
 0.01,
 0.01,
 0.01,
 0.01,
 0.01,
 0.01,
 0.01,
 0.01,
 0.01,
 0.005
 ],
 "last_epoch": 5,
 "_step_count": 0,
 "_is_initial": false,
 "_get_lr_called_within_step": false,
 "_last_lr": [
 0.005000500000000001,
 0.005000500000000001,
 0.005000500000000001,
 0.005000500000000001,
 0.005000500000000001,
 0.005000500000000001,
 0.005000500000000001,
 0.005000500000000001,
 0.005000500000000001,
 0.0025005
 ]
 },
 "metrics": {
 "best_val_acc": 61.802,
 "best_epoch": 4,
 "scale_accuracies": {
 "128": 61.802,
 "256": 70.146,
 "384": 72.66,
 "448": 73.44666666666667,
 "512": 73.876,
 "576": 74.25133333333333,
 "640": 74.45733333333334,
 "768": 74.82,
 "896": 75.262
 },
 "training_history": {
 "epochs": [
 1,
 2,
 3,
 4,
 5
 ],
 "train_loss": [
 4.900288952114611,
 4.0311296205286915,
 3.872760212415066,
 3.772298225203155,
 3.694623793321019
 ],
 "train_acc": [
 51.657460216609806,
 56.824520144524485,
 57.987626385423084,
 58.73678711154231,
 59.30749074866899
 ],
 "val_acc": [
 58.398666666666664,
 60.18533333333333,
 60.89533333333333,
 61.468666666666664,
 61.802
 ],
 "scale_accs": {
 "128": [
 58.398666666666664,
 60.18533333333333,
 60.89533333333333,
 61.468666666666664,
 61.802
 ],
 "256": [
 67.03133333333334,
 68.56666666666666,
 69.374,
 69.826,
 70.146
 ],
 "384": [
 69.55466666666666,
 71.02133333333333,
 71.764,
 72.25133333333333,
 72.66
 ],
 "448": [
 70.34333333333333,
 71.97066666666667,
 72.66733333333333,
 73.1,
 73.44666666666667
 ],
 "512": [
 70.83533333333334,
 72.43066666666667,
 73.07733333333333,
 73.59866666666667,
 73.876
 ],
 "576": [
 71.29266666666666,
 72.69866666666667,
 73.488,
 73.876,
 74.25133333333333
 ],
 "640": [
 71.60266666666666,
 73.09866666666667,
 73.69266666666667,
 74.18333333333334,
 74.45733333333334
 ],
 "768": [
 72.03333333333333,
 73.37733333333334,
 74.138,
 74.52333333333333,
 74.82
 ],
 "896": [
 72.252,
 73.866,
 74.37,
 75.06266666666667,
 75.262
 ]
 },
 "lr": [
 0.00975530705321762,
 0.00904518046337755,
 0.00793913236883622,
 0.00654543046337755,
 0.005000500000000001
 ]
 }
 },
 "train_config": {
 "name": "david_training",
 "run_id": "20251013_004438",
 "dataset_name": "AbstractPhil/imagenet-clip-features-orderly",
 "model_variant": [
 "clip_vit_b16",
 "clip_vit_laion_b32",
 "clip_vit_b32"
 ],
 "num_classes": 1000,
 "preset": "gated_expert_team",
 "custom_config_path": null,
 "num_classes_override": null,
 "use_belly_override": null,
 "belly_expand_override": null,
 "progressive_training_override": true,
 "scale_warmup_epochs_override": {
 "128": 0,
 "256": 0,
 "384": 0,
 "448": 0,
 "512": 0,
 "576": 0,
 "640": 0,
 "768": 0,
 "896": 0
 },
 "num_epochs": 10,
 "batch_size": 1024,
 "learning_rate": 0.01,
 "weight_decay": 1e-05,
 "warmup_epochs": 3,
 "use_rose_loss": true,
 "rose_initial_weight": 0.1,
 "rose_max_weight": 0.8,
 "rose_weight_schedule": "adaptive",
 "use_cayley_loss": false,
 "cayley_weight": 0.01,
 "scale_loss_balance": null,
 "use_mixed_precision": false,
 "gradient_clip": 10.0,
 "scheduler_type": "cosine_restarts",
 "min_lr": 1e-06,
 "freeze_strategy": "never",
 "freeze_threshold": 90.0,
 "unfreeze_on_plateau": true,
 "patience": 10,
 "track_gradients": true,
 "gradient_scale_threshold": 1e-05,
 "gradient_scale_multiplier": 10.0,
 "log_interval": 50,
 "val_interval": 1,
 "save_interval": 5,
 "log_fusion_weights": true,
 "log_loss_components": true,
 "save_format": "safetensors",
 "hf_repo": "AbstractPhil/david-shared-space",
 "upload_to_hub": true,
 "base_dir": "./david_training",
 "num_workers": 10,
 "pin_memory": true,
 "prefetch_factor": 4,
 "persistent_workers": true
 }
}