{
  "epoch": 9,
  "optimizer_state_dict": {
    "state": {
      "0": {
        "step": "tensor(12520.)",
        "exp_avg": "tensor([[-3.3982e-05, -1.0953e-05,  1.7543e-04,  ...,  6.4913e-06,\n          1.6940e-04, -1.4159e-05],\n        [-5.7677e-05, -4.9742e-05, -3.0099e-05,  ..., -8.3677e-06,\n          2.8781e-05,  1.2027e-04],\n        [-8.9894e-05, -2.1777e-04,  1.2677e-05,  ..., -6.1214e-05,\n          3.1293e-05,  7.5520e-05],\n        ...,\n        [ 4.8690e-05,  1.5020e-05, -7.1849e-06,  ..., -1.5127e-05,\n          3.0196e-06, -2.3969e-05],\n        [ 5.5280e-05, -5.0883e-05,  6.9018e-05,  ...,  1.6113e-04,\n         -1.1080e-04, -3.1360e-05],\n        [-9.5261e-05,  4.9204e-05, -7.8318e-05,  ..., -4.3743e-05,\n         -3.8414e-05,  4.0480e-05]], device='cuda:0')",
        "exp_avg_sq": "tensor([[1.7173e-07, 1.5678e-07, 1.1389e-07,  ..., 1.0819e-07, 1.2813e-07,\n         1.4233e-07],\n        [3.3395e-07, 2.9660e-07, 2.0795e-07,  ..., 1.4909e-07, 1.8681e-07,\n         1.4371e-07],\n        [2.1162e-07, 3.3164e-07, 1.1557e-07,  ..., 2.2329e-07, 1.4452e-07,\n         1.1007e-07],\n        ...,\n        [1.7035e-07, 4.0312e-07, 6.6579e-08,  ..., 7.3701e-08, 1.0635e-07,\n         6.7853e-08],\n        [2.1834e-07, 1.3539e-07, 9.7246e-08,  ..., 2.3455e-07, 1.1941e-07,\n         9.8950e-08],\n        [1.3615e-07, 1.5688e-07, 1.0096e-07,  ..., 1.8153e-07, 1.2603e-07,\n         1.1375e-07]], device='cuda:0')"
      },
      "1": {
        "step": "tensor(12520.)",
        "exp_avg": "tensor([ 0.0017, -0.0003, -0.0070,  ..., -0.0002,  0.0029, -0.0005],\n       device='cuda:0')",
        "exp_avg_sq": "tensor([0.0002, 0.0004, 0.0003,  ..., 0.0002, 0.0002, 0.0003], device='cuda:0')"
      },
      "2": {
        "step": "tensor(12520.)",
        "exp_avg": "tensor([[ 1.0975e-05,  1.5755e-05,  1.0744e-05,  ..., -1.3598e-05,\n         -3.8318e-05, -1.6680e-04],\n        [ 3.7216e-05,  3.4361e-05, -5.7702e-05,  ...,  1.1646e-06,\n          1.2002e-05, -5.8342e-05],\n        [-7.8021e-07, -2.0279e-06, -3.0945e-06,  ...,  1.4121e-07,\n          8.4004e-07,  3.1037e-05],\n        ...,\n        [ 2.3329e-06,  1.5408e-05,  3.5999e-05,  ...,  3.0199e-07,\n         -9.6144e-06,  5.6069e-06],\n        [ 6.5681e-06,  5.8676e-05,  1.1507e-05,  ..., -2.4418e-05,\n         -3.7428e-05,  6.0859e-06],\n        [ 9.7367e-06,  1.0336e-05, -1.3461e-06,  ...,  9.1855e-06,\n         -1.1903e-05, -1.5370e-05]], device='cuda:0')",
        "exp_avg_sq": "tensor([[4.9997e-09, 1.5235e-08, 1.5223e-08,  ..., 2.6582e-08, 1.2319e-08,\n         1.6635e-07],\n        [5.6319e-08, 2.4800e-08, 1.6775e-08,  ..., 2.1915e-09, 5.8474e-09,\n         2.6024e-08],\n        [3.3986e-10, 4.8650e-08, 2.7091e-08,  ..., 5.2131e-10, 1.3679e-10,\n         2.2498e-07],\n        ...,\n        [1.2763e-08, 1.4239e-08, 6.6815e-09,  ..., 3.0789e-08, 1.9514e-09,\n         1.4911e-08],\n        [8.7597e-08, 2.8876e-08, 6.7278e-08,  ..., 9.2448e-09, 2.1344e-08,\n         1.5159e-08],\n        [2.1134e-08, 2.2864e-08, 1.8600e-08,  ..., 1.6188e-08, 1.1675e-08,\n         1.9054e-08]], device='cuda:0')"
      },
      "3": {
        "step": "tensor(12520.)",
        "exp_avg": "tensor([-1.7846e-03, -1.6047e-03, -2.7712e-03,  3.5488e-04, -2.0071e-03,\n         3.1915e-03,  7.1355e-03,  5.0039e-03, -2.7097e-03,  2.2581e-03,\n        -5.9356e-04,  6.3242e-04,  3.3306e-03, -1.8915e-03,  4.2513e-03,\n        -7.6795e-03,  5.6051e-03,  1.0274e-02,  3.4063e-03, -1.2994e-03,\n         4.0244e-03, -2.0842e-03, -3.0080e-03,  5.4794e-03,  6.6777e-03,\n        -1.2737e-03, -9.7348e-04,  1.3658e-02,  6.2069e-04,  1.7310e-02,\n        -1.3324e-03,  1.1948e-02, -2.7248e-04,  1.6862e-03,  5.6052e-45,\n         3.9732e-03,  6.9720e-05,  2.4953e-04,  4.1399e-03, -6.0663e-03,\n         7.6706e-03,  2.4724e-03, -4.6812e-03, -2.7210e-03, -3.0743e-03,\n        -9.3073e-03,  1.1034e-03, -2.6380e-03,  6.4732e-03,  7.1869e-03,\n        -1.1779e-03,  7.1797e-03, -9.7685e-04,  1.6353e-03,  1.5711e-03,\n         1.8686e-03,  8.0207e-04, -1.5582e-03,  9.8798e-03, -5.8738e-04,\n         4.3991e-03, -1.5061e-03,  4.7640e-03, -2.0286e-03, -4.2722e-03,\n        -1.5321e-02, -1.2774e-02, -1.5930e-03, -8.2700e-03,  1.1485e-03,\n        -2.5580e-04, -9.8465e-03, -3.5919e-03, -1.8237e-03,  2.2992e-03,\n        -3.4710e-04, -8.5559e-03,  6.9857e-04, -1.5025e-02, -7.1187e-03,\n        -1.8204e-03, -1.5370e-02,  1.2927e-02,  1.2988e-03,  9.5919e-03,\n        -5.3130e-04,  1.9210e-03,  9.4302e-03, -3.9297e-03,  6.7162e-03,\n         1.7777e-02,  2.3698e-03, -5.8565e-03, -7.2391e-04,  3.5072e-03,\n         1.5629e-03, -1.1299e-03, -1.1573e-03, -8.6950e-03,  1.2512e-02,\n         8.1844e-04, -7.5532e-04,  5.6052e-45,  1.7372e-03,  5.6052e-45,\n         8.1108e-03,  5.6052e-45,  7.2543e-03,  3.4145e-03,  2.7394e-03,\n        -2.4233e-03, -4.6987e-03, -9.3740e-04, -7.4051e-03, -7.4401e-03,\n        -1.7840e-03,  3.6646e-04, -5.6052e-45,  1.6272e-03,  3.3879e-03,\n         2.8586e-03, -1.1333e-03,  1.5111e-03, -8.9469e-03,  6.9240e-03,\n        -4.2297e-03, -2.9366e-05,  3.9520e-04,  6.1098e-03,  4.1974e-03,\n        -1.6689e-03,  5.1490e-04,  6.1056e-03,  2.8829e-04,  5.6052e-45,\n        -8.9083e-04,  1.8264e-03,  4.1032e-03, -1.3337e-03,  4.4157e-03,\n        -4.3836e-03, -7.3288e-03,  2.7636e-03, -1.8120e-02,  1.9695e-05,\n         2.5443e-03, -5.7475e-03,  3.6393e-03, -7.3945e-04,  6.6761e-03,\n        -8.4491e-04,  2.8292e-04,  1.5034e-03,  1.6572e-03, -7.1606e-03,\n         3.3926e-03,  5.7012e-04,  5.6052e-45,  9.6786e-03,  1.4984e-02,\n         4.9034e-03, -1.5322e-02,  4.4309e-04, -1.5121e-03,  1.5980e-02,\n        -7.3637e-04, -3.2984e-16,  1.5516e-03, -5.2118e-03, -3.8194e-03,\n         7.8500e-03,  1.1841e-02,  4.4891e-03, -6.2607e-03,  5.8961e-03,\n        -1.1178e-02, -4.3738e-03, -1.0356e-02,  1.2857e-02, -1.9236e-03,\n         4.7227e-03,  3.3029e-03, -3.8216e-03,  2.5929e-03,  1.0788e-02,\n         3.1226e-03, -2.4403e-03, -7.6711e-04, -8.2425e-03, -2.6933e-03,\n         5.6052e-45,  5.4204e-04, -4.8568e-03,  1.6138e-02,  9.3968e-03,\n         1.4599e-03, -2.9716e-03,  1.0287e-02,  1.0910e-02,  2.2804e-03,\n         1.4053e-02, -3.0411e-03,  1.6018e-02,  4.7980e-03,  1.5261e-03,\n         8.2853e-03,  1.0370e-02, -1.1592e-03, -4.0010e-03, -1.1160e-03,\n         4.6853e-03,  5.6559e-03,  5.6052e-45, -2.0821e-02,  7.6858e-03,\n         5.2427e-03, -4.2534e-04, -8.2090e-03,  2.4625e-03,  5.6052e-45,\n        -2.3814e-03, -7.2497e-03,  1.8457e-03, -4.1384e-03, -4.1082e-03,\n         5.6052e-45, -1.3275e-03,  1.8731e-03,  5.9328e-03,  2.7647e-03,\n        -9.2584e-03,  5.6052e-45, -1.1944e-02,  1.9217e-03,  2.4255e-03,\n         5.6052e-45,  1.1038e-02,  1.1073e-02, -1.0236e-02, -1.0783e-02,\n        -1.0531e-02,  8.1953e-03,  1.4172e-02, -4.8396e-03, -2.8708e-03,\n         1.9812e-04, -3.8474e-03, -2.0448e-03,  4.9921e-03,  1.5695e-02,\n         2.0389e-03,  1.4112e-03, -8.7789e-04, -1.7349e-03,  2.1424e-03,\n        -6.3926e-03, -1.3598e-02, -2.1791e-03, -1.1308e-02, -9.0192e-04,\n         4.7054e-04,  1.9523e-03,  5.0861e-03,  2.1705e-03,  1.8581e-03,\n        -8.2226e-03, -5.5658e-03,  1.1281e-03,  5.4278e-03, -9.6263e-04,\n         4.1664e-03, -2.2800e-04,  6.7424e-03,  2.5256e-03,  4.6186e-04,\n        -6.4333e-04, -2.8880e-03, -7.3633e-04, -5.8384e-04, -1.3324e-02,\n        -9.8847e-03,  3.6767e-03, -6.0194e-03,  3.7170e-03,  5.1216e-04,\n        -2.8999e-03,  7.5729e-04, -1.0875e-02,  5.7759e-04, -8.7784e-04,\n         7.5398e-03,  1.8345e-03, -4.2558e-03,  1.6988e-02, -2.5368e-03,\n        -6.2014e-03,  1.3258e-03, -4.7156e-03,  1.3965e-02, -1.5976e-03,\n        -2.2396e-03, -3.3375e-03, -8.3520e-03, -4.4934e-03,  6.8695e-03,\n        -4.2274e-03,  3.0972e-03,  6.0018e-04, -3.5410e-03,  6.7453e-03,\n         3.9454e-03,  2.6813e-03,  8.0046e-03, -3.2923e-03,  6.1874e-03,\n         1.9225e-03, -3.9064e-04, -1.4957e-03,  3.3084e-03, -5.9985e-03,\n         5.6234e-03,  7.3978e-03,  2.7103e-03,  5.1303e-03,  1.1364e-04,\n         5.4041e-03,  5.6052e-45,  6.5872e-04,  3.1756e-03,  6.1702e-03,\n         8.8001e-04,  1.2977e-05,  8.1725e-03, -3.8344e-04, -9.8837e-03,\n        -1.7761e-03,  1.0345e-02,  9.7374e-03, -5.6052e-45,  1.3824e-02,\n         2.8315e-03, -6.9424e-03,  9.5244e-04,  2.4303e-03,  3.6498e-03,\n        -5.6987e-03, -1.1123e-02,  5.6052e-45,  3.1973e-03, -6.5071e-05,\n        -7.9636e-03,  1.2393e-03, -4.1314e-03,  3.1265e-03,  5.2525e-03,\n        -1.3546e-03, -3.9709e-03, -5.6052e-45,  1.3338e-02, -2.1989e-29,\n        -2.2609e-03, -1.0709e-02,  5.0716e-03,  1.1711e-03,  1.1658e-02,\n        -7.2678e-03, -3.2170e-06, -2.4005e-03, -2.5484e-03,  5.6052e-45,\n        -5.3617e-03, -4.5223e-04,  5.6052e-45,  5.6052e-45, -1.4931e-02,\n        -1.9783e-05, -2.2529e-05, -4.5698e-03, -3.6127e-03,  4.1085e-03,\n         5.0706e-03,  1.1036e-02,  5.6052e-45, -4.4641e-05,  6.7493e-03,\n        -1.0962e-02,  1.7158e-04, -2.2724e-03, -9.8607e-03,  5.9968e-18,\n        -1.2894e-02,  5.2329e-03, -4.4552e-03, -8.5987e-04, -6.6962e-03,\n         3.3463e-03,  4.2405e-03, -1.4331e-02, -9.6496e-03, -4.5273e-03,\n         8.7212e-03,  4.4708e-03, -3.2893e-03, -5.4577e-03,  6.9387e-04,\n         3.3922e-04, -4.9561e-03, -9.4396e-04, -1.8302e-03,  8.0187e-03,\n        -9.7200e-03, -2.3200e-03,  2.6197e-03, -1.7248e-02, -1.2497e-02,\n        -1.3093e-04, -7.7391e-03, -2.3416e-04,  8.4954e-04,  4.6423e-03,\n         3.1372e-03,  4.4749e-03,  5.6052e-45,  1.5920e-02, -1.1701e-03,\n        -6.5339e-03, -1.1795e-03,  5.6052e-45,  8.0764e-03,  4.0273e-03,\n         7.0800e-03, -1.3082e-03, -6.4406e-03, -1.3615e-03,  8.9815e-03,\n         2.0612e-03,  3.4377e-04,  1.5672e-03, -7.5054e-04,  4.5349e-03,\n         1.5821e-03,  2.1757e-03, -1.0066e-03, -5.2929e-03,  1.8887e-03,\n        -1.3497e-02, -1.7011e-03,  1.8860e-03,  6.4310e-03, -3.1069e-03,\n         3.2777e-03, -2.6720e-03, -1.4805e-04,  5.6052e-45,  1.4046e-02,\n         1.7020e-02, -3.3264e-03,  1.1043e-02,  1.0253e-03, -1.1932e-02,\n         1.7918e-03,  5.5326e-03,  1.5822e-02,  2.4387e-03,  4.6707e-04,\n        -1.0329e-03,  5.6052e-45,  2.2728e-03,  1.9443e-03, -1.6130e-03,\n         7.5298e-03,  9.6330e-04,  6.8644e-03, -8.3195e-03, -6.3823e-03,\n         2.5839e-04,  2.5512e-03,  5.6052e-45, -1.3421e-04, -9.3446e-03,\n        -3.6109e-03,  8.5299e-03,  1.2627e-02, -2.2063e-03,  7.5478e-03,\n        -2.6124e-03,  1.6786e-02,  3.6803e-04, -5.8538e-03, -6.1590e-03,\n         3.7420e-03, -1.8863e-03,  8.1267e-04,  4.7752e-03,  1.2127e-02,\n        -7.2825e-03,  1.0628e-02, -4.0741e-03,  2.2492e-03,  2.0293e-03,\n         8.3747e-03, -1.9357e-02,  6.5163e-03,  5.6052e-45, -8.7575e-03,\n         1.7605e-03,  1.4652e-03,  1.5523e-03,  8.0858e-04, -5.3864e-03,\n        -2.2068e-04, -1.2220e-03, -2.0594e-03,  1.0054e-02,  2.9236e-03,\n        -3.4266e-03,  3.6974e-03,  6.1451e-03, -5.1123e-03, -7.7712e-03,\n        -4.2555e-04, -4.0185e-03,  2.4883e-04,  8.7303e-03, -8.3862e-03,\n        -2.6914e-03,  4.4365e-03, -2.8428e-03,  7.9516e-03,  5.2749e-05,\n        -5.3275e-03, -2.0583e-03,  2.6644e-03,  3.1597e-04,  5.3829e-04,\n         4.9284e-03,  1.4987e-02,  7.1930e-04, -1.0582e-03, -6.4533e-03,\n        -6.2718e-03,  6.8500e-03,  1.5387e-02,  2.4724e-03,  1.1110e-03,\n         2.5668e-03,  6.1711e-03,  1.1437e-03,  1.3291e-02,  1.3096e-02,\n         1.0245e-05,  7.6930e-05,  1.9434e-03,  6.3460e-04, -4.3982e-03,\n        -1.1942e-02,  1.7599e-03,  8.5021e-03, -6.3966e-03,  3.7549e-03,\n        -3.0948e-03,  1.7638e-02,  1.5639e-03, -1.1246e-02,  5.2671e-03,\n         3.1558e-03,  7.3757e-03, -3.1010e-04, -1.0724e-02,  4.6816e-04,\n        -2.6336e-03, -1.4695e-03, -3.6983e-04,  4.1572e-03, -4.0806e-03,\n         9.5542e-03,  2.9407e-03, -4.6196e-03,  4.6488e-03,  4.5182e-03,\n         7.0508e-03,  3.5219e-04, -1.2896e-02, -9.6335e-04,  5.6052e-45,\n        -1.5052e-02,  7.4414e-04,  1.3310e-02,  3.0258e-03,  1.2382e-03,\n         6.6676e-03, -4.1797e-03, -1.6610e-02, -1.6557e-03,  3.3231e-03,\n        -2.6779e-03, -7.3380e-03,  5.8797e-03,  2.7879e-03,  7.5796e-04,\n         8.4945e-03, -4.8260e-03, -2.0642e-02, -2.2485e-02, -1.2688e-02,\n        -3.5866e-03,  2.7538e-03, -5.2528e-03,  6.0385e-03, -8.2631e-04,\n         6.6555e-03, -6.9195e-03, -2.9720e-04,  5.6052e-45, -7.4422e-04,\n         3.9687e-04,  5.3035e-03, -6.9763e-03, -7.7616e-03,  1.4228e-03,\n         1.3448e-04, -3.5609e-03, -4.7422e-03, -2.2839e-03, -6.2104e-03,\n        -3.0117e-03, -7.6038e-03,  2.2916e-03,  9.7211e-03,  9.6338e-03,\n        -1.3344e-02,  2.5590e-03, -4.8124e-03, -1.1087e-02, -5.1422e-03,\n        -5.5866e-03,  5.0433e-03,  4.7298e-03, -9.7338e-04,  7.2044e-03,\n         1.0018e-05,  5.6052e-45, -3.1809e-03,  4.4608e-03, -1.3074e-02,\n        -1.0556e-02, -1.8553e-02,  3.1812e-03,  5.6052e-45,  6.4300e-03,\n         5.1107e-03,  4.8368e-03,  5.6052e-45,  5.7584e-03,  2.1680e-03,\n         5.4767e-03,  1.7703e-02, -9.1498e-03, -1.0493e-03, -4.4499e-03,\n        -7.1832e-03,  2.1964e-05,  4.5481e-03, -8.3948e-03, -1.6401e-02,\n        -1.2762e-02,  1.1664e-03,  1.1088e-02, -7.0874e-03,  3.7026e-04,\n        -1.4136e-03, -2.3450e-03,  6.1413e-04,  3.3509e-03, -3.2954e-03,\n         5.6798e-03,  2.4177e-03, -1.4481e-02,  2.9683e-04,  5.6052e-45,\n        -1.1816e-04,  2.7979e-04, -7.4796e-03,  1.7980e-03, -6.0244e-03,\n         9.2894e-03, -7.5130e-03,  1.6232e-03,  9.3410e-04,  7.0981e-03,\n        -1.3747e-03, -5.4861e-04, -3.0898e-03, -8.0301e-04,  2.5192e-04,\n         2.6007e-03,  9.3327e-03,  2.6647e-03, -2.3384e-02, -1.7623e-02,\n         1.5172e-03, -1.5493e-02, -6.2929e-03,  1.5672e-02, -5.2833e-03,\n         6.8817e-03, -1.2747e-02, -2.0626e-04,  9.6511e-04, -2.4608e-03,\n        -1.4671e-03, -3.1904e-03, -1.1223e-03,  3.7755e-03,  5.6052e-45,\n        -2.2978e-04, -3.8735e-03, -9.0570e-03,  1.4153e-03, -7.7968e-03,\n        -6.9683e-03, -6.5593e-03, -7.8394e-04, -3.6054e-03, -4.0221e-03,\n         3.4660e-03, -3.4899e-03,  5.6052e-45, -2.5566e-04,  1.6837e-04,\n         1.8946e-03,  5.6052e-45,  2.9651e-03,  5.0739e-03, -3.3126e-03,\n         3.5081e-03, -2.1355e-04, -6.5427e-04, -1.6898e-02,  2.0250e-03,\n        -1.0810e-02,  2.2142e-03, -1.8472e-03, -3.8606e-03,  4.2945e-03,\n        -6.0844e-03,  5.4849e-03, -1.0607e-03, -1.7831e-03,  7.7099e-03,\n        -1.6598e-03, -3.5432e-03, -8.0558e-03,  1.2136e-03, -2.0589e-03,\n        -1.0901e-02,  4.9002e-03, -1.3181e-03, -3.5804e-04,  1.0091e-03,\n        -3.2545e-03,  3.6536e-03,  4.6987e-03, -5.7004e-03,  2.9840e-03,\n         7.3203e-03,  5.5140e-03,  4.0732e-03], device='cuda:0')",
        "exp_avg_sq": "tensor([7.6389e-04, 6.2930e-04, 3.9359e-04, 8.3126e-04, 6.6397e-04, 9.0199e-04,\n        8.0906e-04, 6.3213e-04, 1.1187e-04, 6.4525e-04, 1.8306e-04, 6.3379e-04,\n        2.9524e-04, 3.6001e-04, 2.5619e-04, 6.9754e-04, 2.9509e-04, 6.3137e-04,\n        6.5457e-04, 5.4414e-04, 7.1628e-04, 2.4054e-04, 6.5522e-04, 8.2953e-04,\n        8.1087e-04, 8.4408e-04, 6.1653e-04, 8.0087e-04, 5.3065e-05, 7.3368e-04,\n        4.1091e-04, 4.0316e-04, 2.2510e-04, 6.4559e-04, 4.2491e-13, 6.2817e-04,\n        7.7248e-06, 7.0840e-04, 7.4133e-04, 6.9959e-04, 6.7517e-04, 4.8790e-05,\n        4.7194e-05, 9.3488e-05, 8.0672e-05, 7.2411e-04, 7.7369e-04, 6.3641e-04,\n        6.6155e-04, 4.8919e-04, 7.3146e-04, 6.7845e-04, 6.4218e-04, 7.4808e-04,\n        1.2172e-04, 1.0863e-04, 8.0172e-04, 2.4727e-04, 3.2328e-04, 6.1492e-05,\n        3.7973e-04, 3.3009e-04, 6.6689e-04, 7.0189e-04, 6.2941e-04, 7.9546e-04,\n        8.6591e-04, 5.6402e-05, 7.4854e-04, 5.0912e-04, 7.4868e-04, 7.1801e-04,\n        1.8504e-04, 3.7429e-04, 8.2035e-04, 3.0036e-04, 7.3229e-04, 1.4209e-04,\n        2.5984e-04, 6.7839e-04, 5.6363e-04, 6.1817e-04, 7.5294e-04, 6.7337e-04,\n        6.4296e-04, 5.6102e-04, 8.3828e-04, 6.8215e-04, 8.8707e-04, 6.1227e-04,\n        9.1877e-04, 3.7307e-04, 7.9602e-04, 5.2340e-04, 6.3215e-05, 1.0070e-04,\n        6.4771e-04, 6.7258e-04, 7.7339e-04, 8.1606e-04, 7.1307e-04, 7.9090e-04,\n        3.3826e-10, 1.4305e-04, 3.5769e-13, 5.8773e-04, 4.0797e-12, 6.1067e-04,\n        5.3502e-04, 1.4878e-04, 8.4979e-04, 2.8025e-04, 8.1380e-04, 7.4308e-04,\n        6.2494e-04, 6.4420e-04, 2.1385e-05, 8.3139e-12, 7.1385e-04, 4.2715e-04,\n        8.4790e-04, 9.4110e-04, 6.5731e-04, 6.7878e-04, 7.9119e-04, 4.5059e-04,\n        9.3883e-04, 3.1650e-04, 7.5051e-04, 6.4400e-04, 1.0824e-04, 1.1270e-04,\n        9.8518e-04, 8.3399e-04, 2.4026e-10, 6.2175e-04, 1.0062e-04, 1.4185e-04,\n        6.8768e-04, 8.1338e-04, 7.5125e-04, 7.8656e-04, 6.2653e-04, 8.0538e-04,\n        6.7331e-05, 1.1287e-04, 7.2611e-04, 2.3994e-04, 5.6961e-04, 8.1848e-04,\n        6.5074e-04, 2.7859e-04, 5.7783e-04, 3.0207e-04, 6.7605e-04, 5.8078e-04,\n        4.6711e-04, 7.1945e-11, 9.5715e-04, 7.5028e-04, 3.7962e-04, 7.4244e-04,\n        4.3010e-04, 1.3220e-04, 8.6426e-04, 1.6127e-04, 1.4159e-10, 7.4008e-04,\n        9.4783e-04, 1.6999e-04, 6.9123e-04, 9.7367e-04, 6.4544e-04, 7.2692e-04,\n        5.5665e-04, 5.9228e-04, 6.3189e-04, 2.1932e-04, 9.0218e-04, 1.0911e-03,\n        6.4898e-04, 8.0302e-04, 1.8456e-04, 6.8310e-04, 8.8549e-04, 8.9316e-04,\n        5.7102e-04, 1.0368e-04, 7.1099e-04, 6.0772e-04, 3.7995e-11, 1.9519e-04,\n        2.4161e-04, 8.7751e-04, 1.0113e-03, 6.5432e-04, 6.0749e-05, 8.2079e-04,\n        8.2242e-04, 5.7982e-04, 3.5173e-04, 6.3566e-04, 8.6742e-04, 7.7805e-04,\n        2.1360e-04, 7.0446e-04, 6.4572e-04, 6.8519e-04, 5.7713e-04, 1.9840e-04,\n        2.0806e-04, 2.8931e-04, 1.2658e-10, 8.9748e-04, 5.4446e-04, 1.0457e-04,\n        2.7082e-05, 8.1720e-04, 8.4486e-05, 8.6127e-15, 2.3261e-04, 7.5468e-04,\n        6.7581e-04, 3.7730e-04, 8.0248e-04, 4.7718e-11, 2.4233e-04, 3.6570e-04,\n        6.5876e-04, 1.0792e-04, 6.7860e-04, 7.0985e-11, 9.8734e-04, 4.0604e-04,\n        4.8365e-04, 4.4927e-11, 8.1062e-04, 8.1510e-04, 6.0210e-04, 7.0011e-04,\n        6.6571e-04, 5.8131e-04, 7.9880e-04, 7.1379e-04, 5.8018e-04, 3.8169e-04,\n        5.7999e-04, 5.2115e-04, 6.4053e-04, 7.3489e-04, 1.1077e-04, 2.4936e-04,\n        5.0445e-04, 4.6902e-05, 4.7057e-04, 7.2465e-04, 7.1048e-04, 1.5821e-04,\n        7.0108e-04, 3.6225e-05, 2.4466e-04, 3.7942e-05, 8.4554e-04, 6.9751e-04,\n        5.3003e-04, 3.5728e-04, 5.0472e-04, 6.0583e-04, 3.4872e-04, 7.5739e-04,\n        6.5450e-04, 6.5697e-04, 7.6307e-04, 7.4577e-04, 7.0170e-04, 2.6240e-05,\n        4.6468e-04, 6.4418e-04, 7.2998e-04, 4.1778e-04, 6.1231e-04, 5.6791e-04,\n        7.2101e-04, 1.2973e-04, 6.4293e-04, 9.1344e-05, 1.4890e-04, 2.7588e-04,\n        6.2223e-04, 2.2084e-04, 7.1546e-04, 6.5224e-05, 4.3592e-04, 6.6174e-04,\n        7.1278e-04, 6.4038e-04, 6.2841e-04, 6.2394e-04, 6.8146e-04, 1.9269e-04,\n        4.6200e-05, 1.8406e-04, 6.2188e-04, 7.8931e-04, 2.9880e-04, 7.6677e-04,\n        6.5631e-04, 1.1659e-04, 7.6007e-04, 6.2897e-04, 2.5199e-04, 1.3565e-04,\n        7.8522e-04, 7.7842e-04, 2.0454e-04, 7.7065e-04, 2.8973e-04, 8.1614e-04,\n        7.6097e-04, 1.0271e-03, 6.2267e-04, 5.8958e-04, 1.8616e-04, 7.4284e-04,\n        7.2752e-04, 7.2199e-04, 5.6940e-12, 1.4544e-04, 6.2066e-04, 7.0073e-04,\n        2.7046e-05, 1.0688e-04, 6.4035e-04, 5.2381e-05, 7.1353e-04, 3.2395e-04,\n        9.1634e-04, 2.3842e-04, 2.2554e-10, 6.8496e-04, 7.0886e-04, 6.2083e-04,\n        7.0936e-04, 1.8677e-05, 6.2919e-04, 3.0394e-04, 5.3400e-04, 1.3849e-10,\n        1.5215e-04, 6.0834e-04, 3.4138e-04, 2.5285e-04, 8.0523e-04, 7.0822e-04,\n        7.5417e-04, 3.7534e-04, 6.6911e-04, 2.7929e-10, 6.7549e-04, 1.4758e-10,\n        1.1416e-04, 6.2415e-04, 8.9548e-04, 1.2983e-04, 7.2149e-04, 6.0799e-04,\n        7.0230e-04, 7.4821e-04, 3.3895e-04, 1.8381e-13, 6.0042e-04, 2.1928e-04,\n        4.2566e-14, 1.4595e-11, 7.8914e-04, 3.4166e-04, 7.8093e-04, 8.0792e-04,\n        6.5756e-04, 7.3249e-04, 7.9968e-04, 7.2118e-04, 1.0553e-11, 6.7997e-04,\n        8.5378e-04, 7.1195e-04, 3.8239e-05, 6.0448e-04, 9.1952e-04, 2.8652e-10,\n        7.1237e-04, 7.5980e-04, 8.6492e-04, 1.8565e-04, 7.3788e-04, 2.5985e-04,\n        6.9977e-04, 6.6732e-04, 7.3772e-04, 5.5213e-04, 8.1588e-04, 7.3821e-04,\n        7.5471e-04, 2.5406e-04, 9.5439e-05, 3.8177e-04, 4.2525e-04, 6.7868e-04,\n        4.8435e-04, 2.9597e-04, 7.1962e-04, 6.5727e-04, 6.3389e-04, 6.7024e-04,\n        7.2904e-04, 4.1218e-04, 5.3061e-04, 7.0144e-05, 1.9306e-04, 3.8293e-04,\n        6.3096e-04, 2.4561e-04, 4.8678e-12, 8.1073e-04, 4.8987e-04, 6.0511e-04,\n        7.4441e-04, 6.9738e-12, 7.8233e-04, 1.3911e-04, 4.8862e-04, 6.1144e-04,\n        6.5030e-04, 7.7498e-04, 6.6790e-04, 9.1607e-04, 2.1570e-04, 2.2723e-04,\n        9.1322e-04, 5.2331e-04, 7.6310e-04, 5.3466e-04, 3.3860e-04, 6.8369e-04,\n        1.1797e-04, 7.9129e-04, 1.8162e-04, 7.5853e-05, 7.6290e-04, 7.6995e-04,\n        6.1016e-04, 7.9709e-04, 7.8788e-04, 1.4281e-12, 7.5061e-04, 1.1572e-03,\n        6.2491e-05, 7.4101e-04, 1.1704e-04, 6.5874e-04, 6.3876e-04, 8.0718e-04,\n        6.7325e-04, 6.7855e-04, 1.7822e-04, 6.6901e-04, 1.3885e-13, 1.5757e-04,\n        7.4951e-04, 2.1293e-04, 6.7242e-04, 3.9726e-04, 8.5753e-04, 8.3984e-04,\n        6.3584e-04, 2.5487e-04, 1.1476e-04, 8.2148e-11, 6.7926e-04, 7.1736e-04,\n        6.8942e-04, 7.0649e-04, 1.0039e-03, 7.9935e-04, 6.3103e-04, 1.9420e-04,\n        1.1181e-03, 2.0781e-04, 6.4396e-04, 7.7828e-04, 3.4009e-04, 6.4079e-04,\n        4.3455e-04, 7.3331e-04, 7.3595e-04, 7.0791e-04, 6.5590e-04, 1.8016e-04,\n        2.6968e-04, 1.5008e-03, 3.9010e-04, 1.0269e-03, 9.0026e-04, 5.0616e-11,\n        6.0045e-04, 5.5884e-04, 8.8828e-05, 7.2893e-04, 2.3919e-04, 5.0436e-04,\n        7.0196e-04, 3.2810e-04, 1.1940e-04, 6.2183e-04, 8.2042e-04, 7.6509e-04,\n        9.6628e-04, 7.1208e-04, 5.2910e-04, 6.3214e-04, 1.8616e-04, 8.6343e-04,\n        6.7826e-04, 6.8312e-04, 7.2625e-04, 3.5513e-04, 1.6190e-04, 4.0021e-04,\n        7.8700e-04, 6.1725e-04, 6.9377e-04, 5.6445e-04, 7.6390e-05, 1.4529e-04,\n        6.9734e-04, 7.2948e-04, 8.9032e-04, 6.2911e-04, 3.9013e-04, 8.5722e-04,\n        5.8091e-04, 4.9930e-04, 7.0259e-04, 7.0847e-05, 1.0486e-04, 7.0478e-04,\n        5.4587e-04, 7.5576e-04, 7.4934e-04, 6.9877e-04, 7.1116e-04, 2.2625e-04,\n        1.8930e-04, 3.2511e-04, 7.5564e-04, 5.6319e-04, 2.7223e-04, 6.1314e-04,\n        7.2022e-04, 7.7867e-04, 6.1756e-04, 5.9489e-04, 1.5120e-04, 7.3916e-04,\n        5.7533e-04, 7.0562e-04, 8.0636e-04, 3.0550e-04, 9.0324e-04, 7.1100e-04,\n        5.5805e-04, 2.9576e-04, 6.5177e-04, 7.5922e-04, 7.4781e-04, 5.2061e-04,\n        7.4376e-04, 2.4326e-04, 5.7613e-04, 7.7279e-04, 5.9745e-04, 5.1842e-04,\n        8.5368e-04, 5.6688e-04, 9.7615e-12, 7.6781e-04, 9.0333e-05, 6.9542e-04,\n        6.4695e-04, 1.5766e-04, 7.4659e-04, 2.7356e-04, 7.0783e-04, 4.5100e-05,\n        1.2051e-03, 2.5862e-04, 8.2914e-04, 2.7698e-04, 7.0423e-04, 3.7953e-05,\n        7.9037e-04, 7.0407e-04, 7.7374e-04, 7.7368e-04, 7.8718e-04, 2.8119e-04,\n        1.2416e-04, 6.9369e-04, 4.1722e-04, 2.5379e-04, 7.4603e-04, 6.7123e-04,\n        6.5549e-04, 3.5543e-13, 2.5282e-05, 7.5116e-04, 6.9180e-04, 6.1857e-04,\n        7.3038e-04, 1.7782e-04, 7.2628e-04, 7.1222e-04, 3.7057e-04, 4.6582e-04,\n        7.0127e-04, 7.1274e-04, 6.6520e-04, 8.3481e-05, 1.0132e-03, 2.8619e-04,\n        6.4681e-04, 6.5633e-04, 1.8126e-04, 6.6121e-04, 2.4156e-04, 8.0668e-04,\n        6.9819e-04, 2.6324e-04, 7.8184e-04, 6.3264e-04, 3.3494e-05, 2.6192e-10,\n        7.0218e-04, 6.8167e-04, 9.4174e-04, 6.9423e-04, 3.2654e-04, 2.9492e-04,\n        2.2006e-10, 2.0893e-04, 6.5685e-04, 4.6984e-04, 1.8093e-10, 5.7597e-04,\n        1.0653e-04, 4.5494e-04, 5.9326e-04, 9.0385e-04, 6.5178e-04, 2.7397e-04,\n        4.7323e-04, 1.4211e-06, 7.3844e-04, 6.3098e-04, 6.6084e-04, 7.1947e-04,\n        9.6759e-05, 4.7629e-04, 5.1434e-04, 6.0325e-04, 6.4218e-04, 4.0646e-04,\n        3.8309e-04, 2.8892e-04, 2.7810e-04, 4.8240e-04, 2.5490e-04, 7.1355e-04,\n        1.3808e-04, 4.8294e-12, 5.8311e-04, 5.7030e-04, 7.9360e-04, 7.8234e-05,\n        6.6285e-04, 7.2094e-04, 6.1827e-04, 7.9556e-04, 5.8225e-04, 6.3871e-04,\n        6.6118e-04, 6.7968e-04, 6.0856e-04, 4.2397e-04, 4.5934e-04, 3.3636e-04,\n        7.4754e-04, 7.5814e-05, 8.4236e-04, 5.1316e-04, 5.6795e-04, 4.1442e-04,\n        5.9920e-04, 6.7676e-04, 7.5651e-04, 1.9911e-04, 8.2546e-04, 7.5020e-04,\n        3.3591e-04, 2.4047e-04, 6.0255e-04, 5.9927e-04, 6.7096e-04, 5.9411e-04,\n        5.3072e-11, 1.3700e-04, 6.2286e-04, 6.2719e-04, 6.2809e-04, 6.9425e-04,\n        3.6544e-04, 6.6583e-04, 3.1356e-04, 6.9707e-04, 5.5173e-04, 5.3976e-04,\n        3.4952e-04, 1.4121e-10, 4.7912e-04, 4.3616e-04, 1.4175e-04, 5.1067e-12,\n        9.3013e-04, 8.6911e-04, 6.6490e-04, 1.8963e-04, 7.8568e-04, 1.8229e-04,\n        8.6334e-04, 2.5828e-04, 7.5885e-04, 6.5749e-04, 1.7528e-04, 1.3367e-04,\n        7.1310e-04, 8.7247e-04, 5.7553e-04, 7.9258e-05, 6.0589e-04, 6.3404e-04,\n        7.2600e-05, 7.0482e-04, 5.8263e-04, 2.4720e-04, 5.7573e-04, 8.3341e-04,\n        8.8678e-04, 8.1216e-04, 1.4744e-04, 7.8172e-05, 7.2263e-04, 7.6675e-04,\n        1.4528e-04, 7.3367e-04, 8.0810e-04, 4.8289e-04, 8.6135e-04, 5.8753e-04],\n       device='cuda:0')"
      },
      "4": {
        "step": "tensor(12520.)",
        "exp_avg": "tensor([[ 6.8452e-05,  2.3082e-05, -3.1936e-06,  ..., -5.5797e-05,\n          9.4437e-05, -1.5240e-05],\n        [ 4.5822e-05, -3.5939e-05,  2.8178e-05,  ..., -8.9451e-05,\n         -2.6775e-04,  5.2930e-05],\n        [-7.3490e-05,  1.7058e-05, -2.3277e-05,  ..., -2.2920e-05,\n          1.3489e-05, -5.4111e-05],\n        ...,\n        [ 3.8439e-05, -2.1087e-06,  1.0629e-05,  ...,  7.6991e-05,\n          2.3649e-04, -9.4672e-05],\n        [-8.0361e-05, -9.0796e-06, -1.8367e-06,  ...,  9.5890e-05,\n          1.6946e-04,  2.1059e-04],\n        [ 2.9980e-05, -4.2076e-05,  2.6302e-05,  ...,  2.9310e-05,\n         -3.5005e-04, -4.8777e-05]], device='cuda:0')",
        "exp_avg_sq": "tensor([[7.4598e-08, 5.5570e-08, 4.1993e-09,  ..., 5.0409e-08, 6.3409e-08,\n         6.2175e-08],\n        [1.7390e-07, 1.4145e-07, 7.8123e-09,  ..., 1.0214e-07, 1.4099e-07,\n         1.2915e-07],\n        [1.5727e-07, 1.5091e-07, 1.3544e-08,  ..., 7.7507e-08, 1.0900e-07,\n         9.7301e-08],\n        ...,\n        [1.8693e-07, 9.7148e-08, 8.2130e-09,  ..., 2.1055e-07, 1.2128e-07,\n         1.6212e-07],\n        [1.4476e-07, 1.1073e-07, 4.7141e-09,  ..., 1.5427e-07, 1.1904e-07,\n         1.7328e-07],\n        [1.5464e-07, 1.0617e-07, 1.3367e-08,  ..., 1.0019e-07, 1.7688e-07,\n         1.4292e-07]], device='cuda:0')"
      },
      "5": {
        "step": "tensor(11268.)",
        "exp_avg": "tensor([[ 4.7941e-07, -6.2938e-07,  5.0307e-07,  ..., -1.3382e-07,\n         -8.1753e-07,  1.1447e-07],\n        [ 3.9566e-08,  1.7446e-07,  1.3842e-06,  ...,  7.9154e-07,\n         -1.1687e-06,  4.5255e-07],\n        [-2.0757e-08,  4.1517e-08, -2.9358e-07,  ...,  1.4817e-07,\n          4.1314e-07,  2.0170e-06],\n        ...,\n        [-9.2102e-07, -1.2048e-06,  2.2633e-06,  ..., -5.2388e-07,\n          1.0597e-06, -3.6342e-06],\n        [-1.1328e-06,  2.7723e-07,  6.4734e-07,  ...,  1.6260e-07,\n          2.3260e-07,  2.7690e-06],\n        [-4.0765e-07,  5.8131e-06, -6.2679e-08,  ..., -2.9708e-07,\n          3.9167e-06,  3.8680e-07]], device='cuda:0')",
        "exp_avg_sq": "tensor([[1.3969e-11, 3.5375e-11, 7.3857e-11,  ..., 3.0387e-11, 1.1878e-10,\n         3.3591e-11],\n        [7.3833e-12, 9.1795e-11, 6.0418e-11,  ..., 9.3322e-12, 7.4797e-11,\n         2.2182e-11],\n        [1.7579e-11, 8.1314e-11, 4.3529e-11,  ..., 3.5426e-11, 7.3920e-12,\n         6.9478e-11],\n        ...,\n        [4.2534e-11, 4.0186e-11, 4.4659e-11,  ..., 9.1949e-12, 5.9823e-11,\n         7.9364e-10],\n        [1.0157e-10, 4.9631e-11, 3.7440e-11,  ..., 1.3118e-11, 8.6943e-12,\n         6.0024e-11],\n        [1.2800e-11, 6.4572e-12, 1.7662e-11,  ..., 8.0178e-11, 1.5032e-10,\n         4.0466e-11]], device='cuda:0')"
      },
      "6": {
        "step": "tensor(11268.)",
        "exp_avg": "tensor([ 3.3323e-04, -5.4160e-04, -5.1611e-05,  ..., -6.6685e-05,\n        -1.6798e-04,  2.0086e-04], device='cuda:0')",
        "exp_avg_sq": "tensor([1.9115e-06, 1.5879e-06, 1.6357e-06,  ..., 1.6601e-06, 2.3222e-06,\n        2.2084e-06], device='cuda:0')"
      },
      "7": {
        "step": "tensor(11268.)",
        "exp_avg": "tensor([[ 1.0470e-06,  8.5049e-07, -1.1953e-06,  ..., -1.1290e-06,\n          4.5457e-06, -3.0175e-07],\n        [-3.1635e-06, -2.4050e-06,  3.8834e-06,  ...,  4.7416e-06,\n         -1.5997e-06, -7.6750e-06],\n        [-6.2111e-06,  4.0364e-06,  3.9840e-07,  ...,  9.4759e-07,\n          2.3636e-06,  2.1182e-06],\n        ...,\n        [-1.0404e-06, -4.4427e-06,  6.4047e-07,  ..., -3.9188e-08,\n          3.2633e-06,  3.3230e-06],\n        [-5.6016e-07,  5.0030e-09, -9.4635e-07,  ...,  3.4760e-06,\n         -1.7220e-06, -9.1471e-07],\n        [-4.8749e-07,  1.6041e-06,  2.1476e-06,  ...,  6.4793e-07,\n         -2.9199e-07, -7.3215e-07]], device='cuda:0')",
        "exp_avg_sq": "tensor([[4.8390e-11, 4.0201e-11, 5.3530e-11,  ..., 4.7031e-11, 1.2817e-10,\n         1.0139e-10],\n        [8.8992e-11, 8.4129e-11, 9.8977e-11,  ..., 9.8386e-11, 9.7058e-11,\n         1.4371e-10],\n        [1.0886e-10, 1.1114e-10, 1.0282e-10,  ..., 1.1655e-10, 1.8577e-10,\n         1.3285e-10],\n        ...,\n        [1.9043e-10, 9.8932e-11, 1.3641e-10,  ..., 1.3900e-10, 1.6246e-10,\n         1.8465e-10],\n        [9.6088e-11, 1.0834e-10, 9.9783e-11,  ..., 1.1330e-10, 1.4017e-10,\n         1.4899e-10],\n        [1.1319e-10, 1.0294e-10, 1.2609e-10,  ..., 1.1330e-10, 1.7005e-10,\n         1.1082e-10]], device='cuda:0')"
      },
      "32": {
        "step": "tensor(11268.)",
        "exp_avg": "tensor([-2.0588e-33], device='cuda:0')",
        "exp_avg_sq": "tensor([8.7947e-08], device='cuda:0')"
      },
      "33": {
        "step": "tensor(11268.)",
        "exp_avg": "tensor([ 6.8230e-37, -4.5335e-38, -6.3696e-37], device='cuda:0')",
        "exp_avg_sq": "tensor([1.0779e-10, 1.2453e-12, 9.0779e-11], device='cuda:0')"
      },
      "34": {
        "step": "tensor(11268.)",
        "exp_avg": "tensor([-5.6052e-45, -3.7990e-36, -7.2043e-37, -4.9143e-37, -3.3602e-37,\n        -6.2205e-37, -1.0486e-36, -1.9856e-37,  3.0671e-37,  4.0027e-37],\n       device='cuda:0')",
        "exp_avg_sq": "tensor([4.4056e-08, 6.5507e-10, 5.0068e-10, 4.7524e-10, 5.5830e-10, 5.2521e-10,\n        7.2379e-10, 5.1026e-10, 4.7479e-10, 5.0483e-10], device='cuda:0')"
      },
      "36": {
        "step": "tensor(11268.)",
        "exp_avg": "tensor([[-4.7427e-41, -1.0539e-39, -7.5222e-41,  ..., -1.2288e-41,\n         -9.6369e-41,  1.6731e-39],\n        [-1.4402e-40,  4.4737e-39,  6.3997e-41,  ..., -8.0674e-41,\n          2.4619e-41, -8.3330e-39],\n        [-5.1873e-41, -6.7389e-40, -5.8092e-41,  ...,  1.1237e-41,\n         -2.6137e-41,  3.9760e-40],\n        ...,\n        [-1.2672e-40, -1.4689e-39, -8.6634e-41,  ..., -1.4275e-40,\n         -1.5815e-40,  9.4279e-39],\n        [-1.4561e-41, -1.5151e-40, -5.5554e-41,  ..., -6.1743e-41,\n         -4.6052e-41,  1.5435e-39],\n        [-2.5929e-40,  4.4314e-39,  2.9328e-40,  ...,  8.9186e-40,\n          2.1289e-40,  1.6450e-38]], device='cuda:0')",
        "exp_avg_sq": "tensor([[2.0299e-18, 2.2990e-18, 2.6693e-19,  ..., 1.3822e-18, 2.0990e-18,\n         1.9448e-19],\n        [3.4491e-16, 9.2250e-17, 4.4352e-17,  ..., 3.9549e-16, 1.0160e-16,\n         4.4084e-16],\n        [1.5612e-18, 5.6962e-19, 2.8620e-20,  ..., 4.7748e-19, 6.1520e-19,\n         6.1671e-19],\n        ...,\n        [3.0881e-16, 9.4983e-17, 1.3656e-17,  ..., 8.4600e-17, 4.2030e-17,\n         9.5813e-17],\n        [7.0123e-18, 2.0806e-19, 3.3701e-20,  ..., 4.7197e-19, 3.3706e-19,\n         9.2077e-19],\n        [1.2704e-15, 1.6293e-16, 6.0189e-17,  ..., 5.8333e-16, 2.0816e-16,\n         7.5888e-16]], device='cuda:0')"
      },
      "37": {
        "step": "tensor(11268.)",
        "exp_avg": "tensor([-2.3535e-37, -4.5149e-37, -1.5177e-37, -2.0394e-37,  2.4486e-37,\n        -1.4623e-36,  8.7462e-38, -6.1470e-37,  1.9347e-36,  6.1203e-37,\n        -2.9564e-37, -8.0873e-37,  9.1449e-37,  4.9544e-38, -3.1390e-37,\n         1.2499e-36,  2.1602e-37, -2.7942e-37,  2.2969e-37,  3.2310e-39,\n         5.0112e-38, -1.4170e-36,  5.1284e-38, -1.0740e-36,  1.4717e-37,\n        -1.4073e-36, -1.3635e-37, -3.1259e-37, -2.1145e-36,  4.7628e-38,\n        -4.4145e-38,  3.1924e-37,  6.1798e-37,  1.2339e-36,  1.0043e-37,\n         3.5573e-37,  6.2469e-37, -1.0222e-37,  6.4574e-37, -5.3852e-37,\n        -1.8350e-37, -1.3570e-38,  3.3684e-37, -6.3494e-37, -3.3485e-37,\n         1.2125e-36,  2.7157e-38,  6.4306e-37,  7.6735e-37, -3.0530e-37,\n         5.5661e-37, -8.4917e-37, -2.1492e-37, -1.4758e-37, -1.2277e-37,\n         5.7928e-37,  4.7922e-37,  5.5747e-38,  3.3465e-36,  1.2306e-36,\n         1.9707e-37,  1.4157e-36,  1.8547e-36,  1.8158e-36, -1.7579e-37,\n        -2.3703e-37,  4.8948e-39,  4.5970e-38,  6.6145e-38, -6.2059e-37,\n        -1.5660e-36, -1.2968e-37, -5.3859e-38, -2.2892e-37,  2.7665e-37,\n         4.7336e-37, -1.6634e-36,  4.0351e-37, -3.7592e-37,  2.2684e-38,\n        -1.7764e-36,  2.5115e-37, -3.6092e-37, -9.6234e-37,  1.4326e-36,\n         1.3261e-36, -1.6188e-38,  1.7768e-36,  1.0678e-36, -1.4912e-36,\n         4.9992e-37, -4.8340e-38,  1.9536e-37, -1.0853e-37, -4.2015e-37,\n         2.6833e-38,  2.4018e-37, -1.0378e-37, -1.7739e-37, -1.9391e-37,\n        -4.9544e-37,  4.2862e-37, -2.8670e-38, -1.6501e-36,  7.8944e-38,\n        -2.1413e-37,  7.3351e-37, -6.9793e-38,  1.2700e-36, -5.0189e-37,\n        -3.2405e-37, -8.0018e-37, -1.4883e-36, -1.0057e-37,  8.1124e-38,\n        -5.6120e-37, -8.6468e-37, -1.8798e-37, -5.0240e-37, -1.8881e-38,\n        -4.8397e-37,  2.4783e-37, -2.1937e-36, -1.0550e-36,  3.0735e-37,\n        -2.3015e-37,  4.5230e-39, -2.7304e-37,  7.5998e-38, -9.0881e-37,\n        -2.6648e-37, -1.5608e-36,  2.6354e-37, -4.2955e-37,  6.0339e-37,\n        -1.5903e-37, -1.7887e-38, -2.5715e-37,  1.2557e-37, -1.4370e-37,\n         3.8278e-38, -6.8288e-37, -1.8419e-37,  1.2825e-36, -4.3796e-37,\n        -2.8922e-37, -3.3311e-38, -2.8871e-37, -8.0637e-37,  9.2659e-37,\n        -7.2282e-37,  1.3123e-36,  3.4865e-37, -4.2202e-37,  3.7986e-37,\n        -4.8485e-37, -7.0627e-38,  4.2733e-37, -3.9033e-37,  2.5588e-37,\n        -1.3563e-37, -6.9545e-38, -6.5165e-37, -3.5497e-37, -6.2157e-37,\n         3.9422e-37,  1.9496e-37, -1.1191e-36, -3.4926e-37,  3.6556e-37,\n         3.6482e-38, -3.0481e-37,  8.8422e-37,  4.3857e-37, -1.4162e-37,\n        -1.0882e-38,  2.4828e-38,  1.5394e-36, -1.3362e-36,  3.0258e-37,\n        -4.7196e-37, -8.9831e-38, -1.0638e-37,  4.3749e-42,  9.5637e-37,\n        -6.5088e-37,  7.9104e-37,  2.0420e-37, -3.9232e-37,  2.9035e-37,\n        -8.7271e-37, -4.1207e-37,  5.0169e-37,  1.5205e-37,  5.7748e-37,\n         2.0308e-36, -4.7575e-37,  1.7399e-38, -1.6974e-36, -7.2826e-37,\n        -6.0914e-37,  3.1791e-37,  5.0680e-37,  1.2397e-36,  1.2733e-36,\n         7.8827e-38,  4.7455e-37, -1.3965e-36, -1.8394e-37,  9.1984e-38,\n        -3.4015e-36,  1.0927e-36,  1.0119e-36, -1.2226e-36,  3.9232e-37,\n        -3.3037e-37,  1.3919e-37, -1.5302e-37, -8.9138e-37, -8.2457e-37,\n        -3.1860e-38, -5.1147e-38, -1.2301e-36, -5.4015e-37,  9.7266e-37,\n        -2.2655e-37, -2.4781e-37, -2.2940e-38,  4.3898e-37,  8.6496e-38,\n         4.1434e-37, -6.7621e-37, -7.3590e-38,  3.2572e-36, -1.7986e-37,\n         6.1509e-38,  2.4897e-36, -2.0659e-37,  2.5674e-37, -1.2093e-37,\n         2.2342e-36,  1.9246e-37, -1.6350e-36,  1.4634e-37,  1.5087e-36,\n        -1.5325e-36,  1.6626e-36, -8.5082e-37, -9.5858e-37, -3.5364e-37,\n        -1.1572e-36, -4.5746e-37, -2.9823e-37, -2.6920e-37, -2.4816e-37,\n        -8.1844e-37, -3.7424e-37, -1.0643e-36,  1.5131e-36,  9.3087e-37,\n         1.1848e-36, -1.9054e-37, -9.1914e-37, -3.1727e-37,  7.3580e-37,\n        -2.8355e-37, -5.9555e-38, -1.8414e-37,  3.9001e-37,  8.2337e-37,\n        -3.2784e-37, -2.1290e-37,  5.2241e-38,  1.3069e-36,  2.7948e-38,\n        -4.9550e-37, -8.8185e-37,  3.3189e-37, -2.8837e-37, -2.8013e-37,\n        -4.9530e-37,  4.3772e-37,  8.9313e-37,  6.2551e-37, -8.0964e-37,\n        -2.5709e-38, -1.3130e-36, -6.8973e-37,  1.3306e-36,  5.3818e-37,\n         1.7871e-37,  7.6681e-37, -1.6685e-37, -1.6831e-37, -1.8657e-36,\n        -4.0350e-37,  2.0124e-38,  8.9738e-38,  1.3916e-36, -9.1452e-37,\n        -5.0183e-37, -2.2974e-37, -1.3491e-36,  2.6709e-36, -2.4262e-37,\n        -6.8577e-38,  1.8489e-37, -4.7378e-37, -3.4935e-37,  2.4835e-36,\n        -1.2541e-36, -3.0614e-37, -3.6448e-37,  3.7988e-37, -3.1763e-37,\n        -1.5406e-37,  4.3826e-37, -2.6261e-38, -1.1493e-36, -1.2657e-36,\n         2.2817e-36,  1.4639e-36, -3.6336e-37,  4.7342e-37, -2.3914e-37,\n         1.5832e-37, -1.2504e-36,  6.1016e-38, -8.3712e-37, -1.2275e-36,\n        -9.3483e-38, -3.2234e-37,  1.6725e-37,  1.7456e-38,  1.0822e-36,\n        -1.2319e-37, -1.8230e-38, -2.0647e-37,  2.3845e-36,  8.0109e-38,\n        -8.0407e-37, -3.2146e-37, -1.0711e-36, -8.6994e-38,  1.9067e-37,\n        -1.7953e-37, -1.2868e-36,  6.1865e-37, -2.5326e-37,  8.8377e-37,\n         1.7972e-36,  5.4026e-37,  1.2807e-36,  4.6961e-37,  5.4063e-37,\n        -2.0134e-37, -3.1930e-37,  3.9169e-37,  1.5327e-36, -6.3254e-37,\n         4.8137e-37,  9.8858e-38, -2.4445e-37,  1.0172e-36,  7.3681e-37,\n         5.0068e-37,  5.7107e-37, -1.7328e-39, -2.3026e-37, -1.4775e-37,\n        -3.7130e-37, -3.3247e-37, -3.3419e-37, -3.3044e-37, -1.8092e-37,\n        -5.3782e-37,  1.8980e-36,  1.5024e-36, -3.4236e-37,  5.8561e-38,\n         4.5295e-37,  1.0339e-36, -1.4958e-36,  3.2352e-38, -8.5242e-37,\n        -6.4809e-37, -1.8187e-37,  5.8046e-37,  2.7342e-38, -2.2110e-37,\n        -1.2361e-37,  1.0744e-36,  5.2147e-37,  6.5305e-38, -2.5814e-37,\n        -6.5224e-38,  8.0598e-37, -5.8750e-37, -1.8753e-36,  3.7869e-37,\n        -1.5220e-38, -2.5237e-37, -4.9542e-38,  3.1764e-37, -2.0311e-37,\n         1.6747e-36, -8.4486e-37, -1.1027e-37, -7.1282e-37,  1.3839e-36,\n         9.7054e-37, -3.1830e-37, -5.7797e-37,  1.9895e-37, -1.8596e-37,\n        -2.1063e-37, -6.2426e-37, -8.8990e-37, -2.4055e-37,  1.4243e-38,\n         8.3903e-37, -1.7338e-37, -1.9902e-38,  1.2397e-37,  4.8637e-37,\n        -2.3043e-37, -2.1774e-36, -2.3221e-37, -2.2294e-37,  1.0439e-36,\n         8.2049e-37, -9.6620e-37, -3.1657e-37,  1.4670e-36,  2.3647e-37,\n        -3.5425e-37,  2.6854e-37,  1.9013e-36,  7.7425e-37,  1.6424e-37,\n        -6.0032e-37,  1.1370e-36, -2.8236e-37, -2.9802e-37, -1.3497e-36,\n        -2.9332e-37,  1.3706e-36,  7.5307e-37, -2.8305e-37, -1.3433e-36,\n        -3.7235e-37, -4.8656e-37, -1.8894e-37, -2.9903e-37, -2.3990e-37,\n         5.0878e-38, -9.0530e-38, -1.5136e-36, -1.0315e-36, -2.9782e-37,\n         1.0480e-37,  1.7707e-37,  7.0921e-37,  5.8259e-39,  5.6346e-37,\n         1.7288e-37, -1.7394e-37, -4.1444e-37,  3.3122e-36, -7.4272e-37,\n        -1.6331e-37,  3.8032e-38,  3.7976e-37,  5.1576e-37, -5.1076e-37,\n        -1.1302e-37, -1.3634e-36,  3.5668e-36, -1.2073e-36, -8.3608e-38,\n         1.1667e-36, -6.7371e-37, -1.6746e-38,  3.1604e-37,  5.4687e-38,\n        -2.7778e-37,  1.4760e-37,  9.0901e-37,  3.4855e-37,  3.3812e-37,\n         2.2672e-36, -1.1848e-36, -7.9289e-37, -3.2157e-37, -1.9130e-37,\n         3.9716e-37, -6.5866e-37, -3.6779e-37, -7.5934e-37,  8.6132e-38,\n        -1.6684e-37,  5.6081e-37, -2.0780e-37, -8.7159e-37, -1.7850e-37,\n        -2.9861e-37, -6.2767e-37, -6.9950e-37, -2.4653e-36, -2.3178e-37,\n        -9.8808e-38,  1.3077e-36], device='cuda:0')",
        "exp_avg_sq": "tensor([6.9046e-13, 1.4867e-10, 2.5439e-13, 2.9247e-13, 6.1998e-12, 6.2125e-11,\n        1.2929e-11, 6.5776e-12, 4.1759e-10, 2.1881e-12, 4.4071e-11, 1.7494e-11,\n        2.6360e-11, 8.4305e-13, 2.1174e-11, 7.5485e-11, 5.3767e-11, 3.1411e-12,\n        2.3071e-12, 1.0459e-13, 4.1575e-11, 1.9186e-10, 2.8801e-13, 7.2937e-11,\n        2.1749e-12, 1.4704e-11, 2.4081e-13, 9.8897e-13, 2.6561e-11, 3.5402e-13,\n        5.8245e-12, 1.2497e-11, 1.2282e-11, 2.2159e-10, 6.0268e-11, 1.8704e-12,\n        8.6108e-11, 4.0777e-12, 2.0511e-12, 2.2373e-11, 1.1013e-11, 7.8918e-11,\n        1.5675e-11, 1.6326e-11, 6.7964e-11, 1.1736e-10, 1.3712e-13, 1.4140e-12,\n        5.2313e-11, 8.2603e-12, 3.2234e-12, 3.9801e-11, 4.0315e-12, 9.8580e-13,\n        1.4293e-12, 5.1482e-12, 1.4499e-11, 2.3229e-12, 2.7428e-10, 1.5508e-11,\n        4.9587e-13, 1.3663e-11, 4.7184e-11, 5.5898e-10, 6.7589e-13, 5.3718e-12,\n        2.7169e-11, 3.5869e-10, 2.0380e-13, 3.9382e-10, 8.5795e-12, 5.0374e-13,\n        8.5744e-13, 1.3201e-12, 4.4463e-12, 4.8844e-11, 3.4495e-10, 3.9482e-13,\n        8.3382e-11, 2.3474e-13, 2.2252e-10, 2.3263e-11, 7.2321e-12, 1.5625e-10,\n        6.7312e-11, 1.8745e-11, 2.1386e-13, 2.2725e-11, 2.2312e-11, 2.8345e-10,\n        2.0072e-12, 6.9565e-12, 5.3252e-12, 5.1727e-13, 4.6290e-11, 5.9821e-14,\n        4.7078e-11, 1.7428e-13, 2.1952e-13, 4.0194e-12, 1.8899e-11, 5.4360e-11,\n        2.8423e-12, 3.9046e-11, 1.0119e-12, 1.1120e-12, 2.3646e-12, 2.3790e-13,\n        1.1053e-10, 1.0035e-10, 1.4393e-12, 8.3352e-11, 1.7133e-11, 4.6079e-12,\n        1.2575e-10, 5.5202e-12, 1.2963e-11, 3.0821e-13, 3.7332e-11, 1.8202e-12,\n        3.7925e-10, 8.0247e-13, 3.5437e-11, 9.5112e-12, 3.0677e-12, 5.5657e-12,\n        6.0542e-13, 2.1592e-11, 6.2002e-13, 2.6016e-11, 1.4951e-12, 1.6213e-10,\n        6.7497e-11, 1.5165e-13, 5.4266e-12, 2.0049e-13, 2.9197e-13, 7.1017e-12,\n        8.9081e-12, 4.2749e-12, 4.5927e-12, 1.1963e-10, 9.8713e-12, 2.0423e-10,\n        9.1907e-12, 7.3621e-12, 1.7253e-13, 1.1495e-11, 3.9314e-10, 2.1304e-11,\n        4.0194e-11, 7.5168e-11, 9.7306e-11, 2.6365e-10, 2.0553e-12, 9.1796e-11,\n        1.2773e-12, 4.2582e-12, 6.1419e-12, 7.3718e-13, 1.3868e-12, 3.1915e-13,\n        3.8570e-12, 2.3869e-12, 3.9232e-11, 1.2167e-11, 1.5454e-12, 4.2750e-11,\n        1.4602e-12, 3.5964e-13, 4.9701e-13, 2.5163e-14, 5.8054e-11, 9.2194e-11,\n        5.0313e-12, 1.6183e-13, 7.9373e-13, 1.0377e-11, 2.8831e-10, 3.0632e-12,\n        1.3591e-10, 1.1152e-13, 1.0007e-11, 1.4940e-11, 8.6758e-11, 1.5927e-11,\n        9.2567e-11, 1.2045e-13, 2.3023e-13, 4.0428e-13, 1.0010e-10, 9.6497e-12,\n        6.4513e-13, 2.7683e-14, 1.7350e-10, 1.7058e-10, 2.0600e-12, 1.3187e-12,\n        1.9706e-10, 1.6428e-11, 1.6524e-12, 2.5163e-12, 3.9651e-13, 1.7831e-11,\n        6.2838e-12, 2.2633e-13, 1.7471e-12, 7.6103e-12, 2.8893e-13, 1.4102e-11,\n        7.1326e-11, 2.2692e-10, 2.0047e-11, 7.9440e-11, 8.2936e-12, 5.1404e-13,\n        2.3220e-12, 1.5340e-13, 2.7719e-10, 1.6100e-11, 3.7456e-12, 1.4474e-13,\n        2.3560e-11, 5.3620e-11, 5.2591e-11, 1.0190e-12, 1.3148e-12, 2.0340e-12,\n        3.2972e-10, 2.1235e-13, 3.7328e-12, 5.0604e-11, 6.5070e-13, 5.9636e-10,\n        3.4899e-13, 3.2165e-14, 3.4194e-10, 3.4138e-13, 3.8686e-13, 4.4559e-13,\n        4.3859e-10, 3.8286e-11, 1.3463e-10, 3.9274e-12, 8.1505e-11, 3.9102e-11,\n        6.5389e-11, 3.9216e-11, 6.7174e-11, 3.1010e-12, 1.9261e-10, 7.2692e-11,\n        8.8196e-12, 2.8757e-12, 6.0147e-13, 5.4619e-12, 8.5574e-13, 5.5701e-11,\n        8.4063e-11, 2.1416e-11, 4.1290e-11, 1.0742e-11, 1.5540e-11, 1.0446e-12,\n        5.4504e-12, 6.2967e-12, 2.6723e-13, 1.7509e-11, 8.4585e-13, 9.5613e-11,\n        6.7561e-12, 2.4789e-13, 4.3267e-13, 3.7173e-10, 3.0073e-12, 4.8845e-11,\n        5.9036e-11, 2.5909e-13, 1.1437e-13, 8.1024e-11, 1.8395e-12, 4.6796e-12,\n        2.0801e-11, 4.2655e-11, 1.5302e-10, 1.1891e-12, 1.1708e-10, 3.8211e-11,\n        1.6487e-10, 1.3432e-13, 2.6758e-13, 1.5833e-12, 1.0691e-12, 3.4994e-12,\n        1.7810e-10, 3.5007e-11, 3.3501e-12, 2.1674e-13, 3.5792e-11, 1.2121e-10,\n        1.0291e-11, 1.7458e-12, 2.8047e-11, 1.4086e-10, 2.5500e-11, 5.0936e-13,\n        1.1046e-13, 2.2710e-12, 1.7876e-12, 7.1118e-10, 8.1184e-10, 1.9038e-12,\n        3.9698e-12, 2.3705e-12, 1.4541e-11, 5.4252e-13, 8.8984e-13, 1.1610e-14,\n        1.1461e-10, 2.1954e-11, 3.8466e-10, 4.8267e-12, 4.9957e-11, 1.1541e-10,\n        7.6726e-14, 1.0405e-13, 1.9488e-12, 7.4011e-12, 1.2024e-11, 1.0541e-10,\n        1.4317e-10, 1.3523e-11, 1.3681e-11, 2.4788e-13, 2.9678e-11, 3.5347e-12,\n        7.2079e-13, 7.3102e-13, 6.9952e-11, 3.3003e-13, 1.5799e-10, 6.8611e-12,\n        1.0084e-10, 1.7507e-12, 6.2969e-13, 7.1976e-12, 3.4688e-10, 4.3519e-12,\n        2.1333e-11, 7.8538e-12, 2.2331e-12, 2.6990e-11, 1.1177e-11, 1.5120e-12,\n        9.0502e-11, 1.1912e-12, 2.5352e-12, 4.5409e-12, 3.8278e-10, 2.7620e-10,\n        9.0211e-11, 3.7484e-10, 2.4362e-13, 6.4337e-11, 1.0270e-10, 7.5877e-13,\n        5.0387e-10, 2.2344e-13, 1.9506e-12, 1.3676e-12, 3.3007e-11, 2.0178e-11,\n        1.7661e-11, 1.2590e-11, 1.8392e-12, 3.2426e-10, 1.1703e-10, 6.4080e-11,\n        1.8886e-11, 3.2156e-13, 3.7233e-12, 3.0294e-11, 7.6892e-11, 2.0093e-11,\n        4.1733e-11, 3.6717e-10, 4.3867e-13, 2.2740e-11, 2.2627e-11, 1.8318e-12,\n        2.0927e-12, 4.8386e-11, 1.5765e-12, 5.9119e-13, 7.9968e-13, 2.7442e-12,\n        4.3276e-11, 3.5402e-11, 3.6480e-10, 1.8192e-10, 2.2957e-13, 1.4296e-11,\n        8.3864e-12, 1.2425e-13, 6.3262e-14, 4.1035e-11, 2.6776e-11, 4.6310e-13,\n        9.3127e-11, 1.1052e-10, 4.7692e-11, 6.3475e-12, 3.0469e-11, 1.4252e-10,\n        1.1555e-12, 2.5268e-14, 1.0288e-11, 1.5051e-10, 1.2750e-12, 1.2860e-12,\n        2.6211e-10, 3.0356e-12, 2.8063e-13, 4.8999e-12, 3.2968e-11, 1.4404e-12,\n        1.0447e-10, 1.7774e-12, 6.0992e-12, 5.3680e-11, 4.8910e-11, 8.5042e-11,\n        1.2248e-13, 8.8433e-12, 2.9159e-13, 2.8044e-12, 3.1318e-11, 8.7574e-11,\n        1.1748e-11, 2.4809e-11, 8.4875e-11, 4.6911e-12, 1.9750e-12, 2.0219e-11,\n        4.1635e-11, 2.4276e-12, 3.1446e-11, 3.2388e-11, 9.7953e-13, 2.1683e-10,\n        2.6702e-13, 5.0037e-11, 2.3260e-12, 8.5141e-11, 8.5481e-13, 2.0873e-13,\n        6.1289e-13, 1.4001e-10, 4.0681e-11, 6.1984e-12, 4.9924e-12, 1.2865e-13,\n        1.1062e-11, 8.5927e-13, 1.9110e-11, 7.0619e-11, 1.6733e-13, 3.8986e-11,\n        2.1431e-10, 2.5657e-12, 3.1898e-13, 5.3966e-13, 1.5014e-12, 2.6098e-12,\n        8.5203e-11, 2.4623e-13, 5.3394e-10, 9.7868e-11, 1.7120e-10, 1.7736e-13,\n        5.7493e-12, 1.6269e-10, 1.0772e-14, 1.2316e-10, 3.5512e-14, 6.3174e-14,\n        2.9095e-11, 3.5305e-11, 8.0574e-13, 2.7041e-12, 8.8258e-11, 1.7310e-10,\n        4.2780e-11, 3.2422e-12, 4.9707e-12, 1.2952e-12, 1.3935e-11, 1.6719e-11,\n        1.2748e-10, 1.3361e-10, 2.4201e-12, 3.6577e-12, 5.0381e-13, 2.8596e-10,\n        1.1317e-11, 2.6615e-11, 1.0539e-11, 1.5805e-11, 1.5724e-10, 7.2954e-11,\n        2.9340e-13, 3.2380e-10], device='cuda:0')"
      },
      "38": {
        "step": "tensor(11268.)",
        "exp_avg": "tensor([-1.2918e-39,  9.2553e-38,  7.0973e-39,  2.6749e-39, -1.0423e-39,\n         9.5816e-39,  3.6735e-38,  2.1299e-38,  7.6213e-38,  1.0790e-38,\n         5.7857e-38, -2.1344e-40,  3.9780e-38,  2.4166e-39,  5.5550e-39,\n         7.0933e-38,  1.1588e-38, -1.0156e-39, -2.0760e-39,  2.6136e-39,\n         9.1800e-38,  3.4223e-38,  3.4678e-39,  1.9988e-40,  3.8732e-39,\n         1.9541e-39,  6.8455e-39, -6.8842e-40,  4.4875e-38,  3.6074e-39,\n        -9.9065e-41,  3.1764e-38,  2.0019e-38,  5.8310e-38,  5.0511e-38,\n         4.5839e-39,  8.3036e-38,  3.9822e-41,  1.1782e-38,  8.4768e-39,\n         8.5445e-39,  1.7153e-38,  2.9528e-38, -4.9544e-40,  3.1250e-39,\n         7.6521e-38,  3.4113e-39, -4.8171e-40,  6.5077e-38, -1.0209e-39,\n         1.2316e-38,  2.5751e-39,  1.3252e-39,  1.1847e-40,  2.1726e-39,\n         5.4577e-39,  2.6941e-39,  1.1682e-38,  5.4157e-38,  1.5421e-38,\n         2.1092e-38,  1.2876e-38,  5.5826e-39,  1.4834e-37,  2.4269e-40,\n        -2.1487e-39,  5.9963e-39,  6.1321e-38, -2.4504e-39,  1.5901e-38,\n         5.2409e-38, -2.3404e-39,  2.5987e-39, -1.5952e-39,  1.5153e-38,\n         6.8592e-39, -3.9434e-40,  5.7847e-39, -1.1570e-40,  6.0340e-39,\n        -4.4597e-40,  2.7953e-38, -1.3146e-39,  3.3735e-38,  6.6405e-38,\n         1.1425e-38,  5.2128e-39,  1.4520e-37,  1.7512e-38,  2.9635e-39,\n         5.7132e-39, -1.1016e-40,  7.8258e-39, -2.2550e-40,  6.3002e-38,\n         7.2891e-39,  2.7372e-38,  5.2938e-40,  1.6286e-40, -1.9403e-39,\n         2.0445e-38,  7.6647e-38,  6.4653e-40,  4.5499e-39, -4.2827e-40,\n         1.4142e-39,  1.2773e-38,  5.4255e-39,  3.9634e-38,  1.0860e-38,\n         5.3004e-41,  6.3069e-38,  2.5881e-40,  2.7260e-40,  2.2255e-38,\n         7.2434e-40,  7.7134e-38,  2.6141e-39,  1.7460e-39, -6.0096e-40,\n         1.8925e-38,  9.6489e-39,  7.8650e-39,  4.5483e-38, -1.0058e-39,\n         6.2481e-40,  1.9336e-39, -1.5965e-39,  1.0009e-39,  3.2246e-39,\n        -1.5813e-39,  4.6770e-39,  4.1475e-38, -3.5063e-40,  1.6030e-39,\n         1.2308e-39,  6.1711e-39, -5.9789e-40,  1.9307e-38,  3.0756e-39,\n         4.3618e-40, -6.2279e-41,  6.6399e-40,  6.2155e-38, -7.5354e-40,\n         3.8460e-39,  4.9234e-39, -4.4183e-40,  1.4600e-38,  3.5892e-38,\n        -4.1376e-40,  6.7607e-38,  7.1669e-39, -9.2402e-40,  6.0153e-39,\n        -6.4496e-40, -3.5414e-40,  8.5981e-39, -1.4511e-39,  8.0364e-39,\n         1.9042e-39, -1.8437e-39,  2.2708e-38, -3.8899e-40,  9.6773e-38,\n         1.7541e-38,  3.8018e-40,  2.1370e-40, -1.2669e-39,  7.5780e-39,\n         2.5267e-40, -1.3607e-40,  3.5164e-38,  3.6752e-38, -2.3537e-39,\n         4.9175e-39,  4.0182e-39,  7.0900e-38,  5.9931e-39,  1.1012e-39,\n         6.0912e-39,  2.8642e-39,  6.5374e-40,  3.5664e-39,  7.4039e-38,\n        -5.0257e-40,  3.2548e-38,  2.4561e-39, -3.0116e-40,  6.3508e-39,\n         4.5371e-39,  2.4615e-39,  8.0155e-39,  3.0710e-39,  5.2842e-38,\n         7.7110e-38,  7.8280e-38,  1.8520e-39, -1.1891e-39, -4.0226e-40,\n         7.4709e-38,  4.6829e-39,  6.0849e-39,  5.0668e-38,  6.6146e-38,\n         5.3388e-39,  7.2908e-39, -3.0173e-40,  2.1052e-39,  6.1272e-39,\n        -1.2834e-38,  1.3849e-37,  2.1066e-38,  1.0118e-38,  1.4944e-38,\n        -3.9521e-41, -5.6132e-40,  2.0776e-39,  8.7585e-39,  2.7405e-38,\n         1.4900e-38,  5.1792e-39, -9.4125e-41,  1.5556e-38,  9.5214e-38,\n         4.9463e-40, -1.2324e-41, -1.6920e-39,  1.0576e-37,  8.8774e-39,\n         2.7745e-38, -7.7333e-40,  1.8208e-39,  1.2054e-37,  1.1574e-39,\n         2.3840e-39,  8.7799e-38,  1.7625e-39,  1.0938e-38,  1.4344e-39,\n         1.4229e-37,  1.7859e-38,  5.0827e-39,  1.2917e-39,  5.4703e-38,\n         3.5078e-39,  1.0127e-37,  3.2249e-39,  9.8159e-40, -1.4340e-39,\n         2.4594e-39,  4.2686e-38,  1.0879e-39, -1.0296e-39, -1.4164e-41,\n        -1.9885e-40, -1.1759e-39,  1.0322e-38,  6.6143e-38,  2.5933e-38,\n         2.3724e-38,  3.2414e-39,  5.6931e-40,  2.1160e-41,  2.4030e-39,\n        -1.8627e-39,  7.9200e-39, -2.3503e-39,  8.6521e-39,  2.5410e-38,\n        -1.3031e-39,  5.5783e-40,  5.8101e-40,  7.2383e-38,  2.0954e-39,\n        -5.6681e-40,  1.8979e-38,  2.4653e-39, -1.7807e-40,  1.3480e-38,\n         3.5631e-40, -2.7476e-40,  1.3395e-38,  8.2930e-38,  8.6070e-38,\n         2.2668e-39,  9.7657e-39,  3.5155e-38,  1.2354e-37,  6.5803e-39,\n         1.0013e-39,  1.2407e-38, -4.0076e-40,  6.7742e-40,  3.7235e-39,\n         2.2187e-38,  3.2501e-38,  2.2571e-39,  1.1205e-37,  3.7247e-40,\n         3.3243e-38, -7.7084e-41,  6.0111e-38,  1.1509e-37, -3.1860e-40,\n         1.2706e-39,  1.6151e-39, -1.1426e-39, -8.4747e-40,  1.2918e-37,\n         7.1625e-38, -1.3478e-40,  3.7562e-39,  4.0178e-39,  3.5319e-40,\n        -5.3759e-40,  8.2498e-39,  2.4056e-39,  3.2634e-38,  1.6686e-38,\n         7.0320e-38,  3.3596e-38, -1.0377e-40,  2.3932e-38,  3.3868e-40,\n         6.3092e-39,  1.3011e-38, -2.9779e-39,  1.6598e-38,  4.2432e-39,\n         3.3388e-38, -1.5319e-39,  8.5154e-38,  7.3323e-39,  2.8033e-38,\n        -2.4007e-39,  2.5280e-39,  7.6841e-39,  5.4789e-38,  3.5154e-39,\n        -1.7819e-40, -3.5774e-40,  3.1897e-40,  5.5693e-39,  2.2502e-39,\n         2.1346e-39,  1.7794e-38,  1.4142e-38,  4.7113e-38,  4.4565e-39,\n         5.2933e-38,  1.3708e-38,  5.3321e-38, -1.5567e-39,  3.4509e-38,\n         2.8757e-40, -2.1189e-40,  5.5730e-38,  1.5735e-37,  7.4643e-40,\n         5.9720e-38,  9.2249e-38, -9.0872e-40,  5.2504e-38,  3.4611e-38,\n         6.0079e-39,  1.0048e-37,  3.2296e-39,  3.5232e-40,  3.8145e-39,\n        -7.9758e-40, -1.9498e-39,  1.1378e-39, -8.9692e-40,  3.5658e-40,\n         1.9106e-38,  1.3139e-37,  4.4451e-38, -7.0486e-40,  2.8844e-39,\n         1.9213e-38,  2.0861e-38,  7.3725e-39,  2.5895e-38,  5.1863e-38,\n         4.6045e-39,  6.1044e-39,  1.2363e-37,  4.5061e-38,  1.9735e-39,\n         8.4396e-40,  2.5026e-38, -4.7439e-39,  1.7228e-40, -8.0084e-41,\n         3.7452e-39,  1.0276e-37, -3.7247e-40, -1.7713e-40,  1.6550e-38,\n         8.4843e-39,  1.3391e-40,  1.0600e-39,  1.9483e-39,  6.9659e-41,\n         1.4514e-38,  2.5804e-38,  4.0667e-39,  4.8766e-38,  2.6679e-38,\n         4.8393e-38,  1.3394e-38,  6.8124e-38,  2.1364e-38, -1.8885e-39,\n         2.0573e-39,  2.4072e-39,  7.1775e-39, -8.6076e-41,  1.0910e-38,\n         1.4191e-37,  2.1490e-39,  5.4471e-39,  2.2082e-38,  3.0382e-38,\n         8.7483e-39,  2.5562e-38,  2.0543e-39,  1.9273e-38,  1.1207e-37,\n         4.6358e-38,  9.6004e-39, -4.2220e-40,  4.9051e-38,  9.7213e-39,\n         8.6979e-39,  1.8477e-39,  1.0720e-37, -2.3948e-42,  3.1611e-38,\n         1.4956e-39,  6.9291e-38, -1.7850e-39, -5.0152e-41,  1.3059e-38,\n        -1.7905e-40,  1.0359e-38,  2.1871e-38,  1.1235e-40,  3.1004e-40,\n         1.2541e-40,  6.5789e-39,  1.2388e-39,  1.6619e-38, -5.3602e-40,\n         4.4820e-39,  3.1977e-39,  1.6934e-39,  2.8139e-38, -1.5280e-40,\n         1.8744e-39,  2.6556e-39,  8.8786e-39, -1.3607e-39,  3.1147e-38,\n         4.6679e-38,  1.8448e-39,  5.5846e-38,  9.8939e-38,  4.0817e-39,\n         3.1880e-39,  1.2556e-39,  7.7217e-39,  6.0308e-39,  5.0534e-38,\n         7.7262e-39,  5.2354e-39,  6.1996e-38,  6.6573e-38,  2.3973e-39,\n         1.7073e-38,  2.2020e-38,  3.2484e-39,  3.1714e-38,  1.3568e-40,\n        -1.0925e-40,  4.0764e-38,  3.6033e-38,  3.0726e-40, -9.1799e-42,\n         3.2831e-38,  3.3436e-38, -2.5441e-41,  2.1120e-39, -1.7366e-39,\n         8.1570e-39,  9.3865e-39, -1.1210e-39, -6.5045e-40,  9.5157e-38,\n         1.7763e-39,  1.0058e-38,  1.0603e-40,  1.0432e-38, -1.8600e-40,\n         1.5495e-38, -4.3442e-40,  1.8819e-39, -4.5248e-39,  2.8315e-39,\n         4.8981e-39,  8.7045e-38], device='cuda:0')",
        "exp_avg_sq": "tensor([4.9925e-19, 1.1977e-13, 5.3706e-17, 4.0323e-16, 3.4882e-16, 7.8860e-15,\n        3.5976e-15, 3.7459e-15, 4.7099e-14, 1.9224e-15, 1.9037e-14, 1.0618e-17,\n        4.6138e-15, 7.8288e-17, 1.2980e-16, 2.1227e-14, 6.6268e-15, 5.4462e-18,\n        2.0004e-17, 3.1929e-17, 8.3492e-14, 5.1275e-14, 2.4164e-17, 1.5366e-15,\n        8.5400e-18, 1.2714e-15, 5.5612e-18, 5.0130e-16, 5.4351e-14, 6.4068e-17,\n        1.2787e-15, 1.0471e-14, 5.1746e-16, 3.4661e-14, 2.7343e-14, 2.3927e-16,\n        6.2159e-14, 5.8532e-18, 3.0134e-15, 6.2168e-15, 1.1176e-15, 1.1449e-14,\n        3.2678e-15, 9.4764e-17, 7.0317e-15, 3.1106e-14, 1.6485e-17, 1.4757e-17,\n        2.7917e-14, 3.1032e-19, 5.8746e-16, 3.5246e-16, 3.4668e-18, 1.7274e-19,\n        6.5613e-16, 8.2229e-18, 4.5198e-17, 1.2893e-15, 8.1833e-15, 3.9625e-16,\n        8.6027e-16, 7.2196e-18, 2.6618e-17, 3.3281e-13, 2.0479e-16, 1.8595e-17,\n        3.4301e-15, 8.7634e-14, 3.7135e-18, 7.2445e-14, 2.5154e-14, 2.8864e-18,\n        9.5443e-20, 3.8102e-18, 3.1971e-16, 5.2519e-15, 1.4084e-14, 4.8030e-16,\n        3.9728e-17, 2.0283e-17, 6.1348e-15, 2.6671e-15, 2.0515e-17, 2.2785e-14,\n        3.2036e-14, 1.6891e-18, 4.7371e-17, 5.9223e-14, 1.6125e-16, 3.9273e-14,\n        8.7988e-16, 5.4132e-18, 7.5049e-18, 3.6108e-16, 1.4859e-14, 6.0752e-16,\n        2.9341e-15, 1.2038e-16, 1.6164e-16, 7.1060e-18, 1.9259e-15, 6.8612e-14,\n        1.6115e-16, 8.1719e-16, 8.6782e-18, 2.5661e-16, 3.1208e-15, 1.7711e-17,\n        5.9443e-15, 1.1629e-14, 4.9606e-16, 4.2793e-14, 6.2821e-17, 7.0318e-16,\n        3.4375e-14, 4.4015e-16, 4.2997e-14, 1.7554e-16, 1.5164e-15, 6.2674e-19,\n        1.3013e-13, 8.6845e-16, 4.8569e-15, 1.8690e-14, 2.5158e-17, 2.9328e-20,\n        1.2342e-19, 1.0916e-16, 1.0400e-17, 2.9685e-17, 7.7744e-19, 1.0318e-14,\n        3.0901e-14, 6.7421e-19, 2.8890e-17, 2.9685e-16, 8.7059e-17, 3.1210e-19,\n        8.3513e-16, 7.6388e-18, 8.7733e-16, 4.4916e-15, 4.9338e-16, 1.2000e-13,\n        1.9478e-17, 2.0712e-17, 9.8464e-18, 8.1412e-18, 8.5413e-14, 1.7674e-15,\n        6.5543e-16, 2.1862e-14, 9.4149e-15, 5.0357e-15, 5.7627e-18, 2.7974e-15,\n        2.2436e-18, 1.3367e-15, 1.3290e-19, 4.6681e-16, 1.1328e-17, 2.8270e-18,\n        1.7444e-15, 6.5426e-21, 6.6638e-14, 4.2277e-16, 2.3853e-18, 2.2101e-15,\n        1.2596e-18, 5.8646e-16, 5.3090e-17, 6.8767e-17, 2.9199e-15, 7.7022e-15,\n        1.6096e-18, 1.1974e-17, 2.9665e-16, 4.6477e-15, 2.6021e-14, 1.2983e-17,\n        9.3297e-15, 2.2711e-17, 1.0255e-17, 1.9473e-15, 3.5034e-14, 6.8389e-18,\n        6.0366e-15, 3.8404e-16, 1.9191e-16, 2.8353e-16, 9.1288e-15, 7.1863e-16,\n        6.8085e-16, 1.7104e-17, 3.1877e-14, 2.8781e-14, 3.0135e-14, 8.9256e-17,\n        6.0922e-15, 3.6147e-18, 2.7761e-14, 3.0978e-16, 3.4283e-16, 2.9440e-15,\n        2.8329e-15, 1.3487e-16, 7.7621e-16, 1.6703e-16, 1.8852e-16, 1.0190e-17,\n        2.7072e-15, 2.2284e-13, 5.5217e-16, 3.7365e-15, 5.1437e-15, 2.8106e-16,\n        1.1168e-17, 1.1261e-18, 6.3427e-14, 2.2470e-15, 2.2515e-15, 3.6749e-17,\n        5.5712e-18, 2.4593e-15, 3.7362e-14, 4.8283e-19, 1.8598e-19, 6.9661e-18,\n        2.0024e-13, 3.7609e-16, 4.0510e-15, 4.2294e-16, 1.3159e-16, 1.4818e-13,\n        3.4645e-17, 6.6362e-18, 6.8633e-14, 1.7338e-18, 3.4547e-16, 9.0602e-20,\n        2.1566e-13, 8.6961e-16, 3.0672e-14, 1.3025e-16, 9.8857e-15, 3.1201e-16,\n        3.8137e-14, 3.1463e-15, 2.3257e-17, 1.3305e-18, 1.8056e-14, 1.8451e-14,\n        3.1797e-15, 6.0848e-19, 1.0603e-18, 5.2968e-17, 4.1583e-17, 1.4505e-15,\n        2.2024e-14, 1.4760e-15, 5.1648e-15, 2.1640e-17, 7.9217e-16, 2.1387e-16,\n        1.7833e-15, 1.6902e-17, 9.6339e-17, 2.3638e-16, 6.5437e-16, 3.0984e-15,\n        1.5628e-18, 2.4220e-18, 5.7783e-17, 2.0538e-13, 6.2874e-17, 1.9939e-17,\n        2.2449e-14, 7.4550e-17, 5.0289e-17, 1.3327e-14, 2.6889e-16, 1.5481e-16,\n        1.3564e-17, 5.8070e-14, 1.5058e-13, 2.1013e-16, 1.3836e-14, 3.3528e-16,\n        1.3205e-13, 4.9794e-16, 6.9682e-18, 2.3387e-15, 3.9154e-16, 1.5162e-18,\n        3.7542e-14, 2.2821e-15, 3.4500e-15, 4.6585e-18, 4.1626e-14, 2.8003e-15,\n        6.9224e-15, 3.2751e-16, 5.2068e-14, 4.6818e-14, 8.3485e-18, 6.0826e-20,\n        2.3138e-17, 4.3449e-17, 3.3417e-18, 2.4599e-13, 4.1589e-13, 7.4414e-16,\n        1.1403e-17, 3.4085e-16, 4.7439e-18, 4.7037e-19, 9.3947e-16, 9.4457e-18,\n        2.1627e-14, 9.3970e-16, 2.9589e-14, 7.0196e-15, 1.8127e-17, 3.4401e-14,\n        8.0666e-17, 7.0454e-17, 2.6446e-15, 1.5997e-15, 1.3794e-15, 3.2138e-15,\n        5.2400e-14, 9.0684e-18, 2.8309e-14, 3.6298e-18, 1.7791e-15, 5.3785e-18,\n        2.2790e-20, 8.7149e-16, 1.3589e-14, 3.9313e-18, 3.1432e-15, 2.0613e-18,\n        2.0151e-15, 1.3036e-15, 6.1248e-17, 2.1963e-17, 1.4576e-13, 1.9956e-17,\n        2.9407e-14, 1.1464e-17, 1.8889e-15, 2.3086e-16, 4.2883e-15, 4.4460e-18,\n        5.0373e-15, 6.1328e-16, 8.8958e-16, 5.1724e-15, 3.0271e-13, 1.5731e-14,\n        1.9359e-14, 1.8631e-13, 4.5592e-18, 2.4581e-14, 2.7537e-14, 5.3872e-16,\n        2.2288e-13, 6.4594e-18, 2.3267e-19, 1.0649e-16, 8.2350e-17, 1.8972e-16,\n        2.6308e-17, 2.1368e-16, 5.7501e-16, 1.0164e-13, 1.1897e-13, 1.5957e-14,\n        1.2171e-17, 1.3282e-17, 5.3462e-16, 5.8086e-15, 2.3165e-14, 3.8168e-15,\n        2.9695e-14, 3.9467e-14, 5.5536e-19, 7.8042e-14, 1.9430e-15, 2.5608e-18,\n        1.2175e-19, 3.2788e-15, 3.6802e-17, 1.0012e-19, 4.7599e-16, 2.7925e-17,\n        5.5052e-14, 1.5637e-17, 4.6798e-14, 2.6472e-14, 8.5452e-17, 8.0295e-18,\n        1.2548e-17, 3.3560e-17, 1.7803e-18, 1.8539e-16, 5.2255e-15, 4.9862e-18,\n        2.6363e-14, 7.0853e-16, 6.9711e-15, 1.2589e-15, 2.4624e-14, 2.7878e-14,\n        3.5042e-18, 1.2515e-16, 6.6994e-17, 2.4253e-14, 6.6800e-16, 6.3031e-16,\n        2.5210e-13, 2.2648e-18, 1.0620e-17, 2.0277e-16, 1.1648e-15, 3.0711e-17,\n        2.9491e-14, 7.2035e-16, 1.0520e-15, 3.8165e-14, 1.5964e-14, 6.9280e-15,\n        1.0913e-16, 2.0728e-15, 3.7647e-16, 7.6169e-17, 1.7992e-15, 6.8400e-14,\n        9.4364e-17, 6.9155e-15, 4.9663e-15, 5.8994e-15, 9.3207e-19, 7.6296e-20,\n        8.4968e-15, 1.5202e-20, 1.2987e-18, 8.0557e-16, 1.7861e-18, 3.5196e-15,\n        2.6215e-16, 2.2991e-15, 5.9827e-19, 1.3857e-14, 4.5435e-16, 4.2753e-17,\n        6.0248e-19, 8.5193e-16, 1.9308e-14, 1.2000e-19, 4.1959e-16, 2.3660e-18,\n        2.6229e-17, 6.1954e-18, 1.6358e-15, 2.2617e-15, 4.3425e-19, 2.0129e-14,\n        4.7920e-14, 1.0389e-15, 1.2591e-16, 2.7080e-19, 7.4086e-16, 1.3686e-18,\n        3.4230e-14, 9.6827e-17, 1.1593e-13, 1.0576e-14, 1.0964e-13, 3.3080e-18,\n        8.5569e-17, 4.2661e-14, 1.0222e-17, 7.4324e-15, 2.8519e-17, 7.2230e-17,\n        9.5058e-16, 7.9322e-15, 3.1141e-18, 1.3528e-17, 1.0571e-15, 3.6295e-14,\n        1.2662e-16, 8.3657e-16, 8.6288e-19, 1.0263e-15, 5.5047e-16, 1.4950e-17,\n        9.8707e-16, 1.0033e-13, 2.7008e-18, 2.4027e-15, 2.6899e-16, 5.4192e-14,\n        1.5595e-17, 1.0105e-14, 2.5846e-16, 6.5628e-17, 7.2164e-15, 1.1532e-15,\n        2.2585e-16, 7.7931e-14], device='cuda:0')"
      },
      "39": {
        "step": "tensor(11268.)",
        "exp_avg": "tensor([ 4.2761e-39,  4.8699e-38, -5.8600e-39, -1.8264e-39,  8.5216e-40,\n         1.2469e-38,  3.8263e-38,  2.4958e-38,  6.8407e-38, -6.1608e-39,\n         4.1011e-38,  2.4850e-39,  4.7692e-38, -1.7753e-39,  1.9053e-38,\n         5.9680e-38,  2.7929e-38,  1.0891e-38,  1.7551e-38, -2.2218e-39,\n         5.2862e-38,  2.4482e-38, -2.7977e-39, -1.9856e-39, -2.9712e-39,\n         5.9993e-39, -5.7603e-39,  2.2708e-39,  2.3768e-38, -2.8362e-39,\n         1.3307e-40,  3.8965e-38,  3.7209e-38,  5.6100e-38,  4.2521e-38,\n        -3.2297e-39,  5.6215e-38, -7.0340e-41, -6.6691e-39,  1.8611e-38,\n         2.2292e-38,  2.8448e-38,  3.7333e-38,  6.5885e-39,  1.6804e-38,\n         5.9988e-38, -2.8402e-39,  2.6740e-38,  5.3337e-38,  2.3249e-39,\n         3.3370e-38,  1.0039e-38, -7.8583e-40, -6.5107e-41, -1.4173e-39,\n         2.9850e-38,  2.5854e-38, -8.1570e-39,  8.0357e-38,  4.3091e-38,\n         3.3228e-38,  4.4619e-38,  4.7578e-38,  8.0554e-38,  2.6841e-40,\n         8.0626e-39,  2.2463e-38,  4.4902e-38,  1.4231e-38,  2.2286e-38,\n         2.9153e-38,  9.3471e-39, -2.3018e-39,  3.8714e-39,  3.1378e-38,\n         2.9150e-38, -5.9179e-39, -3.9065e-39,  3.5077e-40, -4.7407e-39,\n        -8.4390e-39,  3.6897e-38,  4.4703e-39,  2.7675e-38,  6.0446e-38,\n         4.2723e-38, -4.2603e-39,  7.9607e-38,  4.1717e-38,  6.2032e-39,\n        -3.7089e-39,  9.0672e-41,  2.6224e-38,  3.0596e-40,  4.1388e-38,\n        -5.4961e-39,  3.5844e-38, -2.3230e-40,  4.6210e-40,  1.1115e-38,\n         2.5805e-38,  5.3050e-38,  1.7764e-38,  6.9566e-39,  3.9479e-40,\n        -7.4297e-40, -6.8537e-39, -4.4371e-39,  5.1774e-38,  1.9892e-38,\n         6.5184e-40,  3.8383e-38,  1.1343e-39, -3.0127e-40,  3.2598e-38,\n         1.0275e-38,  4.1741e-38, -2.2773e-39,  1.3219e-38,  1.6728e-38,\n         2.6067e-38, -6.3967e-39,  7.3418e-39,  3.1201e-38,  8.3777e-40,\n         3.1728e-40, -1.7236e-39,  4.2192e-39, -8.9960e-40,  1.1102e-38,\n         3.9784e-39,  7.1776e-39,  4.1732e-38,  1.4993e-39,  2.6679e-38,\n        -8.3072e-40, -4.8880e-39,  1.5686e-39,  3.2017e-38, -2.7599e-39,\n        -2.6284e-40,  6.8912e-39,  1.5591e-38,  5.8309e-38,  2.4791e-39,\n         1.8079e-38, -4.1206e-39,  1.5634e-39,  1.9796e-38,  4.6678e-38,\n         4.7659e-39,  5.9855e-38,  2.7727e-38,  8.3257e-39,  2.7897e-38,\n         8.6437e-39,  1.6848e-38, -5.4386e-39,  7.3193e-39, -5.5097e-39,\n         1.6658e-38,  1.1484e-38,  2.5886e-38,  9.3719e-40,  4.8433e-38,\n         3.3916e-38, -2.8387e-40,  3.5729e-39,  3.1377e-39, -5.0472e-39,\n        -2.0748e-40,  3.2187e-40,  4.5681e-38,  4.1499e-38,  8.3153e-39,\n        -4.1237e-39, -3.0249e-39,  6.2726e-38,  1.0367e-38,  2.2233e-38,\n         1.6621e-38, -2.3534e-39, -6.2794e-40,  2.1067e-38,  5.7227e-38,\n         7.0466e-39,  4.3950e-38, -1.9114e-39,  1.0130e-39, -4.4790e-39,\n         1.2509e-38,  1.4861e-38, -4.9949e-39, -2.4614e-39,  4.8068e-38,\n         6.9431e-38,  4.4831e-38, -1.5065e-39, -5.1185e-39,  1.6500e-39,\n         4.3069e-38, -3.3355e-39, -3.8865e-39,  5.4545e-38,  5.8711e-38,\n        -4.0323e-39, -4.6755e-39,  9.1801e-40, -1.8747e-39,  2.4109e-38,\n        -1.8452e-38,  7.1477e-38,  4.2041e-38,  1.3972e-38, -8.7819e-39,\n         4.5380e-40,  1.8665e-38, -1.7574e-39,  1.6052e-38,  2.6223e-38,\n         2.7969e-38, -4.2713e-39,  5.8606e-40,  2.3268e-38,  6.2133e-38,\n         3.1732e-40,  5.3690e-40,  1.5340e-38,  5.9069e-38, -6.4602e-39,\n         3.7896e-38,  4.6475e-39, -1.4167e-39,  9.1146e-38, -8.4338e-40,\n        -2.0464e-39,  7.7116e-38, -9.8294e-40, -7.1577e-39, -8.5161e-41,\n         8.3709e-38,  3.1672e-38,  7.4488e-39, -1.0083e-39,  5.8628e-38,\n         6.9872e-39,  7.0442e-38,  1.0658e-38, -3.6751e-39,  4.2103e-39,\n         7.4650e-39,  3.4828e-38, -4.1408e-40,  3.1443e-39,  6.8475e-40,\n         1.1197e-39,  8.5872e-39,  1.5687e-38,  6.1502e-38,  4.3268e-38,\n         4.5675e-38,  1.8397e-38,  5.6627e-39, -6.1256e-41, -1.4815e-39,\n         5.0205e-39, -6.0902e-39,  9.6286e-39, -5.5251e-39,  4.1935e-38,\n         4.0830e-39, -2.6206e-40, -5.2702e-40,  6.0648e-38, -1.7398e-39,\n         1.6067e-39,  2.1554e-38, -1.8892e-39,  1.0001e-39,  2.5024e-38,\n        -1.8305e-39,  2.1910e-40,  3.7845e-38,  5.6207e-38,  4.3858e-38,\n        -1.8457e-39,  1.3359e-38,  2.9959e-38,  7.1417e-38, -4.2210e-39,\n        -8.3537e-40, -6.7175e-39,  3.3870e-40, -5.8199e-40,  4.3271e-39,\n         2.7336e-38,  3.6137e-38,  2.1270e-38,  6.9999e-38,  5.1585e-39,\n         3.1599e-38,  4.4562e-40,  3.3334e-38,  8.4073e-38,  1.1501e-39,\n        -1.1363e-39, -1.3120e-39,  5.2794e-39,  9.6903e-39,  8.4870e-38,\n         3.7367e-38,  2.5365e-41,  1.6304e-38, -2.8097e-39,  1.3132e-38,\n         1.4346e-38, -5.2900e-39, -2.1269e-39,  2.5990e-38,  1.7789e-38,\n         7.0635e-38,  5.2063e-38,  1.6385e-40,  3.7275e-38, -8.9123e-43,\n        -4.8090e-39,  1.5004e-38,  2.1062e-39,  2.1267e-38,  8.5636e-39,\n         3.3860e-38,  5.3696e-39,  5.2360e-38, -5.6746e-39,  4.6029e-38,\n         6.5296e-39, -2.1695e-39,  2.1800e-38,  6.8926e-38,  2.2013e-38,\n         5.1944e-40,  9.2027e-40,  2.0201e-39,  2.1717e-38, -1.8047e-39,\n         1.7328e-38,  1.8658e-38,  3.4739e-38,  3.8115e-38,  3.3373e-38,\n         6.1137e-38,  3.3582e-38,  5.5522e-38,  2.2891e-38,  4.2235e-38,\n         6.3414e-40,  1.1122e-39,  4.6704e-38,  7.9025e-38,  9.5743e-39,\n         4.9335e-38,  5.3306e-38,  3.6222e-39,  5.2482e-38,  4.4126e-38,\n        -3.8765e-39,  5.9580e-38, -2.7249e-39,  3.4376e-40, -3.1139e-39,\n         2.0903e-39,  6.3311e-39,  1.3890e-38,  1.0865e-38, -1.4798e-40,\n         2.5845e-38,  7.8519e-38,  5.5837e-38,  1.6426e-39, -2.4533e-39,\n         3.5357e-38,  4.2546e-38,  1.0124e-38,  3.3628e-38,  3.4586e-38,\n         1.4368e-38, -5.2261e-39,  6.4074e-38,  4.0266e-38, -1.8244e-39,\n        -7.4510e-40,  4.4726e-38,  3.6778e-39, -1.6735e-40,  9.9211e-40,\n        -3.0991e-39,  6.1955e-38,  9.9253e-40, -1.1144e-39,  3.3118e-38,\n        -6.4505e-39,  1.0245e-40, -9.8224e-40, -1.4045e-39,  1.1831e-40,\n         4.8504e-38,  2.5761e-38, -3.4807e-39,  3.4452e-38,  4.8974e-38,\n         5.0988e-38,  2.4174e-38,  4.1501e-38,  3.3601e-38,  9.2253e-39,\n        -1.8354e-39,  1.2625e-38,  1.4426e-38,  9.9944e-40, -7.7950e-39,\n         6.9699e-38, -1.8988e-39, -4.4974e-39,  3.2499e-38,  4.0102e-38,\n         2.1440e-38,  1.6605e-38, -1.3237e-39,  2.7778e-38,  6.6395e-38,\n         4.8081e-38,  1.6532e-38,  1.2975e-39,  5.6356e-38, -6.5927e-39,\n         2.0497e-38,  2.3048e-38,  7.3882e-38,  2.8635e-38,  3.6710e-38,\n         1.0171e-38,  5.7875e-38,  4.8577e-39,  7.6455e-40,  1.5023e-38,\n         8.7646e-40,  4.2435e-38,  3.9582e-38,  5.7648e-40, -3.7634e-39,\n        -3.6418e-40,  1.7645e-38, -1.0652e-39,  2.6052e-38,  1.4778e-39,\n        -3.5042e-39, -2.7988e-39, -8.5789e-39,  2.5055e-38,  7.2484e-40,\n        -1.5389e-39,  2.2783e-38,  3.3054e-38,  1.5211e-38,  4.1152e-38,\n         4.1943e-38, -1.4543e-39,  3.9928e-38,  8.8837e-38,  1.2671e-38,\n        -2.8290e-39, -1.0572e-39, -5.0534e-39,  2.9040e-38,  3.7216e-38,\n        -6.0927e-39,  9.7027e-39,  8.4540e-38,  3.6005e-38, -2.0959e-39,\n         4.2745e-38,  2.4963e-38, -2.6065e-39,  3.8893e-38, -1.1045e-40,\n         5.7754e-40,  4.0182e-38,  4.6253e-38,  2.2162e-38,  2.2425e-38,\n         6.1374e-38,  2.5782e-38,  4.2765e-39,  1.5367e-38,  5.3329e-39,\n        -5.3183e-39,  1.9242e-38,  2.9178e-39,  2.9529e-39,  5.3621e-38,\n        -1.5738e-39, -5.9334e-39,  5.9231e-40,  1.6555e-38,  1.7045e-40,\n         2.5866e-38,  6.9524e-39,  1.0632e-38, -1.2388e-38,  1.6543e-38,\n        -4.0747e-39,  6.3883e-38], device='cuda:0')",
        "exp_avg_sq": "tensor([1.8681e-16, 5.3065e-14, 2.7797e-18, 1.9060e-16, 1.3428e-16, 4.0177e-15,\n        9.4911e-15, 2.4123e-15, 1.0724e-13, 8.8629e-16, 2.0562e-14, 7.9974e-16,\n        1.5908e-14, 3.2882e-17, 3.8195e-15, 3.3052e-14, 9.1589e-15, 1.2487e-15,\n        1.7204e-15, 1.2806e-17, 3.5144e-14, 4.4521e-14, 8.5345e-19, 1.2082e-14,\n        3.5739e-18, 1.3342e-15, 5.1555e-17, 2.7361e-16, 2.2513e-14, 2.8254e-17,\n        5.6376e-16, 5.8662e-15, 5.6387e-16, 6.3518e-14, 3.2897e-14, 9.2528e-17,\n        4.4274e-14, 4.9408e-16, 1.1238e-15, 8.4370e-15, 1.4529e-15, 9.3665e-15,\n        1.9894e-15, 6.0294e-16, 1.3841e-14, 4.4109e-14, 5.9447e-18, 1.2353e-15,\n        3.0789e-14, 1.5275e-15, 4.5221e-15, 8.2250e-16, 6.6203e-16, 5.1383e-17,\n        3.0108e-16, 2.6397e-15, 5.9005e-15, 4.6192e-16, 7.5214e-14, 9.0797e-15,\n        3.4492e-15, 6.9048e-15, 1.4803e-14, 1.6126e-13, 9.7597e-17, 3.2447e-16,\n        5.4343e-15, 8.5676e-14, 5.0261e-16, 5.4768e-14, 1.2128e-14, 2.9614e-16,\n        3.5859e-17, 8.0668e-16, 5.3075e-15, 1.1067e-14, 4.9283e-14, 2.4688e-16,\n        1.2840e-14, 6.0077e-19, 2.4722e-14, 4.1619e-15, 1.6935e-15, 3.1215e-14,\n        3.4453e-14, 8.5117e-15, 1.2006e-17, 2.5566e-14, 9.9740e-15, 3.4856e-14,\n        3.3781e-16, 4.7000e-16, 2.3390e-15, 8.1243e-17, 7.4856e-15, 2.5196e-16,\n        9.0579e-15, 5.5276e-17, 8.1617e-17, 8.5083e-16, 5.1718e-15, 3.6987e-14,\n        8.8642e-16, 1.0549e-15, 3.8676e-18, 1.1532e-16, 1.2582e-15, 2.6409e-19,\n        3.7380e-14, 1.1417e-14, 2.6296e-16, 2.6427e-14, 2.3322e-16, 3.3153e-16,\n        2.7336e-14, 4.1699e-15, 1.9548e-14, 8.0731e-17, 3.9764e-15, 2.0224e-15,\n        7.3698e-14, 4.4773e-16, 3.9039e-15, 1.5133e-14, 9.8921e-18, 4.0614e-16,\n        8.3296e-18, 4.8537e-15, 3.3972e-17, 6.8358e-17, 2.4138e-16, 1.2828e-14,\n        1.6962e-14, 4.9236e-16, 2.3986e-15, 1.3824e-16, 1.3133e-17, 2.1997e-16,\n        9.6673e-16, 5.8098e-16, 3.5748e-16, 2.0494e-14, 4.8387e-15, 5.5253e-14,\n        7.2526e-16, 2.5108e-15, 3.1760e-17, 1.2735e-15, 5.8540e-14, 1.2327e-14,\n        4.3194e-15, 3.5769e-14, 1.7059e-14, 4.4650e-14, 1.9843e-15, 1.3397e-14,\n        1.9631e-15, 5.1213e-16, 1.3966e-16, 2.2456e-16, 2.4262e-15, 4.1102e-16,\n        6.7810e-15, 7.2806e-17, 2.6685e-14, 5.5674e-15, 9.8666e-17, 4.4698e-15,\n        1.1403e-16, 2.8754e-16, 1.7506e-17, 2.4046e-17, 2.2113e-14, 2.1065e-14,\n        3.0488e-16, 6.6558e-19, 1.1595e-16, 7.1253e-15, 2.8445e-14, 1.4671e-15,\n        1.3827e-14, 1.0889e-17, 1.1569e-15, 4.2177e-15, 3.8659e-14, 6.1849e-16,\n        3.1965e-14, 1.6138e-16, 9.3950e-17, 1.4069e-16, 1.0362e-14, 1.2687e-15,\n        2.8634e-16, 6.0858e-18, 5.5944e-14, 5.6869e-14, 1.3361e-14, 4.6204e-17,\n        2.2023e-14, 3.9761e-16, 1.1416e-14, 1.3341e-16, 1.7473e-16, 6.8542e-15,\n        7.4151e-15, 2.4272e-18, 3.4855e-16, 4.1100e-16, 7.5923e-17, 4.9183e-15,\n        1.6073e-14, 9.4067e-14, 1.1124e-14, 3.5365e-15, 2.4812e-15, 1.3677e-16,\n        1.3121e-15, 1.1976e-16, 5.4969e-14, 1.4383e-15, 6.5433e-15, 1.1104e-18,\n        2.7664e-15, 8.5815e-15, 3.0596e-14, 1.0725e-16, 3.0440e-17, 7.5469e-16,\n        1.0170e-13, 1.6612e-16, 7.6509e-15, 3.3991e-15, 5.5058e-17, 1.4988e-13,\n        1.6705e-17, 3.6911e-18, 1.0567e-13, 3.0207e-16, 1.8120e-16, 1.2640e-17,\n        1.3551e-13, 1.3176e-14, 1.9511e-14, 5.2928e-17, 3.2156e-14, 3.2258e-16,\n        3.3474e-14, 5.3021e-15, 6.7475e-15, 1.3290e-16, 2.6336e-14, 2.2597e-14,\n        1.4580e-15, 4.0978e-16, 1.3225e-16, 2.5925e-15, 9.0295e-16, 2.0064e-15,\n        3.6162e-14, 1.1137e-14, 1.8903e-14, 3.3372e-15, 2.4840e-15, 1.1268e-16,\n        7.2839e-16, 1.8617e-15, 7.5206e-18, 3.5465e-15, 2.7981e-16, 2.9613e-14,\n        2.1515e-16, 4.6088e-16, 1.8379e-17, 9.3720e-14, 2.5967e-17, 6.4674e-15,\n        1.2509e-14, 2.9355e-17, 2.4617e-17, 1.0386e-14, 1.4036e-16, 6.0707e-17,\n        7.8788e-15, 3.1432e-14, 5.5990e-14, 7.8887e-17, 8.5628e-15, 1.3569e-16,\n        7.0675e-14, 2.4004e-16, 1.1354e-17, 9.9115e-16, 1.8365e-16, 1.4479e-16,\n        2.2207e-14, 9.3714e-15, 7.4871e-15, 1.0057e-15, 2.7340e-14, 1.1174e-14,\n        1.1600e-14, 1.5412e-16, 2.3603e-14, 4.9929e-14, 3.6996e-15, 2.0636e-18,\n        1.2771e-18, 1.0599e-15, 7.4691e-16, 1.9611e-13, 1.9187e-13, 3.2876e-16,\n        1.9281e-15, 1.5894e-16, 3.8442e-17, 7.9574e-16, 4.7118e-16, 4.5082e-18,\n        1.6082e-14, 6.7687e-16, 9.9302e-14, 5.7428e-15, 6.9172e-15, 2.7907e-14,\n        4.2107e-17, 3.5349e-17, 3.8110e-15, 7.3733e-16, 1.1284e-15, 7.9939e-15,\n        2.6389e-14, 4.0960e-15, 1.5447e-14, 3.8699e-19, 1.4112e-14, 1.7621e-15,\n        2.4254e-17, 3.7691e-15, 3.1143e-14, 1.0908e-15, 2.4643e-14, 1.3559e-15,\n        9.1275e-15, 2.3837e-15, 3.0043e-17, 2.1773e-15, 6.6761e-14, 3.6187e-15,\n        1.9393e-14, 2.8999e-15, 2.8356e-15, 7.5840e-15, 1.1211e-14, 1.6433e-15,\n        2.6890e-14, 2.9792e-16, 4.3036e-16, 8.3716e-15, 1.2859e-13, 4.3439e-14,\n        3.2304e-14, 1.1416e-13, 1.7085e-16, 3.2446e-14, 3.6300e-14, 2.8196e-16,\n        1.3562e-13, 2.0812e-17, 1.9113e-17, 4.2751e-17, 5.8402e-15, 3.8636e-15,\n        3.1404e-16, 1.1841e-15, 2.6128e-16, 5.9365e-14, 5.9951e-14, 2.9400e-14,\n        1.7027e-15, 9.9027e-18, 2.6340e-15, 1.4718e-14, 1.3584e-14, 1.4062e-14,\n        2.0266e-14, 6.0289e-14, 1.9959e-17, 2.8026e-14, 8.2556e-16, 3.1417e-16,\n        1.8580e-17, 2.1141e-14, 1.7421e-17, 1.2762e-18, 1.9599e-16, 1.1112e-17,\n        3.2416e-14, 2.4221e-15, 5.0287e-14, 2.8640e-14, 6.9684e-18, 1.9838e-15,\n        1.1202e-15, 6.9172e-18, 7.0134e-19, 1.6087e-14, 1.1038e-14, 4.7056e-17,\n        2.1901e-14, 3.0986e-14, 2.1687e-14, 1.4100e-15, 1.1395e-14, 2.2225e-14,\n        4.8617e-16, 6.1047e-17, 2.4035e-16, 2.1101e-14, 3.2255e-16, 2.5507e-16,\n        9.8136e-14, 3.9762e-16, 7.3008e-18, 4.2925e-15, 8.8474e-15, 2.3454e-15,\n        1.4907e-14, 3.2420e-16, 7.1509e-15, 2.3828e-14, 7.8647e-15, 5.6135e-15,\n        5.1936e-17, 8.9614e-15, 1.8671e-16, 2.4842e-15, 9.3884e-15, 4.5828e-14,\n        5.5630e-15, 3.4281e-15, 1.2327e-14, 8.2690e-15, 2.6083e-16, 3.4267e-15,\n        4.0637e-15, 1.1902e-16, 1.1880e-14, 1.5105e-14, 1.8759e-16, 3.1175e-14,\n        1.0872e-16, 2.8451e-15, 3.2233e-17, 9.6406e-15, 2.2866e-16, 3.2929e-18,\n        2.7490e-17, 1.6057e-14, 2.4768e-14, 1.5131e-16, 1.8718e-16, 1.0705e-15,\n        5.3324e-15, 7.9000e-16, 4.6700e-15, 1.0704e-15, 1.3699e-16, 1.7576e-14,\n        7.4287e-14, 1.8636e-15, 4.3487e-17, 1.1976e-17, 3.1905e-16, 1.9548e-15,\n        3.1166e-14, 2.2634e-18, 9.1215e-14, 3.7888e-14, 5.4232e-14, 1.5237e-18,\n        3.2120e-15, 2.1609e-14, 4.0202e-18, 3.6778e-14, 3.9649e-18, 3.5716e-17,\n        5.8704e-16, 2.0110e-14, 9.9726e-16, 2.4523e-15, 2.7712e-14, 3.8167e-14,\n        2.5234e-15, 2.0798e-15, 1.7932e-16, 5.5997e-16, 8.9156e-15, 1.2746e-15,\n        1.6240e-14, 5.2941e-14, 2.0655e-16, 1.1137e-15, 1.3742e-16, 4.1688e-14,\n        8.8683e-16, 9.2945e-15, 1.0951e-15, 4.2579e-16, 1.2460e-14, 5.2597e-15,\n        9.7161e-17, 9.2011e-14], device='cuda:0')"
      },
      "40": {
        "step": "tensor(11268.)",
        "exp_avg": "tensor([[-6.6842e-42,  1.5521e-39,  1.5790e-41,  ...,  2.0618e-40,\n         -4.2913e-40,  8.8721e-39],\n        [ 6.8884e-40, -1.0068e-38,  1.3875e-39,  ...,  1.5844e-39,\n          1.2217e-39, -1.9722e-38],\n        [-1.9502e-40,  1.3018e-40, -2.8795e-40,  ..., -3.9152e-42,\n         -1.3304e-40,  3.1059e-39],\n        ...,\n        [-2.7265e-41,  4.0820e-41, -6.6462e-41,  ..., -1.1550e-40,\n         -6.4904e-41, -3.1823e-39],\n        [ 1.0495e-40,  6.4759e-39,  3.7576e-40,  ..., -2.0839e-40,\n         -3.5714e-40,  2.9317e-38],\n        [-5.3574e-40,  5.1125e-39,  6.5337e-41,  ...,  2.7263e-40,\n          2.0840e-41, -4.0834e-39]], device='cuda:0')",
        "exp_avg_sq": "tensor([[4.2681e-17, 2.4850e-17, 2.0058e-17,  ..., 1.3194e-16, 4.0126e-17,\n         1.5350e-16],\n        [1.2738e-16, 1.4891e-17, 1.0550e-17,  ..., 8.0308e-17, 1.9093e-17,\n         4.2145e-17],\n        [2.6005e-17, 3.5981e-18, 1.2939e-18,  ..., 1.9117e-17, 7.9801e-18,\n         8.5467e-18],\n        ...,\n        [1.7571e-17, 3.7226e-18, 3.5156e-19,  ..., 5.2786e-18, 1.4365e-18,\n         6.3986e-18],\n        [3.9411e-17, 7.3949e-18, 3.5086e-19,  ..., 7.3911e-18, 6.0326e-18,\n         2.2367e-18],\n        [2.1346e-16, 1.8839e-17, 1.0925e-17,  ..., 1.7832e-16, 2.1583e-17,\n         4.9292e-17]], device='cuda:0')"
      },
      "41": {
        "step": "tensor(11268.)",
        "exp_avg": "tensor([ 4.0134e-37,  4.5408e-36, -3.4328e-37, -9.6182e-38, -2.8582e-37,\n        -1.7746e-37, -3.7382e-37,  1.2589e-36,  3.6903e-36,  4.0822e-37,\n         1.0268e-36, -4.3702e-36,  5.4967e-37, -3.6424e-37, -6.7205e-38,\n        -1.9454e-37, -6.7740e-37,  1.6874e-37, -3.4398e-37, -2.2864e-37,\n         1.7503e-36,  7.9376e-37, -1.3721e-37, -9.8876e-37, -9.5437e-37,\n        -2.3515e-37,  2.7106e-37,  3.9358e-37, -1.7270e-37, -1.3556e-36,\n        -4.6016e-38, -5.8403e-37, -2.9628e-37,  1.9462e-36,  6.2898e-37,\n        -3.4725e-36,  5.8679e-37, -9.6234e-37,  2.3251e-36, -2.0318e-36,\n        -3.8285e-38, -1.1425e-36,  3.4367e-37,  1.4907e-36,  1.3744e-37,\n         3.3745e-37, -1.3086e-36, -5.3369e-37, -1.7903e-38,  1.0048e-37,\n        -3.9864e-37, -6.7652e-37, -2.9634e-37, -2.1505e-37, -1.8471e-37,\n         1.1446e-36,  2.6166e-36,  4.5888e-37, -6.7752e-37, -2.5713e-37,\n         2.1270e-36,  6.0592e-36,  1.6139e-36, -1.8774e-36, -3.3791e-37,\n        -7.6976e-37, -1.0409e-36, -2.5391e-36,  2.9267e-37, -4.5527e-37,\n        -1.1834e-37,  4.0757e-37,  3.1470e-36, -1.8460e-39,  2.3745e-36,\n        -3.2290e-37, -1.5317e-37, -2.6046e-37,  6.4287e-37,  3.1782e-37,\n        -4.7288e-37,  4.0821e-37, -3.0021e-37, -2.5267e-37, -3.1975e-37,\n         1.6126e-36,  3.5336e-36,  1.0729e-36,  5.4350e-37, -1.1631e-38,\n        -1.5294e-36, -4.4944e-37, -4.6831e-37,  1.0370e-36,  8.6816e-38,\n         9.9174e-37, -2.8665e-37,  1.0125e-36,  2.0512e-37, -7.0608e-37,\n        -3.4508e-36,  2.3878e-36, -1.4844e-36, -2.7889e-36, -5.6586e-37,\n        -2.6534e-37,  5.5123e-37, -7.9267e-37,  1.7587e-36, -3.8155e-37,\n        -4.3976e-36,  1.2227e-37, -6.0834e-37,  6.7252e-38,  1.8001e-36,\n         7.2432e-37,  5.8416e-37,  6.6718e-37, -3.8831e-37,  1.2566e-36,\n         2.7950e-36,  6.2619e-38,  2.4786e-37,  3.6449e-36,  7.3457e-37,\n        -3.9269e-36, -1.7579e-36, -2.7417e-37, -3.0102e-37, -4.8452e-37,\n        -8.2383e-37, -3.6780e-37, -1.0657e-37,  9.0327e-37,  4.1132e-36,\n        -4.9218e-38,  3.9976e-36, -1.1830e-36, -3.1352e-37, -1.2442e-37,\n        -1.3951e-36, -1.1858e-36, -1.4584e-37,  3.6803e-37, -1.1620e-36,\n        -7.0608e-37, -8.7269e-37, -5.7851e-37, -8.3059e-37,  6.4179e-38,\n        -1.1526e-37,  1.7629e-36,  1.2035e-36, -3.5765e-37,  3.7177e-36,\n         1.9860e-37,  1.8146e-36, -7.8313e-37, -4.0879e-36,  1.1594e-36,\n         9.5016e-37, -3.7581e-36, -2.6109e-36, -3.9998e-37,  3.0252e-37,\n        -3.3429e-37, -3.8633e-36, -3.5027e-37, -9.2029e-37,  2.3499e-36,\n         7.8804e-38,  1.3364e-37,  1.6328e-36,  7.2218e-37, -7.1753e-37,\n        -6.3011e-37,  2.6799e-37, -1.1164e-37, -3.8217e-36, -1.5266e-36,\n        -4.5101e-37,  7.5302e-37, -1.6054e-36,  2.2418e-36,  2.6047e-36,\n        -9.4924e-38,  5.8527e-36, -8.4822e-38, -4.4372e-37, -9.3394e-38,\n         4.7919e-37,  9.5932e-37,  8.1124e-37, -1.1191e-36, -8.5554e-37,\n         9.8831e-37, -3.9251e-37,  1.5799e-37, -4.7689e-37, -7.2464e-37,\n         4.2646e-36, -2.7853e-36,  1.1517e-36, -1.6707e-36,  4.6186e-38,\n         1.1137e-36,  4.4808e-37, -1.6502e-36, -3.8855e-37, -2.3779e-37,\n        -4.7205e-36,  4.2681e-36, -2.0681e-37, -4.2376e-36,  6.9884e-36,\n        -5.3474e-37, -1.3153e-36, -7.3827e-39, -1.9594e-36, -4.3083e-37,\n         8.7004e-39,  1.4277e-36, -3.7941e-36, -7.6501e-37, -2.0290e-37,\n        -2.9766e-37, -6.3625e-38, -2.4498e-37,  2.9630e-37, -2.2012e-37,\n         2.4781e-36, -9.0581e-37, -1.7127e-36,  5.5690e-36, -2.0242e-37,\n        -7.4392e-37,  6.7463e-36,  2.2703e-37,  3.3126e-38,  9.5845e-37,\n         1.1959e-36, -2.8778e-36, -3.8711e-37, -3.5623e-36,  1.4578e-36,\n        -2.5833e-36, -3.0661e-38, -1.8885e-36, -9.5712e-37, -5.7798e-37,\n        -1.9660e-36, -3.9502e-37, -8.1591e-37,  3.5127e-37, -7.1486e-37,\n        -1.2860e-37, -3.8921e-37, -2.9895e-38,  1.3371e-36, -8.9164e-37,\n         1.0048e-37, -9.7541e-37, -2.6445e-37, -3.8430e-37, -1.8629e-36,\n         6.4384e-38,  4.7927e-36, -2.6053e-37, -1.3228e-36,  1.3015e-36,\n        -9.0010e-37, -2.6415e-37, -1.9025e-36, -1.5795e-37,  3.0697e-37,\n        -2.6458e-37, -3.7420e-37, -1.4205e-36, -2.6418e-36, -2.6994e-37,\n        -2.6565e-36, -1.0033e-36,  3.3817e-37,  1.7369e-36,  1.5882e-36,\n        -9.2517e-38, -3.0787e-36, -1.4912e-36,  3.0131e-36, -3.0726e-37,\n        -6.3014e-37,  2.0945e-36, -1.7544e-36, -4.0228e-38, -2.1136e-37,\n         1.7752e-36, -4.2005e-37,  8.4076e-38,  1.2050e-36,  2.1172e-37,\n        -1.7053e-36, -3.6127e-36,  2.9839e-36,  1.1565e-36, -5.4491e-37,\n        -3.2585e-37, -5.0957e-37,  4.9114e-37, -7.1889e-37,  5.7910e-36,\n         1.5910e-36,  1.9338e-37, -1.0099e-36, -2.9448e-37,  3.4101e-37,\n        -3.6375e-36,  4.1299e-37,  4.4099e-38, -2.4315e-36,  3.6282e-37,\n         4.4246e-36, -1.1034e-37, -3.3517e-36,  1.5932e-37, -2.2016e-37,\n         2.4581e-36, -2.2352e-37, -9.0711e-36,  1.5199e-37, -1.7408e-36,\n        -1.6169e-37,  1.0556e-36,  1.9949e-37, -2.8966e-37,  1.0531e-36,\n         1.3252e-36, -6.4845e-37, -5.2338e-37,  1.3340e-36,  3.3336e-36,\n        -6.6589e-37, -3.4866e-37, -2.1855e-36,  1.2331e-36,  1.4461e-37,\n        -7.6336e-38, -6.4856e-37, -1.2477e-36, -3.2043e-37, -6.4211e-37,\n         6.1095e-37,  8.7659e-37,  1.2759e-36,  1.1966e-36,  2.0140e-36,\n        -2.0619e-36,  9.1799e-38,  3.8434e-37,  4.9075e-36, -2.1626e-36,\n        -8.2156e-37,  5.8330e-37,  6.0471e-37,  3.5968e-36,  5.6326e-36,\n        -3.7095e-37,  9.2162e-37, -2.1933e-37, -3.9093e-37, -4.5970e-37,\n        -4.2305e-37, -1.5827e-37, -2.7026e-36, -5.0958e-37, -1.4579e-36,\n         8.2727e-37, -1.7491e-37,  1.8323e-36, -1.9598e-36, -1.5564e-37,\n        -3.8676e-37, -2.0492e-37, -2.9135e-37,  6.4842e-36, -1.9692e-36,\n        -3.5773e-36, -3.9261e-37, -1.6920e-37,  6.2708e-37, -7.0886e-37,\n         9.3842e-38, -2.0340e-37, -1.5195e-36, -1.6960e-36, -5.1931e-37,\n        -5.3333e-36,  1.7833e-36, -1.2106e-40, -1.7351e-36,  8.6382e-37,\n         1.1550e-36, -5.3438e-37, -2.4534e-37, -1.6050e-36,  1.7440e-37,\n         1.1299e-36,  5.4315e-38, -2.2573e-37, -1.2666e-37,  1.4062e-37,\n         5.6336e-36,  2.5450e-37, -2.3902e-36, -9.7774e-38,  1.2958e-36,\n         3.2704e-37,  7.2688e-39, -1.3498e-36, -2.3999e-37,  1.5918e-36,\n         6.9003e-38, -3.2881e-37,  1.9805e-36, -6.7562e-37,  6.5738e-37,\n        -2.0312e-36, -4.8859e-37, -6.6243e-37,  6.5034e-38, -1.3715e-36,\n        -1.2683e-37, -3.5510e-37, -8.0676e-38,  6.0828e-36, -2.4102e-37,\n         1.8948e-38,  3.5199e-36, -4.0042e-37,  9.5215e-37,  1.9302e-37,\n        -5.7298e-37, -5.8526e-38,  9.7009e-38, -6.8890e-38, -4.3602e-37,\n         5.6280e-37,  3.2006e-36,  3.0551e-36, -1.8060e-36, -5.5372e-37,\n         2.7061e-37,  4.6745e-38, -4.2869e-37,  2.4627e-37, -1.7655e-36,\n        -4.3842e-37, -1.9734e-37, -1.7822e-37,  2.5143e-36, -8.4939e-37,\n        -1.1876e-36, -2.6263e-37,  3.0294e-36, -2.0928e-36,  2.0628e-37,\n        -1.4895e-37, -1.2320e-36, -3.3361e-36,  7.0408e-36, -2.6159e-36,\n        -3.2264e-37, -1.7228e-37, -3.6383e-37,  6.1041e-36, -3.6647e-38,\n         1.1638e-36, -2.2135e-37, -2.3042e-37,  1.6323e-36, -1.3203e-36,\n        -1.9065e-37, -5.1730e-37, -3.0274e-36,  1.3987e-36, -2.3439e-37,\n        -1.1588e-36,  4.4326e-37, -2.9865e-38,  2.3114e-37, -5.3416e-38,\n         3.3361e-37, -1.9390e-36, -2.3197e-36, -1.2440e-36,  7.1483e-37,\n         8.0200e-37, -2.9607e-36, -1.1523e-37,  5.0840e-37,  1.6115e-36,\n        -2.0842e-37,  2.0865e-37,  5.2024e-38, -1.7955e-36, -4.3167e-36,\n        -1.5005e-36,  3.5441e-38, -2.0118e-36, -8.4678e-37,  1.0679e-37,\n         1.8786e-36,  9.3893e-37], device='cuda:0')",
        "exp_avg_sq": "tensor([6.4391e-11, 3.8973e-11, 1.0908e-11, 4.7298e-11, 3.2435e-12, 9.8872e-13,\n        3.4338e-12, 1.3392e-14, 6.3096e-10, 1.3951e-12, 8.0690e-11, 7.2253e-11,\n        2.0961e-11, 4.7931e-12, 3.1012e-12, 1.2145e-11, 9.6965e-11, 1.9338e-13,\n        3.9273e-13, 4.1645e-12, 1.4098e-11, 3.7105e-12, 2.5350e-13, 4.5814e-11,\n        4.4433e-11, 1.3122e-11, 1.1352e-11, 1.0088e-12, 7.3441e-13, 6.8955e-12,\n        2.6028e-13, 8.2884e-12, 9.5643e-14, 5.1189e-10, 1.9037e-11, 3.1902e-11,\n        4.0631e-11, 2.0913e-11, 4.5558e-11, 3.0480e-11, 8.0084e-13, 4.7008e-11,\n        4.9460e-13, 3.7430e-11, 6.1142e-11, 1.2986e-12, 3.7843e-11, 4.5792e-14,\n        1.3806e-13, 3.3772e-11, 2.7709e-11, 1.5080e-13, 1.5856e-12, 1.2428e-12,\n        6.8261e-13, 1.1287e-10, 3.1273e-10, 6.1026e-10, 4.5806e-10, 2.0127e-13,\n        5.6178e-12, 1.1918e-10, 5.0341e-13, 3.7564e-10, 1.8106e-11, 5.6978e-11,\n        1.9009e-11, 3.9418e-10, 2.3495e-12, 2.1941e-10, 2.8221e-14, 2.1240e-12,\n        1.1526e-11, 8.9193e-14, 5.2773e-12, 2.4400e-13, 5.2452e-11, 2.3800e-13,\n        3.4817e-13, 4.1822e-13, 1.6841e-11, 3.6032e-12, 1.5174e-12, 5.1235e-13,\n        1.5926e-12, 2.6735e-10, 3.2834e-11, 1.0066e-13, 2.7794e-13, 3.1314e-12,\n        1.4091e-10, 1.7717e-13, 6.6778e-13, 3.5512e-10, 2.6106e-12, 6.5487e-11,\n        6.7515e-14, 7.8024e-11, 1.5054e-12, 2.1685e-12, 1.4679e-11, 3.1979e-12,\n        1.0560e-11, 3.5047e-11, 1.0401e-11, 6.0264e-13, 7.0185e-12, 1.3768e-12,\n        2.1681e-10, 2.2897e-13, 4.4697e-10, 2.3609e-11, 2.9410e-11, 5.3592e-12,\n        1.5992e-10, 7.3727e-12, 7.5420e-12, 3.1951e-10, 1.6584e-11, 3.0619e-12,\n        3.2763e-10, 2.1212e-14, 4.9808e-12, 3.6505e-12, 6.2217e-13, 1.4771e-11,\n        6.4194e-12, 8.0816e-13, 4.4361e-12, 7.0563e-13, 2.2712e-11, 1.9620e-12,\n        1.4560e-11, 1.2564e-12, 2.1607e-10, 1.6142e-12, 3.3825e-11, 2.0693e-11,\n        3.1298e-13, 2.3672e-12, 8.8160e-11, 3.1439e-10, 8.1764e-14, 4.1436e-11,\n        7.3929e-12, 1.3040e-12, 3.8744e-11, 1.9771e-11, 1.6502e-12, 2.9274e-11,\n        1.4779e-11, 8.0897e-11, 5.2368e-11, 1.7454e-10, 1.2114e-10, 2.7300e-12,\n        4.2447e-12, 1.5287e-11, 4.9364e-11, 1.4466e-12, 6.0640e-12, 7.7378e-11,\n        1.3075e-11, 4.2250e-13, 1.5420e-12, 7.3838e-13, 2.8901e-10, 3.0468e-14,\n        8.3094e-11, 8.3802e-11, 6.0310e-12, 6.5494e-12, 4.0382e-11, 9.2279e-12,\n        3.7325e-12, 7.1869e-12, 2.0287e-11, 2.0846e-13, 3.1262e-10, 6.1633e-11,\n        2.6616e-10, 1.1628e-11, 2.3556e-10, 1.2886e-11, 5.0871e-11, 2.2553e-12,\n        5.9993e-11, 7.1908e-14, 2.8562e-13, 1.2550e-12, 2.4715e-13, 1.3912e-11,\n        1.3018e-12, 6.1208e-11, 3.9600e-11, 9.1794e-11, 1.9933e-12, 3.7364e-11,\n        2.1918e-11, 1.5227e-11, 1.8242e-11, 8.5843e-11, 1.1235e-11, 2.9636e-11,\n        1.2363e-13, 1.4923e-12, 1.0665e-10, 1.4843e-11, 2.2462e-12, 1.5815e-13,\n        7.0414e-11, 2.3964e-10, 2.4576e-14, 1.9855e-10, 1.4777e-09, 7.8517e-13,\n        4.7559e-11, 1.0098e-13, 4.0199e-10, 1.1622e-13, 1.2081e-12, 7.9881e-11,\n        9.8658e-11, 4.6516e-11, 9.8438e-14, 1.6232e-12, 4.7856e-13, 5.7849e-13,\n        2.4284e-12, 5.7577e-14, 2.2026e-11, 1.7830e-11, 1.6326e-11, 4.9325e-10,\n        5.0409e-12, 9.6926e-12, 3.8283e-10, 1.1634e-13, 9.3428e-13, 8.8374e-14,\n        1.5414e-11, 1.3336e-10, 8.1967e-12, 1.3910e-10, 2.3939e-10, 7.3339e-11,\n        2.0158e-14, 1.8684e-11, 1.6286e-12, 4.1025e-11, 2.2552e-10, 1.9143e-11,\n        5.5621e-10, 7.9564e-14, 7.3241e-13, 1.7489e-12, 1.6456e-11, 3.4694e-12,\n        3.1212e-11, 8.9408e-13, 3.6999e-11, 5.4380e-11, 1.8479e-13, 2.6754e-12,\n        2.7483e-10, 2.8476e-12, 5.5357e-11, 2.7154e-13, 2.0341e-12, 1.4631e-12,\n        1.1526e-11, 5.1703e-13, 5.2735e-11, 1.8439e-13, 1.1943e-10, 2.4007e-11,\n        5.6880e-13, 1.0109e-11, 6.7057e-11, 2.0182e-12, 8.1783e-11, 1.2720e-11,\n        5.7794e-11, 6.6138e-11, 1.9721e-10, 1.1964e-13, 2.2663e-10, 7.6822e-12,\n        4.1516e-11, 1.5309e-14, 6.6551e-12, 4.0666e-10, 2.8989e-10, 7.8744e-12,\n        3.4088e-12, 4.2366e-11, 9.6838e-12, 2.6714e-12, 5.7181e-11, 2.1275e-12,\n        1.8272e-11, 7.2966e-11, 9.7626e-12, 2.0324e-11, 3.1539e-10, 4.5029e-12,\n        6.1453e-13, 6.1291e-11, 5.0371e-12, 8.0997e-10, 3.2132e-10, 6.1873e-13,\n        1.0665e-11, 5.2345e-11, 1.2886e-12, 1.5112e-11, 3.1175e-13, 1.6491e-12,\n        5.4754e-11, 3.8077e-11, 1.1187e-10, 2.5175e-12, 4.2848e-10, 6.0061e-13,\n        2.7583e-12, 1.7128e-11, 1.7976e-12, 4.9028e-10, 3.2962e-12, 7.7420e-11,\n        8.8223e-12, 1.0526e-10, 2.0929e-12, 3.3099e-15, 4.5629e-14, 2.3412e-11,\n        1.2627e-11, 6.5746e-12, 1.8448e-11, 7.9690e-11, 4.4737e-11, 1.0941e-12,\n        1.7104e-10, 3.6324e-11, 3.5669e-13, 8.5125e-11, 8.0604e-11, 3.4257e-12,\n        2.8045e-13, 6.1967e-12, 2.7954e-11, 1.1499e-12, 4.2640e-13, 5.1604e-13,\n        2.6136e-11, 5.4984e-11, 7.1201e-12, 1.8031e-12, 3.9137e-10, 9.3137e-10,\n        3.0426e-12, 1.1469e-12, 9.7754e-13, 1.1304e-10, 6.1972e-11, 5.8364e-11,\n        2.0620e-11, 8.3901e-14, 1.6306e-11, 3.8997e-13, 2.3228e-11, 2.8458e-12,\n        7.3172e-11, 2.8172e-12, 4.9783e-11, 5.0150e-11, 1.6528e-12, 3.9274e-11,\n        1.1129e-10, 2.5089e-13, 4.3162e-14, 1.0396e-13, 1.2036e-10, 8.8124e-11,\n        8.6411e-11, 6.3597e-10, 7.6271e-12, 7.9202e-12, 3.0138e-11, 3.1710e-12,\n        1.7823e-13, 1.5909e-13, 8.1503e-12, 2.1226e-11, 7.0042e-11, 1.2697e-10,\n        2.2929e-11, 1.8282e-12, 2.1243e-10, 1.0505e-10, 2.4301e-13, 3.4898e-11,\n        3.1111e-13, 3.3857e-11, 5.2497e-13, 1.7816e-13, 3.2742e-14, 2.6669e-13,\n        1.5256e-13, 5.8078e-10, 4.4010e-11, 4.6702e-14, 3.2724e-11, 2.2971e-12,\n        1.1983e-11, 1.6072e-12, 5.3948e-13, 1.5318e-10, 1.7918e-13, 3.2962e-10,\n        1.0148e-12, 7.9244e-14, 5.1260e-11, 1.5659e-11, 4.9995e-12, 2.9457e-12,\n        9.8575e-13, 3.0381e-11, 4.1560e-11, 2.0747e-11, 1.0299e-13, 7.3381e-12,\n        1.7922e-13, 2.9441e-11, 4.3489e-13, 4.3433e-14, 3.8455e-11, 1.9487e-13,\n        3.0653e-11, 4.7549e-13, 3.6222e-11, 5.9667e-13, 2.2003e-13, 5.4532e-12,\n        8.6297e-12, 3.0560e-12, 1.0035e-10, 9.4902e-11, 1.1388e-10, 3.4204e-11,\n        3.4786e-12, 3.5532e-12, 1.1391e-13, 2.1238e-12, 1.6163e-10, 1.1109e-12,\n        5.9669e-13, 1.0601e-11, 5.5828e-11, 5.8185e-11, 6.4371e-11, 5.9964e-14,\n        2.8272e-11, 1.4980e-10, 1.3753e-12, 1.9712e-12, 2.5882e-12, 2.4685e-11,\n        2.5227e-10, 1.2137e-11, 2.6802e-12, 9.6491e-13, 2.1362e-11, 3.1409e-11,\n        7.2596e-12, 1.7882e-13, 2.8932e-11, 3.0519e-13, 7.4151e-12, 1.7586e-11,\n        5.3708e-13, 4.4615e-11, 7.2993e-11, 3.7407e-10, 1.2004e-14, 2.7995e-11,\n        4.3999e-11, 9.9764e-13, 8.3832e-11, 2.3721e-13, 1.4066e-13, 2.1476e-10,\n        1.2300e-10, 4.3529e-11, 4.6786e-13, 6.8469e-13, 7.8728e-11, 7.0221e-14,\n        7.7556e-14, 5.9089e-12, 3.6428e-12, 5.2193e-13, 2.9719e-12, 4.2928e-11,\n        3.0377e-10, 1.5385e-11, 2.0483e-13, 3.5669e-11, 1.1426e-15, 4.4930e-12,\n        4.2852e-12, 4.9423e-11], device='cuda:0')"
      },
      "42": {
        "step": "tensor(11268.)",
        "exp_avg": "tensor([ 4.2371e-38,  1.0820e-37,  7.4077e-39, -1.5665e-39, -7.1640e-40,\n         2.4733e-39,  7.9312e-39,  3.3651e-38,  3.1860e-37,  1.7104e-38,\n         9.0282e-38,  3.5098e-39,  4.7657e-38,  1.3144e-40,  9.3280e-39,\n         8.2076e-38,  5.1979e-39,  1.1211e-38,  1.0951e-38,  1.4335e-39,\n         5.8736e-38,  5.5097e-39,  5.4980e-39, -1.9316e-39,  1.0423e-37,\n         1.3788e-39, -6.6516e-39,  1.9598e-39,  6.5278e-39,  1.4499e-38,\n         1.4420e-39, -2.1569e-39, -4.5664e-41,  1.5683e-37,  3.5167e-38,\n         1.0377e-37,  1.2193e-37,  5.6444e-40,  3.9914e-38,  2.5455e-40,\n         2.1648e-39, -1.2563e-39,  3.1698e-40,  8.2507e-38, -6.5340e-40,\n        -2.8043e-39, -9.7678e-41, -9.1936e-40,  1.3472e-38,  1.4900e-38,\n        -3.5325e-39, -5.3300e-41, -8.7241e-40,  1.8195e-39, -2.3899e-39,\n         1.2478e-37,  1.4249e-37,  2.6229e-37,  1.2748e-37,  4.6071e-39,\n         1.2767e-37,  1.0865e-37,  2.9999e-38,  1.8028e-37,  1.3824e-41,\n         6.2069e-38,  2.3002e-39,  1.0100e-37,  1.0859e-38, -8.4884e-40,\n         1.5591e-38,  1.4972e-38,  1.0012e-37,  5.9604e-39,  2.5984e-38,\n        -1.7496e-39, -1.2126e-39,  1.0675e-38, -3.5687e-39,  5.8410e-39,\n        -5.3174e-41,  4.8495e-39,  5.6889e-39,  1.0689e-38,  2.0604e-39,\n         2.4939e-37,  1.0236e-37, -7.0454e-39,  2.3395e-38,  2.4690e-39,\n         1.0262e-37,  6.7513e-41, -2.4422e-39,  2.9414e-38,  9.3806e-39,\n         1.6718e-37,  1.2709e-39,  1.6422e-38,  1.1271e-40, -2.1827e-39,\n         2.8491e-39,  6.9251e-39, -1.5032e-41,  2.2532e-38, -4.8049e-40,\n         8.7967e-40,  2.4702e-39, -1.8414e-39,  2.5849e-37,  6.0175e-40,\n         8.6222e-39,  5.2850e-38, -1.1135e-39,  4.0867e-39,  5.3903e-38,\n         1.0937e-38,  9.9563e-38,  4.2873e-38, -2.4371e-39, -1.1703e-39,\n         3.8348e-38,  2.0836e-38, -4.9986e-39,  4.9040e-38, -5.3866e-39,\n         9.7246e-39,  6.5187e-38,  5.1050e-39,  1.3997e-39, -3.1428e-40,\n        -1.4470e-39,  5.4282e-40,  8.5232e-40,  9.2198e-38,  1.0936e-37,\n         3.9877e-39,  1.4950e-37, -1.3188e-39,  8.2791e-40,  1.3875e-39,\n        -3.0593e-40,  5.2333e-38,  3.8797e-39, -5.9668e-39,  2.1852e-40,\n        -4.6544e-40,  1.3449e-37, -1.6795e-39, -1.8188e-40,  6.6970e-38,\n         1.1448e-39,  1.3330e-37,  3.7672e-38,  4.9633e-40,  2.0720e-37,\n         2.8329e-39,  1.7029e-38, -1.9421e-39,  2.2972e-38,  1.0861e-38,\n         3.3075e-38,  9.7008e-38,  4.4447e-38,  5.2208e-40,  4.8223e-38,\n         1.8075e-39,  1.9204e-38,  8.1037e-40,  2.3847e-38,  2.5394e-37,\n         2.8167e-40,  3.1317e-39,  5.4043e-38,  3.4361e-39, -2.4365e-39,\n        -2.3271e-39,  1.2942e-37,  5.2879e-39, -1.7023e-40,  9.5236e-39,\n         1.9210e-38,  1.7737e-37,  3.7173e-38,  5.3212e-38,  6.6835e-38,\n         4.3132e-40,  6.6281e-38,  5.9715e-39, -1.0055e-39,  3.8494e-42,\n         2.4707e-39,  1.9315e-39,  4.4776e-39,  1.2818e-37,  1.5102e-38,\n         1.9688e-37, -3.2610e-39,  4.6066e-38, -7.3146e-41, -1.5657e-39,\n         1.4019e-37,  1.1916e-37,  6.1696e-38,  1.0275e-37,  3.9543e-39,\n         5.1335e-40,  1.0982e-37,  2.0822e-39, -3.3801e-40, -3.0554e-39,\n        -7.4598e-39,  2.0895e-37,  7.2628e-39, -4.5615e-39,  3.5687e-37,\n        -1.3194e-40,  1.1362e-37,  6.9727e-39,  3.7891e-38, -3.2785e-41,\n         3.6401e-39,  7.5720e-38,  1.2768e-38,  2.3752e-38, -1.3770e-39,\n         8.4825e-38, -5.2422e-39,  3.2109e-39,  2.4380e-38, -3.7111e-39,\n         1.1747e-37, -1.5506e-39,  5.5874e-38,  2.4008e-37, -4.7125e-39,\n         7.8960e-40,  3.7469e-37,  9.0018e-39,  9.2853e-39,  1.3183e-37,\n         1.2884e-38,  1.0271e-37, -1.5336e-40,  1.6696e-38,  2.1746e-37,\n         3.3801e-38,  5.9106e-39,  4.7593e-38,  1.9400e-40,  2.8556e-38,\n         3.3572e-39,  5.2555e-38,  4.2474e-40,  4.1251e-39, -1.7765e-39,\n         3.3661e-40, -2.7648e-39,  2.9935e-39,  1.2616e-37, -1.7541e-39,\n         4.0238e-38,  8.4485e-38,  3.6584e-39,  3.9262e-40,  1.2914e-37,\n         8.1727e-40,  2.1897e-37, -2.0676e-39,  1.8134e-39,  2.2732e-39,\n        -1.5479e-39, -7.9401e-40,  3.1574e-38,  1.8050e-38,  7.5660e-39,\n         4.3432e-40, -9.0301e-41,  2.4558e-38,  1.6232e-39,  3.9328e-39,\n         4.5628e-39, -3.8823e-40,  3.3048e-38,  1.4022e-37,  1.4827e-37,\n         4.1231e-39,  1.5598e-39,  1.8181e-38,  1.9514e-37,  2.4881e-40,\n         6.6857e-38,  1.8364e-37,  2.5855e-39,  1.9118e-39,  3.6827e-39,\n         5.4109e-38,  3.0257e-38, -5.2718e-39,  2.7433e-38,  1.4367e-39,\n         1.0231e-37,  5.9943e-39,  2.9641e-37,  4.1737e-38,  3.0673e-38,\n         1.1671e-39, -1.0470e-39,  7.0469e-38, -1.0652e-39,  3.2318e-37,\n         5.9480e-38, -1.2929e-39,  4.5825e-39,  4.0090e-38,  2.6186e-39,\n         9.5282e-38,  1.6452e-38,  1.6685e-38,  1.0387e-39,  8.8632e-38,\n         5.7314e-38,  3.9954e-39,  3.8564e-39,  4.7789e-39,  1.2636e-39,\n         5.3015e-38,  6.3672e-40, -1.1040e-38,  4.7324e-39,  1.3536e-40,\n         4.6832e-39,  1.5583e-37,  9.3194e-40,  1.6677e-39, -7.2439e-39,\n         7.1174e-38, -3.0094e-40,  6.8469e-38,  3.3827e-38,  1.3326e-37,\n         2.5126e-40,  2.4547e-40,  2.2652e-38,  1.4156e-38,  2.1191e-39,\n         3.6869e-38, -1.3738e-39,  3.0152e-40,  2.0559e-39,  1.4960e-39,\n         1.0909e-37,  6.4647e-39, -1.4784e-40,  2.3785e-38,  4.5636e-38,\n         2.2631e-38,  1.3628e-39,  4.5284e-39,  1.6327e-37,  3.1241e-38,\n         4.5091e-39,  2.2121e-38,  6.4592e-39,  2.0799e-37,  4.8634e-38,\n         6.8132e-38,  1.1086e-38,  2.8548e-39,  2.2707e-38,  2.2311e-40,\n        -1.8620e-39,  5.5820e-39,  2.3304e-38,  7.5614e-38, -1.6124e-40,\n         5.6047e-39, -2.6168e-39,  2.5960e-38, -2.5068e-41,  2.0778e-39,\n        -6.3971e-40,  8.4924e-39,  4.5355e-38,  2.5763e-37,  1.1970e-37,\n        -3.3991e-39, -3.0442e-39,  9.1956e-38,  1.2431e-37,  4.4282e-39,\n         6.0030e-40,  1.1115e-38,  3.5177e-39,  1.8650e-39, -7.4837e-40,\n        -7.2614e-39,  3.8736e-38,  2.3826e-39,  2.2929e-39,  5.0195e-38,\n         2.5177e-38, -2.4054e-39,  7.5243e-40,  7.0395e-39,  3.2564e-39,\n         1.7689e-38,  7.1579e-39,  8.4617e-39,  4.4088e-39,  1.0695e-37,\n         7.6103e-38,  7.3572e-39,  3.7862e-38,  3.4640e-39,  4.3901e-38,\n         2.4216e-39,  2.1520e-39, -3.0111e-41,  1.3570e-39,  1.3876e-37,\n         2.6205e-38,  1.4698e-39,  1.0793e-37,  1.2450e-37,  8.6034e-39,\n         1.0239e-37, -2.3647e-39, -1.7955e-40,  2.3983e-38,  2.2726e-38,\n         9.4125e-39, -1.6222e-40,  4.2295e-39,  7.9911e-38,  7.8030e-39,\n        -5.0577e-39,  1.1162e-37,  6.4301e-39,  3.8622e-38, -8.7410e-40,\n        -1.8497e-39,  4.5489e-39,  5.8391e-39,  5.4718e-39,  9.6587e-41,\n         1.1066e-39,  6.4595e-38,  2.1505e-37,  3.0769e-38,  1.1234e-40,\n         1.3645e-39,  1.5017e-39,  3.0778e-41,  5.2078e-39,  2.1271e-38,\n        -3.2577e-39, -4.5299e-39, -6.1875e-40,  2.1484e-37,  7.6271e-39,\n        -8.6488e-40,  8.0756e-39,  2.4438e-38,  1.4897e-37,  4.2078e-39,\n         1.3968e-39,  1.0411e-37,  3.5187e-38,  2.8316e-37,  1.3885e-39,\n         1.3650e-39,  4.7589e-39,  1.0573e-38,  1.5745e-37,  2.2444e-37,\n         1.0850e-39, -6.2776e-40,  1.7048e-38,  2.3203e-38, -1.8474e-40,\n         1.1629e-39, -3.0373e-39,  6.2168e-38,  2.6646e-37,  6.6476e-40,\n        -1.2400e-39,  6.3666e-38, -3.9868e-39,  2.3696e-37,  9.6901e-39,\n         1.2723e-38,  1.2343e-37,  5.2466e-39,  9.0319e-39,  4.7704e-39,\n         1.2880e-38,  4.1056e-38, -2.7287e-40, -3.4062e-39,  1.3496e-38,\n         3.9589e-39,  1.8459e-38,  2.1941e-39, -3.5243e-41, -6.5722e-39,\n         5.0700e-38,  6.2187e-39,  2.2304e-38,  4.5231e-40,  1.2106e-39,\n         1.5205e-37,  4.1184e-38], device='cuda:0')",
        "exp_avg_sq": "tensor([3.3969e-15, 3.3741e-15, 1.8691e-16, 1.8242e-15, 3.8149e-16, 7.5619e-19,\n        5.3241e-18, 2.9646e-15, 5.2934e-13, 1.6959e-15, 1.1666e-14, 7.0346e-15,\n        2.4195e-15, 4.1041e-19, 1.4537e-16, 7.9362e-15, 5.7642e-15, 2.5996e-16,\n        8.1816e-16, 5.1673e-18, 1.5728e-15, 6.2932e-16, 8.6202e-19, 7.5778e-16,\n        2.2393e-14, 5.6089e-16, 1.4083e-17, 6.1845e-19, 2.2067e-16, 1.5685e-15,\n        7.8841e-17, 8.2256e-17, 1.4443e-17, 1.4116e-13, 1.1422e-15, 6.0817e-14,\n        3.2838e-14, 8.3282e-17, 1.4391e-15, 1.5508e-15, 2.2675e-16, 1.0069e-15,\n        9.4778e-20, 1.6259e-15, 2.1930e-15, 9.2825e-18, 1.3859e-16, 3.1117e-17,\n        3.2085e-16, 4.8739e-15, 1.0773e-16, 4.9113e-17, 3.5161e-16, 1.1361e-19,\n        2.1123e-16, 3.1764e-14, 7.5737e-14, 6.6167e-13, 2.2703e-13, 2.1870e-16,\n        1.4850e-14, 6.1738e-15, 4.8519e-15, 3.6416e-13, 2.6250e-18, 4.4868e-15,\n        1.3620e-15, 1.8254e-13, 3.5970e-16, 4.8787e-15, 9.1250e-17, 9.0587e-16,\n        1.2119e-14, 1.5103e-17, 7.6805e-18, 7.9966e-17, 9.6988e-18, 1.8104e-16,\n        4.5331e-17, 7.1890e-19, 1.8611e-17, 1.5881e-16, 4.4572e-18, 2.6966e-16,\n        1.2609e-17, 2.6105e-13, 8.7805e-15, 1.1969e-18, 1.3543e-15, 5.4586e-18,\n        5.6566e-14, 8.2523e-17, 2.1545e-18, 6.0383e-14, 3.1755e-16, 7.0869e-14,\n        3.7228e-17, 6.7258e-15, 1.7229e-18, 2.0338e-18, 9.0658e-16, 9.7713e-18,\n        2.2181e-17, 5.3343e-15, 6.9624e-19, 2.3404e-20, 3.6432e-17, 7.9055e-19,\n        2.7369e-13, 2.6207e-17, 8.4663e-14, 2.7293e-15, 1.5804e-16, 1.7406e-19,\n        3.4949e-14, 5.1965e-16, 8.5133e-15, 7.4466e-14, 2.0121e-17, 9.6106e-18,\n        6.5399e-14, 1.1891e-15, 2.5633e-17, 3.6698e-16, 2.7381e-19, 3.8170e-15,\n        1.4998e-14, 3.7561e-16, 5.8751e-16, 4.0517e-17, 9.1019e-17, 2.2007e-16,\n        6.6129e-18, 3.7625e-15, 2.0905e-14, 8.9384e-18, 1.1456e-14, 1.0095e-17,\n        6.2330e-18, 2.1567e-18, 1.1430e-15, 9.9621e-14, 3.5569e-17, 9.6711e-17,\n        3.5692e-18, 2.8555e-18, 6.6362e-14, 4.7751e-16, 4.2516e-16, 7.9617e-15,\n        3.9975e-17, 4.4742e-14, 1.1042e-14, 1.5080e-16, 8.7668e-14, 4.5486e-18,\n        1.6892e-17, 3.5384e-18, 6.9766e-16, 3.9737e-18, 6.4464e-16, 9.7803e-14,\n        2.8410e-14, 2.3994e-17, 7.0069e-16, 4.6477e-19, 4.9543e-14, 5.5172e-17,\n        5.6750e-15, 1.4593e-13, 7.9114e-18, 1.5587e-16, 2.1129e-15, 1.2351e-17,\n        6.9469e-18, 1.3547e-17, 3.7915e-14, 9.6856e-19, 2.5048e-14, 4.0311e-16,\n        2.7828e-14, 2.9497e-14, 4.4637e-14, 9.4519e-15, 3.3976e-15, 8.4987e-20,\n        5.0993e-16, 2.4745e-17, 6.0202e-17, 1.0947e-17, 1.1092e-18, 6.7093e-16,\n        1.6512e-17, 4.1322e-14, 4.0706e-16, 8.6729e-14, 1.2435e-17, 1.9263e-15,\n        1.0167e-17, 1.8115e-18, 1.0238e-14, 7.8660e-14, 1.7074e-15, 2.4629e-14,\n        5.4086e-18, 5.8666e-16, 3.3275e-14, 3.8490e-17, 6.0307e-16, 2.3806e-17,\n        1.3636e-15, 1.1102e-13, 3.1732e-19, 1.4418e-14, 8.2421e-13, 1.9843e-16,\n        3.6750e-14, 8.5290e-18, 1.2587e-13, 2.7678e-18, 1.4741e-16, 3.3151e-14,\n        9.8628e-15, 1.5727e-15, 4.1314e-18, 2.2625e-14, 2.8826e-18, 9.1743e-17,\n        2.2447e-15, 8.3913e-19, 8.4056e-15, 3.2945e-18, 2.5032e-14, 1.2768e-13,\n        1.8732e-17, 6.0138e-17, 3.8255e-13, 1.2704e-16, 5.2798e-16, 2.3854e-14,\n        9.5306e-18, 9.3613e-14, 2.7385e-19, 7.8278e-15, 1.7004e-13, 4.2133e-16,\n        8.5829e-17, 1.2528e-14, 2.8063e-16, 2.2719e-16, 1.4947e-14, 5.3168e-15,\n        2.5561e-14, 6.6868e-18, 1.4485e-19, 5.1747e-18, 3.9015e-16, 9.5575e-17,\n        2.4273e-14, 1.5840e-17, 9.1947e-15, 1.9975e-14, 2.5069e-17, 2.3742e-16,\n        1.2236e-13, 4.4765e-16, 3.4965e-14, 7.3470e-17, 1.0064e-16, 1.9337e-17,\n        3.7971e-18, 1.2636e-16, 2.6771e-14, 6.7100e-17, 3.0669e-15, 7.5257e-18,\n        7.6891e-17, 8.0031e-16, 1.4617e-15, 1.6350e-18, 1.0995e-16, 1.4892e-18,\n        7.2959e-16, 5.3116e-14, 6.1707e-14, 2.0567e-18, 2.0591e-14, 9.8832e-16,\n        4.7873e-14, 7.9993e-18, 2.2495e-14, 1.6801e-13, 1.3072e-14, 5.8055e-18,\n        2.4898e-16, 2.2411e-15, 7.6288e-15, 8.3005e-18, 8.4073e-15, 2.0848e-19,\n        2.5204e-14, 3.9616e-15, 7.1056e-14, 3.1189e-16, 5.6697e-14, 2.7900e-18,\n        9.5832e-17, 2.7178e-14, 2.3131e-17, 5.0820e-13, 7.3406e-15, 8.0651e-18,\n        4.9495e-17, 3.1034e-15, 7.3371e-20, 5.4878e-14, 1.6572e-15, 5.5920e-16,\n        4.6256e-16, 2.1488e-15, 2.1856e-15, 4.0426e-17, 6.4645e-14, 3.1517e-19,\n        1.5418e-16, 2.0284e-15, 1.8684e-19, 1.4230e-14, 7.1046e-17, 6.0291e-16,\n        1.8309e-17, 1.2553e-13, 1.4450e-17, 8.8377e-18, 3.3053e-20, 9.8056e-15,\n        8.4732e-18, 2.5207e-14, 1.1204e-15, 2.2039e-14, 2.6883e-18, 1.9855e-16,\n        1.1311e-14, 2.4241e-15, 2.0329e-16, 2.2745e-15, 1.3933e-16, 2.8857e-16,\n        1.3829e-16, 1.3414e-17, 1.5965e-14, 7.9710e-16, 6.2392e-18, 1.8279e-15,\n        6.9445e-16, 1.3343e-14, 7.5062e-16, 2.4326e-16, 7.3234e-14, 2.4092e-13,\n        6.0403e-18, 1.3762e-15, 4.6063e-16, 7.9120e-14, 2.5696e-15, 1.7783e-14,\n        2.7659e-17, 3.0612e-17, 5.1641e-16, 2.9088e-17, 1.1391e-17, 2.2141e-16,\n        8.4892e-16, 1.9041e-14, 2.2317e-15, 3.1822e-15, 6.0102e-18, 2.4095e-15,\n        2.6584e-15, 7.2431e-17, 3.8278e-18, 9.9052e-17, 3.4703e-14, 6.7799e-14,\n        7.5011e-14, 7.0791e-14, 1.8141e-17, 9.6910e-15, 8.0368e-15, 8.3931e-16,\n        6.1365e-18, 1.2867e-17, 1.8905e-18, 4.0039e-17, 1.4998e-16, 9.0401e-15,\n        5.7603e-16, 8.2846e-20, 1.2510e-14, 2.0601e-14, 1.3326e-15, 6.9972e-17,\n        1.3588e-16, 6.7859e-16, 1.0984e-19, 1.7198e-15, 1.6475e-16, 3.5530e-18,\n        1.2415e-16, 1.5384e-13, 1.3825e-15, 1.7121e-17, 6.4582e-15, 3.0711e-18,\n        5.2620e-15, 1.9014e-16, 2.1120e-19, 3.3204e-15, 6.6500e-17, 1.1473e-13,\n        1.2635e-15, 3.4277e-17, 3.8790e-14, 3.7131e-14, 6.6094e-16, 2.6995e-14,\n        8.4617e-19, 1.5883e-17, 8.9598e-15, 7.0908e-16, 6.2430e-18, 1.9219e-18,\n        1.6946e-17, 7.1871e-16, 6.8361e-16, 1.2068e-18, 1.6283e-14, 1.5170e-16,\n        4.2769e-15, 5.7588e-16, 7.1273e-17, 3.2373e-16, 6.2168e-18, 1.0615e-17,\n        2.5597e-19, 2.9469e-16, 4.5237e-15, 7.9544e-14, 9.4104e-15, 5.6417e-18,\n        3.0427e-18, 5.3536e-17, 7.3222e-18, 3.2125e-18, 3.7832e-14, 1.7844e-18,\n        3.8639e-19, 1.2331e-17, 8.9641e-14, 1.0709e-15, 2.5014e-16, 4.7699e-18,\n        1.2165e-16, 1.3774e-13, 2.8529e-16, 3.5584e-17, 2.2216e-14, 9.8913e-15,\n        1.6350e-13, 4.5035e-16, 4.3613e-18, 1.0797e-16, 2.2730e-17, 8.9910e-15,\n        6.6896e-14, 2.0104e-17, 1.0818e-17, 3.8395e-16, 2.8118e-17, 5.8932e-18,\n        1.6369e-18, 2.8644e-16, 3.8089e-15, 3.3305e-13, 1.1048e-17, 2.8895e-16,\n        7.2694e-15, 1.3839e-17, 1.3842e-13, 2.1436e-16, 2.3769e-16, 1.1240e-13,\n        3.2777e-15, 3.3999e-15, 2.2301e-17, 1.0749e-15, 3.0185e-14, 2.1171e-17,\n        6.4858e-17, 3.6559e-18, 3.8055e-18, 6.9490e-16, 4.9547e-16, 9.9592e-16,\n        2.6941e-14, 2.0396e-14, 1.6777e-18, 5.0191e-16, 5.7414e-17, 1.3319e-16,\n        1.2391e-14, 7.2540e-16], device='cuda:0')"
      },
      "43": {
        "step": "tensor(11268.)",
        "exp_avg": "tensor([ 6.3186e-38,  1.2936e-37,  3.4891e-38,  2.8130e-38, -1.6089e-40,\n        -2.2882e-39,  3.4664e-38,  6.8456e-38,  1.5974e-37, -1.2036e-38,\n         8.6901e-38,  4.4382e-39,  6.6934e-38, -1.0148e-41, -7.5898e-39,\n         7.2522e-38,  3.0391e-38, -8.9261e-39, -9.3169e-39, -8.8029e-40,\n         8.4087e-38, -3.9852e-39, -4.4247e-39,  8.0784e-39,  7.3084e-38,\n         2.8833e-38,  2.1735e-38, -1.7543e-39, -4.9084e-39,  3.1822e-38,\n        -1.3989e-39,  1.7705e-38,  3.0048e-39,  1.1331e-37,  6.1722e-38,\n         5.4712e-38,  9.1429e-38, -1.4192e-39,  8.3805e-38,  3.8907e-39,\n        -1.7706e-39,  1.3010e-38, -2.2149e-40,  8.7633e-38,  3.2322e-38,\n         3.1848e-38,  1.6414e-39,  3.5427e-39, -1.0701e-38,  4.6295e-38,\n         1.2558e-38,  1.4770e-40,  3.8998e-39, -1.5397e-39,  8.1858e-39,\n         9.7322e-38,  1.1669e-37,  1.2094e-37,  8.2084e-38, -3.5341e-39,\n         1.0769e-37,  1.4743e-37, -1.5602e-38,  8.5465e-38,  5.4852e-40,\n         6.0421e-38,  2.1598e-38,  6.0183e-38, -8.1931e-39,  2.3300e-38,\n        -1.2335e-38, -1.0871e-38,  1.1098e-37, -5.3070e-39,  7.9203e-38,\n         4.6399e-39,  7.0761e-40, -8.6469e-39,  3.0874e-39, -4.7809e-39,\n        -3.7243e-40, -3.8896e-39, -5.2078e-39, -9.2203e-39, -9.4956e-40,\n         1.2880e-37,  1.1655e-37,  3.7379e-38, -1.5329e-38, -2.1987e-39,\n         6.7952e-38,  9.2992e-40,  9.4015e-39,  6.4851e-38, -7.5804e-39,\n         1.0648e-37, -2.6328e-40,  5.6481e-38, -6.0313e-41,  1.2253e-38,\n         4.1854e-39,  6.9904e-38,  1.1170e-38,  2.5795e-38,  1.3939e-39,\n        -5.8397e-40,  4.1543e-38,  1.0347e-38,  1.3183e-37,  1.5542e-39,\n         8.3689e-39,  6.4975e-38,  1.8832e-38, -3.7208e-39,  8.2644e-38,\n         4.9897e-38,  8.5663e-38,  6.4927e-38,  8.3995e-39,  4.9353e-38,\n         8.9174e-38, -1.5328e-38,  2.7479e-38,  1.0262e-37,  4.8651e-39,\n         1.1275e-38,  5.4029e-38, -3.1521e-39, -2.3285e-40,  1.3859e-39,\n         5.2090e-39, -6.6301e-40, -6.4998e-40,  8.6229e-38,  1.2520e-37,\n        -3.4324e-39,  1.3247e-37,  6.4215e-39,  5.8534e-41, -1.0930e-39,\n         6.0473e-39,  5.3620e-38, -3.2813e-39,  2.1870e-38, -2.7570e-40,\n         1.3167e-39,  8.2050e-38,  2.0020e-38,  9.4451e-40,  6.9985e-38,\n        -9.7360e-40,  1.0620e-37,  7.0054e-38,  1.6107e-40,  1.4078e-37,\n        -2.4963e-39,  6.7751e-38,  1.6919e-38,  1.9358e-38,  5.6099e-38,\n         6.5132e-38,  5.2000e-38,  4.0601e-38,  1.4602e-39,  6.4975e-38,\n        -1.1898e-39,  1.7900e-38, -3.2927e-40,  4.0616e-38,  1.3671e-37,\n        -2.5587e-40, -2.4688e-39,  8.1353e-38,  4.5537e-38,  1.2367e-38,\n         6.7019e-39,  9.0875e-38, -4.8400e-39,  2.5097e-40,  2.4281e-38,\n         4.3012e-38,  1.0641e-37,  4.3883e-38,  8.6859e-38,  9.6424e-38,\n        -3.3351e-40,  1.3494e-37, -5.1333e-39,  3.4395e-39,  3.1339e-38,\n        -2.0008e-39,  4.5999e-38,  4.7947e-38,  7.8585e-38,  3.5143e-38,\n         1.1316e-37,  1.0039e-38,  6.1918e-38, -2.3575e-40,  4.8548e-39,\n         1.3329e-37,  6.3310e-38,  7.8809e-38,  6.6582e-38, -3.1390e-39,\n        -4.2619e-40,  8.6792e-38,  1.3435e-38,  2.0085e-39,  1.6956e-38,\n        -1.2432e-38,  1.4721e-37, -5.9545e-39, -8.2161e-39,  2.0046e-37,\n         2.0489e-39,  7.2734e-38, -6.0232e-39,  3.9654e-38,  2.5650e-39,\n        -2.3549e-39,  8.5308e-38,  1.3769e-38,  4.2796e-38,  2.4968e-38,\n         7.2808e-38,  1.4704e-38, -2.5629e-39, -1.6537e-38,  1.1898e-38,\n         1.0904e-37,  5.5284e-39,  4.9599e-38,  1.6686e-37,  1.5802e-38,\n         2.2352e-38,  1.9937e-37, -7.2468e-39, -7.5981e-39,  9.7571e-38,\n         5.7708e-38,  5.8627e-38,  9.5536e-40,  1.6754e-38,  1.2158e-37,\n         3.4683e-38, -5.0162e-39,  4.4891e-38, -7.3390e-40,  4.7490e-38,\n         1.3377e-38,  5.9267e-38,  2.1895e-38, -3.4094e-39,  5.5601e-39,\n        -3.8344e-40,  2.0229e-38, -2.0750e-39,  9.9876e-38,  7.1431e-39,\n         6.0109e-38,  6.6349e-38, -3.3838e-39,  9.4377e-40,  7.2801e-38,\n        -2.4676e-40,  1.5458e-37,  6.7965e-39,  1.7204e-38,  5.2210e-38,\n         5.9421e-39,  4.3376e-39,  3.7192e-38, -1.3938e-38,  4.3520e-38,\n        -2.2158e-40,  1.4765e-39,  3.6738e-38,  6.3506e-39, -3.6180e-39,\n        -1.8889e-38,  2.5399e-41,  5.9673e-38,  1.0725e-37,  1.0769e-37,\n        -3.7614e-39,  3.8685e-39,  3.2407e-38,  1.3183e-37,  1.7220e-39,\n         6.4817e-38,  1.2052e-37,  1.4939e-38, -1.6441e-39, -2.8524e-39,\n         8.2876e-38,  5.0303e-38,  2.1062e-38,  6.5222e-38, -1.3084e-39,\n         6.6918e-38,  1.0018e-38,  1.4995e-37,  7.0488e-38,  4.8266e-38,\n        -4.9465e-40,  3.4289e-39,  7.4197e-38,  1.9933e-38,  1.8234e-37,\n         8.2697e-38,  5.7705e-40,  2.5112e-38,  5.5893e-38, -2.1203e-39,\n         5.0962e-38, -1.1762e-38, -1.2409e-38,  5.5896e-39,  7.9776e-38,\n         1.1443e-37, -2.8233e-39,  5.6178e-39, -4.1710e-39, -4.4109e-40,\n         9.0260e-38, -4.6178e-40, -9.4716e-38, -3.2853e-39,  6.3812e-39,\n        -4.4309e-39,  1.0469e-37, -5.5322e-40, -1.0896e-39,  3.6172e-38,\n         8.3229e-38,  1.5303e-39,  6.4995e-38,  7.0197e-38,  1.2195e-37,\n        -3.0933e-40,  1.1656e-39,  3.0392e-38,  5.7359e-38, -1.7751e-39,\n         5.6115e-38,  4.5256e-39,  1.3987e-38, -7.1707e-40,  2.2135e-38,\n         8.7999e-38, -4.6808e-39,  4.8648e-38, -1.3863e-38,  8.2459e-38,\n         3.1134e-38, -9.4858e-40, -3.7494e-39,  1.4581e-37,  3.4753e-38,\n         2.6310e-38, -1.4479e-38, -4.9279e-39,  1.4030e-37,  1.2722e-37,\n         6.7030e-38,  5.2872e-38, -2.4736e-39,  4.5731e-38,  2.0170e-39,\n         4.8167e-39, -4.4613e-39,  2.6907e-38,  6.9053e-38,  8.8997e-39,\n         4.7318e-38,  8.0929e-39,  7.4002e-38,  3.9090e-39, -1.3265e-39,\n         2.8786e-39, -7.0889e-39,  5.8967e-38,  1.7944e-37,  6.9445e-38,\n        -8.0015e-39,  1.4085e-38,  7.5903e-38,  9.1853e-38,  2.6297e-38,\n        -5.1581e-40, -9.5308e-39, -9.2286e-39, -5.9911e-39,  2.4333e-39,\n        -7.1667e-39,  7.7042e-38, -2.1857e-39,  1.3022e-38,  7.0778e-38,\n        -1.4631e-38,  6.9659e-39, -3.8519e-40,  2.1151e-38, -2.7981e-39,\n        -1.0981e-38, -6.0852e-39, -7.5238e-39, -3.6942e-39,  8.3511e-38,\n         1.3475e-37, -5.7453e-39,  3.7750e-38, -3.1736e-39,  7.3664e-38,\n        -1.9409e-39, -1.9464e-39,  7.5342e-39, -9.0128e-40,  1.0536e-37,\n        -1.8301e-38, -8.1282e-40,  1.0053e-37,  8.0939e-38, -6.3034e-39,\n         6.4666e-38,  6.4582e-39,  5.1269e-40,  5.2125e-38,  3.6121e-38,\n        -7.7697e-39,  1.4373e-39, -3.7124e-39,  1.4105e-37, -6.4544e-39,\n         2.1663e-38,  1.1878e-37, -5.0421e-39,  6.8168e-38,  3.4102e-38,\n         4.6768e-39, -3.7112e-39, -5.0887e-39, -4.7362e-39,  3.0106e-40,\n        -9.7975e-40,  1.0178e-37,  1.3624e-37,  3.7853e-38, -6.2465e-40,\n        -1.1602e-39, -1.1068e-39,  3.7064e-41, -4.2486e-39,  3.3104e-38,\n         9.7495e-39,  1.4113e-38,  7.5335e-40,  1.3129e-37,  2.9405e-38,\n         3.6174e-39, -6.7885e-39,  8.6383e-38,  7.7267e-38, -3.5079e-39,\n        -3.7375e-40,  7.0370e-38,  2.9857e-38,  1.9001e-37,  5.6905e-39,\n        -3.6157e-40, -3.7806e-39,  3.7170e-38,  1.5775e-37,  1.0942e-37,\n         4.9755e-38,  3.7723e-39, -1.3546e-38,  6.7778e-38,  1.0458e-38,\n        -8.6054e-40,  9.9086e-39,  4.2746e-38,  1.2996e-37, -4.5852e-40,\n         9.3367e-39,  7.1994e-38,  1.0387e-38,  1.1408e-37, -7.9988e-39,\n        -9.4329e-39,  7.0743e-38,  1.2516e-38,  2.7306e-38,  4.4836e-38,\n        -8.7871e-39,  3.5534e-38,  2.8648e-40,  3.0627e-39,  6.3011e-38,\n        -3.4084e-39, -1.3412e-38, -3.2803e-40, -4.6847e-40, -9.0380e-39,\n         5.0329e-38, -5.5218e-39,  3.0110e-38, -3.3538e-39, -8.9438e-40,\n         1.1082e-37,  6.9157e-38], device='cuda:0')",
        "exp_avg_sq": "tensor([2.0542e-14, 1.9379e-14, 1.7822e-15, 1.0373e-14, 1.6762e-16, 6.5371e-17,\n        2.4135e-15, 5.5460e-15, 1.9637e-13, 6.0748e-16, 3.3187e-14, 1.1817e-14,\n        1.4377e-14, 2.7871e-16, 6.6211e-17, 1.4020e-14, 1.2043e-14, 1.3520e-16,\n        3.6949e-16, 5.8809e-16, 1.1725e-14, 2.7729e-16, 1.5309e-16, 7.7452e-15,\n        1.3357e-14, 1.6231e-15, 3.9193e-15, 9.4255e-17, 1.0113e-16, 2.0267e-15,\n        7.5860e-18, 7.0155e-16, 6.4177e-18, 1.4648e-13, 1.2845e-14, 2.1961e-14,\n        3.2071e-14, 2.8404e-15, 2.0712e-14, 5.0242e-15, 1.0975e-16, 4.5509e-15,\n        3.7578e-19, 9.4500e-16, 1.1449e-14, 9.4367e-16, 3.8538e-15, 9.5269e-17,\n        1.3905e-16, 8.5053e-15, 4.5458e-15, 1.4254e-17, 1.7476e-16, 6.0540e-18,\n        1.1035e-16, 4.2897e-14, 1.0270e-13, 2.1437e-13, 1.5323e-13, 9.2678e-17,\n        1.4902e-14, 4.1796e-14, 1.8318e-15, 1.3904e-13, 1.9141e-15, 1.8981e-15,\n        3.3281e-15, 1.0700e-13, 1.4373e-16, 3.1239e-14, 2.0020e-17, 3.7984e-16,\n        7.2529e-15, 4.3348e-18, 4.6855e-15, 3.1712e-17, 6.6926e-15, 6.3817e-17,\n        5.6305e-17, 5.8777e-17, 1.4272e-15, 7.1778e-17, 3.1702e-16, 1.0315e-16,\n        5.9741e-16, 1.1307e-13, 2.2178e-14, 2.5037e-16, 6.6877e-16, 3.4134e-16,\n        5.2125e-14, 2.4263e-17, 3.1840e-16, 5.2477e-14, 1.4778e-16, 4.6056e-14,\n        7.3387e-18, 1.2546e-14, 1.2311e-16, 7.0064e-16, 4.6749e-15, 2.8735e-15,\n        1.6918e-16, 8.2359e-15, 4.2598e-16, 8.8595e-18, 3.5413e-15, 4.6543e-16,\n        1.0268e-13, 1.4619e-17, 3.9976e-14, 8.3674e-15, 1.3076e-15, 4.7603e-16,\n        3.2486e-14, 5.7635e-15, 1.3439e-14, 5.4479e-14, 1.9321e-15, 2.6015e-15,\n        6.3343e-14, 5.6243e-16, 3.5333e-16, 5.6343e-15, 1.7293e-17, 5.2693e-15,\n        6.4245e-15, 1.6808e-16, 2.7061e-16, 1.9219e-17, 1.1148e-15, 9.9216e-17,\n        2.0090e-15, 7.4874e-15, 6.2784e-14, 3.6742e-16, 1.7900e-14, 3.8480e-16,\n        2.7404e-18, 1.8396e-16, 8.2619e-15, 4.6010e-14, 1.6830e-18, 9.3007e-15,\n        1.1734e-16, 2.2432e-16, 3.7408e-14, 2.4981e-15, 2.1235e-16, 2.0751e-14,\n        2.0369e-15, 4.4993e-14, 9.8167e-15, 2.8472e-14, 6.2151e-14, 3.6732e-16,\n        4.1243e-15, 9.4831e-16, 3.3939e-16, 2.0240e-15, 7.3000e-15, 4.4277e-14,\n        1.8477e-14, 1.1454e-17, 4.2858e-15, 4.2011e-17, 2.2401e-14, 2.1446e-17,\n        4.3936e-15, 6.0168e-14, 8.1300e-16, 6.8263e-17, 1.9665e-14, 2.4285e-15,\n        7.5205e-16, 5.6370e-16, 2.5465e-14, 1.8200e-18, 2.6731e-14, 1.8471e-14,\n        2.8435e-14, 1.0428e-14, 2.2731e-14, 8.2210e-15, 2.3327e-14, 3.5200e-17,\n        2.2205e-14, 1.0915e-19, 2.3819e-17, 1.3176e-15, 8.4207e-18, 2.2018e-15,\n        1.6715e-15, 3.0125e-14, 1.5963e-14, 5.3503e-14, 1.3635e-15, 3.2861e-15,\n        1.9186e-15, 3.8367e-16, 1.7923e-14, 3.2851e-14, 1.0534e-14, 1.0122e-14,\n        1.1264e-18, 2.9088e-16, 3.8706e-14, 1.4827e-16, 3.2461e-16, 1.7507e-16,\n        1.9911e-14, 9.5508e-14, 1.3857e-16, 1.4566e-14, 4.1046e-13, 9.7564e-17,\n        2.5974e-14, 5.1977e-18, 6.6149e-14, 1.3141e-18, 6.5541e-17, 2.2918e-14,\n        2.1280e-14, 5.3926e-15, 4.7427e-16, 9.6873e-15, 2.8406e-16, 4.7967e-17,\n        1.0750e-15, 3.4808e-16, 1.8689e-14, 2.9730e-16, 1.0809e-14, 1.3950e-13,\n        5.6614e-16, 2.9603e-16, 1.5350e-13, 2.0211e-17, 2.3199e-16, 1.1744e-14,\n        6.6579e-15, 5.5403e-14, 1.1216e-15, 4.5603e-15, 9.9468e-14, 1.9472e-16,\n        2.4167e-18, 5.5269e-15, 1.4233e-16, 2.6227e-16, 2.6822e-14, 9.6786e-15,\n        8.6391e-14, 4.1534e-19, 5.5244e-16, 4.3731e-16, 1.8321e-15, 4.5743e-17,\n        2.6695e-14, 8.3502e-16, 2.1966e-14, 2.0806e-14, 1.1219e-18, 1.1104e-16,\n        7.3916e-14, 2.0066e-16, 3.5600e-14, 3.6697e-17, 1.5761e-15, 1.2501e-15,\n        8.2457e-16, 6.0837e-17, 1.6138e-14, 3.0807e-17, 8.8521e-15, 3.0716e-15,\n        3.8158e-17, 8.7108e-15, 2.7389e-15, 1.7170e-16, 7.6962e-15, 4.6698e-16,\n        1.9769e-14, 4.0256e-14, 6.9457e-14, 2.3729e-17, 2.6356e-14, 3.6782e-15,\n        3.5192e-14, 3.9461e-16, 1.3803e-14, 1.3771e-13, 3.3125e-14, 5.8457e-16,\n        1.0891e-16, 1.5243e-14, 8.4722e-15, 1.8386e-15, 1.1239e-14, 9.6586e-17,\n        1.9738e-14, 2.5722e-15, 2.6950e-14, 1.0397e-14, 4.0775e-14, 3.1295e-16,\n        4.8474e-17, 1.2912e-14, 1.6292e-15, 2.4697e-13, 7.9661e-14, 1.7251e-17,\n        3.0279e-15, 7.7666e-15, 1.0941e-17, 2.4019e-14, 6.6728e-16, 2.2760e-16,\n        7.2374e-15, 1.1658e-15, 3.6329e-14, 1.7176e-17, 5.1317e-14, 1.0713e-16,\n        6.8338e-17, 1.3081e-14, 9.9759e-18, 6.4628e-14, 3.2118e-17, 4.9398e-15,\n        1.4707e-15, 5.3558e-14, 6.0654e-18, 2.2752e-18, 2.2863e-16, 1.8600e-14,\n        6.7175e-16, 1.1244e-14, 1.2210e-14, 3.7995e-14, 6.0466e-15, 1.0392e-16,\n        8.9262e-15, 4.3289e-15, 1.0774e-16, 2.3859e-14, 1.3540e-14, 1.3900e-15,\n        7.2949e-17, 2.2088e-15, 1.7526e-14, 4.1277e-16, 7.3783e-16, 8.8537e-16,\n        1.0697e-14, 9.1152e-15, 3.1566e-16, 1.0369e-16, 1.2245e-13, 1.3284e-13,\n        1.7618e-15, 7.3865e-16, 2.4033e-16, 6.0572e-14, 2.4268e-14, 2.9332e-14,\n        7.6267e-15, 2.3139e-18, 1.0677e-15, 1.6506e-17, 4.2288e-15, 9.1519e-17,\n        6.5576e-16, 1.1230e-14, 6.7195e-15, 1.1671e-14, 8.9414e-16, 1.7094e-14,\n        1.0341e-14, 2.4134e-17, 1.7553e-18, 1.5037e-17, 2.3230e-14, 5.2128e-14,\n        4.2516e-14, 9.5877e-14, 3.5599e-16, 1.4287e-14, 3.5318e-15, 3.5354e-15,\n        1.2220e-18, 7.2246e-17, 2.3096e-16, 7.5719e-16, 1.2191e-14, 3.6764e-15,\n        1.2912e-14, 4.0729e-17, 2.4516e-14, 1.3551e-14, 6.3145e-16, 4.7982e-15,\n        2.7158e-17, 6.3494e-15, 5.1523e-19, 7.9132e-16, 7.7382e-17, 1.3565e-16,\n        5.2734e-17, 1.6412e-13, 1.9440e-14, 2.2668e-18, 1.2891e-14, 2.7846e-16,\n        4.0745e-15, 8.2613e-17, 1.0430e-17, 2.0454e-14, 6.5730e-18, 1.1327e-13,\n        5.7996e-16, 1.0606e-17, 1.7487e-14, 2.3518e-14, 2.5904e-16, 1.6130e-14,\n        1.8866e-16, 4.2973e-15, 1.4567e-14, 9.2709e-15, 1.6561e-17, 2.7976e-16,\n        3.9308e-18, 1.4610e-14, 3.0331e-16, 3.9466e-16, 2.4137e-14, 4.0830e-17,\n        1.7836e-14, 2.6687e-15, 4.7123e-15, 1.5365e-16, 1.1953e-19, 9.1265e-16,\n        4.6977e-16, 1.5329e-16, 3.8021e-14, 5.5016e-14, 2.3882e-14, 3.4228e-15,\n        3.0212e-16, 2.2958e-17, 2.4790e-18, 2.5050e-16, 2.8075e-14, 3.0094e-16,\n        4.3792e-16, 8.4041e-16, 4.3906e-14, 2.1980e-15, 4.3868e-15, 2.8621e-17,\n        1.2968e-14, 6.3646e-14, 1.3784e-16, 1.7271e-17, 1.4898e-14, 1.4604e-14,\n        1.0052e-13, 1.8174e-15, 8.8013e-16, 4.9558e-17, 7.4445e-15, 1.9708e-14,\n        2.1748e-14, 6.2887e-16, 5.2313e-15, 1.5627e-16, 3.9501e-15, 2.2604e-16,\n        3.1324e-17, 5.9436e-15, 1.3673e-15, 1.3213e-13, 8.2371e-19, 1.7788e-15,\n        1.5688e-14, 7.7439e-16, 5.2164e-14, 9.3307e-17, 2.6358e-17, 5.7899e-14,\n        6.9731e-15, 4.0172e-15, 1.5499e-15, 4.5754e-16, 4.2086e-14, 2.4355e-18,\n        8.0545e-18, 3.7277e-15, 3.1349e-16, 2.6004e-16, 2.2415e-16, 4.7752e-15,\n        2.9163e-14, 1.2165e-14, 1.1541e-17, 5.6471e-16, 2.4491e-18, 6.2356e-17,\n        1.3105e-14, 1.8330e-14], device='cuda:0')"
      },
      "44": {
        "step": "tensor(11268.)",
        "exp_avg": "tensor([[ 3.5700e-40,  1.2081e-38,  5.3138e-40,  ...,  8.3227e-40,\n          2.9738e-40, -1.6824e-38],\n        [ 3.5131e-40,  1.3114e-39,  4.8634e-40,  ...,  8.1717e-40,\n          3.5561e-40,  9.4792e-39],\n        [ 6.8552e-42,  1.0940e-39,  6.9181e-41,  ...,  4.6373e-41,\n         -2.0519e-41, -1.7261e-39],\n        ...,\n        [ 8.9403e-42,  3.6968e-40,  1.0731e-40,  ..., -1.1379e-41,\n          1.1314e-40, -2.8216e-39],\n        [ 2.2517e-40, -3.7158e-39,  2.4279e-40,  ...,  2.7071e-40,\n          1.2104e-40, -7.1617e-39],\n        [ 4.5495e-40,  1.8335e-38,  1.5330e-39,  ...,  1.9585e-39,\n          1.3671e-39, -1.9412e-38]], device='cuda:0')",
        "exp_avg_sq": "tensor([[1.8131e-16, 5.5236e-17, 2.8322e-17,  ..., 2.5296e-16, 7.1938e-17,\n         3.0980e-16],\n        [1.8506e-18, 1.2059e-19, 1.7196e-19,  ..., 3.7659e-19, 6.6933e-19,\n         8.8066e-19],\n        [1.0442e-17, 2.1888e-18, 7.2120e-18,  ..., 1.5164e-17, 5.1490e-18,\n         1.0592e-17],\n        ...,\n        [1.3165e-18, 8.3896e-20, 6.8775e-20,  ..., 1.9553e-19, 2.5561e-19,\n         3.0070e-19],\n        [8.1866e-19, 1.9196e-19, 9.1074e-20,  ..., 1.4462e-18, 5.2722e-20,\n         2.0768e-19],\n        [5.9978e-16, 1.3774e-16, 4.7512e-17,  ..., 4.6147e-16, 1.0019e-16,\n         6.6642e-16]], device='cuda:0')"
      },
      "45": {
        "step": "tensor(11268.)",
        "exp_avg": "tensor([ 3.7749e-37,  1.8246e-36, -1.4719e-38, -1.0165e-37, -3.7428e-37,\n         7.6190e-37,  3.7340e-37, -7.6487e-37,  2.2613e-36,  1.6618e-36,\n        -3.9708e-37,  6.8937e-38,  7.8983e-37,  1.2462e-37, -1.1033e-37,\n         4.1709e-37, -3.4297e-37, -6.5513e-37,  4.2564e-36, -7.0495e-37,\n         3.0411e-37, -3.6101e-36, -1.5888e-36, -2.6395e-37, -3.4519e-37,\n        -6.5005e-37, -3.2484e-37, -2.4677e-37, -7.3971e-38, -5.2145e-37,\n        -6.2306e-37, -3.2245e-37, -2.9657e-37,  1.7521e-36, -3.2494e-37,\n        -5.2938e-37,  2.3803e-36, -2.6049e-37,  2.2110e-37, -1.8727e-38,\n        -6.3731e-38, -9.5794e-37, -1.6033e-37, -8.7187e-38,  7.2811e-37,\n         3.0023e-37, -3.4118e-37,  1.0360e-36,  1.3487e-36, -3.8117e-37,\n        -3.9396e-37, -1.7958e-36, -8.9365e-38, -8.6249e-37, -3.1510e-37,\n         5.7177e-37,  4.2596e-36,  4.2087e-36,  6.5776e-37,  8.2358e-37,\n        -6.7568e-37,  2.0196e-36,  3.7633e-36,  3.8703e-36,  3.5111e-37,\n         5.3695e-38, -3.7075e-37,  6.3813e-37, -1.2841e-37, -4.6145e-37,\n        -6.6995e-38,  2.6309e-36, -4.1370e-37, -4.5296e-37,  3.9173e-36,\n        -1.7068e-37, -3.8168e-36,  8.3924e-37,  2.3249e-37, -8.9102e-37,\n        -1.7405e-37, -3.6198e-37, -2.7616e-37,  8.7596e-37,  1.5644e-36,\n         2.9085e-36,  1.8472e-36,  1.9649e-36,  3.6910e-36, -1.9063e-36,\n         2.1795e-37, -3.3302e-36,  1.5137e-37, -1.5066e-36, -3.5704e-37,\n         1.7280e-36,  9.5285e-38,  1.3377e-37,  1.1862e-38,  9.0343e-38,\n         9.2686e-38,  1.4029e-36, -6.6088e-37, -3.2141e-37,  8.6577e-38,\n         1.1579e-38,  2.2779e-36, -8.3492e-37,  3.1901e-37,  4.2616e-38,\n        -1.8503e-37, -1.3623e-36, -9.2655e-37, -4.2278e-37, -5.0186e-37,\n        -1.6569e-36, -2.0538e-37, -2.5102e-37, -2.0548e-37,  8.2367e-37,\n        -2.9279e-37,  1.1172e-36,  2.9938e-38,  2.8373e-36, -2.5742e-37,\n         4.7162e-37, -1.3756e-36,  5.3242e-37, -3.4940e-37, -1.3505e-37,\n        -6.6751e-37, -4.6441e-37, -2.9747e-37, -3.4469e-37,  1.1749e-37,\n        -1.7163e-37,  1.9033e-37, -1.3655e-36, -1.0945e-37, -3.0684e-37,\n        -7.7784e-37,  1.1152e-37, -5.5140e-37,  9.8975e-37, -5.9188e-37,\n        -1.6523e-36,  1.5323e-36, -3.4772e-37, -1.9573e-36, -2.8321e-37,\n        -1.1970e-36,  3.2818e-37, -5.0881e-37,  2.9115e-37,  1.9097e-37,\n        -4.4425e-37,  1.6502e-36,  1.3338e-36, -3.9231e-37, -3.3949e-36,\n        -3.5436e-37,  1.0694e-37, -1.9651e-36,  1.2037e-37,  9.8818e-38,\n         4.0484e-37, -1.7147e-36, -2.3774e-37, -1.5728e-36,  1.7317e-36,\n        -7.6111e-37, -5.8047e-37, -1.0848e-36, -4.1662e-37, -2.8595e-37,\n        -2.9583e-36, -3.0325e-37,  1.3909e-38, -1.7164e-36, -1.1756e-37,\n        -2.6736e-37, -1.1590e-37, -1.4071e-36, -1.1016e-37, -7.8705e-37,\n        -3.2288e-36,  3.1029e-36,  1.0040e-36, -2.5848e-37, -1.9434e-36,\n        -4.1717e-37, -4.3930e-37,  1.3686e-36, -4.5722e-37,  1.9619e-36,\n         3.9936e-37, -1.6122e-36, -3.7979e-37, -1.5361e-36, -7.5757e-37,\n         6.9943e-37, -8.9145e-38,  8.8806e-37,  1.0423e-36, -3.6014e-37,\n        -7.2752e-39,  7.0999e-39, -3.6551e-36, -3.7826e-37, -1.6636e-36,\n         9.6813e-37,  3.5611e-37,  1.3204e-36,  1.1337e-37,  3.2115e-36,\n        -3.6586e-37, -1.5367e-36,  2.2564e-37, -6.5813e-37, -7.1093e-37,\n        -3.7708e-37, -3.1441e-37, -4.2900e-37, -1.9500e-36,  1.2042e-37,\n         1.2962e-37, -1.8833e-37, -2.5439e-37,  7.8426e-37,  1.0479e-37,\n        -1.2361e-38, -2.3632e-37,  9.0607e-38,  6.8092e-37,  3.2552e-36,\n        -2.3971e-36,  2.0310e-36,  4.6146e-37,  1.7782e-37,  1.5211e-36,\n         1.4357e-37, -7.6358e-37, -6.0362e-37, -3.7250e-37,  4.2525e-36,\n        -1.3689e-36,  1.6556e-36, -1.2324e-36, -5.2924e-36,  7.5388e-37,\n         2.9620e-38, -1.8982e-37,  1.4409e-36, -4.1932e-37, -2.8738e-37,\n        -2.7679e-36, -2.3300e-37, -1.7893e-36, -3.7990e-37, -6.2185e-37,\n         2.2934e-36,  1.6455e-37, -3.6574e-37, -5.4473e-37, -3.0406e-36,\n         1.6833e-36, -3.9272e-37, -3.9808e-37,  1.6052e-36,  2.1390e-36,\n        -1.4623e-36, -3.2703e-37,  1.2983e-36,  2.6552e-36, -3.6640e-36,\n        -5.8291e-37, -4.5412e-37, -5.9103e-37, -9.5908e-39, -7.3136e-38,\n        -2.5111e-37, -1.0429e-36,  1.2034e-36,  1.3671e-36,  8.1329e-37,\n        -3.3476e-36, -3.8386e-37, -8.7474e-37, -3.6168e-37, -1.7468e-37,\n        -1.0422e-36,  3.0316e-36, -1.8176e-36, -2.8502e-36, -4.0309e-37,\n         3.3022e-38, -2.0297e-37, -2.0954e-37,  2.3609e-37, -1.2121e-36,\n        -1.2003e-36, -1.4921e-36,  2.2296e-36,  1.7381e-36, -2.1267e-36,\n         6.8833e-37,  7.1657e-38,  1.7364e-37,  4.5578e-37,  4.8456e-36,\n         5.3863e-37, -1.0895e-36,  4.9395e-37, -3.0426e-36, -9.3256e-38,\n        -1.0359e-37,  1.0455e-36, -1.3849e-37, -2.1070e-36,  5.2456e-38,\n         2.4776e-36, -7.3283e-38, -5.8320e-37, -2.4673e-37, -9.7677e-38,\n         8.6719e-37, -3.1321e-36,  6.1592e-37, -4.6223e-37, -2.3416e-36,\n        -3.9023e-37, -4.9772e-37, -7.2081e-37,  7.0817e-38,  9.6204e-37,\n        -4.0844e-38, -6.9277e-37,  2.0971e-37,  3.1387e-36,  2.7927e-37,\n        -1.1075e-36, -2.0602e-37, -7.3318e-37, -4.1141e-37, -7.6223e-37,\n        -5.8795e-37, -3.6883e-37, -1.5706e-37,  2.1640e-36,  2.8108e-36,\n         3.4493e-36, -2.4996e-36,  1.0771e-36,  4.0456e-36, -1.0569e-36,\n        -1.3946e-36, -1.0505e-36, -3.6163e-37,  3.2272e-36, -1.6696e-36,\n         1.4844e-36, -4.4146e-37,  5.6984e-37, -1.6500e-38, -1.8762e-37,\n         1.0279e-36,  2.3023e-36, -3.2382e-38, -1.2246e-37, -4.4357e-37,\n        -2.1964e-37, -1.5464e-36, -8.7342e-37, -5.0819e-37, -2.0644e-36,\n        -4.2546e-37,  6.8452e-37, -2.1751e-37, -3.0598e-37, -5.2649e-37,\n        -1.5035e-37,  1.1046e-36, -1.2270e-36,  4.8940e-37, -3.4923e-37,\n        -2.1954e-36, -1.3219e-37, -2.3380e-37, -9.9629e-37, -3.6499e-37,\n        -5.8711e-38, -1.6035e-37,  5.5131e-37, -2.9160e-36, -2.1559e-37,\n         1.9686e-36,  3.3477e-36, -8.8308e-38,  3.8813e-37, -3.0211e-37,\n         2.8104e-36, -1.3979e-36, -6.3820e-37, -4.9345e-36, -1.8825e-36,\n         1.3976e-36,  5.8933e-37, -2.2179e-37,  6.6568e-37,  1.9953e-36,\n         5.2797e-37, -4.3441e-38,  2.1789e-36, -3.4197e-37,  7.2166e-37,\n        -3.5724e-36, -5.9933e-38, -6.9719e-37, -3.9913e-37,  1.4634e-36,\n         1.8564e-36,  1.6338e-36, -1.6709e-37, -1.8408e-36,  5.4842e-38,\n        -5.6494e-37, -6.3923e-37,  8.3433e-38, -4.3499e-37,  2.7640e-36,\n         1.7463e-36, -1.7809e-37, -3.0480e-37, -2.7091e-37,  4.1459e-37,\n        -9.7188e-39,  1.9395e-36, -3.5782e-37,  4.3046e-36, -3.8601e-37,\n        -4.5065e-37, -1.9609e-37, -6.0777e-38, -5.4400e-38, -5.2656e-37,\n        -3.7362e-36,  2.8420e-36,  8.0557e-37, -1.1026e-36, -7.0427e-37,\n        -5.5459e-37, -1.0551e-36, -1.2907e-36, -3.2295e-37, -8.3139e-37,\n        -1.7380e-37, -2.3952e-37, -1.1945e-36, -3.4847e-37, -1.3810e-36,\n        -1.3353e-36,  8.2012e-38,  1.0336e-36,  7.7582e-37, -3.8912e-37,\n         1.3146e-37,  1.3752e-36,  1.5406e-37, -3.0692e-37,  2.1923e-39,\n        -9.8397e-38,  8.2140e-38,  1.5997e-36,  4.3858e-37, -2.7878e-37,\n        -5.5344e-37, -2.0433e-37,  9.8071e-37,  5.8729e-37, -1.5033e-36,\n        -2.7343e-37, -3.3375e-37, -6.3808e-37,  1.0977e-36, -4.1725e-37,\n         7.5255e-39, -2.7317e-37, -2.2741e-37,  2.6059e-36,  3.9279e-36,\n         3.3091e-37, -3.3945e-37, -5.0594e-37, -7.5461e-37,  1.6799e-36,\n         2.2558e-36,  3.1979e-37, -1.5149e-36, -3.6449e-37,  1.2873e-36,\n        -6.8045e-37, -1.9674e-37, -3.4451e-37, -4.2245e-37, -3.9306e-36,\n        -4.4582e-37,  9.0114e-38,  3.3627e-37,  1.2780e-37,  1.3569e-37,\n         3.0818e-37,  4.6958e-36], device='cuda:0')",
        "exp_avg_sq": "tensor([9.6322e-11, 9.3545e-13, 5.4566e-12, 1.3673e-12, 1.3988e-11, 1.0226e-11,\n        1.2309e-11, 6.3459e-12, 2.6046e-10, 2.4386e-10, 1.3425e-10, 1.7993e-13,\n        7.1275e-12, 5.1590e-13, 1.8529e-13, 8.4408e-12, 1.0760e-11, 5.8669e-13,\n        1.8112e-10, 4.7371e-11, 2.8345e-11, 3.0009e-10, 5.1846e-11, 2.9126e-12,\n        3.8313e-14, 1.9158e-11, 1.2499e-12, 9.6322e-11, 1.1231e-12, 1.6703e-12,\n        4.9070e-12, 9.5362e-13, 1.5873e-15, 6.1974e-11, 2.4021e-13, 1.3001e-11,\n        5.4962e-11, 2.0687e-13, 1.2323e-10, 1.1368e-12, 2.9030e-13, 6.0347e-11,\n        1.7732e-11, 1.8322e-14, 5.7991e-11, 3.7669e-13, 1.5034e-11, 2.0858e-10,\n        4.0708e-11, 7.2320e-13, 2.3900e-11, 7.9473e-11, 8.3290e-11, 4.1832e-12,\n        1.7955e-13, 1.2953e-10, 1.7757e-10, 2.1290e-10, 5.1695e-10, 5.2821e-12,\n        2.6035e-12, 2.3536e-10, 9.1795e-10, 4.9492e-10, 5.2885e-11, 3.1067e-13,\n        1.3866e-13, 4.1151e-11, 3.4454e-13, 4.4289e-12, 9.5584e-13, 4.6801e-11,\n        1.0448e-13, 4.4569e-11, 6.4686e-11, 1.2420e-11, 6.2507e-10, 5.1614e-11,\n        2.3256e-13, 7.4737e-11, 5.0842e-12, 8.4200e-13, 1.7738e-12, 6.0877e-12,\n        7.7007e-11, 1.4711e-10, 1.7510e-11, 5.3452e-12, 1.8442e-11, 2.5576e-10,\n        4.8647e-13, 2.8203e-10, 1.2130e-13, 4.3353e-10, 2.7838e-13, 2.8810e-11,\n        4.6676e-12, 6.2495e-13, 1.6249e-12, 6.5219e-11, 1.1707e-14, 7.4204e-11,\n        1.2820e-12, 1.2100e-12, 1.0035e-12, 7.2767e-11, 1.3113e-10, 6.5651e-12,\n        1.5034e-10, 5.0012e-13, 4.7467e-12, 2.2352e-11, 5.6865e-11, 2.7652e-11,\n        5.4876e-11, 5.2606e-12, 1.7814e-13, 2.8128e-13, 4.4577e-12, 9.6474e-12,\n        1.3153e-10, 2.3585e-10, 2.5261e-13, 1.6375e-11, 1.4316e-12, 2.3960e-11,\n        2.3864e-13, 6.3437e-11, 1.4622e-11, 1.9550e-12, 4.7716e-12, 2.5165e-12,\n        2.4324e-12, 9.1371e-14, 3.8421e-13, 1.1649e-12, 1.1732e-12, 3.5483e-11,\n        2.6986e-11, 3.2220e-12, 8.4776e-13, 2.2704e-10, 4.6081e-12, 1.6630e-10,\n        9.2222e-12, 3.6098e-11, 5.2989e-11, 4.0022e-13, 2.3988e-10, 5.2227e-14,\n        1.0837e-10, 1.9682e-13, 5.4562e-13, 3.5727e-12, 3.0981e-13, 6.1661e-13,\n        5.1276e-11, 1.3923e-10, 3.8749e-12, 9.0979e-12, 6.7355e-14, 1.0347e-13,\n        7.8748e-12, 1.7299e-12, 1.5913e-12, 5.6261e-14, 3.7572e-10, 5.7379e-11,\n        3.0690e-11, 8.1193e-11, 3.2988e-12, 9.5776e-12, 9.2032e-11, 2.8707e-11,\n        8.9105e-11, 8.5402e-12, 2.9006e-13, 3.6408e-14, 2.9782e-10, 1.2989e-12,\n        1.0752e-11, 1.4800e-13, 1.8025e-10, 1.5824e-12, 1.2955e-10, 6.8054e-11,\n        4.7506e-11, 6.2834e-11, 2.2422e-13, 2.5002e-11, 8.7318e-11, 1.4534e-11,\n        6.8454e-12, 7.8039e-13, 2.2841e-10, 7.3604e-11, 2.2016e-11, 4.0194e-11,\n        1.6324e-10, 5.6247e-11, 1.1458e-11, 5.5298e-13, 6.9746e-11, 5.9140e-12,\n        6.0161e-13, 3.2248e-12, 1.1226e-12, 6.8307e-11, 4.9279e-13, 3.2301e-10,\n        5.1881e-13, 8.6996e-13, 4.6929e-11, 2.7712e-13, 9.8159e-10, 2.4569e-11,\n        8.3368e-11, 4.3455e-14, 8.8576e-11, 1.5869e-11, 2.6008e-14, 1.5604e-12,\n        3.0266e-12, 6.6812e-11, 1.9040e-13, 6.8794e-12, 6.4505e-13, 8.4200e-12,\n        9.6863e-13, 1.8280e-13, 2.1948e-12, 5.4258e-12, 3.7790e-13, 4.1842e-12,\n        1.1450e-12, 3.8337e-11, 3.0728e-10, 3.9871e-11, 1.1224e-13, 4.4363e-12,\n        2.5020e-14, 1.5836e-10, 1.1806e-12, 2.9658e-12, 3.0476e-10, 4.5240e-11,\n        3.3575e-12, 1.2324e-11, 2.9018e-10, 4.5125e-11, 2.6923e-12, 8.2129e-13,\n        6.3601e-10, 5.5718e-13, 1.1248e-11, 4.1531e-11, 4.4975e-11, 5.8966e-11,\n        9.6366e-14, 7.0044e-12, 6.4729e-11, 1.0809e-13, 4.5240e-12, 1.1438e-12,\n        2.7857e-10, 1.9772e-10, 5.9083e-14, 6.7776e-11, 1.2374e-11, 6.2646e-11,\n        7.1547e-11, 4.1721e-12, 7.2900e-11, 3.0414e-10, 1.6297e-10, 2.2465e-10,\n        2.1485e-11, 2.3940e-13, 8.0367e-11, 3.1856e-13, 2.3112e-11, 1.7178e-11,\n        1.6330e-10, 2.8871e-12, 1.0454e-12, 3.4200e-12, 2.1732e-11, 1.4241e-11,\n        6.0791e-14, 1.1515e-13, 3.8493e-11, 1.8224e-10, 2.1800e-10, 1.0572e-10,\n        1.0728e-11, 1.8240e-11, 2.4680e-13, 5.9244e-14, 6.9493e-14, 1.3930e-10,\n        1.0606e-12, 5.5773e-11, 2.0869e-11, 3.7991e-10, 3.0856e-10, 2.4049e-11,\n        4.5588e-13, 3.2454e-13, 6.3952e-11, 8.2871e-11, 4.2571e-12, 1.0457e-10,\n        2.2528e-13, 8.0672e-11, 1.0176e-12, 1.4044e-11, 4.8008e-11, 5.6152e-13,\n        1.1633e-10, 1.9523e-12, 5.6774e-10, 2.1387e-13, 8.2960e-13, 3.9975e-12,\n        2.2266e-12, 5.7042e-12, 2.3576e-12, 6.2209e-13, 1.4918e-11, 2.2963e-10,\n        2.5957e-12, 1.6648e-10, 1.0018e-12, 1.4763e-12, 3.8891e-11, 2.2984e-13,\n        1.6002e-11, 1.6144e-14, 1.2230e-10, 4.9857e-13, 2.1217e-10, 1.1210e-12,\n        3.1344e-13, 5.0013e-12, 3.0730e-12, 2.5695e-10, 3.5690e-11, 2.3506e-13,\n        2.6289e-11, 1.0148e-10, 7.4703e-12, 2.6112e-10, 2.6757e-11, 2.6691e-11,\n        2.2983e-11, 2.6756e-11, 1.2469e-10, 8.4530e-15, 1.9838e-10, 1.0955e-09,\n        2.1985e-11, 2.9314e-10, 1.2117e-12, 1.2305e-13, 1.6010e-13, 3.2805e-11,\n        5.1852e-10, 1.6105e-13, 1.5278e-12, 5.0591e-12, 2.1415e-13, 2.2565e-10,\n        7.3248e-11, 6.8947e-12, 1.3360e-10, 9.4235e-12, 4.9120e-12, 2.0483e-13,\n        4.4385e-14, 3.4692e-11, 2.4317e-15, 3.7681e-11, 8.4235e-11, 1.5948e-13,\n        5.3731e-11, 6.0902e-10, 5.7966e-13, 5.6440e-16, 2.2513e-11, 1.2697e-12,\n        5.5660e-13, 2.3582e-13, 3.3542e-12, 3.7701e-11, 1.5759e-10, 3.4121e-11,\n        2.3723e-11, 4.3958e-12, 2.7501e-12, 5.5689e-13, 5.7498e-12, 2.0998e-10,\n        9.6048e-13, 1.5889e-10, 4.7430e-11, 2.4065e-11, 1.0903e-13, 5.5458e-12,\n        2.2750e-11, 1.1962e-10, 2.3767e-13, 2.4334e-13, 2.0933e-11, 4.0604e-11,\n        2.2070e-12, 6.4127e-12, 3.9964e-11, 5.0625e-11, 2.4841e-10, 1.9617e-11,\n        3.5557e-10, 1.0104e-10, 2.1764e-12, 3.1032e-11, 9.6424e-14, 1.2003e-12,\n        4.3461e-12, 2.9115e-13, 1.7214e-11, 4.0426e-11, 2.5488e-11, 8.1735e-13,\n        1.8407e-12, 2.8157e-13, 4.5489e-14, 2.8825e-14, 6.0254e-11, 8.4663e-13,\n        8.3017e-11, 8.2875e-13, 1.4108e-12, 4.4164e-13, 1.5793e-11, 1.2859e-12,\n        3.4774e-11, 1.2938e-10, 2.2679e-10, 2.1561e-13, 4.5143e-11, 1.1631e-13,\n        3.6028e-12, 2.8623e-11, 3.0608e-11, 1.2541e-11, 1.5014e-10, 7.9226e-14,\n        2.2237e-11, 7.6796e-11, 4.0539e-13, 4.7768e-11, 9.6161e-11, 6.1040e-14,\n        2.1156e-10, 1.3585e-12, 2.4225e-11, 4.6338e-11, 7.1696e-13, 9.5421e-13,\n        1.5032e-13, 1.4587e-12, 1.6973e-13, 6.4679e-13, 2.2721e-11, 2.7802e-11,\n        2.2468e-11, 2.6046e-11, 2.0508e-10, 6.2926e-13, 2.4531e-11, 3.4291e-11,\n        2.7719e-13, 2.5091e-11, 1.4668e-11, 2.7296e-12, 2.0604e-11, 6.6604e-13,\n        1.3849e-11, 1.0523e-14, 4.1160e-11, 8.1021e-11, 2.4572e-10, 2.8013e-11,\n        2.4965e-12, 6.6152e-12, 7.0085e-12, 7.4512e-12, 1.0348e-10, 9.6051e-11,\n        2.2180e-11, 8.3364e-11, 1.7874e-11, 5.6594e-11, 2.1984e-11, 2.0002e-12,\n        2.9357e-10, 2.7595e-13, 1.8520e-12, 4.3466e-12, 2.0164e-12, 2.5394e-13,\n        6.5192e-13, 2.0866e-10], device='cuda:0')"
      },
      "46": {
        "step": "tensor(11268.)",
        "exp_avg": "tensor([ 1.1385e-37,  4.9151e-39, -1.3110e-39,  3.0435e-39, -2.5293e-40,\n         1.5535e-38,  4.8313e-38,  1.1216e-38,  6.5195e-38,  1.8906e-37,\n         1.4705e-37,  1.6486e-39,  2.8070e-38,  4.8821e-40,  2.5035e-39,\n         4.8450e-38,  3.7381e-40,  4.3875e-41,  1.4649e-37,  2.2643e-38,\n         1.0209e-37,  7.9036e-38,  1.0431e-38,  1.6690e-39,  8.0402e-40,\n         5.9897e-39,  5.1452e-39, -2.7027e-39,  2.4106e-39, -2.3143e-39,\n        -3.7406e-41,  4.4321e-39,  3.0512e-39,  3.7989e-38,  3.4812e-39,\n         9.9606e-38,  7.8092e-38,  2.8726e-40,  6.9659e-38,  4.9302e-39,\n         6.3073e-39, -1.3273e-39,  4.0273e-39,  9.7039e-40,  2.7067e-38,\n         1.3884e-38, -1.3244e-39,  1.3253e-37,  9.1905e-38, -1.0225e-40,\n        -2.7598e-39,  6.8708e-40,  1.7700e-38,  3.0976e-40,  3.4937e-39,\n         1.0698e-37,  1.1928e-37,  7.5105e-38,  2.1661e-37, -6.2761e-40,\n        -1.9471e-39,  1.5244e-37,  1.8748e-37,  2.4233e-37,  4.7395e-38,\n         3.1933e-39,  1.0425e-39,  1.8582e-38,  3.1972e-39,  3.8842e-40,\n         5.0081e-39,  3.9411e-38, -5.2384e-40,  2.5849e-38,  1.5809e-37,\n        -4.1064e-39, -5.1952e-39,  3.0782e-38,  2.0171e-40,  3.4981e-39,\n        -1.0252e-39,  6.1149e-40,  2.2498e-39,  1.5693e-40,  1.2674e-37,\n         8.2263e-38,  4.3538e-38,  4.2493e-38,  3.9732e-38,  6.2649e-39,\n         1.1401e-38, -3.3572e-39,  9.0133e-39,  2.4178e-39, -9.6551e-41,\n         7.8805e-38,  7.7556e-39,  1.3155e-39,  1.8200e-39,  2.2296e-38,\n         7.3058e-39,  1.2029e-37,  7.8552e-39,  3.0855e-40,  2.2493e-39,\n         1.1878e-38,  5.6507e-38, -1.0394e-39,  1.7800e-37,  7.1860e-39,\n        -4.2574e-40,  2.3919e-38,  3.0494e-39,  1.4786e-40, -3.0258e-39,\n         2.3233e-38,  5.6035e-39,  3.2490e-39,  5.0173e-39,  2.0163e-38,\n        -2.4571e-39,  2.0170e-37,  1.6909e-39,  7.0463e-38, -6.4150e-40,\n         2.8857e-38,  5.1126e-38,  2.1314e-38, -3.9601e-40,  9.7726e-40,\n         7.7698e-39,  7.4676e-40,  4.0401e-39, -1.4530e-40,  9.4852e-39,\n         3.7461e-39,  8.8567e-39,  4.3538e-38,  4.0902e-38,  2.7913e-39,\n        -9.5055e-40,  1.6455e-38,  2.9930e-39,  4.8591e-39, -4.2322e-40,\n         3.4964e-38,  1.3863e-37,  5.2423e-40,  6.7777e-39,  4.1418e-39,\n         3.1287e-40,  1.9102e-38,  7.2955e-40,  1.2034e-39,  1.0337e-38,\n        -1.7060e-39,  1.0055e-37,  7.5859e-38, -6.4436e-40,  6.3045e-38,\n         2.0991e-39,  6.8323e-39,  7.8231e-38,  3.9755e-40,  8.0663e-39,\n         2.1880e-39,  1.5096e-38,  3.3320e-39,  2.0678e-39,  1.0395e-37,\n         1.3740e-40, -6.7083e-40,  1.0144e-37,  9.1945e-39,  1.0545e-37,\n         5.9916e-38,  6.8585e-39,  1.1989e-38,  8.8440e-39, -3.6722e-39,\n         1.7393e-40,  6.3589e-39,  1.4060e-39,  1.8754e-39,  1.5655e-37,\n         1.5366e-39,  1.8204e-38,  9.3756e-38,  1.1104e-39,  9.0776e-38,\n         1.1380e-38,  1.1533e-38,  2.3345e-38, -2.3497e-39,  1.4859e-37,\n         1.9652e-37,  4.5288e-38,  9.0058e-39,  8.0301e-40,  9.3049e-40,\n         4.2710e-38, -4.3496e-39,  1.8680e-37,  8.6741e-38,  2.0071e-40,\n        -1.4117e-39, -5.6649e-40,  2.0194e-39,  4.3442e-39,  1.4711e-37,\n        -3.9422e-39,  1.9032e-38,  8.9302e-38,  2.2476e-39,  1.0783e-37,\n        -2.4657e-41,  1.2702e-37,  1.0975e-38, -1.3628e-39, -1.9222e-39,\n         5.8392e-40,  1.1240e-39, -2.1285e-40,  6.8324e-38,  2.0239e-38,\n         4.9220e-38, -2.7580e-39,  1.6965e-38,  1.5572e-38,  1.6736e-38,\n         2.1149e-38,  1.8992e-39,  3.7101e-39,  2.1460e-38,  7.7208e-38,\n         2.5963e-39,  2.0690e-37,  9.6217e-38,  6.5422e-39,  3.1260e-38,\n        -4.0674e-39,  1.0945e-37,  3.8740e-41,  1.6736e-39,  2.8662e-37,\n         2.3616e-38,  9.4937e-39, -3.3410e-40, -2.0041e-38,  5.0128e-38,\n         5.2515e-40,  7.9986e-39,  2.3600e-38, -1.2724e-39,  3.3509e-39,\n         2.1325e-39,  6.3829e-38,  2.4301e-39,  6.0542e-39,  5.2771e-38,\n         6.8809e-38,  3.9529e-39,  1.0944e-40, -3.2639e-40,  3.4705e-38,\n         7.5984e-38,  4.0019e-39,  2.6175e-39,  4.9445e-38,  1.0342e-37,\n         6.3350e-38,  4.9547e-39,  3.6988e-38,  9.4281e-38,  8.9940e-39,\n         3.9192e-39, -3.1582e-39, -1.6475e-39,  1.3225e-38,  6.3272e-39,\n        -2.9821e-39,  1.5546e-39,  1.0426e-37,  7.6653e-39,  9.0414e-39,\n         6.0008e-39, -1.3121e-39,  4.0934e-39,  7.7141e-39,  5.6133e-39,\n        -8.3712e-40,  1.2638e-37,  2.1101e-39,  1.6327e-38,  1.5530e-40,\n         2.4284e-38,  6.0386e-39,  4.0691e-39,  6.2528e-39,  1.5026e-40,\n         3.4549e-38,  1.3644e-38,  7.2625e-38,  2.4238e-37, -7.1352e-40,\n         6.1739e-38,  4.5998e-39,  4.9040e-39,  2.9438e-38,  4.5259e-38,\n         1.4032e-38,  3.9827e-40,  4.0062e-39,  2.3783e-38,  1.9066e-39,\n         1.3383e-37,  1.0699e-37,  9.6987e-39,  1.0839e-38,  4.2614e-39,\n         1.1560e-37,  6.9572e-39, -1.8956e-40,  4.5310e-39,  2.6159e-39,\n         1.0203e-37,  5.8771e-38, -7.7627e-39, -1.6917e-39,  1.9394e-38,\n         7.3906e-40,  7.1392e-38, -1.7152e-39,  9.6324e-39,  1.1875e-37,\n         8.2264e-39,  3.3952e-39,  9.0001e-39,  1.3688e-37,  1.5027e-38,\n        -4.3614e-40,  6.3094e-39,  2.1612e-41,  9.6084e-39, -1.8745e-39,\n         1.1517e-37,  1.2461e-40,  3.8878e-39,  1.0729e-37,  5.5508e-38,\n         7.2117e-38,  9.6569e-38,  2.3752e-37,  3.9945e-38,  1.3272e-38,\n         3.0080e-39, -1.4275e-39,  2.5619e-39,  8.8808e-38,  3.0462e-38,\n         4.5716e-38,  1.1695e-37,  6.6490e-39,  6.1761e-39,  5.5124e-39,\n         5.3157e-38,  1.3429e-37,  2.7838e-39,  1.1613e-40, -1.4046e-40,\n         1.3107e-39,  3.6573e-39,  5.3618e-39, -2.0148e-39, -1.0488e-40,\n        -2.4629e-39, -2.0692e-39,  7.3058e-39,  6.2122e-39, -1.9735e-39,\n         2.9769e-39,  5.5498e-38,  1.2334e-38,  7.8256e-39,  5.9614e-38,\n         1.1390e-38,  6.0507e-39, -2.0344e-39,  2.0514e-39,  6.7856e-40,\n         2.8746e-39,  1.1467e-38, -4.7377e-39,  2.1910e-39,  5.0082e-40,\n         9.8917e-38,  1.1076e-37,  2.3172e-39, -6.3703e-42,  2.6845e-41,\n         2.3959e-38,  4.7528e-39, -9.4149e-40,  2.4223e-39,  1.9188e-39,\n         1.8429e-38,  7.4322e-39, -2.6456e-39,  2.2995e-38,  2.8927e-38,\n        -2.2875e-39,  4.8590e-39,  7.0122e-38, -2.6759e-39,  2.2258e-38,\n         5.7873e-38,  8.1174e-38, -7.2279e-40,  6.7526e-38,  6.7565e-39,\n         1.0702e-37,  6.1088e-38, -1.3009e-39,  1.1585e-37,  6.9697e-39,\n        -1.7955e-39, -1.3340e-39,  5.4481e-39, -1.7075e-39,  6.3516e-38,\n         4.7686e-38,  8.0531e-39,  1.0111e-39,  8.6168e-39,  1.2367e-38,\n         6.2909e-39,  1.6305e-38,  6.1598e-39,  1.7618e-37,  5.0567e-38,\n         8.0154e-42,  8.3612e-39, -9.8481e-40,  5.1416e-39,  4.1111e-39,\n         5.5039e-38,  2.3931e-37, -5.4181e-39,  6.9690e-39,  3.7123e-40,\n        -6.9363e-41,  1.4806e-38,  1.4186e-39,  9.1650e-40,  5.3603e-38,\n         5.2608e-39,  3.8961e-38,  7.2194e-40,  2.1065e-39,  6.7828e-39,\n         8.4779e-40,  1.3544e-38,  1.4716e-37,  1.1892e-38,  4.0549e-38,\n         2.0271e-39,  2.9222e-38,  1.0899e-38,  9.9631e-39, -2.8806e-40,\n         5.4048e-39,  2.2059e-39,  2.9436e-38,  1.1048e-37,  1.3552e-37,\n         1.4741e-37,  1.7881e-40, -7.4753e-39,  3.4821e-38,  1.4612e-38,\n         3.0399e-39, -7.0368e-40, -2.0902e-39,  3.1098e-39, -8.1577e-41,\n         5.5290e-39,  2.7151e-40,  2.1037e-39,  5.8866e-38,  1.2435e-37,\n         1.8804e-37,  1.0257e-38,  1.1379e-40, -1.7321e-39,  1.2167e-37,\n         3.0297e-39,  9.9684e-38,  4.0226e-38,  9.3189e-41,  1.2005e-37,\n        -9.5774e-40,  1.6923e-38, -8.1616e-40,  1.2635e-39,  8.9286e-39,\n        -1.4388e-40,  2.6734e-39,  4.9545e-39, -1.7347e-39,  4.0444e-39,\n         3.1065e-39,  1.2006e-37], device='cuda:0')",
        "exp_avg_sq": "tensor([3.8094e-14, 8.9377e-19, 9.8335e-18, 6.7749e-19, 2.2540e-18, 1.3272e-15,\n        1.8622e-15, 1.0871e-15, 1.1181e-14, 1.6999e-13, 9.9602e-14, 3.5565e-17,\n        9.1857e-17, 1.1514e-19, 1.6417e-17, 2.2662e-15, 1.2878e-19, 8.2801e-18,\n        8.2166e-14, 1.0972e-14, 2.9789e-14, 2.1832e-13, 1.1665e-14, 5.0885e-16,\n        7.3071e-18, 5.2561e-16, 4.7908e-18, 1.6474e-15, 2.0223e-16, 3.8835e-18,\n        9.4577e-16, 1.0437e-16, 2.0977e-17, 5.0724e-16, 6.9286e-16, 2.1197e-14,\n        1.0361e-14, 7.0439e-17, 3.1600e-14, 1.3833e-18, 9.3446e-19, 7.8598e-16,\n        9.3840e-17, 4.8713e-18, 1.2989e-14, 9.6864e-16, 1.9091e-17, 8.7505e-14,\n        3.0842e-14, 2.3395e-16, 1.4221e-16, 1.1267e-15, 1.4282e-14, 2.5043e-19,\n        3.7121e-17, 4.4594e-14, 2.6139e-14, 8.0608e-15, 5.1673e-13, 1.4437e-18,\n        3.5158e-17, 9.4493e-14, 3.1349e-13, 2.5890e-13, 6.1147e-15, 1.2307e-18,\n        3.9066e-17, 2.4308e-16, 4.1859e-19, 1.0017e-15, 3.2409e-16, 5.2989e-16,\n        1.5867e-17, 1.3229e-14, 4.1343e-14, 1.1187e-16, 7.6927e-14, 9.6945e-15,\n        8.6864e-17, 4.3260e-15, 8.5708e-16, 1.0628e-17, 2.7112e-16, 1.2394e-15,\n        6.4796e-14, 9.2031e-15, 4.3336e-15, 3.2885e-16, 9.2320e-17, 2.3787e-14,\n        5.6338e-16, 1.7824e-14, 1.5753e-16, 4.1262e-14, 2.2481e-18, 6.2354e-15,\n        2.6638e-16, 4.1866e-18, 2.0437e-18, 5.1285e-16, 4.0101e-17, 5.4778e-14,\n        3.6217e-16, 3.7222e-19, 3.4800e-20, 3.9577e-15, 7.7270e-15, 3.8783e-19,\n        1.9721e-13, 4.8829e-19, 4.5407e-16, 2.2983e-15, 1.3167e-15, 4.1551e-18,\n        1.8761e-16, 1.0525e-14, 2.5199e-18, 1.1148e-16, 9.5163e-19, 3.4001e-16,\n        1.8101e-15, 2.4878e-13, 6.5917e-18, 2.4607e-15, 9.0397e-17, 8.3509e-16,\n        1.6647e-14, 8.1517e-15, 4.2225e-19, 2.4645e-18, 7.7497e-16, 3.1748e-16,\n        5.6106e-16, 5.5896e-19, 4.4700e-16, 2.1070e-18, 4.1776e-16, 3.3127e-15,\n        8.2783e-16, 2.6523e-18, 3.0109e-16, 2.8752e-14, 3.2304e-16, 1.0809e-14,\n        2.7558e-18, 5.9735e-15, 3.5853e-14, 1.0698e-16, 3.1464e-14, 2.0381e-18,\n        5.3036e-15, 1.0951e-15, 1.1185e-16, 2.7717e-17, 7.4440e-16, 2.4515e-16,\n        2.0751e-14, 1.5563e-14, 6.5517e-19, 4.6232e-14, 3.2711e-16, 1.1770e-16,\n        3.7751e-14, 4.5442e-19, 4.5017e-16, 1.0783e-16, 5.2484e-14, 3.2400e-15,\n        7.2368e-16, 2.7058e-14, 9.8632e-16, 3.3948e-19, 6.8865e-14, 2.1282e-17,\n        3.9032e-14, 2.5678e-14, 2.8733e-18, 2.6115e-17, 1.8044e-14, 1.9867e-18,\n        5.7588e-18, 1.3798e-17, 1.0754e-14, 1.2124e-18, 1.7391e-13, 3.3472e-15,\n        1.8989e-17, 2.8167e-14, 4.7943e-17, 6.4564e-14, 5.5659e-15, 1.8561e-16,\n        2.4243e-17, 1.9351e-18, 8.7280e-14, 1.3147e-13, 2.2683e-14, 3.4031e-16,\n        7.8809e-16, 3.6618e-16, 4.2267e-16, 2.9087e-18, 1.0062e-13, 7.1627e-15,\n        4.2582e-19, 9.8728e-18, 3.2195e-18, 5.8009e-16, 2.1250e-16, 2.7446e-13,\n        3.0762e-16, 1.9061e-15, 3.1471e-14, 9.3293e-18, 1.3458e-13, 6.2788e-18,\n        9.1876e-14, 1.1369e-16, 2.6809e-16, 1.5813e-18, 1.2240e-19, 2.9894e-16,\n        3.9195e-19, 1.0700e-14, 2.5364e-16, 2.3342e-15, 1.1711e-18, 6.4975e-17,\n        1.2988e-15, 2.0434e-16, 1.2195e-16, 4.3104e-16, 1.3933e-18, 3.0008e-15,\n        7.7436e-16, 1.5141e-15, 2.7368e-13, 2.9255e-14, 1.0784e-16, 1.3165e-15,\n        3.2934e-17, 8.6222e-14, 3.0568e-16, 3.3667e-16, 3.1792e-13, 1.2311e-15,\n        1.9154e-18, 9.4816e-17, 2.0760e-14, 1.2046e-15, 7.4188e-19, 3.1868e-16,\n        8.2590e-14, 5.8680e-17, 3.8813e-16, 1.0609e-14, 2.1207e-14, 5.2837e-16,\n        1.9929e-17, 4.0140e-15, 6.4021e-15, 4.4420e-17, 1.4892e-18, 1.3879e-16,\n        6.7604e-14, 7.8823e-14, 7.9930e-20, 3.2517e-15, 1.0203e-15, 1.6664e-14,\n        1.2921e-14, 7.4063e-16, 1.2008e-14, 9.6374e-14, 9.7614e-15, 1.0778e-14,\n        2.1425e-16, 3.3378e-18, 2.9259e-15, 2.5933e-18, 9.8545e-18, 2.6784e-18,\n        4.0678e-14, 7.6772e-18, 1.1073e-15, 8.5525e-15, 1.1129e-17, 1.4577e-16,\n        3.6827e-17, 8.7425e-18, 1.4967e-15, 4.3391e-14, 9.5820e-15, 2.6420e-15,\n        8.8110e-18, 5.8283e-16, 9.0976e-20, 4.9799e-17, 2.7922e-17, 2.4984e-15,\n        4.2574e-15, 2.2467e-15, 2.6141e-15, 3.7030e-13, 1.5236e-14, 6.7044e-15,\n        7.2117e-19, 2.6052e-18, 8.9889e-16, 7.3934e-17, 2.4557e-15, 2.9569e-16,\n        2.3232e-16, 1.3209e-14, 1.1721e-19, 4.0546e-14, 3.4338e-14, 4.6498e-17,\n        4.6291e-15, 1.3950e-16, 7.9609e-14, 3.3973e-18, 4.1698e-16, 9.0326e-18,\n        1.1901e-18, 1.0920e-14, 2.5170e-14, 7.1425e-17, 9.3799e-18, 2.1535e-14,\n        6.6902e-16, 1.1585e-13, 3.5710e-18, 1.4569e-16, 4.1957e-14, 1.1827e-17,\n        1.6875e-17, 1.6048e-16, 4.2548e-14, 1.0979e-15, 6.8130e-15, 5.4266e-19,\n        8.3150e-17, 8.4317e-16, 1.2503e-18, 1.1635e-13, 9.0104e-18, 2.2606e-18,\n        3.5464e-14, 1.6329e-15, 6.5014e-16, 1.7519e-13, 1.2482e-13, 5.2103e-17,\n        2.7099e-16, 3.1495e-15, 2.1747e-15, 9.6580e-17, 1.4405e-14, 3.2703e-13,\n        1.1136e-15, 1.3053e-13, 8.7657e-16, 4.1080e-16, 5.6240e-17, 3.1944e-15,\n        1.2010e-13, 2.1786e-17, 1.9624e-17, 3.8402e-20, 8.0066e-17, 2.2128e-14,\n        9.6994e-16, 8.7170e-17, 5.7159e-15, 2.1418e-17, 2.1528e-18, 9.4268e-16,\n        5.8602e-18, 6.2211e-16, 1.2737e-17, 7.6188e-15, 1.4739e-14, 3.9302e-16,\n        9.9016e-15, 1.1925e-13, 1.3503e-16, 2.0668e-18, 7.5523e-18, 1.7444e-16,\n        2.8994e-19, 6.2918e-17, 4.2418e-17, 7.7044e-16, 6.6452e-15, 3.5790e-15,\n        1.1288e-14, 2.7121e-16, 5.9689e-18, 1.8401e-16, 1.1839e-18, 2.0234e-14,\n        3.6154e-16, 5.4912e-14, 6.0281e-16, 1.8432e-17, 2.8781e-16, 2.3256e-18,\n        4.0993e-16, 3.4351e-16, 1.5775e-18, 5.3248e-18, 3.9652e-15, 3.0191e-16,\n        3.0394e-15, 4.0583e-14, 2.3222e-15, 1.0961e-16, 1.1864e-13, 1.5572e-17,\n        5.5097e-14, 2.3522e-14, 1.2756e-16, 7.0681e-14, 1.0624e-16, 8.8184e-18,\n        4.2365e-18, 2.6020e-17, 3.1949e-16, 1.7073e-15, 7.8838e-15, 1.2113e-19,\n        2.5246e-16, 2.7506e-16, 2.2984e-16, 2.8734e-16, 5.3490e-15, 6.6427e-16,\n        8.8041e-14, 1.7846e-14, 2.0515e-16, 3.4514e-16, 2.9100e-16, 2.3604e-19,\n        2.3079e-15, 5.8691e-14, 2.4329e-13, 7.4648e-19, 6.9589e-17, 1.5760e-16,\n        6.0950e-16, 2.6025e-17, 7.9406e-17, 8.2723e-19, 6.5669e-14, 5.7335e-17,\n        3.7875e-15, 6.4750e-17, 2.4549e-16, 8.7405e-16, 1.1013e-15, 6.9575e-17,\n        1.3114e-13, 1.4201e-15, 1.7657e-15, 9.8085e-16, 1.1504e-16, 5.3564e-16,\n        3.3402e-17, 1.2081e-17, 1.9662e-17, 5.0669e-20, 4.2589e-17, 2.4863e-14,\n        4.0169e-14, 8.1856e-14, 8.4219e-15, 1.1290e-18, 1.4178e-15, 1.6276e-16,\n        4.6587e-18, 2.0519e-17, 1.8686e-18, 7.3373e-18, 9.1466e-19, 2.7630e-19,\n        1.6065e-18, 1.8210e-19, 2.1514e-15, 2.4171e-14, 2.2892e-13, 7.2740e-17,\n        2.2817e-16, 1.9227e-16, 1.4511e-14, 2.0315e-17, 6.0528e-14, 1.6174e-14,\n        1.0827e-17, 3.2658e-14, 4.0502e-17, 3.7508e-16, 7.3481e-19, 3.9109e-16,\n        4.5790e-14, 9.2542e-17, 1.2411e-18, 3.1686e-17, 5.5237e-18, 1.1914e-17,\n        5.9562e-18, 2.6151e-14], device='cuda:0')"
      },
      "47": {
        "step": "tensor(11268.)",
        "exp_avg": "tensor([ 7.8782e-38,  5.5883e-38,  2.3540e-38, -2.7381e-39,  1.9870e-42,\n         4.7673e-38,  5.9393e-38,  2.8107e-38,  8.3985e-38,  1.0666e-37,\n         7.9733e-38, -1.4505e-39,  5.4460e-38, -3.9390e-40, -2.2559e-39,\n         5.9503e-38,  1.0646e-39,  2.8828e-40,  1.2596e-37,  3.6557e-38,\n         7.4778e-38,  3.8848e-38,  2.0737e-38, -1.2543e-39,  1.0493e-39,\n         2.3427e-38, -4.1805e-39,  1.6050e-38, -2.0630e-39,  9.6512e-39,\n        -9.4712e-40, -2.9087e-39, -2.5993e-39,  6.9808e-38, -2.7035e-39,\n         6.6641e-38,  8.9294e-38, -6.2474e-41,  6.4951e-38, -4.4166e-39,\n        -5.4022e-39,  7.8797e-39,  2.6755e-38, -9.2060e-40,  5.4531e-38,\n        -9.8168e-39,  3.7007e-39,  8.8999e-38,  8.1850e-38,  1.6264e-39,\n         9.2295e-39,  4.0984e-39,  4.0280e-38, -8.7387e-40, -2.9994e-39,\n         7.8810e-38,  1.2025e-37,  1.0973e-37,  1.0299e-37,  3.8982e-38,\n         7.5676e-39,  1.0304e-37,  1.2786e-37,  1.3794e-37,  5.6729e-38,\n        -2.7738e-39, -2.9611e-40,  4.8185e-38, -2.7769e-39,  3.0670e-40,\n        -4.3241e-39,  8.0612e-38,  2.5259e-39,  4.1085e-38,  1.2369e-37,\n         1.3116e-38, -1.1784e-38,  5.6786e-38, -1.9243e-40,  2.0268e-38,\n         5.7218e-40, -1.4655e-40, -1.5579e-39, -1.7040e-40,  9.3195e-38,\n         9.6304e-38,  7.3650e-38,  7.4209e-38,  9.3729e-38,  1.5015e-38,\n        -8.3842e-39, -9.6974e-39, -7.1392e-39,  1.1821e-38,  2.7586e-39,\n         8.2486e-38, -6.0753e-39, -1.1678e-39, -1.5621e-39,  4.3821e-38,\n        -5.6348e-39,  9.0091e-38,  2.7523e-38, -2.4010e-40, -1.8629e-39,\n         3.7681e-38,  8.2269e-38,  4.3221e-39,  9.2530e-38, -6.0243e-39,\n         4.7995e-40,  3.1253e-38,  1.7419e-38, -1.1281e-40,  8.6043e-39,\n         2.9331e-38, -4.6996e-39, -2.6989e-39, -4.7508e-39,  5.1788e-38,\n         1.8559e-38,  1.0481e-37, -1.3768e-39,  9.2116e-38,  7.9301e-41,\n         5.1445e-38,  4.4283e-38,  4.7893e-38,  1.7144e-39, -6.2154e-40,\n         2.6201e-38, -1.1107e-39, -2.6121e-39,  2.0643e-38, -7.1589e-39,\n        -3.3344e-39, -6.8544e-39,  4.1492e-38,  5.0993e-38, -2.3767e-39,\n         2.8233e-39,  4.0721e-38,  2.3117e-38,  4.5285e-38,  1.2779e-39,\n         3.5102e-38,  9.5374e-38,  2.1102e-39,  1.4969e-38, -3.3025e-39,\n         1.2409e-38, -1.2570e-38,  5.1664e-40, -1.1273e-39, -7.9534e-39,\n         3.9698e-39,  8.7305e-38,  7.7609e-38,  2.0763e-39,  3.4370e-38,\n        -1.3540e-39, -5.4792e-39,  4.9510e-38, -4.0573e-40, -6.3823e-39,\n        -1.8175e-39,  2.2593e-38,  2.6759e-38,  1.1127e-38,  8.9331e-38,\n        -7.0336e-40,  1.8438e-39,  6.2796e-38,  3.0577e-38,  7.0525e-38,\n         3.6617e-38, -6.2713e-39, -9.1495e-39,  1.7445e-38,  9.7318e-39,\n        -3.9753e-41, -5.5657e-39,  1.2612e-38, -1.6496e-39,  7.8830e-38,\n         2.6483e-39,  8.0105e-38,  7.9429e-38, -9.6266e-40,  5.3298e-38,\n         3.2323e-38,  3.0324e-38,  5.9755e-38,  1.3386e-38,  1.0139e-37,\n         9.6579e-38,  3.9119e-38,  3.0266e-38, -3.8176e-39,  1.7018e-38,\n         6.0332e-38,  1.6057e-38,  9.9740e-38,  7.8002e-38,  1.2495e-39,\n         2.4613e-38,  2.4497e-38,  1.2098e-39, -3.8808e-39,  6.9340e-38,\n         2.9291e-39, -1.2560e-38,  8.1167e-38, -1.9497e-39,  1.0564e-37,\n         8.4873e-40,  6.5781e-38, -8.1455e-39,  3.9139e-39,  1.1205e-38,\n         8.0548e-40, -1.6789e-40,  1.9903e-40,  4.6746e-38, -1.3763e-38,\n         5.6404e-38,  7.8371e-39,  3.8191e-38, -9.5871e-39, -1.1894e-38,\n         4.2731e-38, -1.5561e-39, -3.2323e-39, -1.2804e-38,  9.8885e-38,\n         7.9149e-39,  1.1399e-37,  7.4587e-38, -5.1067e-39,  6.4580e-38,\n         1.8018e-38,  6.7401e-38, -7.2699e-42, -8.5922e-40,  1.4834e-37,\n         3.2309e-38,  5.6355e-38,  6.6005e-39, -2.9538e-38,  6.3072e-38,\n        -5.2840e-40, -6.8625e-39,  6.0072e-38,  3.5124e-39,  2.6783e-38,\n         4.7285e-39,  5.7741e-38,  9.7885e-39, -4.9486e-39,  5.1140e-38,\n         8.5532e-38, -3.0712e-39,  1.1462e-39,  2.2546e-39,  2.5590e-38,\n         8.0223e-38, -2.8635e-39,  2.4154e-38,  7.2426e-38,  9.2768e-38,\n         4.7546e-38,  2.6348e-38,  6.3923e-38,  9.7081e-38,  8.3483e-39,\n         2.4155e-38,  1.0552e-38,  5.9622e-39,  3.7276e-38, -5.4432e-39,\n         1.3246e-39, -3.7848e-39,  8.3817e-38,  5.1918e-38, -5.8663e-39,\n         7.2813e-39,  3.1861e-39,  2.0660e-38, -6.5416e-39, -4.6885e-39,\n         8.0212e-39,  1.0792e-37,  8.8147e-39,  1.7737e-38,  7.5031e-41,\n         4.5218e-38, -5.2832e-39, -3.4734e-39, -4.8594e-39,  8.6479e-39,\n         3.8864e-38,  2.3631e-38,  8.6317e-38,  1.1669e-37,  2.5353e-40,\n         6.6365e-38, -3.9791e-39, -4.1516e-39,  5.1743e-38,  1.1035e-37,\n        -9.2740e-39, -9.2747e-40, -3.0000e-39,  2.0688e-38, -1.7705e-39,\n         7.9097e-38,  8.2924e-38,  3.4245e-38,  1.7109e-38, -3.4377e-39,\n         1.0005e-37, -5.8531e-39,  4.3617e-40, -3.7978e-39, -2.1361e-39,\n         7.9918e-38,  3.5319e-38,  6.9873e-39,  3.9704e-39,  2.2299e-38,\n         2.3280e-40,  5.8619e-38,  6.0127e-39, -7.2149e-39,  8.5583e-38,\n        -6.6619e-39,  2.1591e-38, -6.8764e-39,  1.1111e-37, -1.0488e-38,\n         8.9906e-39, -5.5170e-39, -1.1557e-40,  3.2352e-38,  7.1363e-39,\n         7.0525e-38,  3.8584e-40, -3.5398e-39,  9.3731e-38,  8.7591e-38,\n         9.9460e-38,  5.1041e-38,  1.1027e-37,  9.7769e-38,  2.6979e-38,\n         1.3931e-38,  7.0729e-39, -1.8335e-39,  1.0138e-37,  3.1677e-38,\n         6.9580e-38,  7.2226e-38, -4.6773e-39, -5.1440e-39, -4.7356e-39,\n         6.7268e-38,  1.0247e-37, -2.4228e-39,  2.4543e-38,  8.8774e-40,\n        -8.6111e-40,  1.5854e-38,  2.1756e-38,  1.3009e-38,  1.5711e-41,\n         8.4865e-39,  3.4973e-38, -6.0641e-39, -5.6720e-39,  1.4286e-38,\n        -2.7377e-39,  6.9034e-38,  2.4747e-38, -5.6086e-39,  5.5654e-38,\n         1.4775e-38, -4.8167e-39,  6.5411e-39,  1.6346e-38, -2.8013e-40,\n        -2.2633e-39, -9.3135e-39,  3.8210e-39,  4.1179e-39,  2.3922e-38,\n         8.9390e-38,  1.0796e-37, -1.5072e-39, -3.4349e-41,  1.8323e-39,\n         7.7900e-38,  1.5337e-38,  2.9693e-39,  2.6531e-39,  7.3458e-39,\n         5.7687e-38, -5.1965e-39,  6.3030e-39,  5.1290e-38,  7.0061e-38,\n         3.0399e-38, -4.2624e-39,  8.4524e-38,  1.5287e-38,  5.0347e-38,\n         3.2209e-38,  6.4844e-38,  1.5614e-39,  5.7633e-38,  5.3065e-38,\n         9.0970e-38,  7.6252e-38,  2.1506e-38,  6.0897e-38, -5.7760e-39,\n         1.0352e-38,  5.0622e-39, -4.6426e-39,  1.5983e-38,  8.9556e-38,\n         7.3005e-38, -6.9590e-39,  8.1747e-40, -7.0289e-39, -8.5696e-39,\n        -5.3653e-39,  6.2631e-38, -4.7628e-39,  1.3032e-37,  5.2460e-38,\n         5.6171e-40, -7.1280e-39,  2.2960e-38, -4.6839e-39,  2.5182e-38,\n         3.1363e-38,  1.2749e-37,  3.2844e-38,  2.0565e-38, -1.2091e-39,\n        -2.6634e-40,  2.8098e-38,  1.3392e-38, -4.4077e-40,  4.9444e-38,\n        -4.6832e-39,  4.8572e-38, -2.3918e-39, -1.4703e-39,  1.8326e-38,\n         9.9567e-39, -1.0071e-38,  9.2390e-38, -7.6116e-39,  4.8387e-38,\n         3.2071e-38,  6.2139e-38, -8.2983e-39, -8.5038e-39,  3.4829e-40,\n        -4.8243e-39, -1.9292e-39,  6.4632e-38,  7.8614e-38,  7.7973e-38,\n         7.8442e-38,  2.3796e-38,  3.2524e-38,  5.5847e-38,  2.4570e-38,\n        -2.3647e-39,  2.3916e-39,  9.1810e-39,  4.5357e-38,  7.1995e-40,\n        -4.7119e-39,  2.4168e-38, -1.1402e-39,  8.6391e-38,  1.1736e-37,\n         9.5744e-38,  3.2689e-38, -3.2046e-40,  6.6830e-39,  9.2959e-38,\n         6.2529e-38,  7.5084e-38,  3.8674e-38, -2.3854e-40,  8.8727e-38,\n         3.1324e-39,  3.8441e-38,  2.8231e-39,  2.6251e-40,  8.1839e-39,\n         1.6909e-39, -2.3980e-39,  3.6524e-38,  1.5508e-39, -3.4023e-39,\n         3.3679e-38,  1.2510e-37], device='cuda:0')",
        "exp_avg_sq": "tensor([3.0406e-14, 1.2021e-15, 1.3544e-15, 2.2579e-16, 8.1439e-16, 2.1894e-15,\n        8.9510e-15, 1.7851e-15, 6.9410e-14, 9.3418e-14, 5.3723e-14, 5.9794e-19,\n        5.7161e-15, 1.4162e-17, 1.0752e-19, 9.4688e-15, 1.3399e-15, 1.0618e-16,\n        7.0015e-14, 9.5774e-15, 2.6561e-14, 7.7641e-14, 1.5878e-14, 2.2189e-16,\n        1.2345e-18, 9.8704e-16, 3.7176e-16, 1.3685e-14, 8.8177e-17, 4.6499e-16,\n        4.8536e-16, 4.7375e-17, 7.2344e-18, 2.0024e-14, 2.9849e-16, 1.1268e-14,\n        2.8143e-14, 3.1882e-17, 5.1458e-14, 1.3581e-16, 6.2716e-19, 6.9216e-15,\n        2.5484e-16, 1.2984e-18, 1.4591e-14, 4.2180e-16, 1.7033e-15, 7.0245e-14,\n        2.8897e-14, 1.1676e-16, 4.5244e-15, 4.7177e-15, 1.6432e-14, 9.5198e-17,\n        3.2921e-18, 4.6429e-14, 6.1058e-14, 6.3359e-14, 1.7883e-13, 3.6166e-15,\n        7.1231e-16, 8.3279e-14, 2.5547e-13, 1.5178e-13, 3.4364e-15, 3.2479e-19,\n        1.5401e-17, 1.1541e-14, 1.7462e-17, 4.8246e-16, 1.4599e-16, 1.5406e-14,\n        8.4399e-18, 1.8375e-14, 3.9597e-14, 2.4077e-15, 7.6494e-14, 2.3701e-14,\n        3.4743e-17, 1.1417e-14, 4.2589e-16, 5.1299e-18, 1.1567e-16, 4.8563e-16,\n        4.5483e-14, 4.9575e-14, 1.3418e-14, 6.0502e-15, 8.7254e-15, 2.6328e-14,\n        2.6858e-16, 2.9803e-14, 7.9063e-17, 5.9706e-14, 1.1415e-16, 1.9354e-14,\n        1.1416e-16, 6.8860e-17, 1.7014e-16, 1.7010e-14, 6.0918e-18, 4.4214e-14,\n        2.1556e-15, 4.6677e-17, 3.6243e-19, 6.2292e-15, 4.4882e-14, 1.5892e-17,\n        7.7815e-14, 3.7988e-17, 2.0849e-16, 7.7264e-15, 2.8445e-15, 2.2203e-15,\n        9.2638e-15, 1.0260e-14, 4.5308e-17, 1.3777e-17, 3.6011e-16, 7.5989e-15,\n        2.3854e-14, 1.0503e-13, 6.2531e-18, 1.2661e-14, 4.1923e-17, 8.9841e-16,\n        9.6157e-15, 1.5227e-14, 1.5451e-15, 1.4237e-18, 1.6761e-15, 1.4652e-16,\n        2.4560e-16, 9.3239e-16, 1.0288e-16, 2.0964e-16, 2.0235e-16, 2.9597e-15,\n        5.8650e-16, 4.1213e-16, 1.2162e-16, 3.6608e-14, 4.5143e-15, 3.5439e-14,\n        6.4187e-16, 1.3290e-14, 3.5124e-14, 5.3086e-17, 3.0189e-14, 6.9679e-17,\n        1.2645e-14, 5.2566e-16, 5.5103e-17, 5.9309e-16, 3.3118e-16, 1.0013e-16,\n        3.1046e-14, 3.6626e-14, 2.1496e-17, 2.1689e-14, 1.5329e-16, 1.2597e-17,\n        2.0125e-14, 6.7200e-17, 1.8204e-16, 3.0936e-17, 4.0359e-14, 6.6865e-15,\n        1.4867e-15, 4.0182e-14, 4.3983e-16, 2.5301e-16, 4.5864e-14, 7.8772e-15,\n        2.5353e-14, 1.4552e-14, 5.4022e-17, 7.9061e-18, 2.5653e-14, 1.5072e-16,\n        9.7444e-16, 1.0535e-19, 1.9309e-14, 1.0607e-16, 6.8910e-14, 7.0070e-15,\n        1.5334e-14, 3.5182e-14, 8.2331e-18, 3.1180e-14, 6.9370e-15, 4.0582e-16,\n        4.9013e-15, 5.7103e-16, 7.8696e-14, 5.1305e-14, 9.1580e-15, 1.8061e-15,\n        1.8775e-14, 1.9990e-15, 8.1125e-15, 2.8739e-16, 4.6180e-14, 6.8202e-15,\n        1.7141e-16, 1.1914e-15, 6.8262e-16, 3.7108e-16, 5.2371e-17, 9.8507e-14,\n        1.5362e-16, 9.2307e-16, 3.0966e-14, 4.5779e-19, 2.4712e-13, 3.4139e-15,\n        4.2107e-14, 4.2622e-17, 1.5010e-14, 3.1003e-17, 1.4265e-16, 1.3748e-16,\n        7.1127e-17, 4.0471e-15, 1.0346e-16, 8.2079e-15, 2.1361e-16, 3.7022e-15,\n        6.1722e-16, 5.3448e-17, 4.0486e-15, 2.0144e-16, 2.3921e-17, 1.5226e-15,\n        4.7007e-15, 1.5871e-15, 1.2249e-13, 2.9810e-14, 2.3431e-17, 2.2761e-15,\n        1.3109e-16, 5.1153e-14, 1.4638e-16, 1.5943e-16, 1.2110e-13, 1.4474e-15,\n        2.5936e-15, 4.5510e-16, 2.3797e-14, 1.5439e-15, 2.2010e-16, 1.5572e-16,\n        9.3967e-14, 2.8793e-17, 1.0441e-15, 1.5023e-14, 8.8293e-15, 1.8406e-15,\n        1.4786e-17, 9.8116e-15, 2.8843e-14, 4.5095e-19, 6.9492e-16, 6.9611e-17,\n        6.1730e-14, 4.0953e-14, 1.3753e-16, 1.2118e-14, 7.8864e-15, 2.9080e-14,\n        7.4442e-15, 3.7027e-15, 1.1588e-14, 8.9867e-14, 4.4216e-15, 2.7755e-14,\n        4.2394e-15, 9.2804e-16, 4.6357e-15, 1.2086e-18, 1.7738e-15, 6.4775e-16,\n        5.5114e-14, 2.8051e-15, 5.6773e-16, 1.0302e-14, 2.4447e-15, 1.3672e-15,\n        1.8144e-17, 2.6160e-17, 6.0627e-15, 6.6914e-14, 2.0206e-14, 1.4211e-15,\n        1.1566e-15, 6.6306e-15, 5.0583e-17, 1.6525e-17, 4.0458e-18, 1.0565e-14,\n        7.7526e-15, 1.9691e-15, 1.3583e-14, 1.3207e-13, 4.1550e-14, 3.4822e-15,\n        3.6096e-17, 3.5422e-17, 1.7359e-14, 2.5471e-14, 1.1992e-15, 1.0974e-14,\n        1.0049e-16, 1.6719e-14, 3.1091e-18, 1.9751e-14, 3.2958e-14, 1.9605e-15,\n        1.1279e-14, 6.2007e-17, 1.4870e-13, 4.7917e-19, 1.5419e-16, 7.3230e-16,\n        7.3940e-17, 1.2757e-14, 1.3829e-14, 3.7796e-17, 8.6094e-16, 1.4582e-14,\n        3.4215e-16, 5.9695e-14, 5.9419e-16, 6.3617e-17, 3.1180e-14, 2.0301e-17,\n        6.9875e-17, 7.1701e-17, 5.3410e-14, 4.8623e-16, 2.8536e-14, 1.6633e-16,\n        2.8158e-17, 1.7208e-15, 5.0049e-16, 7.3855e-14, 5.9755e-15, 9.9507e-18,\n        1.8368e-14, 2.8637e-14, 5.1611e-15, 7.1811e-14, 3.7356e-14, 1.1387e-14,\n        7.8208e-15, 7.4769e-15, 1.5680e-14, 4.5121e-17, 6.2053e-14, 1.6027e-13,\n        1.1272e-14, 9.6266e-14, 4.5938e-16, 1.7304e-16, 3.4653e-18, 1.6742e-14,\n        1.3948e-13, 8.9195e-19, 1.1068e-15, 1.4513e-16, 5.6118e-18, 3.1923e-14,\n        3.1568e-15, 7.4958e-16, 1.6555e-14, 2.0653e-15, 3.4755e-15, 4.2938e-16,\n        1.2490e-18, 7.2575e-15, 4.2089e-18, 2.2115e-14, 1.4724e-14, 1.8392e-16,\n        2.3090e-14, 8.8885e-14, 6.2614e-17, 3.6206e-16, 3.5685e-17, 8.1933e-17,\n        5.2396e-19, 1.1241e-17, 1.9391e-17, 5.6554e-16, 2.6614e-14, 1.5679e-15,\n        1.9791e-14, 1.3243e-16, 3.2106e-16, 9.5356e-17, 4.2486e-15, 2.8406e-14,\n        1.6038e-16, 3.6368e-14, 1.1760e-15, 1.0664e-14, 1.2850e-16, 1.4661e-15,\n        7.2841e-15, 3.2526e-14, 3.6742e-16, 4.2380e-19, 8.7809e-15, 4.8278e-15,\n        4.0044e-15, 1.2964e-14, 9.1355e-16, 6.5592e-15, 4.3527e-14, 7.6874e-15,\n        1.0447e-13, 1.3419e-14, 2.5189e-15, 3.1117e-14, 4.1987e-17, 6.6473e-16,\n        3.1490e-16, 2.0386e-17, 3.8046e-15, 1.4413e-14, 6.1660e-15, 3.6725e-17,\n        1.1746e-16, 1.2476e-16, 1.0277e-16, 1.4167e-16, 1.5265e-14, 3.1336e-16,\n        4.0243e-14, 9.2298e-15, 1.0192e-16, 1.5970e-16, 1.6043e-15, 1.8143e-16,\n        5.1117e-15, 2.3700e-14, 1.0227e-13, 4.2912e-16, 1.0026e-14, 2.2574e-17,\n        2.9913e-16, 4.6245e-17, 2.3961e-16, 1.5311e-15, 2.5152e-14, 4.0604e-19,\n        2.4168e-15, 7.0750e-15, 1.0964e-16, 1.0915e-15, 6.2063e-15, 1.9673e-17,\n        8.7769e-14, 6.4698e-16, 1.2268e-15, 2.1346e-15, 3.1626e-15, 2.2539e-16,\n        3.3877e-17, 5.6597e-18, 5.0050e-18, 4.8180e-18, 8.7807e-15, 1.6634e-14,\n        2.3004e-14, 3.3721e-14, 3.4100e-14, 6.8601e-16, 1.1848e-14, 2.2165e-16,\n        2.4823e-17, 3.0920e-15, 7.7771e-17, 1.5445e-15, 1.0304e-15, 2.1744e-17,\n        8.9849e-16, 1.6750e-16, 1.8372e-14, 3.9811e-14, 9.6345e-14, 6.3570e-15,\n        1.1498e-16, 1.9348e-15, 7.2811e-15, 3.9115e-15, 5.2933e-14, 6.9064e-15,\n        2.1558e-15, 3.7729e-14, 1.5500e-15, 1.8547e-14, 3.4277e-15, 1.7954e-16,\n        1.8104e-14, 4.9500e-17, 1.8144e-16, 1.2250e-15, 7.3167e-17, 1.2345e-18,\n        1.1156e-15, 6.4004e-14], device='cuda:0')"
      },
      "48": {
        "step": "tensor(11268.)",
        "exp_avg": "tensor([[ 2.3043e-41,  1.7776e-39,  1.6963e-40,  ...,  2.1850e-40,\n          3.5348e-41,  3.2114e-39],\n        [-7.9674e-41,  1.3201e-40, -2.4586e-41,  ...,  6.1223e-42,\n          1.0552e-42,  1.0122e-40],\n        [ 1.0818e-40, -9.6405e-40,  1.4244e-40,  ...,  2.5077e-40,\n          6.5680e-41, -1.4371e-38],\n        ...,\n        [ 9.0075e-42, -9.2916e-39, -5.1866e-40,  ..., -5.3144e-40,\n         -3.8418e-40,  3.1924e-39],\n        [ 3.1454e-40,  8.1712e-39,  4.8508e-40,  ...,  5.4529e-40,\n          2.5744e-40, -2.9360e-39],\n        [-2.9413e-41, -1.1447e-39, -8.5066e-41,  ..., -8.8315e-41,\n          1.2026e-41,  1.7313e-39]], device='cuda:0')",
        "exp_avg_sq": "tensor([[3.9345e-18, 2.1973e-19, 2.5192e-19,  ..., 2.2490e-18, 1.4051e-18,\n         4.7690e-20],\n        [3.5847e-20, 1.3082e-19, 2.3602e-19,  ..., 5.6164e-18, 7.4208e-19,\n         5.2379e-18],\n        [2.0007e-17, 6.5595e-18, 3.8112e-18,  ..., 4.9353e-17, 1.7342e-17,\n         3.8939e-17],\n        ...,\n        [4.9620e-16, 1.0571e-16, 1.4676e-17,  ..., 1.4078e-16, 2.6995e-17,\n         1.4468e-16],\n        [2.4640e-19, 4.9006e-18, 3.0941e-19,  ..., 5.2780e-18, 5.7443e-19,\n         2.7614e-18],\n        [3.8185e-19, 5.5530e-20, 5.2289e-20,  ..., 3.9157e-19, 6.3018e-19,\n         8.7380e-19]], device='cuda:0')"
      },
      "49": {
        "step": "tensor(11268.)",
        "exp_avg": "tensor([ 4.5714e-37,  1.0550e-37, -4.4217e-37,  1.0573e-37, -6.2966e-37,\n        -1.4236e-36, -7.5030e-38, -6.5645e-37,  1.7020e-36,  2.4340e-36,\n         1.3082e-36, -1.7724e-36,  3.4107e-37, -1.0760e-36, -6.0617e-37,\n         1.0105e-37, -5.9388e-37, -2.4372e-36,  5.4138e-37, -2.8602e-37,\n         1.8614e-36,  1.5387e-37, -8.2665e-37, -7.8567e-37, -1.0803e-36,\n        -3.1141e-37, -2.7235e-37, -4.0053e-37,  6.4385e-37, -9.3403e-37,\n        -4.2618e-37, -3.0845e-37,  1.3375e-36,  1.0581e-36, -1.1905e-36,\n         7.0530e-37, -2.9043e-37, -1.8757e-36,  2.4717e-36,  5.9886e-39,\n         4.1904e-37,  1.8599e-37, -7.2926e-38, -5.8059e-37, -2.7854e-37,\n         1.5823e-36, -2.3084e-37,  7.1355e-37,  4.9341e-37,  2.2588e-37,\n        -2.4000e-37, -3.8786e-37,  9.4725e-37, -8.9081e-37,  2.3835e-37,\n         5.3496e-37,  6.9163e-37,  1.9408e-36,  2.9515e-37, -8.0153e-38,\n        -9.7541e-38,  1.5190e-36,  1.0979e-36,  1.6139e-36, -1.7022e-37,\n        -1.7248e-37, -8.1671e-39,  8.9797e-37, -6.9601e-37, -1.1150e-36,\n        -2.5172e-37,  6.9965e-37,  5.5582e-37,  4.5460e-38,  6.3335e-37,\n         3.1080e-38, -4.7101e-36,  1.1470e-36,  5.4633e-37, -2.3231e-37,\n        -1.4325e-37, -2.5970e-38,  7.0600e-37,  1.2333e-37, -2.3634e-37,\n         1.8539e-36,  1.3834e-36, -6.5818e-38,  6.6195e-37, -4.4347e-37,\n        -4.2106e-37, -3.0573e-36,  6.0007e-37, -9.6233e-38, -1.2179e-36,\n         1.8899e-36, -4.7578e-37,  1.1968e-37, -1.0949e-36,  6.2872e-39,\n        -2.7979e-37, -1.8499e-37,  4.2320e-37,  9.7049e-38, -1.0592e-36,\n        -1.9104e-37,  1.5094e-36, -3.6836e-36,  2.7650e-36,  2.3906e-37,\n        -7.9623e-37, -1.0899e-37, -5.3114e-37, -1.9771e-36,  2.4569e-38,\n        -2.2150e-37, -2.2704e-37, -5.8415e-37, -7.6326e-37,  1.2609e-36,\n         1.5629e-36,  2.4020e-36, -2.1676e-36,  1.1964e-37,  3.1686e-37,\n        -1.1109e-36, -4.7311e-38, -2.6416e-37, -7.2143e-37, -1.5001e-37,\n         4.9703e-38,  3.4691e-37,  1.7482e-38,  9.0072e-38,  8.8128e-37,\n         1.8086e-37,  1.1106e-37,  4.9102e-38, -5.0850e-37,  1.9995e-37,\n        -2.0093e-36, -8.3538e-37,  1.6039e-37, -1.9809e-37, -1.3659e-36,\n         1.1919e-37,  5.6300e-38,  6.9431e-37, -2.7121e-37,  6.3620e-37,\n         2.7590e-37,  2.5381e-36,  9.2754e-37, -1.5650e-36, -4.5419e-37,\n        -5.7759e-37, -2.8890e-37, -2.7904e-37, -1.8816e-37, -7.6220e-37,\n        -2.7221e-37,  4.8651e-37, -1.8056e-37,  1.4512e-37,  5.1327e-37,\n        -4.3137e-38, -5.4319e-37, -7.7221e-37, -4.2089e-37,  1.7745e-36,\n         2.3094e-37, -7.3023e-37, -7.4942e-37,  1.4495e-37,  6.0423e-37,\n        -6.4739e-37,  1.0561e-36,  1.1662e-36, -1.3949e-37, -2.9262e-37,\n        -1.3511e-36, -4.4002e-37, -3.9441e-37, -3.2839e-37,  1.4424e-36,\n        -1.0852e-36, -3.2235e-38,  3.3030e-37,  2.2642e-37, -2.6231e-37,\n         7.4158e-38,  1.2850e-37, -1.1086e-37, -2.1449e-37,  6.7348e-37,\n         3.0664e-36, -1.7917e-37, -8.9945e-37, -5.6482e-37, -2.2668e-36,\n         4.9573e-39,  2.7644e-37, -1.8860e-37,  1.0273e-36,  1.3004e-36,\n        -3.5093e-37,  2.1575e-37,  1.4045e-38, -2.6169e-37,  4.0671e-37,\n        -2.8928e-36,  1.4785e-36,  2.6045e-36, -1.3031e-36,  5.0220e-36,\n        -2.9975e-37, -9.2809e-37,  5.4241e-38, -3.2105e-37, -3.3847e-37,\n        -4.5763e-37,  1.5633e-37, -5.1278e-37, -1.8598e-36,  2.4218e-36,\n        -2.7571e-36,  2.8412e-37, -1.3732e-37,  6.9826e-37, -1.3981e-36,\n         1.2590e-37, -2.3120e-36, -1.9509e-37,  1.6771e-36, -3.3356e-37,\n        -7.1492e-38,  4.1732e-36,  7.8963e-39,  7.1700e-37, -3.5475e-37,\n         8.6976e-37,  5.2212e-37, -6.4250e-39, -1.2513e-36,  4.9411e-37,\n        -4.5927e-38, -1.0936e-36,  1.6116e-37,  4.5979e-37, -1.0521e-37,\n         1.7608e-37, -2.0347e-39, -5.7975e-37, -9.1031e-37, -4.7090e-37,\n        -2.1764e-37, -2.7888e-37, -1.0222e-36,  1.6589e-36, -2.7405e-37,\n        -6.7256e-37,  1.3276e-36, -2.6693e-37,  2.9725e-37,  1.7030e-37,\n        -3.5959e-37,  2.4759e-36,  5.2461e-38, -2.3339e-37,  1.0817e-37,\n        -3.4316e-37, -9.5280e-38, -1.6274e-37,  1.0219e-37, -1.3489e-36,\n        -3.1495e-37, -6.2050e-37, -1.2533e-36,  8.6315e-38,  6.6399e-38,\n        -2.8487e-36,  1.2772e-37,  2.8017e-36, -9.3545e-39,  4.5884e-37,\n        -1.0524e-36, -1.3144e-37, -6.3551e-37, -1.6566e-37,  1.2982e-36,\n        -1.5089e-36,  2.2764e-36, -6.1474e-37, -1.3035e-36, -4.8691e-37,\n         2.8470e-37,  8.6940e-37,  1.3572e-36,  1.8711e-36,  1.3285e-37,\n        -2.5644e-37,  2.3326e-37,  1.1321e-36,  4.1569e-37, -1.0210e-36,\n        -2.9330e-37, -8.0391e-37,  5.0199e-38, -1.6393e-36,  1.6489e-37,\n        -1.0472e-36, -3.1714e-36, -1.1333e-37, -7.4982e-37, -7.9845e-37,\n        -1.4325e-37,  1.5446e-37,  8.9416e-38, -2.9990e-36, -2.0270e-37,\n         1.9902e-36,  2.0225e-37,  2.2314e-37, -5.1259e-37, -5.4174e-37,\n        -3.8730e-38, -3.0393e-37, -4.6821e-36, -5.9273e-37, -5.7822e-37,\n         9.1351e-39,  2.1861e-36, -1.0065e-36,  1.3258e-37,  6.6108e-37,\n        -2.8890e-37, -2.2214e-37, -2.6060e-37,  1.1330e-36,  1.1089e-37,\n        -7.6406e-38, -5.5793e-37, -5.3809e-37,  5.8924e-38, -2.9371e-36,\n        -1.3920e-36, -1.5573e-39, -2.3033e-36, -2.0139e-37,  7.2659e-37,\n         9.0386e-37,  4.4726e-38,  1.1891e-36,  2.5448e-36,  3.3887e-37,\n        -6.0091e-37, -2.9867e-37, -5.3755e-37,  2.2309e-36, -7.2492e-37,\n        -2.0052e-37,  1.6643e-36,  3.6758e-37,  3.4183e-36,  1.6837e-36,\n        -1.4039e-37,  4.0190e-37, -1.2444e-37,  6.8039e-37, -3.0799e-37,\n        -1.0701e-37,  1.1805e-37, -1.4273e-37, -2.3920e-37, -2.5452e-36,\n         9.6398e-37,  3.5006e-36,  3.4981e-36, -3.8263e-37, -9.4440e-37,\n         1.2000e-37, -9.3987e-38, -3.3085e-37, -1.6356e-37,  7.7194e-37,\n        -1.3188e-36,  1.2623e-37,  3.1955e-37, -2.1189e-37, -1.0305e-36,\n        -2.4641e-37,  3.0718e-36,  6.1631e-37, -3.7300e-37, -3.0715e-37,\n        -8.4527e-37, -2.5184e-37, -5.2915e-37, -1.1512e-36, -5.0754e-37,\n        -2.7384e-37, -6.7866e-37, -8.9905e-37, -2.0527e-36, -1.3471e-37,\n         3.0681e-36, -1.4623e-36,  2.8105e-37,  1.4483e-37,  1.0403e-36,\n         2.0555e-36, -5.6837e-37, -2.5788e-37, -8.1943e-37,  2.7491e-37,\n        -2.3399e-37, -1.8780e-37, -1.1044e-36, -3.7005e-37,  9.0780e-37,\n         1.2082e-36, -2.2078e-37, -1.6627e-37,  4.9100e-39, -1.4023e-36,\n         1.0022e-37, -1.9459e-37, -9.0612e-37,  6.8159e-37, -1.1634e-37,\n        -1.0258e-37, -5.0877e-37, -6.0281e-38,  9.6916e-37,  1.1549e-36,\n         5.5353e-38,  3.0621e-36,  2.2920e-37, -3.1068e-37, -3.6535e-37,\n        -1.2493e-36,  2.4307e-37, -3.2399e-37, -2.4697e-37, -6.8764e-37,\n        -1.3479e-36,  3.4502e-36,  1.2997e-36,  9.2363e-37, -1.6140e-37,\n        -1.8250e-37, -3.0074e-37, -1.3283e-36, -4.8118e-37, -1.0911e-36,\n         8.7414e-38, -2.4477e-37, -1.1004e-37, -6.4598e-38,  1.0682e-38,\n        -3.6901e-37,  8.3333e-37,  1.7630e-36,  2.1448e-36, -5.6121e-37,\n         1.8973e-37, -4.6750e-37, -1.8137e-36, -2.7323e-37, -6.4185e-40,\n         6.8591e-37, -1.8826e-36, -8.5372e-38,  6.5887e-37, -1.6031e-38,\n         2.9602e-36, -6.9220e-38,  1.1226e-37,  2.9586e-37, -4.9753e-37,\n        -1.2004e-37, -4.0055e-37, -2.9277e-37,  1.3444e-36, -5.5275e-38,\n        -8.8843e-37,  6.3752e-37, -2.6561e-37, -7.6699e-37,  2.1889e-36,\n         5.5414e-36,  1.8658e-37, -3.8005e-37, -1.3507e-37, -1.0401e-37,\n         1.3878e-37, -1.8220e-37, -9.3137e-37, -1.9190e-36,  1.1934e-36,\n        -4.6179e-37,  1.0242e-36,  7.9535e-37, -5.3005e-37, -6.1568e-37,\n        -3.1159e-37, -1.5726e-37, -1.1443e-37, -2.6909e-36, -1.3189e-36,\n         1.2651e-36, -2.0987e-37], device='cuda:0')",
        "exp_avg_sq": "tensor([6.2517e-13, 1.3426e-12, 1.0049e-11, 1.8607e-13, 1.2227e-12, 3.7127e-11,\n        1.0120e-12, 1.0303e-11, 5.9965e-11, 1.8540e-10, 2.1815e-11, 2.6466e-11,\n        1.8117e-12, 1.5530e-11, 8.0237e-11, 5.6450e-13, 7.1739e-11, 4.0682e-11,\n        2.1336e-14, 1.8013e-13, 3.5231e-12, 3.6379e-13, 2.1585e-11, 4.1845e-11,\n        4.3895e-11, 1.5674e-13, 8.4510e-13, 1.9539e-11, 3.4633e-12, 3.8297e-11,\n        4.1428e-11, 3.2947e-12, 1.1994e-11, 1.7536e-12, 8.2749e-11, 6.1764e-12,\n        6.3962e-15, 6.1027e-11, 1.5685e-10, 1.0331e-12, 6.9810e-12, 8.8572e-13,\n        1.1486e-12, 4.7986e-12, 4.7144e-13, 4.9941e-11, 2.8582e-12, 2.9156e-13,\n        1.9802e-11, 4.5009e-11, 2.1866e-13, 6.5789e-12, 3.8461e-11, 1.3068e-12,\n        6.7149e-11, 3.2904e-13, 4.0608e-10, 4.1558e-10, 2.2082e-13, 6.2965e-13,\n        1.0744e-12, 1.0213e-10, 2.3717e-12, 3.8328e-10, 3.0557e-13, 8.2685e-14,\n        1.4735e-12, 8.5417e-11, 4.9975e-11, 4.2864e-10, 9.8710e-14, 9.7505e-11,\n        1.6780e-11, 5.0991e-11, 7.7607e-11, 6.1932e-13, 6.9259e-10, 1.2086e-11,\n        3.1655e-13, 2.6156e-13, 2.0879e-11, 1.5993e-12, 8.8447e-11, 1.8155e-12,\n        4.0419e-13, 1.8353e-11, 7.6432e-11, 5.6383e-14, 4.0643e-13, 5.3394e-11,\n        2.0169e-12, 2.2018e-10, 1.1724e-10, 3.2083e-13, 1.6446e-11, 5.5309e-11,\n        8.7811e-11, 4.9493e-13, 5.7162e-11, 2.6427e-11, 1.3699e-12, 2.7565e-13,\n        2.0374e-11, 2.9035e-12, 9.2018e-12, 9.7764e-13, 2.8218e-10, 3.0574e-11,\n        1.7677e-11, 3.0374e-11, 1.1025e-12, 2.1451e-13, 1.5720e-13, 2.9916e-10,\n        6.9204e-11, 1.2863e-13, 4.1627e-13, 3.6477e-11, 4.6098e-11, 5.6051e-12,\n        3.7433e-10, 6.5759e-11, 6.2500e-11, 3.7367e-11, 2.6422e-12, 1.4466e-11,\n        1.9570e-12, 5.9748e-13, 6.5757e-11, 3.7241e-12, 8.1128e-13, 1.0976e-11,\n        6.2023e-12, 5.1075e-12, 8.6955e-11, 2.6674e-12, 2.3889e-11, 1.8354e-12,\n        1.4582e-12, 4.4962e-13, 1.1981e-10, 3.6596e-10, 2.1257e-11, 2.9715e-11,\n        1.7058e-11, 7.0937e-12, 7.2567e-13, 1.0865e-10, 1.6336e-11, 2.6689e-12,\n        1.3121e-12, 1.1856e-10, 1.6245e-10, 7.0570e-10, 1.2064e-10, 9.3685e-11,\n        1.1651e-13, 1.1831e-11, 1.4146e-12, 4.5640e-11, 2.4888e-13, 1.6033e-12,\n        2.1660e-13, 5.2171e-13, 2.5442e-11, 1.5910e-11, 1.9689e-12, 1.0026e-10,\n        1.3317e-13, 2.4858e-11, 1.0564e-11, 2.2845e-11, 6.1807e-11, 5.3160e-11,\n        1.9585e-11, 3.4524e-12, 6.4207e-12, 2.9430e-12, 1.5249e-11, 2.5439e-10,\n        4.7530e-10, 6.1720e-13, 4.7553e-11, 1.4280e-12, 1.1619e-10, 2.8397e-11,\n        4.0722e-10, 3.7459e-11, 9.3349e-12, 1.7411e-13, 3.6703e-13, 3.0604e-12,\n        5.8326e-14, 1.9572e-13, 8.1943e-11, 3.0996e-11, 1.9210e-12, 7.0349e-11,\n        2.1392e-12, 8.0150e-11, 6.2985e-13, 1.2060e-12, 2.2169e-13, 2.1901e-11,\n        3.4773e-11, 6.9768e-13, 1.6684e-10, 1.2603e-12, 1.1050e-12, 7.9322e-11,\n        1.0409e-11, 1.7561e-10, 6.1450e-11, 1.8125e-10, 8.9527e-10, 1.1562e-11,\n        3.6966e-11, 2.7003e-12, 3.9292e-11, 2.5950e-12, 1.8140e-11, 2.9344e-12,\n        5.9005e-12, 7.3820e-11, 4.0592e-12, 3.0544e-11, 1.2265e-11, 1.7195e-12,\n        2.3634e-12, 2.3710e-11, 8.7511e-11, 1.7945e-10, 1.8561e-11, 7.0730e-10,\n        5.7221e-12, 3.0272e-12, 9.0464e-11, 1.3977e-13, 1.0091e-11, 6.5439e-13,\n        2.1898e-10, 3.5926e-12, 6.1521e-12, 5.4371e-11, 2.0907e-13, 1.4176e-11,\n        1.4838e-11, 1.0209e-12, 9.0849e-12, 3.4938e-12, 7.5282e-12, 1.1570e-11,\n        6.2248e-10, 1.0200e-11, 3.5696e-12, 5.6759e-13, 1.1370e-13, 4.3419e-11,\n        8.6906e-11, 3.1692e-12, 4.8001e-11, 1.7148e-11, 1.5649e-11, 9.0937e-13,\n        9.1404e-11, 3.0135e-11, 5.6213e-13, 4.1734e-12, 3.7229e-11, 3.3352e-13,\n        1.2568e-14, 8.3716e-12, 3.1609e-12, 3.1344e-11, 8.9128e-11, 2.9597e-11,\n        3.3653e-11, 4.4479e-12, 7.5172e-13, 5.9241e-12, 9.1192e-11, 2.1960e-12,\n        1.3834e-10, 8.0011e-13, 3.6026e-12, 2.2899e-12, 1.3200e-11, 6.8397e-12,\n        2.5723e-14, 1.7141e-11, 1.6052e-10, 1.4198e-11, 6.8464e-11, 1.0793e-10,\n        2.0097e-11, 2.6891e-12, 1.1832e-11, 1.6511e-11, 1.4877e-11, 2.5049e-12,\n        2.4522e-13, 5.2641e-13, 6.4116e-11, 1.5519e-10, 1.1797e-10, 4.9538e-13,\n        7.6142e-11, 5.0208e-12, 3.9792e-10, 6.9437e-10, 6.7296e-10, 2.7828e-10,\n        1.5106e-13, 1.9014e-11, 2.7014e-11, 3.3974e-13, 1.4980e-10, 2.3678e-12,\n        1.5879e-10, 6.7842e-13, 1.1946e-10, 6.7908e-13, 3.3521e-13, 5.5551e-11,\n        9.4287e-12, 3.6204e-11, 2.8827e-13, 2.8123e-10, 2.5896e-11, 1.8003e-11,\n        1.9026e-12, 5.0435e-11, 8.0590e-12, 1.4247e-12, 4.0747e-12, 1.7591e-13,\n        3.6115e-12, 4.1227e-12, 9.7698e-11, 2.9130e-12, 9.2309e-12, 3.2070e-11,\n        2.5315e-11, 4.4321e-12, 2.6244e-11, 2.2517e-10, 4.2046e-10, 2.8794e-11,\n        5.2101e-12, 1.5361e-12, 1.8159e-11, 1.6098e-11, 1.9109e-13, 1.5101e-10,\n        5.5930e-11, 7.8775e-11, 4.3083e-11, 1.1104e-12, 6.9746e-11, 1.4485e-10,\n        3.6349e-13, 2.8486e-12, 2.7268e-12, 8.1620e-11, 1.1402e-10, 1.0256e-13,\n        5.9649e-13, 7.8728e-13, 7.4710e-12, 1.0874e-12, 3.6546e-11, 1.0624e-10,\n        1.9542e-12, 9.8789e-13, 3.4563e-10, 4.7135e-11, 1.0077e-10, 1.0296e-10,\n        4.5026e-13, 3.3265e-11, 7.9467e-12, 5.8667e-11, 3.2182e-11, 1.9419e-13,\n        4.1640e-11, 2.4620e-10, 4.6222e-11, 4.7534e-12, 1.9054e-12, 4.6008e-11,\n        3.7270e-12, 5.3438e-11, 3.1375e-12, 1.9215e-13, 1.2515e-10, 1.2375e-10,\n        2.3434e-13, 9.7198e-13, 9.7659e-11, 7.8184e-11, 1.4854e-13, 6.2930e-11,\n        2.3164e-10, 5.0425e-11, 2.4143e-11, 2.0030e-10, 1.8976e-11, 1.0404e-11,\n        7.6260e-11, 3.1232e-11, 1.6437e-10, 6.1164e-12, 1.5933e-12, 1.5770e-10,\n        3.9976e-11, 2.6376e-13, 1.8861e-12, 1.9133e-10, 2.1945e-11, 3.2634e-10,\n        4.9990e-13, 2.4904e-13, 2.1705e-12, 4.1130e-13, 9.4989e-11, 7.3111e-14,\n        8.6983e-13, 9.5060e-11, 1.3733e-11, 1.7198e-14, 5.4123e-13, 1.0140e-10,\n        1.0447e-12, 2.4441e-13, 2.4285e-11, 9.4974e-13, 4.1384e-11, 7.1712e-11,\n        4.4984e-12, 1.4118e-12, 1.8198e-10, 4.3427e-12, 4.5472e-11, 3.4033e-12,\n        2.7087e-11, 1.2415e-10, 3.2777e-10, 1.1458e-11, 4.9848e-12, 6.4142e-12,\n        6.4384e-12, 1.2212e-12, 3.2317e-11, 4.7586e-11, 1.4738e-10, 5.8956e-13,\n        1.0200e-11, 1.8068e-11, 4.8432e-14, 2.6526e-12, 9.9522e-12, 2.4777e-13,\n        9.1599e-12, 1.0935e-10, 1.8719e-11, 5.7600e-11, 3.1864e-13, 6.1441e-11,\n        4.5519e-13, 1.5068e-12, 4.8985e-11, 5.2795e-12, 1.4660e-12, 1.0756e-10,\n        8.6052e-13, 5.2580e-11, 9.4410e-11, 4.6070e-13, 1.2068e-11, 1.0224e-11,\n        4.4494e-11, 3.1472e-11, 2.5360e-13, 3.6096e-12, 1.3063e-12, 3.8448e-11,\n        1.0541e-11, 2.8074e-13, 7.2280e-11, 5.4369e-11, 2.5007e-10, 1.1311e-13,\n        4.8195e-14, 1.2359e-12, 4.2924e-13, 1.9823e-10, 1.2834e-12, 3.9514e-11,\n        2.4277e-10, 4.3700e-11, 2.0501e-11, 3.7118e-11, 1.2771e-10, 1.1733e-11,\n        2.6079e-12, 4.1906e-13, 2.7443e-12, 8.4800e-13, 8.5405e-11, 1.1647e-10,\n        1.6765e-12, 4.5985e-14], device='cuda:0')"
      },
      "50": {
        "step": "tensor(11268.)",
        "exp_avg": "tensor([ 5.3538e-39,  1.5943e-38,  7.7234e-39,  2.8402e-39,  3.9969e-40,\n         1.5222e-38,  7.6526e-39,  1.4196e-39,  1.3800e-38,  1.5163e-37,\n         4.5678e-38,  8.6243e-40,  8.7165e-39, -8.3709e-40,  4.6967e-38,\n         6.8477e-39,  6.2847e-39,  6.2564e-39, -6.5298e-39,  5.8744e-40,\n         2.0662e-38,  7.2745e-39,  3.6467e-38, -1.1931e-39,  5.4192e-38,\n         3.9139e-40,  5.5406e-39, -1.1106e-39,  4.0188e-38,  4.3386e-38,\n         1.6927e-40,  8.3030e-40,  1.1557e-37,  1.2799e-38,  9.8479e-38,\n         4.4260e-38,  2.9375e-39,  3.6839e-40,  5.1802e-38,  3.1078e-39,\n         5.9310e-38,  4.1057e-39,  1.2350e-39, -1.0372e-39,  1.9176e-39,\n         2.1762e-38,  2.5720e-39,  1.0926e-38,  3.3945e-38,  1.5010e-39,\n         2.8215e-39,  1.4783e-40,  6.6698e-38, -3.5051e-40,  4.8368e-38,\n         1.1542e-38,  1.9738e-37,  2.1908e-37, -4.4685e-39,  4.1729e-39,\n         7.6247e-38,  2.0284e-37,  2.0098e-38,  1.5581e-37,  1.6686e-39,\n         1.5255e-39,  4.3911e-39,  2.7904e-38,  1.2795e-37,  1.9581e-38,\n         1.5771e-39,  6.3488e-38,  4.3810e-38,  8.8381e-39,  1.5095e-37,\n         7.9062e-39, -1.0815e-38,  1.0402e-38, -3.0254e-40,  1.7090e-39,\n        -8.9681e-40,  6.7669e-39,  1.8434e-38,  5.4040e-39,  2.8720e-39,\n         1.9751e-38,  1.0040e-37,  1.0242e-38, -5.0638e-39, -7.7666e-40,\n         9.1809e-40, -4.5431e-40,  6.4375e-38,  2.2461e-39,  5.7232e-38,\n         6.4637e-38,  8.8565e-38,  2.8084e-39,  9.8740e-40,  4.9998e-39,\n         6.4384e-40,  1.1992e-38,  8.1499e-38,  1.3441e-39, -9.7716e-40,\n         1.7466e-39,  1.2343e-37,  5.3967e-38,  3.5147e-38,  4.5130e-38,\n         1.9238e-40,  2.0299e-39, -4.3750e-40,  2.3019e-38, -3.5839e-39,\n         2.6186e-39, -3.3270e-39, -1.8131e-39, -1.9507e-39, -2.9906e-39,\n         3.3027e-38,  6.0070e-38,  4.0507e-38,  1.5173e-37, -1.9349e-39,\n         7.4239e-39,  3.0328e-39,  5.2519e-40, -1.1693e-39,  1.4808e-39,\n         2.2067e-39, -5.5235e-40,  3.6547e-39,  3.6269e-38,  2.0627e-38,\n         2.5264e-39,  1.1999e-37,  1.1355e-39, -1.1323e-39,  5.5562e-39,\n        -3.5676e-40,  2.0664e-38,  1.8265e-38, -2.8533e-39, -1.6912e-41,\n         5.6309e-39,  5.0287e-39,  4.4728e-38,  1.7484e-40,  5.2804e-39,\n         2.0967e-39,  1.9123e-37,  2.8511e-38,  1.1428e-38,  1.1410e-37,\n        -1.0402e-39,  7.7643e-40,  3.0973e-39,  1.6666e-39,  1.5067e-37,\n         4.6247e-39,  5.9963e-40,  5.5813e-39,  4.1227e-40,  2.3089e-38,\n         1.9439e-38, -2.0199e-40,  2.6216e-38, -8.2422e-40,  6.3904e-38,\n         1.9403e-40, -6.8924e-40,  1.2525e-37,  9.8522e-39,  1.1620e-38,\n         2.1991e-40,  8.7349e-38,  6.2642e-38,  8.0686e-40,  1.0818e-37,\n         2.2147e-38,  3.9938e-38, -1.7559e-40, -2.7138e-39,  1.5992e-37,\n        -6.4798e-40,  1.6806e-37,  5.9272e-38,  1.1207e-39,  3.6040e-39,\n         2.9691e-39,  2.7414e-39,  4.5501e-39,  8.4933e-40,  1.6253e-37,\n         1.0740e-37,  1.5274e-39,  3.4616e-39,  1.2563e-40,  2.5359e-38,\n         8.9397e-39,  5.7626e-39,  7.1866e-39,  1.5092e-38,  1.1704e-37,\n        -4.2316e-40,  1.3635e-37,  2.2598e-39,  1.2641e-39,  2.3233e-38,\n        -1.8070e-39,  1.5173e-37,  1.3452e-37,  1.4858e-38,  1.5071e-37,\n        -2.0347e-40,  4.7543e-38,  6.6269e-39,  4.8971e-40, -3.6356e-40,\n        -1.8115e-39,  2.8953e-39, -3.3926e-40,  4.0333e-38,  2.5442e-38,\n         7.4657e-38,  5.3886e-38,  4.3148e-39, -7.2773e-40,  9.8781e-38,\n         1.2415e-37,  1.8439e-38, -3.3546e-39,  2.1714e-37,  4.7696e-39,\n         1.7478e-39,  4.3219e-38,  8.5500e-39,  1.8647e-38, -2.4181e-39,\n         1.0626e-37,  1.1060e-38,  1.9755e-40, -1.1306e-39,  2.1967e-38,\n         1.1816e-38,  7.5053e-38,  2.8923e-39, -2.9042e-39,  3.4165e-39,\n         1.5013e-40,  6.0069e-39,  2.8147e-39,  1.8201e-39, -1.4024e-39,\n        -4.5647e-40,  6.0954e-40,  2.6563e-38,  1.3060e-37,  2.3012e-38,\n         7.2275e-38,  3.5993e-38,  7.9137e-40,  2.5965e-39,  2.0798e-38,\n         9.2826e-41,  2.0359e-38, -3.6375e-39,  5.1999e-38, -4.4737e-39,\n        -1.2429e-39,  5.6739e-38,  1.9726e-39, -5.1427e-39,  1.0462e-38,\n         2.8285e-41, -1.3091e-39,  3.4654e-39,  1.4774e-39,  5.2456e-38,\n         2.3935e-39, -1.9948e-39,  7.7479e-38,  9.7539e-39,  9.5756e-39,\n         1.8485e-39,  1.4286e-40, -1.8136e-39,  1.6084e-38,  4.8639e-38,\n         2.3827e-38,  1.1705e-38, -1.5032e-39,  1.5073e-38,  4.0535e-41,\n        -2.6849e-39,  7.6894e-38,  1.0487e-37,  1.1331e-37,  9.0305e-40,\n         3.3986e-39, -1.5971e-40,  1.3627e-37,  1.4036e-37, -2.2357e-40,\n         1.7235e-39,  3.1482e-38,  8.9063e-38,  5.8291e-38,  1.3823e-37,\n         1.1698e-37, -2.3625e-39,  1.2139e-39,  6.5903e-39,  3.4284e-38,\n        -2.3181e-39,  1.6816e-37,  6.5758e-39,  2.5188e-38,  3.1028e-39,\n         3.3065e-38,  2.9416e-39, -1.9441e-40,  4.1692e-38, -8.7423e-40,\n         1.5656e-37,  1.5332e-40,  3.0937e-39, -1.5237e-39, -4.8667e-42,\n         5.3406e-39,  2.7016e-38,  5.5057e-38,  4.1016e-38,  6.6341e-39,\n         2.0989e-39,  5.4875e-40, -2.3890e-39,  1.1895e-37,  2.8467e-39,\n         9.8436e-40, -1.8020e-39,  3.2622e-42,  1.1075e-39,  3.0465e-38,\n         9.9065e-38,  5.1942e-39,  1.0451e-37, -2.5141e-39,  1.2471e-38,\n         5.9507e-38,  2.8682e-39, -1.5196e-39,  1.6764e-37,  1.5716e-37,\n        -1.3494e-39,  5.0084e-40, -5.1034e-40,  2.5225e-38, -4.4783e-40,\n        -2.6230e-39,  5.8711e-40,  9.0665e-39,  1.5759e-37,  1.0851e-37,\n         4.8461e-39,  1.6124e-38, -7.0527e-40,  1.1473e-37,  4.4284e-40,\n        -2.9072e-39,  1.3230e-38,  1.3937e-39,  1.8784e-39,  1.8256e-38,\n         6.2366e-39,  1.7990e-37,  1.1220e-37,  7.6194e-41, -1.1687e-39,\n         4.6565e-38,  7.5119e-38, -2.1018e-39,  5.4804e-39,  5.9110e-38,\n         3.9190e-40,  2.3577e-38,  1.9389e-38,  9.1512e-40,  1.4017e-40,\n         5.1755e-40,  6.9551e-38, -4.9126e-39,  7.4377e-41, -2.8132e-39,\n         4.0289e-39,  2.4858e-39, -5.7096e-40, -4.3394e-40,  4.4300e-38,\n         4.9260e-39, -1.2912e-39, -1.0899e-39,  5.8489e-39,  4.5723e-38,\n         1.0527e-37,  5.8764e-38, -8.0023e-40,  7.8411e-38,  6.5398e-39,\n         2.2007e-37,  5.3859e-38, -4.5513e-40,  2.8230e-40,  8.5297e-38,\n         1.1298e-39,  2.3972e-39,  1.9917e-39, -8.6041e-40,  1.2623e-37,\n         1.4932e-38,  2.5808e-39,  2.0185e-39,  1.2152e-39,  8.8083e-38,\n         2.1418e-38,  1.3854e-39, -1.2193e-39,  2.0811e-38,  4.1841e-39,\n         4.1024e-39,  1.4478e-38,  3.6562e-39,  5.7229e-39,  1.6068e-37,\n         2.3148e-38,  6.8381e-38,  7.3255e-38,  2.5754e-39,  5.9092e-40,\n         1.3879e-39,  7.5365e-39,  1.2712e-39,  2.7793e-39, -1.4485e-39,\n         3.2767e-38,  2.3248e-37,  1.6081e-38,  2.4330e-39, -9.4666e-40,\n        -5.9972e-40,  3.3496e-40, -1.7762e-40,  4.9980e-39,  2.3687e-38,\n         4.2828e-39,  1.1290e-38, -5.9185e-40,  1.1649e-38,  1.7513e-39,\n         2.1480e-40,  1.1627e-38,  1.4617e-38,  6.7245e-38,  4.2425e-38,\n         3.3582e-38,  1.1170e-39,  1.1010e-37,  2.9671e-39,  2.3451e-39,\n         3.7670e-38,  6.7738e-38, -2.6769e-39,  1.8235e-37, -3.7624e-39,\n         1.3993e-37, -2.7569e-39,  1.5674e-38,  4.2978e-38, -1.1577e-39,\n         3.7726e-38, -5.4240e-40, -1.8189e-40, -7.6344e-40, -1.2299e-40,\n        -6.2360e-40,  1.1173e-38,  4.6317e-40,  1.2314e-37,  7.5411e-38,\n         1.6720e-37,  6.5657e-39,  1.8468e-40,  1.8330e-39,  3.5652e-39,\n         1.3253e-37, -1.2878e-39, -6.4595e-40,  1.5608e-40,  4.7459e-38,\n        -1.0548e-39,  1.0932e-38,  1.4048e-38,  1.3614e-40,  8.4311e-41,\n         1.7447e-39,  8.6021e-40,  1.3569e-39, -1.6918e-39,  5.3125e-39,\n         1.2171e-38,  4.8246e-39], device='cuda:0')",
        "exp_avg_sq": "tensor([4.8831e-16, 9.4440e-16, 1.6280e-17, 1.1824e-17, 1.5347e-16, 2.9081e-15,\n        3.3972e-16, 5.1061e-16, 2.4598e-17, 6.4515e-14, 1.5562e-15, 9.6011e-17,\n        8.8417e-16, 3.3715e-16, 1.0886e-14, 6.0447e-16, 5.4172e-15, 1.5923e-15,\n        9.1740e-19, 2.0129e-17, 8.4387e-17, 3.0868e-16, 1.1995e-14, 6.1353e-16,\n        9.9993e-15, 6.7863e-18, 3.1619e-16, 1.5261e-17, 1.1955e-15, 6.0759e-15,\n        1.2666e-17, 1.9598e-19, 1.3228e-14, 2.5615e-15, 9.4464e-14, 1.8624e-15,\n        2.2150e-18, 1.7960e-15, 4.7183e-15, 3.1343e-18, 5.4631e-15, 1.0768e-18,\n        7.5609e-21, 4.0353e-21, 1.0104e-16, 2.7370e-16, 1.5078e-16, 8.9636e-16,\n        2.4548e-15, 2.9525e-15, 3.7599e-17, 4.1189e-19, 2.9239e-14, 4.5060e-18,\n        2.6061e-14, 8.3955e-16, 4.0601e-13, 3.4323e-13, 9.4840e-17, 4.9127e-16,\n        8.1555e-15, 1.4667e-13, 4.6744e-15, 1.7096e-13, 5.1561e-17, 6.8575e-18,\n        3.3897e-18, 1.1095e-15, 5.7635e-14, 8.5324e-14, 2.0075e-17, 1.1263e-14,\n        4.4291e-15, 5.9219e-15, 1.0767e-13, 6.9621e-19, 1.7885e-13, 1.0301e-16,\n        6.1452e-17, 7.1918e-18, 2.5532e-17, 3.9482e-17, 1.2418e-14, 4.4158e-16,\n        1.1306e-16, 6.1303e-17, 3.6372e-14, 1.4638e-16, 8.3096e-18, 7.5871e-17,\n        1.5147e-17, 2.1134e-14, 1.5533e-14, 3.5612e-17, 8.8164e-15, 6.7568e-15,\n        2.5642e-14, 3.0135e-17, 2.7277e-15, 5.2543e-17, 3.8149e-20, 6.7834e-16,\n        7.3231e-15, 5.5599e-17, 2.6046e-17, 5.6914e-20, 9.6856e-14, 4.1825e-14,\n        2.4039e-16, 1.0894e-14, 2.4351e-16, 1.3545e-17, 1.1282e-18, 3.5053e-14,\n        1.1285e-15, 6.5204e-17, 3.2312e-18, 2.5243e-16, 1.1342e-15, 6.2350e-18,\n        6.7871e-14, 7.0049e-15, 2.2866e-14, 8.0541e-14, 1.6223e-17, 2.5262e-16,\n        2.2350e-18, 1.7346e-16, 5.1568e-16, 4.1406e-19, 6.9043e-19, 8.1284e-16,\n        7.1427e-16, 1.0450e-14, 7.9293e-16, 4.5730e-16, 2.3879e-14, 1.6800e-17,\n        1.1883e-18, 3.9929e-19, 4.6677e-15, 7.3034e-14, 1.9182e-15, 2.9783e-17,\n        1.3839e-16, 7.5419e-17, 6.2459e-16, 2.0353e-14, 1.4718e-18, 6.2294e-18,\n        2.2126e-18, 1.3301e-13, 3.0862e-14, 1.4478e-13, 7.3001e-14, 2.6210e-15,\n        8.5580e-18, 5.1138e-18, 1.6602e-17, 1.1357e-13, 4.6337e-16, 6.9667e-18,\n        1.0809e-19, 1.7427e-17, 6.6752e-16, 2.0988e-16, 4.2974e-16, 1.2220e-14,\n        6.9909e-19, 4.7227e-15, 1.2165e-17, 3.9074e-18, 8.7506e-14, 4.3762e-17,\n        1.3346e-16, 1.9217e-16, 1.9034e-14, 4.6771e-15, 2.6016e-18, 1.0264e-13,\n        8.6444e-14, 4.6238e-15, 3.6526e-17, 3.2296e-17, 9.6610e-14, 3.6325e-17,\n        3.7105e-13, 1.5090e-14, 8.8799e-16, 5.2208e-17, 4.0118e-19, 2.3203e-16,\n        7.0614e-17, 7.8567e-17, 1.0217e-13, 1.0184e-14, 1.9327e-18, 8.9118e-16,\n        4.6262e-16, 6.9825e-16, 5.9529e-16, 3.6609e-16, 3.5711e-16, 1.3271e-16,\n        2.0885e-14, 5.4882e-18, 1.0776e-13, 2.3787e-19, 2.4500e-16, 7.6443e-16,\n        1.5079e-17, 1.3074e-13, 3.8734e-14, 1.7595e-14, 1.5462e-13, 9.9460e-19,\n        1.0930e-14, 2.0124e-16, 7.1008e-17, 7.0266e-20, 2.0778e-16, 2.8549e-16,\n        1.9842e-18, 3.4391e-15, 4.2574e-17, 4.6103e-14, 2.8609e-15, 9.7670e-17,\n        4.2404e-18, 4.3753e-14, 7.0132e-14, 1.4150e-14, 1.4580e-16, 5.5911e-13,\n        2.3645e-16, 5.9767e-17, 3.1702e-16, 1.5581e-16, 1.6690e-16, 5.2880e-19,\n        7.6607e-14, 7.8345e-16, 7.4349e-16, 3.0220e-16, 1.0479e-15, 2.4047e-16,\n        2.2692e-14, 6.4371e-20, 7.7115e-16, 9.2285e-17, 7.8894e-16, 5.4273e-17,\n        4.5929e-14, 6.3584e-16, 7.3456e-19, 1.2254e-16, 1.5665e-17, 2.5793e-15,\n        6.7027e-14, 6.0197e-16, 4.7761e-14, 1.5221e-15, 1.2457e-18, 2.0255e-18,\n        2.2895e-15, 2.4483e-17, 3.4153e-18, 9.9239e-17, 8.9330e-15, 3.1439e-18,\n        2.9915e-17, 1.8093e-14, 2.8195e-18, 1.7994e-17, 2.1924e-15, 2.9196e-18,\n        1.5571e-15, 2.0881e-15, 9.3711e-19, 8.8338e-15, 5.8446e-16, 9.2687e-17,\n        8.5036e-15, 5.8483e-16, 7.4346e-16, 4.4308e-16, 9.0599e-18, 5.1614e-19,\n        1.4343e-15, 1.2079e-15, 3.7204e-14, 8.2624e-18, 2.1516e-16, 3.3837e-15,\n        3.5899e-18, 1.3637e-17, 2.3568e-14, 3.2375e-14, 1.3698e-14, 6.6174e-19,\n        1.8483e-19, 3.4414e-18, 4.6338e-14, 1.1786e-13, 1.1930e-15, 1.5367e-17,\n        1.1050e-14, 2.4329e-14, 9.0501e-14, 3.7575e-13, 2.9902e-13, 1.0716e-14,\n        3.5103e-17, 1.9643e-17, 3.7273e-15, 3.1561e-18, 2.0080e-13, 1.8190e-16,\n        1.7475e-14, 4.3320e-20, 1.0895e-15, 1.1031e-16, 5.1092e-17, 3.8130e-14,\n        5.9569e-18, 7.9379e-14, 2.0103e-17, 4.8644e-15, 8.7672e-17, 3.6193e-18,\n        7.1975e-19, 2.7877e-15, 1.1329e-14, 2.2149e-15, 4.3308e-17, 2.4707e-17,\n        7.8191e-20, 1.9650e-17, 6.0192e-14, 4.0028e-18, 1.1994e-17, 3.1303e-16,\n        6.3781e-18, 6.8897e-17, 1.0778e-14, 1.1439e-13, 4.0972e-14, 7.2396e-14,\n        2.0596e-16, 1.5725e-15, 3.0463e-15, 6.6363e-18, 1.3506e-17, 1.0893e-13,\n        6.0111e-14, 7.0843e-16, 1.8939e-17, 1.2083e-18, 5.4483e-16, 3.3276e-16,\n        1.0931e-18, 2.5997e-17, 6.9294e-16, 5.0050e-14, 7.0638e-14, 3.0796e-16,\n        8.7958e-16, 1.2856e-16, 1.8291e-14, 4.2520e-19, 8.4848e-16, 1.2640e-14,\n        4.8339e-17, 6.4252e-19, 9.7630e-14, 4.2128e-15, 9.5036e-14, 3.6784e-14,\n        7.6595e-17, 7.8971e-16, 3.5520e-15, 3.2342e-14, 1.2768e-16, 5.4402e-17,\n        4.6874e-15, 1.3760e-14, 5.8022e-15, 1.4096e-16, 3.4948e-20, 2.4482e-15,\n        3.0148e-19, 7.4933e-15, 6.7734e-17, 2.6331e-17, 2.2612e-15, 4.8942e-15,\n        1.6995e-16, 5.9468e-17, 1.1463e-15, 2.7430e-14, 6.6434e-17, 1.2897e-15,\n        7.1719e-15, 8.5457e-16, 1.3003e-15, 4.7494e-14, 1.3001e-14, 3.2695e-16,\n        1.6768e-14, 3.1655e-17, 1.8207e-13, 8.7932e-15, 2.3849e-19, 6.1181e-15,\n        1.4270e-14, 1.7006e-19, 1.4643e-20, 1.1523e-14, 5.6553e-17, 1.4585e-13,\n        2.2030e-15, 4.5287e-18, 3.1361e-16, 3.3543e-18, 4.7983e-14, 3.3061e-15,\n        6.4441e-17, 1.7613e-15, 4.0265e-15, 1.2451e-16, 9.9637e-20, 7.6248e-15,\n        3.1523e-18, 7.4263e-18, 5.4375e-14, 2.0227e-16, 1.3815e-14, 4.9239e-14,\n        1.1063e-17, 2.9875e-19, 1.3902e-14, 2.6687e-16, 1.7969e-15, 6.4124e-16,\n        2.8602e-16, 1.3078e-14, 2.7782e-13, 3.0901e-19, 1.1559e-15, 9.8295e-16,\n        8.9615e-16, 2.0471e-17, 2.2081e-16, 3.5871e-15, 3.9011e-14, 3.9210e-16,\n        3.3665e-15, 4.0231e-17, 4.8923e-18, 7.8277e-18, 2.8530e-18, 2.0253e-19,\n        3.5380e-18, 7.0949e-15, 3.7574e-15, 7.7628e-16, 1.9625e-18, 7.9491e-14,\n        1.5825e-16, 1.7056e-17, 1.6786e-14, 2.4451e-14, 7.5690e-18, 1.2481e-13,\n        1.3246e-18, 2.9017e-14, 1.7142e-15, 9.5837e-16, 1.3109e-15, 6.8777e-19,\n        4.4290e-15, 1.6329e-17, 6.0236e-18, 1.0492e-17, 1.6383e-17, 2.6277e-16,\n        7.1787e-17, 4.8000e-17, 8.8347e-14, 1.2986e-14, 6.1118e-14, 2.6741e-16,\n        2.4659e-17, 1.5091e-19, 6.9997e-18, 1.7256e-13, 4.8764e-18, 3.8855e-16,\n        7.0428e-15, 2.2266e-15, 8.0123e-17, 2.0405e-17, 1.4095e-14, 6.7283e-18,\n        4.8317e-16, 6.7464e-17, 3.6181e-16, 1.1060e-16, 1.1599e-15, 2.9409e-15,\n        2.9447e-19, 9.5869e-17], device='cuda:0')"
      },
      "51": {
        "step": "tensor(11268.)",
        "exp_avg": "tensor([-4.1581e-39, -1.1354e-38,  2.6611e-38, -2.2629e-39, -1.4279e-39,\n         2.3939e-38, -6.1774e-39,  1.8737e-38,  5.5798e-38,  1.0294e-37,\n         6.4644e-38,  4.1598e-39, -6.4343e-39,  5.6188e-39,  4.7011e-38,\n        -5.3755e-39,  2.5062e-38,  1.0104e-38,  2.2146e-38, -2.0328e-40,\n         6.1209e-38, -5.6531e-39,  4.0868e-38,  4.6687e-39,  4.5891e-38,\n         5.4569e-40, -4.5423e-39,  3.0304e-39,  5.5030e-38,  4.2048e-38,\n        -4.3152e-40,  3.4601e-41,  8.4157e-38, -7.6665e-39,  5.8773e-38,\n         5.8015e-38, -1.9764e-39, -4.2480e-39,  7.8998e-38, -2.7880e-39,\n         5.9989e-38, -3.4398e-39, -9.5459e-40,  3.1231e-39, -9.8704e-40,\n         5.8326e-38, -1.4756e-39, -7.1557e-39,  5.1724e-38,  2.9275e-38,\n        -2.3727e-39,  1.0937e-40,  6.7488e-38,  2.3447e-39,  5.5520e-38,\n        -7.8094e-39,  9.5596e-38,  1.0976e-37,  1.8183e-38, -2.7666e-39,\n         6.1316e-38,  1.0367e-37, -1.1069e-38,  9.5706e-38, -1.3544e-39,\n        -9.1435e-40, -3.6735e-39,  5.3515e-38,  6.9716e-38,  2.9982e-38,\n        -7.7850e-40,  6.4465e-38,  5.5740e-38,  3.3958e-38,  8.5898e-38,\n        -6.4480e-39, -1.2730e-38,  4.7039e-38,  3.1730e-40, -1.4763e-39,\n         6.5546e-40, -5.3000e-39,  4.5958e-38, -4.4533e-39, -5.0904e-40,\n         5.9675e-38,  8.0915e-38, -8.2171e-39,  2.6666e-38,  1.9069e-39,\n         1.9840e-38, -7.9994e-40,  6.3749e-38, -1.9522e-39,  4.6266e-38,\n         7.6901e-38,  6.1128e-38, -2.3246e-39,  1.1895e-38,  3.0083e-38,\n        -2.0515e-40, -9.6050e-39,  6.6569e-38, -9.9801e-40,  2.5661e-39,\n        -1.4575e-39,  8.7626e-38,  2.9018e-38,  7.6806e-38,  5.2939e-38,\n        -4.1108e-40, -1.6845e-39,  9.9741e-40,  2.4868e-38,  1.8026e-38,\n        -2.1041e-39,  1.1229e-38,  5.9272e-39,  1.1052e-38,  4.0582e-38,\n         6.2845e-38,  8.0882e-38,  3.2626e-38,  8.1778e-38,  1.4740e-39,\n         2.0747e-38, -2.6721e-39,  1.7948e-39,  4.0476e-39, -1.1189e-39,\n        -1.7794e-39,  3.1553e-40, -2.9596e-39,  4.8933e-38,  4.8524e-38,\n        -1.8964e-39,  7.4775e-38, -6.4599e-40,  3.2253e-39, -4.5667e-39,\n        -3.4420e-39,  3.2740e-38,  4.0658e-38,  7.0582e-39, -2.0678e-40,\n         3.2348e-38, -3.8537e-39,  5.7997e-38,  2.0092e-40,  3.8022e-38,\n        -1.7604e-39,  1.1140e-37,  5.3365e-38,  1.9947e-38,  6.8327e-38,\n         1.3269e-38,  5.4894e-40,  2.4201e-38, -1.1560e-39,  7.4579e-38,\n        -3.3249e-39,  3.2502e-38, -5.0843e-39, -3.5794e-40,  4.6504e-38,\n         3.9047e-38,  8.8322e-40,  3.6700e-38,  2.1432e-39,  7.5036e-38,\n        -1.5846e-40,  2.6927e-39,  6.8513e-38,  3.5731e-38,  4.1535e-38,\n         1.4502e-38,  7.5090e-38,  6.8556e-38, -7.6442e-40,  6.7964e-38,\n         2.9721e-38,  4.4840e-38,  2.0202e-39,  1.0203e-38,  9.5120e-38,\n         5.5256e-39,  8.3094e-38,  5.9333e-38, -9.4131e-40, -3.1144e-39,\n        -2.6730e-39,  2.8225e-38, -3.5962e-39, -1.7042e-40,  8.8667e-38,\n         9.9516e-38, -1.2734e-39,  1.8811e-38, -6.3804e-40,  2.4836e-38,\n        -6.8070e-39, -4.5681e-39, -5.9956e-39,  4.7896e-38,  8.4605e-38,\n         2.4664e-39,  7.8954e-38, -2.0304e-39,  1.3859e-40,  4.5707e-38,\n        -1.1541e-38,  9.3805e-38,  1.0104e-37,  2.4377e-38,  1.2913e-37,\n         1.0187e-39,  4.4092e-38, -4.9560e-39,  5.8334e-40,  2.3791e-39,\n         6.8062e-39, -2.1907e-39,  1.0399e-39,  3.4524e-38,  6.9824e-38,\n         4.0678e-38,  5.7174e-38, -3.6456e-39,  3.2551e-38,  5.7436e-38,\n         7.5219e-38,  2.0067e-38,  1.1204e-38,  1.0772e-37,  2.7220e-38,\n        -1.3157e-39,  9.6514e-38, -6.7158e-39,  4.7011e-38,  7.2495e-39,\n         7.8144e-38, -7.6088e-39, -7.5447e-41,  4.2503e-39, -1.3551e-38,\n         3.4760e-38,  5.2794e-38, -2.4510e-39,  2.2442e-39, -2.7664e-39,\n        -1.0605e-40,  3.0772e-38,  2.0610e-38,  1.5508e-38,  5.8020e-39,\n         5.0168e-40,  1.2658e-40,  3.4260e-38,  9.0890e-38,  3.8979e-38,\n         5.5281e-38,  6.1383e-38, -2.9010e-40, -2.1142e-39,  4.2317e-38,\n         9.2371e-40,  6.8715e-38,  1.9323e-38,  5.1910e-38,  1.6808e-38,\n         3.2997e-39,  5.4195e-38, -1.1996e-39,  1.4989e-38,  2.1876e-38,\n         6.6325e-40,  1.2663e-38,  1.5962e-38, -1.2408e-39,  5.4281e-38,\n        -1.9752e-38,  1.3599e-39,  8.9699e-38, -7.4416e-39, -6.7843e-39,\n         1.4118e-38,  7.5279e-41,  6.0805e-39, -1.2092e-38,  6.5524e-38,\n         2.8136e-38,  6.2051e-38,  4.9776e-39,  2.5075e-38,  1.2763e-40,\n         2.5667e-38,  7.0137e-38,  8.1934e-38,  8.8966e-38, -8.0087e-40,\n        -3.1290e-39,  1.6755e-40,  8.7963e-38,  8.2364e-38,  7.6867e-40,\n        -1.0691e-39,  3.7844e-38,  6.5826e-38,  4.2949e-38,  7.9945e-38,\n         6.4195e-38, -1.4025e-38, -7.9129e-40,  2.2318e-38,  3.9765e-38,\n         1.8472e-38,  8.5347e-38, -5.1538e-39,  1.9825e-38, -2.6405e-39,\n         6.7363e-38,  2.9668e-38,  1.6008e-40,  4.5726e-38,  2.6766e-39,\n         8.1340e-38,  1.4789e-40, -4.2907e-38,  3.4928e-39,  2.8585e-40,\n        -4.6354e-39,  6.7324e-38,  4.6177e-38,  5.1436e-38,  3.8940e-38,\n        -6.4698e-40, -2.5263e-40,  7.3524e-39,  8.3663e-38,  2.7452e-38,\n        -8.5769e-40,  6.3178e-39,  3.2264e-40,  2.5992e-38,  2.3722e-38,\n         5.7026e-38,  2.9153e-38,  5.2388e-38,  1.7326e-38, -8.1186e-39,\n         6.5032e-38,  2.8869e-38,  4.0894e-38,  1.0665e-37,  8.4739e-38,\n         4.9083e-39, -6.0395e-41,  2.1451e-39,  6.7707e-38,  1.7098e-39,\n         7.8780e-39,  4.7834e-38, -6.6328e-39,  1.1313e-37,  8.6206e-38,\n        -3.8401e-39, -1.0719e-38,  1.3120e-39,  7.8284e-38,  8.5286e-41,\n         1.8263e-38,  3.6767e-38, -8.4869e-40, -1.1793e-39,  1.8282e-38,\n         4.2148e-38,  1.1813e-37,  1.0538e-37,  9.6850e-40,  5.7437e-39,\n         5.3356e-38,  6.1062e-38,  5.5916e-39, -4.1201e-39,  6.2997e-38,\n         7.7412e-39,  4.3358e-38,  4.2658e-38, -5.9616e-40,  1.0223e-38,\n         1.7111e-40,  9.0921e-38,  3.5867e-39, -3.2885e-40,  1.3550e-38,\n         1.9649e-38, -2.0662e-39,  1.6896e-39,  2.3322e-39,  4.6213e-38,\n        -2.9650e-39,  1.0145e-38,  7.0454e-39,  1.1911e-38,  4.9822e-38,\n         9.9859e-38,  4.4371e-38,  2.7770e-38,  6.3879e-38,  4.4257e-38,\n         1.1165e-37,  4.9579e-38,  2.1252e-39,  1.3312e-38,  6.6688e-38,\n        -5.0430e-40, -1.9744e-39,  1.3703e-38,  3.0274e-39,  8.2918e-38,\n        -8.3798e-39, -2.2286e-39, -1.1585e-39,  2.5786e-38,  5.4425e-38,\n         4.2425e-38, -7.8414e-40,  5.7800e-39,  4.7701e-38, -3.3017e-39,\n        -3.6749e-39,  3.2342e-38, -3.2172e-39,  4.1284e-38,  9.2582e-38,\n         4.2463e-38,  9.0871e-38,  6.3788e-38, -1.9938e-39,  2.5953e-40,\n         1.1254e-38, -5.7667e-39,  2.0533e-38, -1.3673e-39,  7.5335e-39,\n         3.4846e-38,  1.2700e-37,  5.2470e-38, -1.6712e-39,  6.7985e-40,\n         7.0283e-40,  3.2357e-40,  3.1406e-39,  2.4341e-38,  3.2455e-38,\n        -3.5147e-39,  3.2834e-38,  3.3049e-40, -9.1566e-39, -1.0743e-39,\n         1.0985e-40,  4.4238e-38,  5.6575e-38,  7.9535e-38,  4.5218e-38,\n         4.7187e-38, -5.1973e-40,  5.6523e-38, -1.4132e-39, -1.7759e-39,\n         5.4759e-38,  4.4468e-38,  1.8222e-38,  9.2333e-38,  1.4883e-38,\n         1.0547e-37,  1.5948e-38, -1.1116e-38,  5.3055e-38,  3.4582e-39,\n         4.7154e-38,  1.8389e-39,  1.1223e-39,  4.2823e-38,  3.3059e-40,\n         2.0833e-39,  4.2213e-38,  8.8765e-40,  6.9094e-38,  8.2949e-38,\n         1.3780e-37, -5.1668e-39, -1.4942e-40, -1.5599e-39, -3.2450e-39,\n         7.7664e-38,  2.0545e-38,  5.1596e-39, -5.4866e-40,  6.3389e-38,\n         3.0109e-39,  4.6231e-38,  4.4909e-38,  1.9395e-40, -1.4071e-40,\n        -1.0077e-39, -2.0055e-40, -9.8525e-40, -6.7302e-39,  1.6155e-38,\n         4.9604e-38, -3.3562e-39], device='cuda:0')",
        "exp_avg_sq": "tensor([2.4916e-16, 3.7171e-16, 3.2672e-15, 6.6332e-18, 7.0269e-17, 2.1381e-15,\n        1.4958e-16, 1.3878e-15, 1.7165e-14, 6.8244e-14, 9.7549e-15, 2.8740e-15,\n        3.2423e-16, 1.5875e-15, 1.7624e-14, 2.6349e-16, 7.8936e-15, 9.6308e-15,\n        1.6576e-16, 4.1124e-18, 4.4606e-15, 1.4362e-16, 1.8291e-14, 7.0877e-15,\n        1.0992e-14, 3.1836e-18, 1.5343e-16, 2.6742e-15, 6.5512e-15, 2.6919e-15,\n        5.2272e-15, 2.5635e-16, 7.6155e-15, 9.7618e-16, 5.2349e-14, 6.2833e-15,\n        2.2431e-16, 9.4997e-15, 4.8841e-14, 1.9045e-16, 4.1601e-15, 9.8688e-17,\n        2.5587e-18, 3.4932e-17, 4.2235e-17, 1.7562e-14, 7.1553e-17, 4.6191e-16,\n        1.3930e-14, 1.0486e-14, 3.7463e-18, 1.0977e-16, 1.5047e-14, 4.7792e-16,\n        1.9630e-14, 4.0912e-16, 1.4447e-13, 1.4978e-13, 1.3404e-16, 2.3699e-16,\n        9.4153e-15, 6.0164e-14, 2.0827e-15, 1.2144e-13, 5.8005e-18, 1.4229e-18,\n        2.2346e-16, 2.2496e-14, 3.1200e-14, 5.3501e-14, 4.4098e-18, 2.7256e-14,\n        2.8618e-15, 1.7306e-14, 5.2131e-14, 1.0890e-16, 8.1636e-14, 7.4344e-15,\n        9.3658e-17, 1.2948e-18, 1.9390e-15, 1.8209e-17, 1.5919e-14, 2.3231e-16,\n        2.6495e-17, 8.6367e-15, 4.1041e-14, 5.4409e-17, 2.6254e-16, 6.5671e-15,\n        1.0371e-15, 1.9138e-14, 3.4860e-14, 6.1507e-18, 5.3753e-15, 2.6928e-14,\n        1.3766e-14, 2.7458e-18, 6.5175e-15, 6.8237e-15, 7.2406e-18, 3.2141e-16,\n        2.8400e-15, 2.6367e-17, 2.8829e-16, 2.0761e-17, 9.8845e-14, 1.9211e-14,\n        9.5924e-15, 5.6636e-15, 9.5809e-17, 6.0017e-18, 7.3383e-19, 2.1494e-14,\n        1.3607e-14, 1.7700e-17, 3.2288e-16, 7.8257e-15, 4.4985e-15, 3.7872e-15,\n        7.4894e-14, 2.9881e-14, 2.0764e-14, 3.5928e-14, 6.9380e-18, 6.8248e-16,\n        2.8455e-16, 7.1731e-17, 8.0580e-15, 2.1745e-16, 2.4715e-18, 3.3606e-16,\n        2.6849e-16, 8.0405e-15, 2.4547e-14, 1.9063e-16, 1.1845e-14, 8.5006e-18,\n        1.0582e-16, 7.1614e-17, 1.0622e-14, 5.4608e-14, 1.1862e-14, 7.6700e-15,\n        9.8089e-16, 2.4911e-15, 2.6131e-16, 1.2323e-14, 2.3900e-15, 2.1894e-15,\n        1.7918e-16, 6.6398e-14, 2.4946e-14, 1.0969e-13, 5.3967e-14, 1.1100e-14,\n        2.8141e-17, 3.4816e-15, 7.9632e-18, 4.1171e-14, 2.1894e-16, 9.8426e-16,\n        4.8541e-17, 4.9812e-18, 1.1385e-14, 6.4962e-15, 2.0149e-16, 7.5711e-15,\n        3.2190e-19, 1.7123e-14, 1.3898e-15, 9.6632e-16, 4.4032e-14, 1.2863e-14,\n        3.7829e-15, 1.0047e-15, 1.3801e-14, 5.4809e-15, 1.2347e-15, 7.1172e-14,\n        5.5106e-14, 5.4815e-15, 6.0750e-15, 1.7601e-15, 5.9803e-14, 1.0647e-15,\n        1.4269e-13, 2.5388e-14, 3.8101e-16, 1.1419e-17, 1.9017e-17, 1.1899e-15,\n        6.7047e-18, 3.8599e-17, 4.9777e-14, 2.1336e-14, 3.6432e-16, 2.8128e-15,\n        2.1583e-16, 3.7091e-16, 2.5025e-16, 1.8979e-16, 1.7331e-16, 2.2760e-16,\n        2.1654e-14, 1.1291e-16, 5.6085e-14, 2.1475e-17, 1.2666e-16, 2.2548e-14,\n        2.9981e-15, 7.9514e-14, 3.8207e-14, 1.4245e-14, 2.3599e-13, 2.1950e-15,\n        1.7984e-14, 8.2893e-17, 7.6013e-15, 6.5072e-17, 3.8307e-15, 1.1657e-16,\n        1.7822e-16, 1.7725e-15, 3.5907e-15, 2.2335e-14, 6.7686e-15, 4.8439e-17,\n        1.1030e-15, 2.5386e-14, 4.5076e-14, 8.4744e-15, 3.2624e-15, 1.9935e-13,\n        8.8039e-16, 2.6164e-17, 2.8180e-14, 2.5350e-17, 6.0695e-15, 4.3385e-16,\n        7.9615e-14, 3.0185e-16, 3.3326e-16, 2.8508e-15, 5.1358e-16, 6.5586e-16,\n        1.9195e-14, 7.5440e-17, 3.5048e-16, 4.4258e-17, 3.3198e-16, 2.1210e-15,\n        9.6173e-14, 1.5302e-15, 1.2158e-16, 5.8506e-17, 1.0768e-18, 3.9546e-15,\n        4.9592e-14, 4.5705e-15, 3.3839e-14, 6.0298e-15, 2.3253e-15, 5.1664e-17,\n        1.9890e-14, 5.1708e-15, 1.9348e-15, 2.6553e-15, 1.4437e-14, 2.2657e-16,\n        8.8909e-18, 9.1385e-15, 6.7059e-16, 8.9519e-15, 2.9334e-15, 3.9046e-15,\n        7.6213e-15, 5.2163e-15, 7.1318e-17, 5.0088e-15, 7.2370e-15, 3.8598e-17,\n        4.2738e-14, 2.4324e-16, 2.6993e-16, 3.9319e-15, 1.8892e-15, 5.0242e-17,\n        6.1768e-16, 1.0919e-14, 2.1517e-14, 7.0880e-15, 6.5249e-15, 3.4879e-15,\n        2.3698e-15, 1.0517e-15, 1.7583e-14, 2.1764e-14, 1.7736e-14, 1.5098e-16,\n        3.6371e-17, 3.7881e-18, 3.7112e-14, 6.6705e-14, 1.6770e-14, 7.4658e-18,\n        5.4483e-15, 1.5205e-14, 1.0121e-13, 2.0734e-13, 1.6817e-13, 2.9974e-14,\n        6.5062e-18, 4.4901e-15, 8.1385e-15, 6.3205e-16, 7.9445e-14, 7.6685e-17,\n        9.7071e-15, 9.9392e-19, 3.4327e-14, 1.4444e-15, 1.0432e-17, 2.2324e-14,\n        3.6290e-16, 3.6076e-14, 9.4360e-18, 3.3480e-14, 2.0661e-15, 9.8740e-16,\n        4.6422e-16, 2.0335e-14, 5.6339e-15, 5.4363e-15, 4.0388e-15, 9.6471e-18,\n        3.0688e-17, 9.1098e-16, 5.1445e-14, 1.8926e-15, 1.2068e-15, 4.9695e-15,\n        1.7195e-15, 5.8928e-16, 4.3480e-15, 5.9466e-14, 7.4449e-14, 2.7882e-14,\n        3.6056e-15, 6.9049e-16, 1.7613e-15, 4.1849e-15, 7.8090e-16, 7.1171e-14,\n        3.0666e-14, 1.2351e-14, 5.9065e-15, 3.1142e-16, 2.3618e-14, 2.3251e-14,\n        1.8165e-16, 1.4423e-15, 2.7371e-16, 4.5820e-14, 4.9974e-14, 1.4923e-16,\n        4.0735e-16, 6.4481e-17, 8.2721e-15, 9.7661e-17, 8.4196e-15, 1.8845e-14,\n        2.4747e-17, 1.0646e-16, 4.6034e-14, 1.2187e-14, 5.3583e-14, 4.7884e-14,\n        3.4633e-17, 6.7734e-15, 4.6003e-15, 3.5435e-14, 5.5009e-15, 3.3114e-18,\n        1.7680e-14, 3.9301e-14, 5.4759e-15, 4.8735e-15, 1.3077e-18, 7.4344e-15,\n        2.6684e-17, 2.6264e-14, 3.2459e-17, 6.0076e-18, 2.2990e-14, 9.8443e-15,\n        8.3505e-17, 2.7117e-17, 1.0959e-14, 1.1632e-14, 3.6963e-18, 9.6052e-15,\n        3.4290e-14, 1.2142e-14, 8.8758e-16, 7.4293e-14, 1.4142e-14, 5.6269e-15,\n        2.0248e-14, 9.5616e-15, 7.7441e-14, 5.4752e-15, 5.2490e-17, 2.1691e-14,\n        2.0406e-14, 9.0001e-17, 2.7751e-18, 2.7540e-14, 4.7345e-15, 1.1026e-13,\n        9.8225e-16, 1.1388e-17, 1.4908e-16, 9.6849e-16, 2.6706e-14, 5.4188e-15,\n        3.0238e-17, 1.5666e-14, 9.2270e-15, 4.9852e-17, 2.0077e-17, 8.2176e-15,\n        1.6388e-16, 9.5706e-16, 2.9009e-14, 2.9348e-15, 1.9664e-14, 4.0733e-14,\n        1.4302e-15, 2.5620e-16, 2.5945e-14, 9.5139e-17, 6.5278e-15, 2.9438e-16,\n        2.2165e-15, 2.1796e-14, 1.2841e-13, 6.2837e-15, 5.2242e-16, 5.0383e-16,\n        4.4132e-16, 9.5098e-18, 1.1269e-15, 6.1635e-15, 2.2384e-14, 1.8724e-16,\n        4.2373e-15, 1.5514e-15, 6.4943e-17, 3.8807e-18, 4.8375e-16, 1.2565e-15,\n        5.0812e-15, 3.4873e-14, 7.4797e-15, 8.2415e-16, 3.9675e-16, 3.6455e-14,\n        2.4601e-17, 7.5750e-18, 1.5792e-14, 9.8516e-15, 5.2858e-16, 5.0850e-14,\n        4.6676e-16, 3.2874e-14, 1.5913e-14, 4.7217e-16, 6.8544e-15, 6.0989e-17,\n        1.5157e-14, 4.5541e-15, 2.6957e-18, 1.5399e-15, 7.9944e-18, 2.9732e-15,\n        3.3579e-15, 9.1355e-18, 4.3832e-14, 2.8958e-14, 8.2704e-14, 9.6187e-17,\n        2.7574e-18, 1.5182e-17, 2.1605e-18, 8.9278e-14, 1.6477e-15, 3.2641e-15,\n        2.9049e-14, 1.8455e-14, 2.8568e-15, 1.1935e-14, 2.6083e-14, 1.0658e-15,\n        2.2466e-16, 2.7055e-17, 1.7149e-16, 4.9794e-17, 5.7822e-15, 6.5519e-15,\n        2.1396e-15, 6.3542e-18], device='cuda:0')"
      },
      "52": {
        "step": "tensor(11268.)",
        "exp_avg": "tensor([[ 5.3772e-40,  1.1823e-38,  6.8009e-40,  ...,  4.1371e-40,\n          7.0794e-40, -1.4708e-38],\n        [-1.1351e-43,  1.1083e-39, -1.7555e-40,  ..., -5.0902e-40,\n         -1.0033e-40,  2.2729e-39],\n        [-3.6560e-42,  4.3278e-39,  4.3162e-40,  ...,  7.7439e-40,\n          3.9672e-40,  9.0079e-40],\n        ...,\n        [-2.3360e-40, -6.8722e-39, -4.5091e-40,  ..., -3.1320e-40,\n         -3.5695e-40,  8.4294e-39],\n        [ 1.8880e-40,  4.7337e-39,  4.0038e-40,  ...,  8.3372e-41,\n          2.8980e-40, -1.3447e-38],\n        [ 1.3231e-40,  2.5581e-39,  2.6120e-40,  ...,  2.7989e-40,\n          8.0638e-41, -5.1220e-39]], device='cuda:0')",
        "exp_avg_sq": "tensor([[2.0622e-16, 1.0902e-16, 3.7446e-17,  ..., 2.7419e-16, 7.3973e-17,\n         3.2348e-16],\n        [1.1182e-16, 9.0400e-18, 6.1030e-18,  ..., 9.4566e-17, 1.8174e-17,\n         5.3385e-17],\n        [1.0097e-16, 1.3348e-17, 1.8913e-17,  ..., 1.1209e-16, 1.3033e-17,\n         1.1296e-16],\n        ...,\n        [6.3303e-16, 1.2460e-16, 1.4883e-17,  ..., 1.8404e-16, 5.5181e-17,\n         2.5168e-16],\n        [3.1260e-17, 2.9322e-18, 2.6962e-18,  ..., 9.6044e-18, 3.3077e-18,\n         2.7160e-17],\n        [4.5165e-18, 1.4338e-18, 7.5225e-19,  ..., 6.8025e-18, 6.3567e-19,\n         9.2080e-18]], device='cuda:0')"
      },
      "53": {
        "step": "tensor(11268.)",
        "exp_avg": "tensor([ 2.2437e-36, -3.3961e-37,  1.4547e-36,  4.1124e-37, -2.0618e-36,\n         8.9137e-38,  7.7877e-37, -6.1414e-37,  1.5151e-36,  1.6509e-36,\n        -9.3032e-37, -2.5353e-36,  7.4255e-37, -4.1311e-37, -1.1017e-36,\n         1.3491e-36, -2.7704e-37, -3.3651e-37,  1.3013e-37, -9.1022e-37,\n         1.0888e-36,  1.5293e-37, -2.5899e-37, -1.3586e-36, -5.7150e-37,\n        -2.9591e-36,  3.1440e-36, -4.6099e-37,  3.6666e-37, -1.8937e-36,\n         1.4160e-37,  3.8045e-37,  1.0057e-37,  4.4127e-37, -9.6633e-38,\n        -4.5945e-38, -6.1383e-37, -2.0450e-36,  2.8569e-36, -2.7369e-37,\n        -1.5818e-36, -1.4009e-36, -3.4827e-37, -2.9524e-37,  2.8214e-37,\n        -9.3343e-37, -6.0194e-37,  3.3928e-37,  3.5244e-36,  8.1482e-37,\n        -3.5072e-37, -2.4103e-37, -1.1029e-37,  1.7294e-37,  2.5982e-37,\n         1.1665e-36,  2.6136e-36,  3.4019e-36, -3.3266e-38,  1.8039e-36,\n        -2.1549e-37,  1.6721e-36,  3.8960e-36,  2.6569e-36, -3.2330e-36,\n        -4.3619e-38, -3.2072e-37,  4.3395e-38,  1.1032e-36, -6.1258e-37,\n         9.0290e-39,  8.1988e-37, -3.6090e-36, -7.4654e-38, -2.2430e-37,\n         1.6156e-37,  2.2832e-37, -1.1739e-37, -2.8712e-36, -1.3319e-37,\n        -2.1421e-36,  3.7831e-38, -3.7874e-37, -7.9098e-37,  1.8738e-36,\n         1.0715e-37, -1.3278e-37,  5.3154e-38,  3.7756e-37, -2.4006e-37,\n         2.1807e-37, -4.1539e-36, -1.7515e-37, -2.1866e-37, -9.2000e-37,\n         1.5619e-37, -1.4081e-36, -5.6416e-38, -4.2525e-37, -2.0343e-37,\n        -2.7497e-38,  6.5765e-37, -2.8887e-37,  6.9666e-38, -1.9200e-37,\n        -7.4796e-37,  4.4491e-36,  7.1232e-37, -3.1068e-37, -4.4232e-37,\n         6.0226e-38,  1.7584e-38, -1.8213e-37,  1.4248e-37, -3.1242e-37,\n        -3.1716e-38, -1.6605e-37,  2.1670e-38, -2.7835e-37,  2.7437e-36,\n        -2.0310e-37,  2.0313e-36, -2.7466e-36,  4.5649e-37, -1.3972e-36,\n         3.3534e-37, -6.6837e-37, -2.1342e-37, -2.5590e-36,  1.1572e-37,\n         1.0981e-37, -7.3866e-37,  1.3772e-36,  2.5889e-37, -2.6723e-37,\n        -2.3434e-37,  1.3017e-36,  8.0801e-38,  6.0621e-38, -3.2932e-37,\n        -1.3342e-36, -5.4423e-37, -1.3339e-36, -3.6637e-37, -7.7856e-38,\n        -2.6847e-37,  3.9886e-37, -1.8987e-37, -4.2726e-37, -8.3473e-37,\n        -1.5777e-36,  7.5780e-37, -3.3443e-37, -4.2834e-37, -1.8371e-36,\n        -3.9804e-37, -2.7494e-37, -6.3079e-37, -1.8656e-36,  5.2596e-37,\n         5.7153e-38, -1.2397e-37, -1.0038e-36, -1.5599e-36,  4.2509e-37,\n         2.1068e-37,  7.5114e-38, -1.2245e-38, -3.9166e-37,  7.3079e-38,\n        -5.4772e-37, -1.3341e-37, -2.3025e-38,  5.3013e-37,  3.8450e-37,\n         6.1502e-37, -9.7212e-37, -5.0459e-37,  1.4385e-37,  1.6325e-36,\n         4.6454e-37, -5.4239e-37,  1.1679e-37, -7.2345e-37, -1.4660e-37,\n        -4.4937e-37,  4.0653e-37,  2.7014e-37, -1.5284e-36,  9.0081e-37,\n        -9.6112e-38, -5.2440e-37,  6.0919e-38, -1.2574e-36,  4.9192e-37,\n         8.1516e-37, -2.9783e-37, -3.2562e-36, -2.5715e-36,  1.1793e-37,\n        -3.4488e-38,  1.7436e-37, -1.4864e-37,  3.7323e-37, -1.3200e-37,\n         2.0107e-36, -1.9107e-36,  2.4154e-37, -1.2406e-36, -5.6302e-37,\n        -4.0950e-36,  3.4225e-36,  2.2432e-36, -3.8473e-37,  8.7620e-37,\n        -2.8273e-37, -8.5045e-37,  1.7284e-37, -6.4420e-37, -5.5257e-37,\n         8.5585e-37, -4.0424e-37, -2.2344e-36, -1.1493e-36,  1.3214e-36,\n         6.0798e-38,  3.5077e-37,  8.8003e-38, -2.2017e-37, -1.6350e-38,\n        -4.2854e-37, -2.1890e-36, -2.1540e-37,  9.0498e-37,  2.2542e-37,\n        -1.2948e-36,  3.8111e-36, -1.3151e-36, -3.7191e-37,  2.1231e-37,\n         5.7879e-36,  1.0553e-36, -4.7507e-37, -1.0453e-36,  5.0257e-37,\n        -3.8857e-37,  1.9908e-36,  1.1676e-37, -7.2055e-37, -2.6709e-37,\n        -1.4449e-36, -2.6338e-37, -1.9844e-36, -6.3463e-37,  3.1428e-37,\n        -1.2136e-36, -1.1323e-36, -6.7189e-37,  2.0171e-37, -4.4295e-37,\n         2.8341e-36,  3.9359e-37, -9.0486e-37, -1.4651e-36, -9.9715e-37,\n        -1.6514e-37, -1.6927e-36, -2.5645e-37, -1.0085e-36, -4.1860e-37,\n        -3.1653e-37, -1.2989e-36,  7.9725e-38,  7.0572e-37, -1.0720e-36,\n        -2.8168e-37, -3.0383e-38, -1.9447e-37, -2.7756e-37, -2.1413e-36,\n         3.3082e-37,  3.2896e-37,  1.5008e-36,  1.3088e-37,  4.2117e-37,\n        -3.4456e-37,  3.2884e-38, -7.0670e-37,  3.4366e-36,  8.8295e-37,\n        -2.7999e-37,  7.3471e-37,  8.2875e-38, -4.2124e-37, -8.9444e-37,\n        -5.9347e-37, -7.7272e-37, -2.0019e-37, -4.5340e-37, -2.8274e-36,\n        -7.0890e-38, -3.5959e-38,  5.5188e-37,  8.8753e-37, -5.8914e-37,\n         1.6040e-36,  2.2270e-37,  4.4327e-38,  9.9875e-37,  6.0835e-36,\n        -2.2706e-36,  2.2101e-37, -6.6769e-37,  2.2190e-37, -1.7128e-36,\n        -3.4530e-38,  2.0229e-36, -1.7617e-36, -2.7157e-37, -3.2908e-38,\n         7.3426e-37,  6.5151e-37,  1.6425e-37, -1.2810e-37, -7.8514e-37,\n        -2.1341e-38,  2.7752e-37, -1.5585e-38, -3.6049e-37, -3.1769e-36,\n        -5.0840e-37, -2.3938e-37,  3.2191e-39,  1.8949e-36, -2.0661e-37,\n        -1.7840e-37, -1.2971e-36, -3.2504e-37, -1.1632e-37,  5.7557e-37,\n        -2.9840e-37, -1.1585e-36, -1.0363e-36, -3.3835e-38, -8.6918e-37,\n        -1.1686e-36,  2.7464e-37,  6.2332e-37,  7.5842e-37,  1.4138e-36,\n         3.9350e-38,  8.2507e-38, -3.3586e-39,  9.8933e-37,  9.4332e-37,\n         1.1679e-37, -4.4274e-37, -2.7233e-37,  3.6526e-36, -1.6288e-36,\n         2.6398e-36,  3.6245e-36,  5.2104e-37,  2.8490e-37, -4.5250e-38,\n        -5.3653e-38,  1.4339e-36,  1.2213e-36, -2.3283e-37, -2.5631e-37,\n        -1.4388e-36, -2.2969e-37, -3.0551e-37, -2.2335e-36, -8.2354e-39,\n        -6.1743e-39,  2.5557e-36,  2.9296e-36, -5.6217e-37, -1.1342e-36,\n         8.9604e-37,  1.3926e-36, -3.6061e-37, -1.1589e-36,  2.4588e-37,\n        -8.3311e-38,  1.0374e-36,  1.8050e-36,  2.3773e-38, -9.9390e-40,\n        -2.6990e-36,  8.0764e-37, -3.8300e-37, -7.1574e-37, -2.7641e-37,\n         1.8163e-37,  2.5846e-37, -9.0398e-37, -2.5376e-36,  1.8254e-38,\n         6.2687e-37, -3.5959e-37, -6.5272e-38,  1.2863e-37, -6.6005e-38,\n         2.7981e-36, -1.2544e-36, -2.5473e-37, -2.7425e-36,  2.8332e-36,\n         3.6137e-37, -5.8554e-37, -7.9577e-37,  2.8295e-37, -2.9488e-36,\n         1.5617e-36, -1.1144e-36, -3.8319e-37, -1.1944e-36,  1.3776e-36,\n         2.8638e-36, -1.6426e-37, -1.3447e-37,  2.9589e-38, -5.0532e-37,\n        -6.0409e-38, -1.3665e-36,  3.3869e-37,  4.3662e-37,  5.1244e-37,\n        -2.8792e-37, -3.5054e-37, -2.7761e-37, -2.6894e-37,  1.0640e-36,\n        -3.1889e-36, -9.1448e-38,  5.0645e-36,  1.9738e-36,  1.7471e-37,\n        -3.2907e-37,  2.7678e-36,  1.3524e-37,  9.5149e-39, -1.8504e-37,\n         2.0689e-37,  2.3919e-36, -2.1557e-37,  2.3855e-37, -2.8017e-36,\n        -5.3591e-37, -3.2594e-37, -2.5897e-37, -3.8979e-38, -1.3801e-36,\n         4.8001e-37, -2.6453e-37,  9.8770e-38,  8.0466e-37, -2.5504e-38,\n         2.6073e-37,  2.7490e-37,  8.8887e-37,  5.9682e-37,  1.7039e-36,\n        -6.9337e-37, -9.2271e-37,  6.6101e-37,  4.4959e-36,  4.5124e-37,\n        -4.5868e-37, -6.2802e-37,  5.2775e-37, -8.0718e-38, -2.8516e-37,\n         1.2590e-36, -2.4728e-37,  5.7951e-36,  3.2290e-37,  4.2227e-38,\n         1.0491e-36, -5.0687e-37, -3.4319e-37,  1.5141e-36, -5.7924e-37,\n         1.3992e-37, -8.3449e-38,  4.5713e-37, -7.6877e-37,  1.1441e-36,\n         8.5249e-37, -8.6437e-37, -5.0244e-37,  3.6660e-38, -9.0706e-38,\n         1.8131e-36,  2.3066e-37, -1.6624e-37, -4.2232e-37,  4.3410e-37,\n        -7.3628e-37,  1.4632e-36,  2.4869e-38, -4.9991e-37, -8.7046e-37,\n        -3.2255e-38,  4.9098e-37,  6.3309e-38, -1.0946e-36, -1.5441e-36,\n         6.6479e-37,  7.1850e-37], device='cuda:0')",
        "exp_avg_sq": "tensor([1.2807e-10, 3.3579e-11, 3.0407e-11, 7.6997e-11, 8.7586e-11, 4.8275e-13,\n        7.5141e-12, 2.9544e-12, 4.7170e-12, 1.7001e-10, 1.5512e-11, 8.5857e-11,\n        1.4954e-12, 4.8899e-12, 4.4697e-11, 1.2072e-11, 1.0210e-13, 4.9313e-12,\n        1.6521e-13, 1.1645e-11, 4.8927e-12, 7.9262e-14, 1.4537e-11, 6.9155e-11,\n        9.4238e-12, 5.5464e-12, 1.0206e-10, 9.7818e-11, 7.2060e-13, 1.7994e-11,\n        2.8765e-13, 2.6814e-11, 3.7875e-14, 2.4911e-11, 3.4249e-13, 2.5545e-11,\n        2.2812e-11, 6.3064e-11, 2.6368e-10, 1.8502e-13, 8.6258e-12, 1.0341e-10,\n        5.9917e-12, 2.0534e-11, 2.5319e-11, 2.2598e-10, 8.2851e-11, 5.9953e-13,\n        2.3414e-11, 1.6878e-11, 1.1255e-11, 3.4507e-12, 4.5668e-11, 8.9678e-13,\n        1.0182e-10, 1.2781e-12, 1.8246e-10, 4.5319e-10, 1.5200e-12, 6.1382e-11,\n        2.7453e-13, 6.6695e-11, 8.6315e-10, 5.2926e-10, 1.8799e-10, 1.4724e-11,\n        5.0131e-13, 6.8099e-11, 8.2199e-11, 1.4924e-12, 4.3652e-14, 2.7668e-12,\n        6.3724e-11, 1.8343e-13, 6.2494e-12, 7.3536e-11, 2.3166e-13, 8.6541e-11,\n        3.0768e-10, 2.6785e-12, 2.1401e-10, 4.3582e-13, 5.1960e-13, 5.3037e-12,\n        4.8992e-11, 6.2038e-14, 1.4277e-13, 5.8947e-13, 1.5640e-12, 1.1168e-11,\n        1.4839e-10, 2.8767e-10, 6.6085e-12, 4.5030e-11, 4.6643e-12, 9.5731e-14,\n        6.3726e-11, 6.8140e-12, 3.5302e-11, 4.2173e-11, 9.2961e-14, 1.5245e-11,\n        2.7497e-13, 6.2414e-14, 2.5331e-13, 1.5286e-11, 3.3040e-10, 9.9433e-12,\n        1.3411e-11, 1.2060e-13, 4.8091e-13, 1.8478e-13, 3.3883e-12, 9.8548e-13,\n        5.8088e-13, 1.2243e-13, 3.4698e-12, 2.1289e-13, 9.2944e-14, 1.2814e-11,\n        4.9996e-12, 2.4791e-10, 3.2078e-11, 5.1945e-12, 3.4945e-11, 3.1007e-11,\n        2.7609e-12, 6.1121e-13, 2.7060e-10, 5.6469e-13, 1.2506e-12, 1.0883e-10,\n        8.9801e-11, 3.4260e-12, 4.0724e-10, 1.6265e-11, 1.6310e-11, 1.5941e-11,\n        7.2137e-13, 4.7342e-12, 9.0180e-11, 1.2646e-10, 1.7012e-11, 4.3578e-11,\n        9.4975e-13, 5.4088e-13, 2.5968e-12, 8.6866e-12, 4.5408e-11, 2.9705e-11,\n        1.4353e-11, 1.7240e-13, 1.0360e-11, 7.7380e-11, 4.6377e-11, 2.6951e-11,\n        3.7848e-14, 9.2815e-12, 6.2669e-11, 5.2016e-13, 1.2008e-13, 3.8412e-11,\n        2.8589e-11, 8.9582e-12, 3.5423e-11, 6.3037e-12, 3.7656e-13, 2.4294e-13,\n        2.0887e-12, 1.1017e-13, 9.0969e-13, 2.0995e-12, 4.8423e-13, 1.0896e-11,\n        2.5700e-14, 2.3709e-11, 3.5058e-12, 1.5273e-11, 1.0056e-11, 1.7117e-10,\n        6.3352e-13, 1.6012e-11, 2.2944e-13, 6.4373e-12, 4.4772e-14, 1.0833e-11,\n        8.7745e-13, 1.9144e-13, 9.0046e-11, 1.8701e-11, 1.9955e-12, 9.2773e-12,\n        1.3060e-12, 1.5408e-11, 7.4749e-13, 2.2709e-13, 5.8003e-12, 9.3486e-11,\n        3.9821e-10, 8.3831e-13, 1.0135e-13, 8.7421e-13, 2.8656e-14, 2.9869e-11,\n        1.9744e-13, 9.0976e-11, 1.2628e-10, 2.0318e-13, 1.6538e-10, 2.2225e-12,\n        1.8501e-11, 5.4757e-11, 2.1881e-11, 4.3613e-13, 6.9504e-12, 1.5309e-12,\n        2.1374e-10, 1.2086e-13, 4.5141e-10, 8.5726e-11, 3.4608e-12, 1.1451e-11,\n        5.8976e-11, 2.4108e-11, 9.3626e-12, 1.9775e-12, 1.8451e-14, 1.4474e-11,\n        7.2706e-15, 4.7942e-11, 3.5099e-13, 1.8502e-10, 4.1814e-12, 3.6591e-12,\n        8.8591e-13, 1.6983e-11, 4.3440e-10, 3.8932e-11, 6.6113e-13, 1.4549e-11,\n        4.3481e-10, 1.2523e-11, 1.4005e-11, 5.7482e-11, 9.3179e-12, 5.6330e-12,\n        1.4711e-11, 6.1431e-13, 1.1478e-12, 7.3112e-14, 1.3272e-10, 5.8943e-12,\n        7.5691e-10, 1.0192e-12, 1.9799e-12, 9.7207e-12, 2.8679e-12, 1.3462e-11,\n        1.7356e-13, 4.2907e-13, 8.5512e-11, 2.2945e-12, 1.0068e-10, 4.9490e-11,\n        5.6680e-12, 7.1062e-11, 4.6030e-12, 1.6038e-13, 1.7982e-11, 9.0345e-12,\n        4.0692e-14, 6.2353e-12, 2.4203e-13, 2.5202e-11, 3.5317e-11, 9.1619e-11,\n        1.9887e-13, 7.2913e-14, 2.6699e-13, 4.4230e-11, 2.2142e-12, 1.2837e-14,\n        7.0582e-11, 2.6530e-11, 1.1864e-13, 7.1798e-12, 1.8892e-13, 5.0319e-12,\n        7.9976e-11, 1.0482e-10, 1.7740e-12, 8.6563e-13, 2.6566e-11, 7.3808e-12,\n        3.5682e-11, 3.7877e-11, 2.1431e-11, 1.4470e-13, 1.2057e-13, 2.3117e-10,\n        1.1839e-12, 1.1146e-12, 2.3672e-13, 1.1315e-10, 3.1269e-11, 2.7776e-12,\n        3.2929e-13, 2.6697e-13, 3.3343e-12, 7.9460e-10, 9.8527e-10, 2.9035e-12,\n        8.1342e-11, 1.6386e-12, 3.8972e-11, 1.3810e-13, 3.7309e-11, 8.9694e-12,\n        5.2361e-13, 5.8535e-13, 8.4895e-11, 2.3957e-11, 2.0269e-13, 2.0779e-13,\n        1.9317e-11, 6.9693e-13, 1.7540e-12, 6.9419e-12, 9.1238e-15, 2.2428e-10,\n        7.4381e-12, 2.5612e-13, 6.4673e-13, 2.5167e-12, 8.4455e-11, 1.2320e-13,\n        2.5531e-11, 2.3640e-13, 4.1182e-13, 1.5952e-13, 3.6110e-11, 9.2731e-11,\n        3.5171e-11, 9.7166e-12, 8.5545e-12, 3.3186e-11, 2.7306e-10, 1.0530e-11,\n        3.1927e-11, 1.4504e-10, 1.5687e-12, 1.8977e-10, 1.1194e-12, 4.1931e-13,\n        3.0829e-11, 1.4157e-10, 3.9474e-11, 9.9037e-14, 1.0423e-10, 3.5235e-10,\n        1.8717e-11, 5.2904e-11, 5.6951e-11, 1.0151e-13, 3.0525e-13, 1.5902e-13,\n        8.6589e-12, 1.4575e-11, 1.4651e-12, 1.4173e-12, 1.0318e-10, 9.3030e-13,\n        8.4401e-14, 2.2459e-12, 3.5360e-13, 1.6545e-13, 9.9511e-11, 4.1906e-11,\n        3.4596e-11, 7.7441e-11, 6.6010e-12, 4.9890e-11, 2.3729e-13, 7.2580e-11,\n        3.9638e-12, 1.4906e-12, 3.0699e-12, 1.3567e-11, 3.2650e-12, 3.2483e-13,\n        7.7628e-11, 9.1052e-11, 1.1319e-14, 2.0061e-13, 1.9134e-11, 4.6977e-13,\n        2.0850e-10, 6.5892e-11, 4.0356e-10, 8.5443e-14, 1.6067e-11, 2.8830e-12,\n        3.0411e-10, 2.3008e-13, 2.1318e-13, 4.3765e-10, 3.3071e-12, 3.1867e-13,\n        9.1134e-11, 2.1046e-10, 3.8088e-13, 1.4949e-11, 1.4208e-11, 1.1639e-10,\n        4.3086e-12, 1.2426e-12, 1.3796e-11, 6.9151e-13, 2.5563e-10, 4.3389e-11,\n        1.2166e-10, 1.7496e-12, 4.4695e-13, 8.7845e-11, 2.8226e-12, 1.9383e-13,\n        5.3657e-11, 1.2264e-10, 5.5059e-11, 8.3590e-11, 1.3508e-12, 1.1420e-10,\n        3.9016e-13, 3.4511e-14, 7.5266e-13, 2.1593e-12, 9.1471e-13, 6.9398e-11,\n        8.3029e-11, 4.2361e-14, 5.3227e-13, 5.3498e-12, 3.1625e-11, 1.1024e-10,\n        1.0443e-12, 1.0794e-12, 8.0798e-11, 6.4666e-12, 8.2699e-13, 4.2455e-10,\n        5.0151e-11, 1.8014e-14, 2.8811e-12, 4.3253e-13, 1.3789e-10, 1.0457e-10,\n        7.7138e-12, 4.0699e-12, 1.1216e-11, 1.9989e-11, 4.9281e-12, 8.3175e-12,\n        2.3856e-10, 7.8093e-13, 3.1071e-11, 5.1520e-11, 1.6347e-12, 4.7790e-12,\n        1.5559e-10, 7.0174e-13, 5.2043e-11, 1.8621e-12, 4.9538e-13, 7.6878e-12,\n        1.7033e-13, 4.6689e-12, 4.1111e-12, 1.7988e-10, 1.0974e-13, 3.1719e-12,\n        5.8560e-11, 5.4762e-13, 5.4363e-13, 2.4835e-11, 2.8914e-13, 1.3412e-13,\n        1.1604e-11, 2.2270e-11, 3.1548e-11, 4.4518e-11, 2.9478e-11, 2.4097e-10,\n        1.4711e-13, 3.2206e-13, 2.0034e-12, 2.7273e-11, 3.3567e-12, 2.1061e-12,\n        1.0175e-12, 7.9486e-13, 2.4492e-12, 3.2927e-10, 9.0434e-11, 2.0427e-11,\n        5.0568e-11, 3.8681e-13, 2.4771e-11, 1.6169e-14, 2.2964e-11, 1.4131e-10,\n        9.1028e-12, 2.2607e-12], device='cuda:0')"
      },
      "54": {
        "step": "tensor(11268.)",
        "exp_avg": "tensor([ 6.8552e-38,  4.9115e-38,  7.2303e-38,  5.7176e-38,  7.6344e-40,\n         1.3710e-39,  4.2553e-38, -3.6227e-40,  5.6536e-39,  1.5817e-37,\n         1.8729e-38,  3.6811e-38,  3.0039e-39, -1.5797e-39,  1.9649e-38,\n         1.2495e-37,  4.6179e-39,  1.9166e-40, -3.7681e-39,  6.2251e-40,\n         1.8982e-38,  7.6187e-39, -2.9104e-39,  5.0505e-40, -1.1734e-39,\n         2.5537e-38,  1.8230e-37, -1.4726e-39,  5.1033e-39,  1.0837e-38,\n         1.6281e-40,  1.9598e-38,  2.6395e-39,  1.2928e-38,  9.8281e-39,\n         3.3276e-38,  1.1396e-37,  1.3288e-39,  1.5827e-37,  1.0505e-39,\n         1.2077e-39,  1.6391e-38, -2.9750e-39,  1.8739e-38,  4.4744e-39,\n         1.3219e-37,  1.1193e-38,  1.0261e-38,  4.1409e-38,  1.6606e-38,\n        -1.2497e-39,  9.7292e-40,  1.3573e-39,  4.2376e-40,  5.0168e-38,\n         2.0297e-39,  1.3483e-37,  1.1211e-37,  2.6508e-38,  8.4420e-38,\n         3.1826e-39,  6.6167e-38,  2.8207e-37,  2.2842e-37,  6.9777e-39,\n        -3.6408e-39, -1.0572e-39,  2.0001e-38,  1.3876e-37, -2.1556e-41,\n         2.5503e-39,  6.9873e-39, -5.6933e-40,  5.8316e-39,  2.1710e-38,\n         2.0269e-38, -1.0781e-39,  8.7851e-38,  1.1326e-38,  1.9480e-39,\n        -6.3317e-40,  4.4799e-39, -1.6269e-42, -8.2073e-40,  6.7773e-38,\n         9.0250e-39,  7.6788e-39, -3.2729e-39,  4.5894e-39,  5.8775e-40,\n         8.3013e-38, -6.8573e-39,  1.1012e-38,  5.1808e-40, -2.5390e-40,\n         8.3718e-39,  3.7974e-38,  1.4177e-39, -1.4088e-39,  1.3150e-38,\n         3.2484e-39,  1.2658e-38,  2.4844e-40,  1.6259e-39,  1.4840e-39,\n         2.1457e-38,  2.4655e-37,  3.8369e-38,  3.7115e-38, -2.4689e-40,\n         1.0131e-39,  4.1067e-39,  2.0742e-40, -1.1320e-39, -4.1857e-41,\n         2.6425e-39, -3.4276e-39,  7.2158e-39, -3.4459e-40,  1.8961e-38,\n         7.3359e-40,  1.8762e-37,  1.0382e-38,  2.4135e-38, -2.4818e-41,\n         5.0839e-38, -1.5912e-39,  4.7793e-39,  8.7785e-39,  1.8671e-39,\n         6.8393e-39,  2.1161e-38,  4.9606e-38,  3.7444e-38,  1.3655e-37,\n        -2.4749e-39,  6.8192e-38, -2.4195e-39,  2.0584e-39, -1.2559e-39,\n         8.4048e-41, -1.3219e-39,  3.3408e-38, -2.4817e-39,  9.0898e-40,\n         1.9071e-39, -5.9927e-39, -2.6485e-40, -1.4188e-39,  1.3648e-37,\n        -2.5050e-40,  1.1662e-38,  5.8519e-40, -3.5544e-41,  1.0069e-37,\n         1.6961e-41,  3.5404e-39,  9.9632e-40,  4.1621e-38,  1.4289e-38,\n         8.0946e-39,  3.3465e-38,  1.3963e-37,  1.9540e-38,  1.4614e-37,\n         3.4375e-38,  1.1204e-39,  1.8117e-39, -1.4624e-40,  1.4538e-38,\n         8.5911e-41, -7.5822e-40,  1.2055e-38,  2.5149e-39,  3.3740e-39,\n         2.1565e-38,  4.1694e-39,  1.0645e-37,  2.3835e-39,  8.3514e-38,\n         7.1527e-40,  5.5109e-38,  2.4371e-39,  1.0581e-38,  6.0832e-39,\n        -1.2487e-39,  1.5889e-38,  4.3453e-39,  1.8765e-39,  9.3448e-38,\n         2.7971e-40, -1.9836e-39,  6.0620e-39,  5.7708e-38,  1.5436e-38,\n         1.3641e-39, -6.0593e-40,  4.1179e-40, -8.5102e-40, -8.7613e-40,\n         4.0570e-39,  5.5401e-39, -2.7287e-39,  1.7719e-38, -2.4745e-39,\n         6.2931e-38,  8.9952e-38,  1.5971e-39,  3.4639e-38, -8.3326e-40,\n        -7.5185e-39,  3.4167e-38,  4.3451e-38, -1.7621e-40, -2.7635e-39,\n         2.1545e-40,  1.3043e-37,  7.4093e-39,  5.1345e-38,  2.3308e-38,\n         1.8935e-38, -1.7384e-40,  7.7498e-40,  4.5606e-39,  2.1087e-38,\n         4.4479e-39,  4.6035e-39,  4.0856e-38,  1.1075e-39,  7.3928e-38,\n        -4.0218e-40,  5.7125e-39,  2.3383e-39,  1.1723e-38,  3.4260e-38,\n         3.1480e-38,  2.3135e-37,  8.7403e-38,  1.8153e-38,  6.2473e-38,\n         2.3510e-37,  2.2929e-38, -4.3445e-40,  1.2241e-38,  1.6108e-38,\n        -4.3877e-40,  2.3080e-38,  2.2175e-39,  8.2911e-40,  1.2137e-39,\n         6.0041e-39,  1.8337e-39,  1.8606e-38, -1.9840e-39,  1.3812e-39,\n         3.8894e-41,  2.2596e-38,  5.7243e-39,  1.1702e-38, -2.3043e-39,\n         2.0835e-37,  3.2790e-39,  1.4694e-38, -8.5118e-41, -5.5415e-40,\n         5.2122e-38,  9.3761e-38,  1.9727e-39,  6.9598e-38,  7.7667e-39,\n         7.2449e-41,  2.9783e-38,  4.4909e-39, -3.8806e-39, -1.5330e-40,\n        -2.5228e-39,  6.0474e-39,  9.2159e-40, -2.1800e-39,  2.5976e-38,\n        -3.6729e-39, -1.6427e-39,  9.7890e-38,  8.8626e-38,  1.1385e-38,\n        -4.0180e-40,  2.5916e-39, -7.6417e-40,  7.6290e-38,  1.1768e-37,\n         2.0789e-40,  1.5975e-38, -1.4040e-39, -9.8522e-40,  9.7246e-41,\n         3.6323e-38,  3.4483e-40,  1.1031e-38,  1.1751e-38,  4.4321e-40,\n         2.3657e-39,  3.2058e-39,  9.9471e-39,  9.5523e-38, -1.1631e-39,\n         4.3566e-38,  2.8993e-39,  9.9406e-40,  5.5534e-39,  3.2141e-37,\n         7.7842e-38, -1.6463e-39,  9.0408e-38,  3.1929e-39,  3.0641e-38,\n         6.4143e-39,  5.9152e-38,  5.4206e-39,  2.2918e-40,  1.6325e-39,\n         1.9521e-38, -4.1456e-39, -3.5327e-41,  5.7177e-39,  2.0369e-40,\n         1.3129e-38,  5.2805e-39, -2.2281e-39, -4.1795e-40,  1.4311e-38,\n        -6.7324e-41,  2.1639e-39,  6.8854e-40,  6.5822e-38,  1.3133e-37,\n         9.2191e-39,  5.9676e-38, -5.1786e-40, -2.6801e-39,  1.4340e-38,\n         2.8570e-40,  3.3229e-38, -6.6809e-40, -1.4672e-39, -6.4611e-40,\n         7.4229e-39,  5.9767e-39,  5.3685e-38,  6.6884e-38,  9.5163e-38,\n        -4.1665e-39,  6.4379e-38,  4.3633e-39, -1.9709e-40,  2.9292e-38,\n         9.8740e-39,  1.1774e-40, -2.4070e-39,  6.9916e-38,  7.2999e-39,\n         7.4603e-38,  3.7801e-38,  7.8234e-38,  1.3417e-38,  6.6029e-39,\n        -3.1417e-39,  5.4804e-39,  5.3251e-38, -2.3282e-39,  7.7160e-40,\n         1.8632e-38,  1.2930e-39, -1.4121e-39,  2.7708e-38,  4.2657e-39,\n         5.9136e-39,  1.3997e-37,  3.0221e-38, -1.4572e-39,  2.7338e-39,\n         5.0178e-38,  7.5670e-38,  6.9280e-42,  1.2651e-37,  5.4184e-39,\n         3.6015e-39,  5.4564e-38,  1.6979e-37,  5.4010e-39,  2.5816e-39,\n        -2.0220e-39,  9.0792e-38, -1.9699e-40,  2.5740e-40,  1.0395e-39,\n         1.1692e-39,  1.5006e-37, -8.3293e-40,  1.7610e-38,  7.7015e-39,\n         1.6888e-38, -2.3742e-40,  4.3565e-38,  3.5161e-39,  2.8707e-39,\n         1.7912e-37,  3.6674e-38,  4.3896e-39,  2.9524e-38,  6.7428e-38,\n         1.4484e-38,  1.6134e-38,  6.1164e-39,  8.7157e-39,  8.7577e-39,\n         1.1056e-37, -1.8187e-40, -8.6347e-40,  1.9505e-38,  2.4457e-38,\n         6.8309e-38,  2.0306e-39,  6.1045e-39,  1.3841e-37, -1.2747e-39,\n         7.6696e-39,  5.6186e-38,  3.2089e-39,  6.4270e-38,  1.5871e-37,\n         1.0765e-39,  1.5286e-38, -7.3066e-40,  4.9049e-40,  1.0184e-38,\n         6.0457e-39,  3.8600e-39,  1.1353e-37,  1.2314e-37,  3.3915e-39,\n         2.1763e-40,  1.4120e-37,  2.6947e-39,  6.1714e-39,  1.2052e-39,\n         1.8951e-39,  4.4805e-38,  6.2451e-39,  4.2169e-39,  6.8033e-39,\n         1.0395e-40, -8.6452e-41,  1.0119e-39,  3.1806e-39,  3.4900e-38,\n         5.9921e-38, -1.9185e-39, -2.3932e-39,  4.2715e-39,  1.3756e-38,\n         3.8460e-39,  7.0416e-38,  1.1472e-37,  1.1799e-38,  1.3489e-37,\n         3.1459e-38,  9.8762e-38,  2.1808e-38,  7.7673e-38,  8.4662e-40,\n        -2.2453e-39,  1.5380e-38,  6.5665e-39,  3.6298e-38, -2.4480e-39,\n         3.2092e-38,  7.5363e-41,  2.4144e-37,  5.2074e-39,  4.9959e-39,\n         1.4732e-37, -4.8570e-40,  5.2194e-41,  1.8438e-38,  5.4595e-41,\n         4.8239e-40,  6.6906e-40,  2.5699e-38,  1.3212e-37,  1.4361e-37,\n         1.6600e-38,  1.1986e-37,  2.4748e-40,  4.3050e-39,  5.1728e-39,\n         1.7557e-38,  4.6351e-39,  2.2984e-39, -1.3535e-39,  1.3656e-38,\n         6.0478e-38,  1.6800e-37,  4.8547e-39, -3.4009e-40, -8.7179e-40,\n         2.6981e-39,  1.3804e-38,  1.5460e-39,  9.8807e-40,  1.3330e-38,\n         5.0943e-38,  1.8874e-38], device='cuda:0')",
        "exp_avg_sq": "tensor([6.4513e-15, 7.3089e-15, 7.8975e-15, 5.0782e-14, 9.0458e-16, 8.9525e-20,\n        1.1354e-15, 6.1001e-20, 1.8358e-17, 9.6488e-14, 9.3348e-16, 1.9161e-14,\n        6.3006e-18, 8.0487e-19, 1.7784e-15, 2.0759e-14, 4.1512e-17, 9.6610e-18,\n        2.4249e-19, 9.4040e-16, 6.3204e-17, 6.5692e-17, 6.9825e-17, 5.1837e-15,\n        2.8441e-18, 1.0787e-14, 9.8681e-14, 1.8465e-15, 3.8851e-16, 1.6275e-15,\n        6.7010e-17, 4.8471e-15, 8.4454e-19, 1.5563e-17, 1.3036e-16, 1.4359e-15,\n        6.2511e-14, 6.5810e-15, 1.1177e-13, 1.0306e-16, 2.0484e-16, 1.4459e-14,\n        7.3532e-18, 1.2589e-16, 1.8341e-15, 2.1415e-13, 7.6230e-15, 5.9939e-16,\n        9.4641e-16, 3.3431e-15, 7.6497e-18, 2.1350e-18, 3.0481e-15, 1.8462e-16,\n        3.9934e-14, 4.7229e-18, 6.4404e-14, 6.4024e-14, 1.7040e-15, 1.8079e-14,\n        7.0098e-17, 9.3616e-15, 7.3288e-13, 3.7623e-13, 4.1480e-14, 1.3928e-16,\n        8.7456e-17, 8.5370e-16, 7.2003e-14, 5.9387e-16, 9.4830e-17, 1.1973e-15,\n        9.4432e-15, 6.1507e-17, 1.4585e-16, 1.4728e-14, 6.0338e-17, 7.6853e-14,\n        8.4550e-14, 4.3761e-18, 5.5807e-15, 9.7191e-17, 1.8539e-16, 3.7577e-18,\n        1.4616e-14, 6.4514e-16, 4.3871e-18, 9.9789e-17, 1.1604e-17, 1.9380e-17,\n        4.0468e-14, 7.9868e-14, 3.3248e-17, 8.4769e-18, 9.7384e-18, 1.6733e-16,\n        1.2695e-14, 4.7905e-18, 8.9665e-17, 3.0178e-16, 4.2088e-17, 3.2631e-16,\n        1.3146e-17, 4.7636e-18, 9.6793e-17, 1.5917e-15, 2.3820e-13, 1.4958e-15,\n        4.4110e-15, 5.7893e-17, 3.7187e-17, 2.0464e-17, 1.5909e-18, 6.0857e-18,\n        2.6028e-16, 8.7093e-19, 1.8928e-17, 2.2671e-17, 6.9551e-17, 2.8749e-17,\n        1.7874e-15, 2.4748e-13, 3.8097e-15, 2.2091e-16, 6.9862e-16, 4.4467e-16,\n        2.8815e-18, 3.2269e-18, 6.7463e-14, 1.7944e-18, 4.3966e-16, 5.6705e-15,\n        3.2977e-14, 9.3433e-16, 2.9206e-13, 1.7287e-16, 3.3732e-15, 8.5604e-18,\n        4.2757e-17, 1.5596e-18, 9.3127e-16, 3.4979e-15, 1.6939e-14, 4.2763e-17,\n        2.1827e-19, 2.3284e-19, 4.8152e-18, 2.9314e-16, 5.1898e-17, 7.3882e-14,\n        2.0446e-16, 8.0617e-16, 6.6377e-18, 1.7552e-17, 7.3142e-14, 3.1264e-18,\n        5.8677e-18, 1.4012e-17, 4.7213e-15, 1.1102e-15, 1.1984e-16, 2.0952e-15,\n        9.2813e-14, 3.6857e-15, 5.6524e-14, 9.2604e-16, 5.3199e-17, 1.4294e-17,\n        2.3370e-19, 3.9567e-16, 4.7380e-16, 1.8121e-16, 7.9560e-16, 1.6094e-17,\n        1.7214e-17, 3.0160e-15, 9.9062e-16, 2.5991e-14, 8.8407e-16, 2.5158e-14,\n        4.1807e-17, 6.4992e-15, 8.9781e-18, 3.2621e-15, 2.0368e-17, 4.5802e-19,\n        7.8143e-16, 5.2084e-17, 2.3763e-15, 1.5645e-14, 1.5275e-16, 3.1571e-17,\n        2.7299e-16, 1.1763e-14, 1.0799e-15, 5.8694e-18, 4.4454e-16, 5.2820e-15,\n        3.7461e-14, 7.1866e-19, 1.9919e-17, 3.5080e-16, 1.3693e-18, 3.5106e-16,\n        1.5381e-18, 4.7030e-15, 8.9656e-14, 1.1745e-18, 6.2891e-14, 3.1989e-18,\n        9.7521e-18, 3.5222e-16, 2.0116e-15, 2.9925e-17, 1.3763e-17, 5.3197e-16,\n        1.7841e-13, 1.9364e-17, 1.9474e-13, 6.4924e-15, 3.0568e-16, 1.2472e-18,\n        9.6827e-16, 1.8923e-16, 1.9031e-17, 8.5975e-17, 1.5254e-17, 2.1271e-15,\n        1.3536e-16, 1.7103e-14, 3.0735e-18, 1.4124e-14, 4.9477e-18, 2.7633e-15,\n        8.4285e-16, 5.7313e-15, 2.8418e-13, 5.5967e-14, 9.4042e-17, 6.7081e-15,\n        1.6233e-13, 6.6727e-17, 4.7614e-18, 7.8934e-16, 5.8860e-17, 2.7730e-20,\n        3.9218e-17, 4.0918e-19, 3.2524e-16, 8.9204e-18, 1.2359e-14, 9.4370e-18,\n        2.0178e-13, 3.0308e-17, 8.8598e-17, 2.8436e-16, 7.5033e-15, 1.0728e-16,\n        1.8332e-16, 5.3618e-17, 1.2041e-13, 3.5899e-16, 1.5821e-14, 9.0442e-16,\n        1.1708e-17, 4.5040e-14, 3.8155e-14, 6.3367e-17, 1.8033e-14, 6.9602e-18,\n        3.2953e-17, 1.0447e-14, 3.6833e-18, 9.9671e-18, 1.7114e-16, 1.3801e-15,\n        1.9161e-17, 5.3486e-19, 9.4425e-17, 3.7525e-15, 2.1039e-18, 1.3314e-17,\n        2.0444e-14, 2.7293e-14, 1.2360e-16, 5.1063e-19, 1.3753e-17, 7.8761e-19,\n        6.5502e-15, 5.5059e-14, 3.7261e-16, 1.5981e-15, 1.5833e-17, 4.8081e-19,\n        1.6588e-16, 5.1297e-15, 1.5726e-15, 4.2560e-17, 1.0894e-17, 1.8617e-14,\n        2.2653e-18, 1.9817e-19, 4.5497e-16, 3.2616e-14, 4.4680e-17, 2.9681e-15,\n        2.3069e-18, 8.0071e-18, 1.9355e-15, 5.1359e-13, 4.3488e-13, 6.4229e-18,\n        4.1006e-14, 2.5541e-16, 4.7367e-16, 1.5510e-16, 4.3413e-15, 7.3538e-16,\n        7.6368e-17, 6.6976e-19, 2.2406e-16, 6.9769e-16, 9.4353e-17, 1.0573e-17,\n        2.7910e-17, 5.3603e-16, 2.6778e-16, 1.4726e-15, 9.3845e-18, 2.9549e-14,\n        2.9204e-18, 1.4242e-16, 1.4196e-18, 2.1077e-15, 1.1783e-13, 4.0081e-17,\n        6.0982e-15, 5.9700e-17, 6.4063e-18, 3.6808e-16, 2.3324e-17, 3.2562e-14,\n        9.3339e-17, 6.8198e-17, 2.1049e-19, 2.1247e-16, 1.9738e-14, 2.7206e-15,\n        3.4618e-14, 2.4518e-14, 4.5043e-19, 2.7816e-14, 1.5885e-18, 6.6957e-18,\n        7.8554e-16, 1.3946e-14, 9.8805e-18, 2.8376e-18, 4.4574e-15, 4.6991e-14,\n        3.1094e-15, 5.1655e-16, 1.2941e-14, 2.7773e-16, 4.9886e-16, 1.8606e-18,\n        4.5214e-17, 1.0334e-14, 7.9705e-19, 1.3059e-16, 3.3349e-14, 2.0624e-16,\n        4.7790e-18, 1.3218e-14, 9.8147e-18, 1.4591e-17, 8.7620e-14, 2.3160e-15,\n        3.3521e-16, 5.0573e-15, 1.7760e-15, 1.0213e-14, 1.1619e-16, 1.4093e-13,\n        9.6874e-18, 1.1916e-17, 9.9451e-15, 4.6479e-14, 1.8679e-18, 3.5772e-19,\n        1.6871e-15, 7.1743e-14, 6.5629e-18, 9.3086e-17, 3.4764e-17, 4.4326e-18,\n        1.8189e-13, 2.5179e-16, 1.0770e-13, 1.9590e-17, 2.4789e-16, 8.9772e-16,\n        7.3240e-14, 1.3971e-16, 1.2575e-18, 2.4246e-13, 3.6433e-15, 8.2687e-19,\n        2.3782e-14, 7.0938e-15, 4.1971e-16, 8.5891e-16, 1.7708e-16, 8.1764e-15,\n        7.2386e-15, 1.6220e-14, 6.4897e-18, 3.1740e-16, 5.8499e-14, 2.2239e-16,\n        8.0209e-15, 1.9217e-16, 5.5203e-19, 9.6670e-14, 5.8809e-18, 5.9839e-18,\n        1.5832e-14, 6.7050e-15, 3.6079e-14, 7.5144e-14, 1.4056e-18, 8.2733e-15,\n        1.2702e-16, 9.3967e-20, 1.1422e-18, 3.0096e-15, 1.3818e-18, 1.9206e-14,\n        9.1239e-14, 4.5901e-18, 1.4032e-16, 1.1423e-14, 2.4372e-15, 1.0793e-14,\n        5.4924e-19, 2.7349e-16, 4.4342e-15, 1.8528e-18, 3.9263e-16, 9.0007e-14,\n        1.4541e-17, 1.4817e-17, 1.5417e-19, 3.5543e-18, 5.7033e-14, 1.2069e-14,\n        1.1924e-16, 6.7636e-19, 1.2145e-16, 8.3350e-17, 3.0099e-16, 1.2057e-14,\n        1.3602e-13, 7.5360e-16, 1.9292e-14, 5.6382e-16, 2.5240e-14, 1.2833e-16,\n        2.3949e-14, 1.8268e-17, 1.3466e-15, 2.3091e-15, 4.2921e-16, 1.7443e-15,\n        4.4712e-18, 1.2868e-16, 1.3524e-15, 1.3416e-13, 1.1243e-16, 2.2134e-18,\n        4.1931e-14, 2.5599e-16, 3.2931e-17, 3.2539e-17, 5.4715e-17, 2.6276e-17,\n        2.1953e-18, 2.3367e-15, 7.3556e-14, 7.2909e-14, 1.0481e-16, 1.4081e-13,\n        4.2604e-17, 5.2518e-19, 2.8543e-18, 2.4935e-17, 3.8210e-18, 3.7820e-18,\n        3.2834e-16, 9.5640e-16, 1.6190e-14, 2.0722e-13, 5.5819e-15, 2.5650e-18,\n        1.0578e-16, 1.0008e-18, 4.4401e-15, 1.9853e-17, 1.5578e-18, 8.1905e-15,\n        1.4964e-15, 1.9017e-15], device='cuda:0')"
      },
      "55": {
        "step": "tensor(11268.)",
        "exp_avg": "tensor([ 8.1280e-38,  4.8377e-38,  7.3913e-38,  5.9391e-38, -7.5108e-39,\n        -1.2251e-39,  5.7197e-38,  1.0989e-39,  5.0027e-38,  9.5694e-38,\n         2.9110e-38,  2.6492e-38,  3.6334e-38,  4.6726e-39,  2.8237e-38,\n         8.5716e-38, -4.2245e-39,  1.8568e-38,  1.3948e-38,  1.2122e-38,\n         5.1391e-38, -5.9198e-39,  1.2944e-38,  6.3662e-39,  1.2044e-38,\n         1.9019e-38,  1.1560e-37,  1.2367e-38, -3.8220e-39,  1.6843e-38,\n        -6.7882e-41,  4.3385e-38, -2.2899e-39,  4.0407e-38, -8.0628e-39,\n         4.4790e-38,  6.5871e-38,  2.7938e-39,  1.0810e-37, -5.4998e-40,\n         7.0954e-39,  2.4168e-38,  1.0379e-38,  3.4834e-38,  3.2271e-38,\n         6.7203e-38,  2.7684e-38, -7.3848e-39,  8.9817e-38,  4.6597e-38,\n         4.0462e-39, -7.8912e-40,  2.4256e-38, -3.7819e-40,  5.4654e-38,\n         4.2871e-38,  1.0109e-37,  1.0507e-37, -1.7250e-38,  8.0889e-38,\n        -2.3813e-39,  7.4562e-38,  1.3880e-37,  1.1813e-37,  7.3386e-39,\n         1.7412e-38,  3.7474e-39,  3.9625e-38,  8.6253e-38, -6.6957e-41,\n        -2.3020e-39, -4.5522e-39, -2.3789e-40, -4.9956e-39,  3.7142e-38,\n         4.1287e-38,  9.7319e-40,  6.2740e-38,  1.0771e-38, -1.7904e-39,\n        -3.5316e-39, -3.6624e-39,  1.0484e-39,  2.6967e-39,  7.6831e-38,\n        -7.1811e-39, -6.3431e-39,  2.0635e-38,  3.2913e-38, -5.0389e-40,\n         6.4409e-38, -7.3870e-39,  3.1641e-38, -3.5267e-40,  9.3943e-39,\n        -6.5112e-39,  3.5309e-38, -1.2971e-39,  4.9345e-39,  3.2885e-38,\n        -2.8535e-39,  4.3168e-38,  7.7453e-40, -1.4386e-39, -9.8115e-40,\n         3.2791e-38,  1.3982e-37,  5.4598e-38,  4.3864e-38,  1.2177e-39,\n        -9.5809e-40, -3.4711e-39, -8.0708e-41,  1.0569e-39,  1.2758e-39,\n        -2.3164e-39,  1.0380e-38, -6.0127e-39,  1.3361e-39,  7.2494e-38,\n         1.4555e-40,  1.0474e-37,  1.0841e-38,  4.6067e-38,  3.4200e-39,\n         5.5691e-38,  5.3738e-39, -4.3143e-39,  1.0245e-38, -1.6809e-39,\n         3.1983e-38,  3.2423e-38,  6.5960e-38,  5.0246e-38,  7.3310e-38,\n         1.5272e-38,  7.1102e-38,  2.1261e-38, -1.6828e-39,  3.2968e-39,\n        -5.3827e-40,  1.2279e-38,  3.4044e-38,  1.0023e-38, -8.2618e-40,\n        -1.0022e-39,  2.1492e-38,  2.1000e-38,  3.4914e-39,  6.8289e-38,\n         1.3520e-39, -7.3506e-39,  5.1442e-42,  3.2688e-40,  5.2103e-38,\n         4.9166e-40, -2.7296e-39,  1.5774e-38,  3.2651e-38, -9.2376e-39,\n        -6.5535e-39,  4.3927e-38,  6.7659e-38,  2.4738e-38,  8.2031e-38,\n         4.7998e-38, -9.5216e-40, -1.6522e-39,  7.4951e-40, -1.0580e-38,\n        -6.3543e-40,  5.7706e-40, -9.3274e-39,  3.4718e-38, -2.6549e-39,\n         4.5807e-38,  1.7185e-38,  6.3963e-38, -1.9606e-39,  7.8963e-38,\n        -5.7669e-40,  4.8754e-38, -2.1147e-39,  2.6284e-38, -4.7391e-39,\n         3.1534e-39, -1.0406e-38, -3.5076e-39,  7.7156e-39,  7.3886e-38,\n        -2.7895e-40,  1.0811e-38, -5.0536e-39,  4.3650e-38, -9.9748e-39,\n         3.8312e-38,  1.7961e-38,  7.8080e-42, -2.2864e-39,  7.3367e-40,\n        -3.5005e-39, -4.4807e-39,  9.2055e-39,  4.2204e-38,  9.9045e-39,\n         7.7535e-38,  4.8570e-38, -1.3250e-39,  3.5085e-38,  1.3463e-38,\n        -2.8530e-38,  8.6438e-38,  7.3690e-38,  7.9452e-40,  3.3428e-38,\n         9.2400e-40,  6.6983e-38, -5.8683e-39,  4.6383e-38,  3.5558e-38,\n         4.8356e-38,  8.7363e-40,  2.4669e-39,  1.5771e-38,  5.4287e-38,\n        -3.5803e-39, -3.6224e-39,  4.9411e-38,  4.6054e-40,  5.9410e-38,\n         1.2274e-39,  9.3158e-39, -1.8992e-39, -7.3774e-39,  4.7730e-38,\n         3.3235e-38,  1.3007e-37,  5.3801e-38,  3.3748e-38,  5.8227e-38,\n         1.5253e-37,  5.2296e-38,  9.2335e-40,  2.3571e-38,  4.1575e-38,\n         1.5016e-39,  6.3635e-38, -1.9590e-39, -2.4001e-39, -8.0660e-40,\n         1.4948e-38,  2.2720e-38,  2.0906e-38,  7.9394e-39, -1.1304e-39,\n         9.6627e-41,  3.0389e-38,  2.1274e-38, -8.6703e-39,  1.0721e-38,\n         1.1634e-37, -2.5571e-39,  2.6530e-38,  2.8002e-39,  3.4942e-39,\n         5.1304e-38,  5.1224e-38, -1.5434e-39,  4.9568e-38,  2.5977e-38,\n         3.7883e-40,  3.2216e-38, -3.8217e-39,  3.1262e-38,  7.6021e-39,\n         1.4079e-38, -5.1902e-39, -5.3845e-40,  6.1908e-39,  2.3357e-38,\n         3.3605e-39,  1.4044e-39,  8.1099e-38,  6.5208e-38, -7.7937e-39,\n         1.2531e-39, -2.2566e-39,  1.1828e-38,  9.7608e-38,  7.9536e-38,\n        -4.3861e-41, -9.6611e-39,  1.2984e-39,  2.7057e-39,  2.0655e-42,\n         4.1120e-38,  1.5375e-38, -9.0291e-39, -1.0409e-38,  9.4612e-40,\n        -2.0192e-39, -2.9765e-39, -6.7452e-39,  7.3948e-38,  3.4076e-39,\n         6.6105e-38, -2.4586e-39, -8.5290e-40, -3.5395e-39,  1.6732e-37,\n         4.2698e-38,  1.5446e-39,  5.8746e-38, -2.5546e-39,  2.9192e-38,\n        -5.4313e-39,  7.6925e-38,  1.1822e-38,  1.4927e-41, -1.4818e-39,\n         4.7567e-38,  3.0890e-38,  4.1052e-41, -5.1116e-39, -5.2393e-40,\n         3.4966e-38, -4.0848e-39,  7.2925e-40,  1.1907e-39,  1.2069e-38,\n         5.8269e-40, -1.3995e-39,  2.4300e-38,  7.6967e-38,  7.2546e-38,\n        -7.6983e-39,  4.3533e-38,  2.1183e-39,  7.5119e-39, -9.1638e-39,\n        -2.1857e-40,  3.4859e-38,  4.5149e-39,  1.9926e-38,  4.0439e-39,\n         1.8723e-38,  3.3255e-38,  5.9178e-38,  6.5065e-38,  7.9075e-38,\n         1.6669e-38,  5.7493e-38,  2.8577e-38,  3.8234e-38,  5.4318e-38,\n         3.3804e-38, -6.3158e-41,  1.2847e-38,  9.8551e-38,  1.4062e-38,\n         8.7690e-38,  8.9484e-38,  6.5895e-38, -9.4523e-39, -5.6707e-39,\n         1.4417e-38,  4.9215e-38,  6.5221e-38,  5.7192e-39,  2.5387e-40,\n         2.4656e-38, -4.0817e-40,  3.7153e-39,  2.4532e-38, -3.8477e-39,\n        -5.0289e-39,  1.0171e-37,  7.9042e-38,  1.0947e-38,  1.2300e-38,\n         6.1118e-38,  7.3868e-38,  2.3226e-40,  6.3495e-38,  3.2267e-38,\n        -3.2578e-39,  6.3574e-38,  9.9089e-38, -4.3258e-39, -2.2206e-39,\n        -9.0520e-39,  7.2487e-38, -2.6103e-40, -4.9914e-40, -2.7797e-40,\n        -1.0632e-39,  8.1067e-38,  3.0400e-39,  1.6592e-38, -6.3031e-39,\n         4.4216e-38,  1.0010e-39,  4.7736e-38, -2.9494e-39, -2.6503e-39,\n         1.1109e-37,  3.5410e-38, -3.9668e-39,  2.2186e-38,  8.7770e-38,\n        -9.7956e-39,  3.0744e-38,  2.0391e-38,  3.4357e-38,  8.3218e-39,\n         8.4477e-38,  2.3100e-40,  2.3897e-39,  2.6961e-38,  5.6672e-38,\n         8.8765e-38, -1.2913e-39, -5.3880e-39,  7.6147e-38,  1.3051e-38,\n        -6.3570e-39,  4.1978e-38,  3.1362e-38,  6.0449e-38,  8.4890e-38,\n        -6.3231e-40,  3.2633e-38,  2.3866e-39,  1.1292e-39,  4.7113e-38,\n         5.9560e-39, -3.5112e-39,  1.2443e-37,  9.1664e-38, -2.9091e-39,\n         1.5171e-40,  1.0338e-37,  2.9358e-38,  3.0476e-38, -1.0897e-39,\n        -1.6016e-39,  7.6426e-38,  2.7873e-38, -3.3715e-39,  7.5538e-39,\n        -6.8686e-40,  6.5613e-40, -4.8996e-40, -2.9091e-39,  3.3604e-38,\n         6.0478e-38,  1.4820e-38,  2.0771e-39,  3.9019e-38,  3.5002e-38,\n        -3.0718e-39,  6.1361e-38,  7.9225e-38, -7.6787e-39,  9.1128e-38,\n         3.8593e-38,  5.9010e-38,  4.7398e-38,  1.1087e-37,  3.1219e-38,\n         1.3648e-38,  3.0259e-38, -4.7004e-39,  4.5269e-38,  1.4403e-38,\n         5.8484e-38,  4.1734e-40,  1.5475e-37, -4.0824e-39, -4.0572e-39,\n         8.7161e-38,  1.4784e-39,  8.3148e-40,  5.5889e-38, -7.3642e-41,\n        -3.4395e-40,  2.3367e-38,  4.7008e-38,  6.8694e-38,  8.7858e-38,\n         4.6657e-38,  6.4473e-38, -6.7964e-40, -3.5960e-39, -4.2154e-39,\n         5.9698e-38,  3.1866e-38, -2.0257e-39,  4.2119e-39, -9.1404e-39,\n         4.9466e-38,  9.5373e-38,  2.9052e-38,  9.6916e-40,  3.4995e-39,\n        -2.2698e-39,  4.1218e-38, -1.3842e-39, -3.5522e-39,  2.0779e-38,\n         5.8517e-38, -1.1025e-38], device='cuda:0')",
        "exp_avg_sq": "tensor([3.4410e-14, 1.6863e-14, 1.0375e-14, 2.7453e-14, 6.8529e-15, 1.2849e-18,\n        5.9603e-15, 1.3398e-17, 2.3400e-15, 6.6861e-14, 4.3514e-15, 1.4092e-14,\n        1.3859e-15, 4.6328e-16, 4.3256e-15, 1.7148e-14, 1.6027e-17, 1.7246e-15,\n        3.2450e-16, 3.0766e-15, 4.8437e-15, 9.9482e-18, 4.0817e-15, 1.4263e-14,\n        3.3788e-16, 6.4449e-15, 5.5272e-14, 1.4566e-14, 1.6343e-16, 1.3537e-15,\n        1.5204e-18, 5.1802e-15, 1.4427e-19, 8.9640e-15, 3.9652e-17, 1.0579e-14,\n        2.8613e-14, 8.9509e-15, 9.6086e-14, 4.4515e-17, 5.7711e-16, 9.3676e-15,\n        2.4180e-16, 1.6165e-16, 8.8736e-15, 8.9762e-14, 8.4556e-15, 2.5840e-16,\n        1.3233e-14, 6.5944e-15, 1.9959e-15, 8.1026e-17, 1.0045e-14, 8.4243e-17,\n        2.2070e-14, 9.0171e-16, 6.8632e-14, 1.2915e-13, 8.5975e-16, 3.2337e-14,\n        3.4116e-17, 2.9320e-14, 2.7013e-13, 1.6357e-13, 1.6553e-14, 9.6397e-16,\n        4.5400e-17, 1.7651e-14, 4.2798e-14, 2.8340e-16, 4.6229e-17, 4.8912e-16,\n        4.2026e-15, 1.7858e-17, 5.7132e-15, 1.4338e-14, 1.8801e-17, 4.8759e-14,\n        3.4495e-14, 3.2209e-16, 2.2613e-14, 4.6692e-17, 9.8948e-17, 5.3342e-16,\n        2.6193e-14, 2.8842e-16, 1.9480e-17, 1.8490e-15, 1.6122e-15, 1.2816e-15,\n        4.9498e-14, 2.8911e-14, 2.2463e-15, 7.2200e-15, 1.2401e-15, 7.2419e-17,\n        1.4278e-14, 9.9660e-16, 4.5094e-15, 1.0971e-14, 2.1629e-17, 9.2039e-15,\n        6.8784e-18, 2.3241e-19, 4.8846e-17, 1.2944e-15, 1.1990e-13, 4.5866e-15,\n        1.2764e-14, 2.6477e-17, 6.8422e-18, 1.7906e-18, 8.4925e-17, 3.3879e-17,\n        1.1983e-16, 3.7512e-17, 6.6445e-16, 1.3910e-17, 2.4470e-17, 7.4087e-15,\n        7.6257e-16, 1.0611e-13, 6.4687e-15, 5.6358e-15, 2.4860e-15, 2.7632e-16,\n        2.1700e-16, 2.4855e-16, 3.5421e-14, 1.3614e-17, 1.7811e-15, 7.6917e-15,\n        1.9836e-14, 5.7070e-15, 1.2001e-13, 4.5097e-15, 3.8408e-15, 2.2526e-16,\n        2.0146e-17, 7.9432e-16, 1.0923e-14, 2.0507e-14, 1.5623e-14, 9.2014e-15,\n        3.9490e-17, 5.6355e-17, 1.5795e-15, 1.9361e-15, 6.6546e-15, 2.9672e-14,\n        8.4166e-16, 4.0793e-16, 1.5074e-15, 1.2966e-14, 3.4426e-14, 3.4652e-15,\n        2.9599e-16, 2.3642e-15, 2.3999e-15, 5.3401e-16, 4.5845e-17, 1.5415e-14,\n        3.1811e-14, 2.4250e-15, 2.5213e-14, 5.5155e-15, 5.9021e-18, 3.2454e-18,\n        2.7072e-17, 2.0523e-16, 2.0265e-16, 9.3100e-17, 3.1680e-16, 3.4208e-15,\n        2.5076e-18, 3.3784e-15, 3.4022e-15, 1.0737e-14, 3.4426e-16, 4.5171e-14,\n        5.2906e-17, 1.0459e-14, 1.6662e-17, 5.3833e-15, 3.3938e-17, 2.9843e-16,\n        3.2819e-16, 8.8509e-18, 7.3083e-15, 1.8921e-14, 6.6831e-17, 3.1165e-16,\n        1.0886e-16, 4.7762e-15, 4.5213e-16, 7.2049e-16, 3.2400e-15, 6.0496e-15,\n        4.8546e-14, 6.9920e-18, 2.0598e-18, 1.8037e-16, 3.6460e-16, 5.2094e-16,\n        2.6347e-16, 2.6007e-14, 4.3162e-14, 1.6280e-20, 3.5045e-14, 1.0814e-15,\n        5.0645e-15, 2.0286e-14, 1.3465e-14, 1.4741e-17, 2.8248e-15, 2.6377e-16,\n        7.3278e-14, 1.8035e-18, 8.4975e-14, 4.5452e-15, 4.9579e-15, 1.9454e-15,\n        7.5082e-15, 2.3466e-15, 5.3160e-15, 3.4331e-17, 3.3016e-18, 6.9105e-15,\n        5.1012e-17, 2.5503e-14, 2.9958e-16, 1.3204e-14, 5.4315e-16, 1.2333e-15,\n        3.3309e-15, 3.0658e-15, 1.5171e-13, 3.3348e-14, 2.3043e-15, 3.0483e-15,\n        1.3434e-13, 4.9624e-15, 1.7521e-15, 9.4712e-16, 6.0955e-15, 5.6086e-17,\n        7.5816e-15, 3.9976e-17, 1.5159e-16, 3.1468e-18, 1.4629e-14, 1.3379e-15,\n        1.1158e-13, 7.9748e-16, 3.7075e-17, 3.4973e-15, 7.7844e-15, 1.2086e-15,\n        8.0378e-17, 1.1155e-15, 5.4647e-14, 1.5977e-16, 1.5064e-14, 2.7492e-15,\n        1.0338e-15, 2.2843e-14, 1.7450e-14, 2.6535e-17, 1.5103e-14, 4.7577e-15,\n        7.5070e-18, 1.1460e-14, 1.6806e-17, 8.8176e-15, 1.0176e-15, 1.1746e-14,\n        1.0593e-18, 9.5534e-17, 4.7274e-17, 1.3888e-15, 9.8331e-17, 1.0081e-18,\n        2.9273e-14, 2.4273e-14, 3.9405e-17, 1.1450e-15, 3.9576e-18, 6.7857e-16,\n        3.1996e-14, 4.8468e-14, 1.7432e-16, 7.5199e-16, 2.9949e-15, 1.1681e-16,\n        4.3108e-15, 1.3194e-14, 5.6151e-15, 4.3077e-19, 5.6031e-18, 1.8539e-14,\n        1.3171e-16, 8.1599e-17, 2.3618e-16, 4.4054e-14, 3.9576e-15, 3.1115e-15,\n        1.2596e-18, 1.4108e-18, 9.2742e-16, 2.3121e-13, 2.3839e-13, 2.4049e-16,\n        2.7588e-14, 1.3165e-16, 2.8597e-16, 7.7313e-17, 1.9530e-14, 1.0288e-15,\n        3.2314e-17, 5.3525e-18, 2.5586e-14, 3.7805e-15, 1.8754e-18, 1.0278e-17,\n        1.2832e-15, 3.6630e-15, 1.0504e-16, 6.6557e-16, 4.0342e-18, 1.2128e-14,\n        1.3426e-15, 3.0049e-17, 8.7197e-16, 3.8609e-15, 5.2011e-14, 1.5932e-18,\n        2.3142e-15, 3.0573e-17, 5.5757e-16, 2.0185e-16, 5.0682e-15, 1.7491e-14,\n        1.5557e-15, 5.2595e-16, 8.0058e-17, 7.3599e-15, 5.0911e-14, 1.0100e-14,\n        1.8625e-14, 4.3676e-14, 3.1339e-16, 5.0324e-14, 1.8497e-15, 7.6011e-16,\n        1.2487e-14, 2.5067e-14, 5.2775e-15, 9.3987e-16, 3.7003e-14, 5.3670e-14,\n        1.1761e-14, 1.9191e-14, 1.9632e-14, 1.2270e-16, 2.5773e-16, 3.0975e-16,\n        3.6705e-15, 1.0799e-14, 1.2832e-16, 6.3699e-17, 2.4805e-14, 8.9841e-17,\n        2.2842e-18, 6.3282e-15, 2.9244e-17, 1.5766e-17, 4.7257e-14, 1.8755e-14,\n        2.1324e-15, 1.3737e-14, 4.9594e-15, 2.6818e-14, 6.2169e-17, 5.2323e-14,\n        2.5039e-15, 2.3268e-16, 6.3200e-15, 2.3127e-14, 7.1825e-19, 1.1156e-17,\n        4.3438e-15, 4.3472e-14, 1.2036e-18, 4.2988e-17, 3.3809e-15, 7.4892e-18,\n        8.7419e-14, 5.8208e-15, 4.6855e-14, 2.0814e-18, 8.7405e-15, 4.5771e-16,\n        4.2443e-14, 5.4438e-17, 1.2809e-19, 1.4650e-13, 6.7403e-15, 1.0609e-16,\n        1.9505e-14, 5.7624e-14, 2.1213e-16, 8.8660e-16, 1.6690e-15, 1.5943e-14,\n        4.5550e-15, 1.0797e-14, 2.5893e-16, 1.5272e-16, 4.4441e-14, 1.5890e-14,\n        4.2118e-14, 7.9130e-17, 4.0422e-17, 4.4728e-14, 6.9649e-16, 1.4422e-18,\n        9.5628e-15, 1.9969e-14, 1.7155e-14, 3.1025e-14, 1.0973e-16, 9.4865e-15,\n        6.6752e-17, 1.4761e-16, 1.5483e-15, 6.7797e-15, 1.7002e-16, 3.2065e-14,\n        4.1937e-14, 1.6051e-18, 6.6664e-17, 7.9315e-15, 5.5033e-15, 2.0616e-14,\n        4.8006e-17, 1.3932e-16, 3.2053e-14, 4.4405e-15, 1.8696e-16, 5.0148e-14,\n        6.0603e-15, 6.0237e-18, 5.8090e-17, 3.9221e-17, 2.3083e-14, 2.8455e-14,\n        1.3988e-15, 2.3894e-16, 6.3789e-15, 1.6444e-16, 1.2937e-16, 1.3889e-14,\n        9.3358e-14, 4.0229e-16, 1.0367e-14, 4.5867e-16, 1.2724e-14, 3.7820e-15,\n        5.3177e-14, 1.0197e-15, 1.0477e-14, 3.0680e-15, 2.2177e-16, 7.1852e-15,\n        4.1841e-16, 4.8034e-15, 6.2003e-16, 8.0620e-14, 5.8319e-17, 1.0585e-18,\n        2.4306e-14, 1.1344e-16, 1.5972e-17, 9.5465e-15, 2.6244e-17, 2.9598e-18,\n        7.5561e-16, 1.3189e-14, 3.0840e-14, 3.7046e-14, 1.1670e-14, 6.2186e-14,\n        1.5836e-17, 1.1579e-17, 1.3273e-18, 1.1439e-14, 2.8338e-15, 1.7372e-16,\n        1.5635e-16, 4.0433e-16, 1.1390e-14, 1.1066e-13, 1.6972e-14, 2.2958e-15,\n        4.6712e-15, 1.2002e-16, 6.7996e-15, 7.0001e-18, 1.3736e-15, 7.6520e-15,\n        8.1308e-15, 9.1875e-16], device='cuda:0')"
      },
      "56": {
        "step": "tensor(11268.)",
        "exp_avg": "tensor([[ 1.0140e-40, -6.1427e-39, -3.7580e-40,  ...,  1.5855e-40,\n         -2.0659e-40,  8.3407e-39],\n        [ 7.5749e-41,  4.2727e-39,  5.0839e-40,  ...,  2.6199e-40,\n          2.5393e-40, -2.4359e-39],\n        [ 1.5137e-40,  2.5644e-39,  3.3615e-40,  ...,  3.3320e-40,\n          2.2883e-40, -1.2494e-38],\n        ...,\n        [-2.7500e-40, -1.2158e-39, -3.8930e-40,  ..., -7.2251e-40,\n         -1.3168e-40,  1.5525e-38],\n        [-3.1211e-41, -9.2121e-40, -6.8983e-41,  ..., -1.2729e-40,\n         -7.7085e-41,  1.1951e-39],\n        [ 4.9313e-40,  8.7516e-39,  7.9359e-40,  ...,  1.2650e-39,\n          8.1983e-40,  2.4350e-38]], device='cuda:0')",
        "exp_avg_sq": "tensor([[4.3996e-16, 1.0730e-16, 8.8895e-17,  ..., 6.2241e-16, 2.0098e-16,\n         7.7722e-16],\n        [6.6433e-17, 2.7101e-18, 7.2086e-18,  ..., 8.9856e-18, 8.3183e-18,\n         2.3743e-17],\n        [1.9896e-16, 3.4970e-17, 4.6338e-17,  ..., 2.1240e-16, 6.1713e-17,\n         2.7572e-16],\n        ...,\n        [6.5729e-16, 1.0626e-16, 1.1173e-17,  ..., 1.6530e-16, 3.1292e-17,\n         1.9230e-16],\n        [5.4303e-19, 6.3541e-20, 8.3315e-20,  ..., 1.2731e-19, 1.3215e-19,\n         3.6896e-19],\n        [8.6542e-16, 1.2996e-16, 6.3334e-17,  ..., 4.1764e-16, 1.7945e-16,\n         4.8002e-16]], device='cuda:0')"
      },
      "57": {
        "step": "tensor(11268.)",
        "exp_avg": "tensor([-3.1840e-37,  8.7286e-37, -4.6733e-37, -1.0053e-37,  1.9599e-37,\n         1.6314e-37,  7.0110e-37, -2.0687e-37,  2.1198e-36,  5.4241e-37,\n        -7.3699e-38, -2.1996e-36, -6.0241e-38, -1.3586e-36, -8.3019e-37,\n         2.7353e-36, -6.7160e-39, -4.2394e-37,  8.9274e-37, -1.6987e-36,\n        -1.2727e-36, -5.8772e-37, -4.6906e-38, -9.7357e-37,  1.4088e-37,\n        -3.1825e-37,  4.6317e-36, -3.4361e-37, -1.0248e-36, -4.2481e-37,\n        -1.5462e-36, -2.2772e-37, -4.6233e-38,  1.1767e-36, -8.2528e-38,\n        -4.9802e-39,  1.7831e-36,  3.3412e-37,  1.6058e-36, -3.6666e-37,\n        -1.4248e-37, -2.2332e-37, -1.2406e-37, -4.3119e-37, -1.1535e-37,\n         3.1219e-36, -3.2293e-37,  5.4758e-37,  2.2773e-36,  3.3094e-37,\n        -1.9194e-37, -2.9803e-36, -4.9395e-38, -2.2429e-36, -2.4668e-37,\n         5.1140e-37,  3.3602e-36,  3.2732e-36,  3.3756e-36, -7.2446e-37,\n        -2.2748e-37, -6.1811e-37,  7.6254e-38,  2.5410e-36, -1.4099e-36,\n         1.4713e-36, -7.3775e-40, -5.2812e-37, -2.5408e-37, -2.9069e-37,\n        -2.2809e-37, -7.4183e-38,  4.8707e-39, -2.4755e-37, -1.8371e-36,\n        -1.6385e-37, -2.7814e-36,  5.3467e-37, -1.3858e-36,  3.8783e-37,\n        -8.5446e-37,  2.9529e-37,  9.9651e-38, -3.9925e-36,  1.6373e-36,\n         1.2232e-36,  2.7191e-36,  8.6904e-37,  2.4241e-37,  5.8973e-38,\n         5.6320e-37, -1.9713e-36, -2.2014e-36,  2.3448e-37,  2.6275e-37,\n         8.9553e-37, -1.0107e-36, -7.7623e-37, -6.0187e-37, -1.5252e-36,\n        -1.9859e-37, -2.3596e-37, -1.6693e-38, -3.7859e-36,  5.1467e-37,\n        -1.3530e-36,  1.0803e-36,  2.0099e-36, -1.5847e-37, -6.3614e-37,\n        -2.0546e-37,  1.3840e-38, -2.1467e-37, -5.2784e-36, -2.8618e-37,\n        -2.9460e-37, -2.9721e-37,  4.7065e-37, -1.9131e-36,  8.5526e-37,\n        -8.4934e-37,  1.0286e-36, -3.3631e-38,  3.7815e-37, -3.4327e-37,\n        -1.4451e-37, -2.8437e-37, -3.0990e-37,  5.1131e-38, -9.2114e-37,\n        -4.8137e-37,  5.8541e-37, -8.8341e-38, -4.0006e-37,  2.3080e-36,\n         7.7183e-38, -2.9017e-37,  7.9827e-37, -6.3370e-37,  1.1348e-38,\n        -4.2205e-37, -4.6489e-37, -7.4178e-37, -2.0332e-37, -1.1819e-36,\n        -8.1229e-37, -5.5630e-37, -3.7375e-37, -1.0895e-36, -5.6763e-37,\n        -1.9813e-37, -8.3299e-38, -2.2120e-37, -2.6253e-36, -6.8165e-38,\n        -1.5285e-36,  4.8578e-37,  2.2988e-37, -2.3132e-37,  8.3495e-37,\n        -2.3942e-37,  3.3678e-37,  2.2205e-37, -4.1736e-37,  2.5302e-37,\n         1.2306e-36,  3.0764e-37, -2.8167e-37, -9.5005e-37,  1.4133e-37,\n        -1.7471e-36, -2.4186e-36,  9.5028e-37, -2.1751e-37,  3.4286e-37,\n        -9.5599e-37, -5.2767e-37,  3.7070e-37, -2.2299e-36,  7.5913e-37,\n        -1.4056e-36,  5.0862e-37,  3.0639e-37,  2.0584e-38,  4.4555e-37,\n        -9.9974e-37,  8.4374e-37,  3.9442e-37, -2.7476e-37,  2.5389e-37,\n        -5.2910e-37, -2.9000e-37,  9.2232e-37, -9.9176e-38, -7.2447e-37,\n         1.6753e-36,  9.0174e-37, -1.0896e-36, -6.6378e-37,  4.5126e-37,\n        -2.4010e-36, -1.5067e-36,  6.0516e-37,  1.9626e-37, -6.5231e-38,\n         6.8589e-37,  9.2718e-37, -2.2989e-36, -3.6888e-37, -2.5645e-37,\n         1.3155e-36,  2.5091e-36,  6.6454e-37,  4.6089e-38,  5.7975e-36,\n        -3.9418e-37, -1.3454e-36,  6.2426e-37, -3.3090e-37, -2.3950e-37,\n        -3.8797e-38, -3.7514e-37,  9.9258e-38, -1.1693e-36,  1.5281e-36,\n        -8.9159e-38, -1.9859e-36, -1.0111e-36,  7.4579e-37, -8.3767e-37,\n        -1.7930e-36, -1.3651e-37, -2.3192e-37,  2.6012e-36, -9.0641e-37,\n        -3.2296e-37,  2.2311e-36, -2.0478e-37,  1.6592e-36,  1.2252e-36,\n         3.6723e-36,  2.3069e-38, -4.0036e-37, -5.0681e-37,  3.1579e-36,\n         1.1347e-37,  2.6929e-36, -8.5331e-38, -4.3135e-36, -1.4775e-36,\n        -1.6923e-37,  1.3711e-36, -2.2792e-36, -2.0051e-37, -6.5823e-37,\n        -2.1227e-37, -5.1580e-37, -4.2403e-37,  1.1491e-36,  8.5203e-37,\n         1.2765e-36, -4.6614e-38, -3.2832e-37, -2.4149e-36, -9.2175e-37,\n        -1.9267e-37,  1.0297e-36, -1.8897e-37, -9.5253e-37, -1.7679e-37,\n        -2.1828e-36, -5.4937e-39, -6.5674e-38, -3.4710e-37, -1.9475e-37,\n        -4.5811e-37, -1.6275e-37, -3.0743e-37, -3.0622e-37, -5.1318e-37,\n        -4.8555e-38, -2.2558e-36,  2.6547e-37,  1.1244e-36,  4.0521e-37,\n        -1.7950e-37, -3.7758e-37, -8.8758e-37,  1.7231e-36, -2.7819e-37,\n        -1.8198e-36,  5.9662e-37, -4.1560e-37,  1.1715e-37, -2.1407e-36,\n        -8.9132e-38, -1.4695e-37,  4.9837e-37,  4.3456e-36, -3.3360e-37,\n        -1.2377e-37, -2.2981e-37, -2.3349e-36,  3.8438e-37,  7.0168e-37,\n        -4.9333e-37,  3.3022e-38, -1.1627e-36, -1.1395e-36,  5.9250e-36,\n         1.5143e-36, -9.0273e-37,  1.7224e-37,  1.3699e-37, -3.0732e-37,\n         8.0755e-38,  4.2251e-37, -1.0274e-36, -3.1216e-37,  2.3952e-37,\n         3.5279e-36, -7.1293e-38, -5.3258e-37, -1.3444e-37, -1.4143e-36,\n         7.1034e-37, -1.2315e-36, -2.6735e-37, -1.0117e-36,  5.4175e-37,\n         3.5342e-39, -1.5902e-37, -6.7541e-37,  2.6102e-36,  2.1555e-36,\n         2.1579e-36, -1.8067e-37, -9.4677e-38,  1.2755e-36,  1.3147e-36,\n        -1.1434e-36, -4.9135e-38, -3.3768e-37, -2.1096e-37, -2.0956e-36,\n        -7.6156e-37, -2.2209e-37, -1.1386e-37, -2.1121e-37,  1.5032e-36,\n         2.7780e-36,  3.7642e-37,  3.9432e-37,  2.3806e-36,  1.3135e-36,\n        -1.6672e-37, -1.3778e-36, -2.8518e-37,  6.3033e-37, -2.2937e-36,\n         2.6546e-36,  1.3301e-36,  3.4879e-37,  2.0903e-36,  1.4581e-36,\n         6.4437e-37, -1.7522e-37, -1.5026e-37, -3.2387e-37,  4.9343e-37,\n         1.8088e-37,  2.3914e-38,  2.4926e-37, -8.2964e-37, -3.3527e-38,\n        -4.4753e-37,  8.1924e-37,  1.8037e-36,  1.5218e-37, -5.8626e-39,\n        -4.2058e-37, -2.4270e-37, -1.3087e-36,  1.2753e-36, -4.4126e-37,\n        -1.0161e-36, -2.7912e-37,  1.2463e-36,  1.5126e-36,  9.1711e-37,\n         3.4185e-38,  1.0733e-36, -3.3359e-36,  3.5921e-37, -3.0697e-37,\n        -2.3206e-37,  8.6169e-37, -1.2551e-36,  2.4364e-37, -2.7903e-37,\n        -5.3653e-37, -1.7579e-36, -1.7494e-37, -3.3412e-37, -4.3777e-37,\n         2.7976e-36, -2.7137e-37,  5.4524e-37, -1.1745e-37,  9.7857e-37,\n         1.1541e-36, -1.9624e-37, -4.7484e-37,  3.0113e-37, -1.4418e-36,\n        -1.7335e-36, -9.6447e-38,  3.9212e-37, -8.6046e-37,  3.8348e-36,\n         4.8650e-36, -8.6305e-37,  1.0099e-36, -1.0946e-36, -7.1687e-37,\n        -3.1343e-37, -2.3261e-37, -8.6668e-37, -3.0338e-37, -2.5077e-37,\n        -2.4333e-37, -1.6035e-36,  6.4465e-37,  1.9637e-36,  4.4736e-37,\n        -8.9966e-38, -9.7734e-38,  3.3232e-36,  6.4263e-37, -3.8774e-37,\n        -1.2491e-36,  1.4005e-37,  3.7872e-38, -3.4416e-37, -7.9490e-37,\n         3.8701e-37,  2.7117e-36, -3.2455e-37, -1.8490e-36, -5.2208e-36,\n         2.1333e-37, -3.3633e-37, -8.5383e-37, -3.3011e-37,  6.5699e-38,\n         7.3727e-38, -3.3033e-37, -9.8990e-37, -2.2109e-37, -5.3808e-37,\n        -6.3502e-37, -2.6063e-37,  1.0372e-36, -5.3397e-37,  1.3166e-36,\n        -1.0025e-37, -3.1820e-37, -4.9392e-38,  2.5908e-36, -3.0837e-37,\n        -4.2801e-37, -3.8330e-37, -6.0219e-37,  1.4530e-36, -2.4400e-38,\n        -1.8007e-36,  3.1439e-37,  4.1526e-36, -2.5266e-38, -2.9399e-36,\n         6.5216e-37, -9.1615e-37, -4.9460e-37,  6.7202e-37, -6.3587e-37,\n        -5.0226e-37,  3.5652e-37,  7.7943e-37,  2.6861e-37,  5.4581e-37,\n         1.7279e-37,  2.8900e-37,  2.3162e-37, -2.4461e-37, -3.3226e-37,\n         1.2084e-36, -2.0498e-37, -3.7821e-37, -2.0509e-36,  6.1575e-37,\n         1.7643e-36,  5.7642e-37,  5.4344e-37, -5.5037e-37, -1.3808e-36,\n        -2.2703e-37,  1.2287e-37, -6.4065e-37, -4.7557e-37, -1.0739e-36,\n        -2.7775e-37,  3.6905e-36], device='cuda:0')",
        "exp_avg_sq": "tensor([2.6007e-10, 8.2397e-12, 7.8578e-11, 1.5796e-13, 2.6756e-12, 1.1118e-13,\n        2.8833e-11, 6.5349e-13, 5.3145e-10, 1.7857e-13, 1.8830e-11, 7.0261e-11,\n        1.8624e-10, 5.1073e-12, 1.0594e-11, 9.6504e-12, 4.0522e-13, 8.9453e-11,\n        6.4805e-11, 6.4466e-11, 1.2807e-11, 1.2279e-11, 1.0496e-13, 8.9056e-11,\n        1.6271e-12, 1.4802e-12, 7.2677e-11, 2.9097e-11, 1.4710e-12, 7.5539e-12,\n        2.0604e-10, 2.1877e-13, 1.2779e-11, 1.6459e-12, 4.3901e-13, 1.0941e-12,\n        2.3234e-11, 6.9616e-13, 2.5137e-10, 1.1008e-11, 1.8170e-12, 1.6338e-12,\n        1.7844e-11, 3.1981e-11, 8.8593e-11, 7.0775e-11, 7.8963e-13, 1.8199e-11,\n        1.1701e-11, 3.4094e-11, 6.6631e-12, 1.1826e-10, 9.2074e-11, 2.9719e-11,\n        1.4463e-11, 1.3185e-10, 3.4580e-10, 6.4176e-10, 4.0533e-10, 5.9494e-11,\n        1.4912e-13, 1.3507e-10, 2.7284e-12, 3.2065e-11, 7.1561e-11, 6.1587e-12,\n        2.9572e-11, 2.2638e-11, 2.2061e-13, 2.3126e-11, 7.2705e-14, 4.3843e-11,\n        2.4191e-12, 5.5203e-13, 3.0935e-11, 8.4332e-13, 2.8126e-10, 5.3538e-11,\n        1.0924e-10, 4.8816e-11, 7.9380e-11, 1.7706e-12, 7.9132e-13, 1.9789e-10,\n        4.6179e-11, 3.9051e-11, 3.0234e-11, 3.9794e-11, 7.9157e-13, 1.2362e-11,\n        2.7639e-14, 1.4640e-10, 1.1389e-10, 6.3917e-12, 7.9637e-12, 3.7059e-11,\n        2.8903e-11, 2.7356e-10, 1.8466e-11, 5.6709e-11, 1.1663e-11, 6.0004e-13,\n        2.0547e-13, 7.7406e-11, 5.7157e-12, 6.1127e-12, 2.3944e-12, 2.0560e-12,\n        6.2469e-14, 3.0987e-11, 2.4261e-12, 3.0418e-14, 4.5009e-12, 2.4004e-10,\n        3.9356e-11, 6.1612e-15, 4.9129e-13, 1.0357e-10, 8.8275e-11, 2.5917e-11,\n        2.3771e-10, 4.1417e-12, 8.9681e-13, 1.1774e-12, 1.2830e-13, 3.4278e-13,\n        7.1736e-14, 8.7854e-13, 1.9746e-13, 2.3153e-11, 6.8773e-12, 1.0335e-11,\n        6.8265e-11, 1.6239e-13, 5.4341e-10, 3.4883e-12, 3.3029e-13, 3.1133e-11,\n        2.0064e-11, 1.0537e-11, 2.7659e-11, 7.6721e-11, 3.5056e-11, 2.4583e-12,\n        6.5064e-11, 5.4603e-12, 3.3255e-11, 5.5249e-12, 1.4886e-10, 2.1802e-11,\n        3.5433e-12, 6.0316e-13, 7.8941e-12, 9.6027e-10, 1.2902e-13, 1.2938e-10,\n        3.5479e-11, 1.4006e-10, 5.1766e-13, 6.1686e-11, 3.6652e-14, 2.9383e-12,\n        1.0703e-13, 1.9547e-12, 6.6675e-13, 7.7374e-12, 9.8242e-12, 1.2529e-13,\n        1.1559e-11, 3.4262e-11, 2.4575e-10, 4.8549e-11, 6.8718e-11, 5.4565e-14,\n        1.8463e-12, 7.0245e-13, 4.9385e-12, 2.7705e-12, 3.6179e-10, 7.2319e-11,\n        3.6226e-10, 3.3502e-12, 3.2059e-13, 1.3232e-11, 2.7333e-13, 2.4344e-11,\n        6.0382e-13, 3.5479e-14, 7.8255e-12, 4.0608e-11, 8.1234e-13, 1.8536e-12,\n        2.3134e-11, 7.7823e-14, 6.7975e-11, 2.0418e-10, 3.3469e-12, 3.2049e-11,\n        2.7123e-13, 4.1645e-12, 7.1978e-11, 4.4124e-11, 1.4217e-14, 8.8321e-12,\n        7.1140e-13, 1.7442e-10, 2.9371e-11, 1.4682e-11, 1.0099e-11, 7.7259e-14,\n        1.7403e-12, 1.9844e-10, 3.6016e-11, 3.4751e-12, 1.1989e-09, 2.6046e-12,\n        1.9654e-10, 9.2151e-12, 5.7462e-12, 1.5584e-12, 6.0884e-12, 5.5679e-11,\n        1.8992e-12, 1.9124e-11, 3.0452e-11, 5.9284e-13, 2.3068e-11, 1.6204e-11,\n        4.5055e-10, 3.8353e-11, 2.6722e-12, 3.6371e-12, 3.2520e-13, 5.9139e-10,\n        6.3146e-12, 1.7835e-13, 4.2400e-10, 4.2549e-14, 3.7548e-11, 1.1268e-11,\n        3.3151e-10, 1.6070e-11, 5.7319e-13, 2.7934e-13, 7.6181e-11, 3.2713e-12,\n        3.6417e-11, 3.2736e-11, 2.5235e-10, 4.3190e-11, 2.9922e-12, 2.5448e-11,\n        8.3144e-10, 3.9803e-12, 1.0603e-12, 5.2625e-13, 2.3813e-12, 6.4406e-11,\n        3.6056e-12, 6.0741e-12, 3.6651e-12, 4.9126e-13, 2.3527e-13, 3.8147e-10,\n        5.8517e-12, 1.2406e-13, 1.8726e-13, 8.0121e-13, 1.7623e-11, 3.6411e-12,\n        4.3797e-11, 1.5094e-13, 2.3962e-13, 5.6725e-12, 1.2993e-12, 6.8852e-13,\n        4.5657e-11, 1.1498e-13, 9.4389e-12, 3.6960e-11, 2.9187e-11, 3.0737e-11,\n        4.1148e-12, 1.9222e-12, 2.6953e-13, 9.9563e-13, 1.7528e-12, 1.2420e-11,\n        7.9106e-11, 7.4326e-14, 1.5966e-10, 2.9937e-10, 2.0311e-12, 2.0414e-13,\n        1.4199e-10, 1.7699e-13, 1.0241e-13, 2.1717e-12, 9.0885e-12, 1.7456e-11,\n        2.9105e-13, 1.3889e-12, 2.9562e-11, 2.0353e-10, 3.9177e-10, 2.8461e-12,\n        1.7212e-12, 5.8083e-12, 1.8651e-12, 7.1687e-10, 6.4392e-12, 8.6681e-11,\n        1.1147e-12, 3.6137e-12, 3.6674e-15, 1.5388e-12, 5.4346e-11, 4.7029e-12,\n        4.1928e-13, 2.5644e-12, 5.5075e-10, 9.6210e-13, 6.3442e-11, 9.8593e-12,\n        1.0382e-10, 1.5688e-12, 5.2889e-12, 3.1303e-11, 5.8028e-11, 9.3077e-12,\n        3.0001e-12, 3.0183e-12, 2.4914e-12, 1.3138e-11, 3.1301e-11, 2.5373e-11,\n        5.2461e-12, 7.2218e-12, 2.9691e-11, 6.7618e-12, 2.3730e-10, 1.7401e-12,\n        1.0839e-13, 3.6651e-11, 7.4584e-11, 9.7365e-12, 3.7806e-13, 4.3436e-14,\n        2.6429e-12, 9.6545e-11, 1.2245e-11, 1.8247e-12, 1.6397e-13, 2.8708e-12,\n        3.0614e-12, 5.6219e-12, 1.2377e-10, 1.4718e-13, 7.1458e-13, 5.8925e-10,\n        1.8197e-11, 1.2250e-12, 5.6212e-14, 4.4454e-11, 3.7764e-11, 6.9010e-12,\n        2.9687e-10, 1.3935e-13, 7.5639e-12, 7.6495e-13, 6.6250e-12, 1.4114e-10,\n        6.5377e-11, 3.7644e-12, 1.4786e-12, 3.2092e-11, 2.1316e-12, 8.2078e-11,\n        5.3668e-12, 2.5171e-12, 3.6281e-12, 4.9614e-13, 9.3474e-11, 1.1099e-10,\n        3.1961e-13, 1.5848e-10, 1.0454e-12, 4.8863e-13, 2.4145e-11, 5.1383e-12,\n        2.8645e-13, 3.7255e-11, 1.3229e-11, 4.9879e-13, 1.6784e-10, 1.9395e-12,\n        3.3581e-14, 1.0193e-10, 4.2025e-13, 4.4770e-12, 6.7750e-11, 1.6183e-10,\n        3.0096e-11, 1.5542e-12, 4.1087e-12, 2.4328e-10, 1.1201e-13, 3.4975e-11,\n        3.0728e-11, 2.7108e-12, 7.6265e-12, 2.5511e-14, 6.0554e-12, 1.3684e-10,\n        4.9288e-13, 5.3905e-12, 1.5214e-12, 1.0199e-13, 3.8639e-11, 1.9395e-10,\n        3.0561e-10, 7.2274e-11, 5.9324e-11, 3.1950e-12, 1.0166e-11, 8.3383e-14,\n        4.9319e-13, 1.4977e-10, 1.3004e-11, 6.8066e-13, 2.4747e-13, 1.4512e-10,\n        6.5034e-11, 4.1847e-11, 4.8133e-12, 5.1804e-13, 1.5699e-13, 7.1620e-11,\n        6.7928e-11, 1.0975e-11, 7.5003e-11, 4.9836e-13, 4.9650e-11, 1.6097e-11,\n        2.0237e-11, 2.4624e-12, 3.3658e-11, 2.6652e-14, 2.7337e-10, 5.6424e-10,\n        1.2339e-12, 5.1057e-11, 4.2616e-11, 3.0858e-13, 6.2201e-12, 3.5290e-11,\n        1.6719e-12, 3.9984e-11, 5.2187e-11, 2.3699e-11, 5.8036e-11, 1.2193e-11,\n        1.2054e-11, 9.3812e-11, 1.4024e-11, 1.6797e-12, 1.0810e-12, 2.2015e-12,\n        2.5059e-11, 7.3891e-14, 7.5434e-11, 3.1705e-12, 6.2248e-12, 3.1826e-11,\n        7.9275e-13, 3.0290e-11, 8.7121e-13, 1.3499e-10, 9.9948e-12, 6.3437e-11,\n        3.9575e-12, 8.4201e-11, 2.6111e-11, 1.1386e-12, 1.4542e-12, 1.1976e-11,\n        3.2318e-12, 1.5535e-12, 5.4587e-12, 1.1866e-12, 2.1315e-10, 4.7913e-14,\n        7.4657e-13, 3.9517e-12, 4.1665e-14, 3.9839e-10, 1.1946e-13, 2.5540e-13,\n        2.3779e-10, 9.8841e-11, 1.9314e-11, 2.1640e-12, 2.3224e-10, 1.7356e-11,\n        5.2159e-11, 2.0231e-13, 6.0171e-11, 1.2544e-11, 1.7476e-11, 1.3656e-10,\n        1.0039e-13, 2.0446e-10], device='cuda:0')"
      },
      "58": {
        "step": "tensor(11268.)",
        "exp_avg": "tensor([ 1.1051e-37,  1.5914e-38,  9.8373e-38,  1.0454e-38, -2.2174e-39,\n         3.1580e-39,  5.1024e-38,  9.1296e-40,  2.1715e-37,  1.0910e-38,\n         7.7273e-38,  1.1975e-38,  1.5674e-37,  1.3408e-38,  3.9497e-41,\n         2.3084e-38,  6.4657e-39,  1.5170e-37,  5.0850e-38,  9.3758e-39,\n         4.3818e-38, -1.0294e-39,  3.6922e-39, -9.0406e-40,  4.1024e-39,\n         5.7841e-40,  9.7553e-38, -2.4309e-40,  5.2350e-38,  6.9438e-38,\n         9.4643e-40,  1.8681e-39,  6.0798e-38,  1.3914e-38,  6.4463e-39,\n         1.7913e-39,  4.4743e-38,  1.4674e-39,  2.3694e-37, -2.7281e-40,\n         8.8094e-40,  2.1856e-39, -3.8048e-40, -2.8871e-40,  2.4802e-39,\n         8.7262e-38,  1.6109e-39,  8.8983e-39,  3.6798e-38,  2.8498e-38,\n        -3.6304e-39,  1.6336e-38,  2.8857e-38,  2.4439e-38, -1.6763e-39,\n         1.7341e-37,  1.4643e-37,  1.5848e-37,  2.4687e-37,  1.1450e-37,\n         4.7370e-39,  1.7215e-37,  2.6837e-38,  2.0904e-38,  2.8252e-40,\n         8.7686e-38,  5.1911e-38,  4.3177e-39, -2.6674e-39,  6.4984e-40,\n         1.2360e-39,  2.1263e-38,  7.7657e-40,  4.3549e-39,  8.1116e-38,\n         2.4878e-39, -1.0448e-39,  5.1196e-38,  8.4987e-40,  4.0296e-38,\n        -6.1364e-41,  7.1979e-39,  4.4040e-39,  4.8657e-38,  2.3045e-38,\n         2.1593e-38,  1.0033e-37,  5.6130e-38, -3.0693e-39,  1.0364e-39,\n         8.8279e-39,  1.7813e-40,  1.0112e-37,  2.0540e-39,  1.9558e-38,\n         7.4340e-38,  1.3031e-38,  3.2954e-38, -1.1408e-39,  1.1741e-38,\n         1.1140e-38, -1.4058e-39,  2.7435e-39,  3.2997e-38,  3.7765e-39,\n         7.9626e-39, -2.6191e-40,  7.3545e-38, -1.2588e-39, -9.7170e-40,\n         3.0490e-40,  4.1867e-39,  4.0526e-40, -1.8467e-38, -3.2520e-39,\n         8.2190e-40, -2.3249e-39,  1.8538e-38,  1.6213e-38,  3.1372e-38,\n         1.2047e-38,  1.6497e-39,  2.1766e-39,  8.0490e-39, -5.0133e-40,\n         5.7710e-39,  4.6693e-40, -3.0268e-40,  1.3088e-39, -7.9982e-40,\n         1.0447e-38,  4.1450e-39,  6.5027e-39,  1.5512e-39,  1.6965e-37,\n         4.7780e-39,  3.4270e-39,  3.0757e-38,  7.4051e-39, -4.2503e-39,\n        -2.7647e-40, -2.1773e-39,  8.4945e-38,  8.2118e-40,  1.1931e-38,\n        -3.5566e-41,  1.1481e-37, -4.8566e-40,  6.1921e-40,  1.3621e-37,\n         1.5556e-39, -4.1462e-39,  3.7854e-39,  1.1359e-38, -2.7998e-39,\n        -2.0272e-40,  1.1675e-37,  1.0768e-37,  1.8441e-40,  8.7149e-38,\n         1.7161e-39,  9.6236e-39,  3.5116e-39,  4.0688e-38,  1.0216e-38,\n         2.9812e-38,  1.4955e-40, -3.5517e-41,  2.9172e-38,  1.2024e-37,\n         6.8467e-40, -2.4328e-40,  1.2462e-37,  4.5062e-39,  4.7065e-39,\n         5.5501e-38, -1.2535e-39,  2.9279e-38,  1.8673e-38,  2.4121e-38,\n         2.6924e-38,  1.3558e-37,  3.4803e-39,  5.4906e-39,  1.3236e-38,\n         1.5234e-39,  1.7654e-38,  6.5017e-39,  4.8490e-40,  1.1534e-37,\n        -3.5610e-41,  6.7271e-40,  4.9798e-38,  3.9893e-39,  8.7421e-38,\n         1.1838e-37,  7.0700e-38,  8.1502e-40,  4.7389e-40, -1.1621e-41,\n         8.9330e-38,  4.7114e-38,  7.0530e-39,  9.1567e-39,  6.8289e-39,\n         1.0373e-37,  3.4405e-38, -2.1683e-40,  1.2307e-40,  1.1913e-39,\n        -7.0888e-39,  1.9751e-37,  1.3731e-37,  1.9333e-39,  2.0468e-37,\n         2.0977e-40,  1.4173e-37,  7.0917e-38,  5.3357e-40,  1.0254e-39,\n        -2.6185e-39,  3.9118e-38,  5.9937e-40, -5.6008e-40,  5.0526e-38,\n         4.9439e-39,  5.7312e-38,  1.1143e-37,  1.3393e-37,  6.8664e-38,\n         1.1611e-37,  1.8914e-39,  8.2407e-40,  2.4670e-37,  5.8265e-38,\n        -6.8748e-40,  2.5659e-37,  4.4598e-39,  7.7341e-38,  5.0074e-38,\n         1.6945e-37,  1.4878e-38,  3.6766e-40, -7.5829e-40,  1.2000e-37,\n         8.9803e-40,  9.0919e-38,  2.1543e-38, -8.6852e-39,  3.2392e-38,\n         5.9084e-40,  4.8712e-38,  2.4971e-38,  3.0753e-39, -1.4411e-39,\n        -6.4853e-40, -1.7536e-39,  5.9938e-40,  1.6049e-38,  9.9046e-38,\n        -2.0467e-39,  2.8833e-39,  4.5060e-40,  6.9544e-40, -6.6987e-40,\n         4.4865e-39,  2.9179e-39,  2.3216e-39,  5.5047e-38,  1.0001e-39,\n         1.0100e-39,  3.0034e-39,  2.8974e-39,  2.8946e-39,  2.0026e-39,\n        -4.3966e-40, -1.3396e-39,  4.2091e-40, -6.1882e-40,  5.6921e-42,\n        -2.3702e-39, -4.2343e-40,  3.2909e-39,  5.7619e-39,  1.1787e-38,\n         4.2925e-39, -5.9333e-40,  1.0783e-38,  1.6813e-37,  1.4782e-39,\n         4.4159e-39,  1.8139e-37, -7.0561e-40,  2.1461e-39,  6.3618e-39,\n         5.5911e-39,  2.9621e-39, -2.3038e-39,  9.3795e-38, -1.9213e-40,\n         1.3085e-39, -5.2179e-41,  2.2815e-38,  1.3222e-37,  6.0806e-38,\n        -1.4927e-39,  2.1524e-39,  2.6953e-39, -6.1032e-40,  2.7940e-37,\n         8.1878e-39,  3.7913e-40,  6.2732e-39,  4.0379e-39,  2.6932e-41,\n         4.9373e-39,  1.4574e-37,  1.5639e-38,  8.6897e-40,  2.0291e-39,\n         1.7323e-37,  3.0345e-39, -8.3003e-41,  2.2500e-39,  3.7879e-40,\n         4.0461e-39,  2.9480e-39, -2.2630e-39, -7.9826e-40,  7.6008e-40,\n         2.8503e-39,  5.1981e-39,  3.1808e-39,  8.9718e-38,  1.2694e-37,\n         7.2365e-38,  5.9298e-40,  6.1229e-39,  4.9497e-38,  3.1954e-38,\n         6.9788e-40,  2.3940e-39,  5.9584e-40, -1.1441e-39,  1.7131e-38,\n         2.6496e-40,  1.7871e-39,  5.8157e-39,  3.3626e-39,  4.9969e-38,\n         8.2034e-38,  8.0789e-39,  9.9484e-39,  1.1861e-38,  4.2391e-39,\n         3.0237e-39, -5.3660e-40,  6.1153e-40,  2.2895e-38,  2.7970e-39,\n         1.2276e-37,  1.4013e-38,  9.3968e-39,  5.7406e-38,  6.2223e-39,\n         4.1539e-38,  1.3804e-37,  3.4965e-39,  3.6170e-38,  3.7010e-39,\n         6.5027e-40,  5.2883e-38,  5.9944e-38, -1.9181e-39,  2.8733e-39,\n        -2.2998e-39, -4.0624e-39,  7.6262e-38,  3.8972e-39,  1.8403e-39,\n        -2.4509e-40,  6.2628e-39,  3.0148e-39,  1.6306e-37, -1.7547e-39,\n        -9.2602e-40,  1.1364e-39, -1.4853e-39,  8.0597e-38,  4.3662e-38,\n         1.5467e-40,  2.3254e-38, -2.3270e-40, -9.1010e-40, -3.1005e-39,\n         8.0721e-40,  1.7290e-38,  6.2532e-39,  1.2493e-39,  4.5591e-39,\n         9.5835e-38,  1.5190e-38,  8.0101e-40,  8.9659e-41, -2.6684e-41,\n         1.2427e-37,  1.0130e-39,  9.5456e-38,  1.9313e-38,  2.0866e-38,\n         1.7496e-38,  2.0678e-39,  2.5097e-39,  3.7153e-38,  1.5144e-38,\n         6.7012e-38,  6.6447e-41,  3.0233e-39,  6.1314e-41,  8.6657e-38,\n         1.7288e-37, -1.9140e-40,  1.8953e-38,  1.3440e-37, -2.8162e-40,\n        -3.4953e-40,  1.2706e-39,  1.1844e-39, -1.8941e-39,  5.7831e-39,\n         3.8417e-39,  1.3053e-38,  2.9536e-40,  8.1412e-38,  3.9294e-38,\n         1.0400e-39,  3.5253e-39,  1.1301e-37,  1.0249e-37,  4.3606e-38,\n         1.2599e-39,  9.0211e-39,  2.0149e-38,  3.2374e-41, -1.1505e-39,\n         9.1462e-40,  1.7905e-38, -2.3363e-39,  9.6592e-38, -1.8012e-38,\n        -1.4060e-39,  1.9188e-39,  5.2717e-38,  8.1643e-40,  3.6436e-39,\n         5.8313e-38, -8.0628e-40,  1.2981e-39,  9.4561e-38,  3.6519e-38,\n        -1.3326e-39,  1.1250e-37,  3.7969e-39,  3.5747e-38,  2.1557e-38,\n         6.2713e-39, -4.2679e-40,  1.2410e-38, -3.0133e-40,  4.4344e-40,\n         2.2823e-38,  3.7565e-38,  1.6113e-39,  1.0694e-37,  5.8249e-39,\n         1.3237e-37,  1.8723e-39,  1.5387e-37,  3.4907e-38,  1.7248e-38,\n         2.7880e-39, -2.7040e-40,  6.9037e-39,  1.5761e-38, -2.7560e-40,\n        -1.3003e-39,  5.5836e-39, -1.8394e-39,  8.9481e-39,  1.4624e-41,\n         1.1514e-37,  2.4206e-39,  1.2837e-39, -2.4088e-39,  2.3796e-39,\n         1.7498e-37,  2.9100e-39, -7.4209e-40,  1.4474e-39,  1.4243e-37,\n         7.5965e-38, -3.6639e-39,  6.4126e-38, -4.0616e-40,  3.6853e-40,\n         2.0127e-39, -1.3315e-39, -1.8398e-39, -6.4271e-41,  2.7781e-38,\n         3.8596e-39,  2.2937e-37], device='cuda:0')",
        "exp_avg_sq": "tensor([8.7980e-14, 8.5724e-18, 3.1691e-14, 5.3551e-18, 1.6055e-16, 2.0575e-18,\n        3.3366e-15, 2.7106e-19, 3.4538e-13, 1.1049e-15, 1.0006e-14, 2.8089e-15,\n        1.8133e-13, 2.3574e-15, 2.8095e-18, 7.7146e-17, 2.0693e-19, 8.5146e-14,\n        7.0497e-15, 1.1949e-14, 9.9307e-15, 3.3587e-18, 2.1799e-18, 1.0827e-15,\n        2.6510e-16, 4.8364e-17, 2.5966e-14, 7.6921e-18, 1.3801e-14, 9.1751e-15,\n        9.2701e-15, 2.7084e-17, 5.3006e-15, 2.1896e-15, 5.5151e-16, 5.2829e-17,\n        6.2929e-15, 5.1987e-19, 2.8172e-13, 1.6302e-17, 1.1909e-19, 1.5863e-16,\n        6.5067e-17, 3.1541e-16, 4.1220e-15, 8.3072e-15, 8.3177e-17, 2.2799e-17,\n        3.8938e-16, 1.2331e-14, 6.7641e-17, 4.2814e-15, 2.3491e-14, 1.1675e-15,\n        1.1421e-20, 1.2262e-13, 6.9030e-14, 1.6438e-13, 3.0556e-13, 8.9091e-14,\n        4.9664e-18, 2.0790e-13, 2.4505e-15, 1.0287e-17, 1.4237e-15, 5.3402e-15,\n        1.4139e-14, 1.5809e-17, 7.5375e-19, 1.5835e-17, 2.1319e-17, 6.3043e-16,\n        2.1035e-16, 3.6226e-16, 3.9658e-14, 1.9386e-18, 5.0762e-15, 2.4493e-14,\n        1.3363e-15, 2.0175e-14, 3.5114e-17, 2.6715e-16, 1.4827e-18, 1.0255e-13,\n        5.6944e-15, 2.6816e-16, 1.1396e-14, 6.0866e-15, 5.8991e-18, 1.3336e-17,\n        1.3124e-16, 5.4007e-15, 9.6466e-14, 1.1512e-17, 4.6410e-16, 2.3944e-14,\n        4.6848e-16, 7.7014e-14, 4.9380e-17, 8.6875e-16, 3.4793e-17, 4.9017e-18,\n        3.9048e-18, 1.9684e-14, 1.3586e-16, 8.8327e-16, 7.9626e-18, 6.8676e-15,\n        9.4027e-18, 9.7109e-17, 3.1246e-16, 1.5387e-17, 7.3782e-21, 1.7566e-14,\n        3.5600e-16, 1.3395e-18, 1.0948e-16, 1.4626e-14, 9.9694e-15, 3.8318e-15,\n        4.4706e-14, 1.4289e-17, 1.1782e-16, 4.0087e-18, 4.6119e-17, 5.2459e-18,\n        6.6735e-18, 2.5934e-16, 1.3885e-17, 1.3720e-16, 5.3427e-16, 4.4072e-16,\n        6.7882e-15, 2.6354e-16, 2.1040e-13, 5.5032e-16, 1.3831e-16, 9.7617e-16,\n        1.1433e-16, 3.2621e-16, 1.1732e-17, 5.9580e-16, 5.8066e-14, 7.2551e-16,\n        4.3786e-15, 1.5925e-17, 5.0136e-14, 2.7526e-18, 7.2539e-15, 5.4676e-14,\n        3.6470e-18, 8.0705e-18, 2.6985e-18, 2.6359e-13, 4.2727e-17, 4.6465e-15,\n        5.0767e-14, 4.3495e-14, 4.8199e-17, 2.1040e-14, 4.4066e-17, 8.0971e-16,\n        1.7089e-17, 4.7313e-15, 6.5632e-16, 2.1000e-16, 1.4749e-15, 2.0807e-17,\n        1.9353e-15, 5.0564e-14, 4.7389e-15, 3.2800e-16, 5.1551e-14, 1.5402e-16,\n        3.4317e-16, 1.4189e-14, 1.3669e-17, 7.4599e-16, 4.4634e-14, 5.3017e-16,\n        6.1715e-14, 2.0716e-14, 1.1903e-17, 2.5415e-15, 1.0674e-15, 7.2790e-17,\n        1.8337e-15, 2.4511e-16, 5.9044e-18, 4.8598e-14, 4.4872e-17, 2.1193e-19,\n        1.4897e-15, 1.9136e-17, 4.2145e-14, 4.8288e-14, 1.6149e-14, 1.2891e-16,\n        1.7014e-16, 3.9509e-17, 8.6531e-14, 1.1351e-14, 3.2456e-16, 1.9596e-16,\n        2.8700e-16, 4.7479e-14, 7.9890e-16, 4.0471e-17, 7.8387e-18, 3.8123e-17,\n        8.1622e-16, 1.5527e-13, 6.4233e-14, 3.0140e-18, 2.8312e-13, 4.8587e-16,\n        1.9965e-13, 1.1051e-14, 1.4262e-15, 5.5823e-20, 3.4929e-16, 2.3242e-14,\n        1.2402e-16, 1.7418e-17, 1.5615e-15, 7.8064e-17, 1.3163e-14, 2.5876e-14,\n        1.5172e-13, 1.3836e-14, 3.7727e-14, 1.4101e-18, 4.6150e-17, 4.0897e-13,\n        6.7310e-15, 1.3820e-17, 4.3421e-13, 4.5859e-16, 6.3702e-15, 5.6797e-15,\n        1.0782e-13, 2.4818e-16, 1.4175e-16, 6.2714e-17, 2.0582e-14, 2.4809e-17,\n        7.5136e-15, 3.0082e-15, 1.4043e-14, 3.5487e-15, 3.8388e-16, 1.2730e-15,\n        2.6172e-13, 1.0254e-18, 5.3758e-20, 1.3416e-16, 9.8752e-18, 9.1562e-16,\n        2.2027e-17, 8.8950e-15, 1.9568e-18, 2.1245e-16, 4.8477e-17, 2.9532e-14,\n        3.0047e-17, 4.2727e-17, 3.8529e-18, 1.3068e-16, 5.2042e-15, 1.3590e-17,\n        3.2171e-16, 1.2174e-17, 1.5009e-17, 1.2028e-17, 1.8108e-19, 2.2898e-16,\n        1.8348e-15, 4.3781e-17, 5.6477e-19, 1.1016e-15, 2.3305e-18, 1.3604e-16,\n        1.2818e-17, 6.4841e-18, 5.8964e-16, 1.3007e-18, 2.4406e-16, 2.9208e-16,\n        7.7561e-14, 1.4829e-18, 2.0080e-14, 2.5775e-13, 4.5367e-16, 2.5784e-18,\n        1.9823e-14, 1.5429e-17, 1.3568e-18, 3.0715e-17, 2.5657e-15, 1.7757e-17,\n        1.9124e-20, 1.8432e-16, 4.5264e-15, 1.0042e-13, 1.2540e-13, 3.5269e-19,\n        2.1086e-18, 1.1126e-15, 1.2944e-18, 3.1294e-13, 4.6409e-15, 2.7456e-16,\n        4.6881e-16, 1.6409e-16, 1.7276e-17, 1.1598e-16, 7.4286e-14, 6.1969e-16,\n        6.0377e-17, 2.9702e-17, 1.7094e-13, 4.2822e-17, 1.5412e-17, 6.2153e-16,\n        3.1314e-15, 8.4981e-17, 1.0501e-15, 2.3504e-17, 1.5346e-15, 3.7175e-16,\n        4.3473e-16, 1.0238e-15, 3.5906e-16, 7.7430e-15, 2.1887e-14, 7.4282e-15,\n        5.0790e-19, 1.4188e-15, 6.8867e-15, 1.3254e-16, 1.0396e-14, 2.2046e-18,\n        5.5695e-17, 1.0751e-15, 2.5435e-15, 1.3554e-17, 1.8958e-16, 2.7060e-17,\n        2.8623e-18, 3.6107e-15, 2.6174e-15, 6.8282e-16, 7.6362e-16, 5.7661e-19,\n        3.7406e-18, 3.8366e-18, 2.5861e-15, 6.9866e-19, 1.5437e-15, 6.6850e-14,\n        1.0434e-14, 2.9043e-15, 2.2296e-16, 5.4481e-15, 1.0939e-15, 1.2229e-15,\n        1.5914e-13, 6.4336e-18, 2.6591e-15, 1.1474e-16, 1.0971e-15, 5.1724e-14,\n        1.3608e-15, 1.6226e-16, 3.8691e-18, 5.2209e-16, 2.1031e-17, 2.7409e-14,\n        1.7551e-16, 3.1538e-16, 3.3220e-17, 3.5195e-16, 1.1576e-14, 1.0530e-13,\n        5.6217e-18, 4.4511e-15, 3.2257e-19, 4.4092e-18, 1.8647e-15, 2.7110e-15,\n        2.8457e-18, 2.8607e-15, 2.1017e-17, 2.8289e-19, 3.1548e-15, 5.2266e-17,\n        9.4991e-16, 2.8377e-15, 2.3845e-17, 2.2992e-18, 4.8767e-14, 4.1200e-14,\n        2.7746e-18, 7.5799e-20, 1.0159e-19, 6.0224e-14, 3.6504e-18, 4.3743e-14,\n        5.2799e-16, 2.8939e-15, 3.5240e-17, 9.4966e-18, 7.8475e-18, 2.9254e-14,\n        5.8763e-15, 1.3743e-14, 2.2565e-20, 4.4981e-17, 8.2581e-17, 1.4053e-14,\n        7.8237e-14, 3.7520e-15, 5.3683e-15, 5.3655e-14, 4.0594e-18, 5.4734e-17,\n        1.0286e-17, 1.1629e-14, 1.3085e-17, 7.6733e-17, 1.8237e-19, 1.2928e-14,\n        2.7004e-15, 8.7635e-15, 1.2840e-15, 7.5532e-19, 2.0717e-17, 3.0745e-14,\n        5.7499e-14, 5.2165e-15, 4.1210e-15, 4.9471e-16, 9.6436e-15, 9.0990e-20,\n        1.5420e-16, 3.9139e-16, 9.7115e-17, 7.5645e-18, 1.2763e-13, 1.0301e-13,\n        1.7615e-17, 1.8531e-15, 2.7869e-16, 5.1499e-17, 8.5639e-16, 5.8960e-15,\n        2.7810e-18, 2.7553e-19, 4.0114e-14, 1.0039e-15, 2.1384e-16, 2.8626e-14,\n        1.1867e-17, 3.0715e-15, 6.0671e-16, 1.6143e-19, 1.2481e-18, 6.2755e-17,\n        4.9506e-17, 4.7960e-17, 1.7885e-14, 2.9640e-15, 8.7440e-18, 1.5735e-14,\n        2.5393e-16, 1.0642e-13, 3.8140e-18, 5.5493e-14, 1.6710e-15, 1.3228e-16,\n        4.6404e-18, 2.9079e-15, 1.9558e-16, 1.6473e-15, 1.5766e-16, 4.6099e-18,\n        5.9088e-17, 1.0113e-17, 3.2228e-18, 1.6887e-17, 7.7754e-14, 1.4716e-16,\n        3.5145e-18, 8.4994e-17, 1.6149e-18, 2.6687e-13, 2.9360e-17, 7.7406e-17,\n        1.2242e-14, 6.6777e-14, 1.2326e-14, 7.4691e-18, 7.3249e-14, 3.5301e-18,\n        1.7250e-16, 3.9620e-17, 1.7987e-15, 5.1860e-18, 4.7063e-18, 6.8484e-15,\n        1.8025e-17, 1.6960e-13], device='cuda:0')"
      },
      "59": {
        "step": "tensor(11268.)",
        "exp_avg": "tensor([ 7.0073e-38,  4.8698e-38,  6.6084e-38, -8.7061e-39,  1.5478e-39,\n        -2.6237e-39,  6.1847e-38, -7.2365e-40,  1.1431e-37, -7.5615e-39,\n         6.2958e-38,  1.7000e-38,  8.3246e-38,  2.3413e-38,  1.3579e-38,\n         7.4374e-38, -5.4977e-39,  7.9327e-38,  6.3729e-38,  1.8110e-38,\n         4.1814e-38,  1.3781e-38, -3.1284e-39,  3.2805e-39, -3.3183e-39,\n         9.8581e-42,  1.1665e-37,  1.9474e-39,  4.6694e-38,  5.7797e-38,\n         8.1736e-39, -1.3595e-39,  5.8276e-38, -8.0875e-39, -5.0595e-39,\n        -1.4123e-39,  7.0294e-38, -1.2607e-39,  1.1277e-37,  1.0501e-39,\n        -7.4689e-40, -9.1534e-40,  2.3484e-38,  1.9259e-38,  2.6693e-38,\n         9.7563e-38, -1.1320e-39,  4.0507e-38,  7.3230e-38,  4.9217e-38,\n         1.2487e-38,  1.6594e-38,  4.5115e-38,  2.5732e-38,  5.4462e-39,\n         9.1698e-38,  1.1328e-37,  1.1467e-37,  1.3130e-37,  6.6999e-38,\n        -4.2509e-39,  8.2030e-38, -1.7491e-38,  7.1386e-38,  3.0765e-39,\n         8.0405e-38,  5.5424e-38,  2.4217e-38,  9.2737e-39, -6.1503e-40,\n         7.4859e-40,  4.1402e-38, -3.9464e-40, -2.9302e-39,  5.0195e-38,\n        -1.9327e-39, -1.4024e-38,  6.0212e-38, -3.1038e-39,  5.4307e-38,\n         5.4901e-41, -5.4715e-39, -3.7181e-39,  2.6692e-38,  6.0586e-38,\n         5.4965e-38,  9.6428e-38,  6.5202e-38,  2.4107e-38, -8.6988e-40,\n        -6.1903e-39,  2.3804e-39,  5.3718e-38, -1.6907e-39,  4.3380e-38,\n         7.1645e-38,  2.7122e-38,  4.1857e-38,  3.4739e-39,  2.1266e-38,\n         3.3647e-38,  1.9392e-38, -2.4601e-39,  2.1029e-38, -2.8363e-39,\n         1.9799e-38,  3.9054e-38,  8.2248e-38,  7.9852e-39,  2.5608e-39,\n         5.9117e-40, -3.3892e-39, -2.7910e-40, -2.7675e-38,  8.8810e-39,\n         3.2918e-40,  1.6006e-38,  4.4912e-38,  2.2715e-38,  5.5944e-38,\n         2.7396e-38,  4.1188e-38, -1.8276e-39,  3.7600e-38,  1.1621e-40,\n        -5.0709e-39,  9.1690e-41,  2.2251e-39, -1.1081e-39,  6.8375e-39,\n         2.9894e-38, -2.9891e-39,  3.1391e-38,  2.2032e-38,  1.0704e-37,\n        -3.4029e-39, -2.6960e-39,  5.5151e-38,  2.6653e-38,  1.7388e-38,\n         7.5902e-40,  9.6112e-39,  5.9837e-38,  2.2469e-39,  2.3613e-38,\n         1.4117e-38,  6.9335e-38,  1.8046e-39,  1.1673e-38,  7.4377e-38,\n        -1.4104e-39,  1.4611e-38, -3.2245e-39,  1.4588e-38,  1.3182e-38,\n         4.2901e-39,  7.8791e-38,  7.4159e-38,  3.6490e-40,  7.4407e-38,\n        -1.0587e-39, -7.2686e-39, -2.7840e-39,  4.6887e-38, -7.6472e-39,\n         5.9035e-38, -9.0384e-42,  1.6726e-40,  3.7117e-38,  7.6908e-38,\n        -3.9560e-39, -2.4270e-39,  8.5052e-38, -3.5784e-39, -3.6481e-39,\n         4.8765e-38,  3.2683e-39,  5.0303e-38,  2.1607e-38,  5.1497e-38,\n         3.2057e-38,  8.3579e-38, -2.8771e-39,  3.1849e-38, -9.1541e-39,\n         1.4655e-38, -1.0713e-38, -4.9544e-39, -2.6972e-40,  7.6267e-38,\n         4.0031e-40,  9.0504e-41,  6.3397e-38, -3.3782e-39,  6.0690e-38,\n         9.0909e-38,  7.0051e-38,  1.3369e-38, -1.0106e-39, -4.7976e-41,\n         4.8786e-38,  4.1304e-38, -5.0945e-39,  3.6915e-38, -5.8240e-39,\n         7.7313e-38,  5.7535e-38, -3.0991e-39,  6.8490e-40,  5.9959e-40,\n         5.1029e-39,  1.1446e-37,  8.5312e-38, -1.6977e-39,  1.4972e-37,\n         7.6611e-40,  6.9844e-38,  6.7570e-38, -1.2945e-40, -4.2876e-40,\n         2.0818e-38,  4.6002e-38, -4.5379e-40,  7.3749e-39,  6.9893e-38,\n        -3.6405e-39,  4.1276e-38,  6.4887e-38,  8.5023e-38,  5.2840e-38,\n         6.0350e-38, -1.7566e-39, -3.7456e-40,  1.2351e-37,  5.0009e-38,\n         2.8267e-39,  1.2164e-37, -3.5628e-39,  7.9616e-38,  6.7210e-38,\n         1.2074e-37,  3.8556e-38,  1.3596e-40,  1.7939e-39,  1.0502e-37,\n        -7.4452e-40,  9.4354e-38,  4.0526e-38, -2.2815e-38,  3.4409e-38,\n        -1.8341e-40,  6.7887e-38,  2.5260e-38, -2.5934e-39,  4.0519e-39,\n         8.7734e-40,  4.5338e-39,  2.0753e-38,  5.0271e-38,  7.7679e-38,\n         4.1923e-38, -2.2861e-39,  2.3704e-41,  1.8157e-41,  7.1627e-39,\n        -3.8892e-39,  4.2318e-38, -1.1498e-39,  4.8660e-38,  2.4655e-38,\n         3.7691e-39, -2.6337e-39, -2.6499e-39, -1.7653e-39, -1.7617e-39,\n         1.5598e-39,  1.9851e-38,  3.4463e-40,  2.4638e-39,  1.9024e-38,\n         1.8046e-39, -2.5610e-39,  3.2990e-38,  4.6058e-38, -8.3894e-39,\n        -3.9490e-39,  2.3852e-39,  2.6538e-38,  1.0159e-37,  1.0200e-39,\n         1.2615e-38,  9.3614e-38,  1.0591e-40, -1.8822e-39,  1.1537e-38,\n        -4.7913e-39, -2.5754e-39,  2.8940e-38,  1.1267e-37,  4.0355e-40,\n        -1.0419e-39,  1.6358e-39,  2.3551e-38,  8.2341e-38,  6.5827e-38,\n         3.9711e-39, -1.9235e-39,  1.4920e-38,  3.7324e-39,  1.6206e-37,\n        -4.6249e-39, -1.0788e-39, -4.9592e-39, -3.2771e-39,  8.5671e-40,\n        -3.7303e-39,  8.5024e-38,  2.7932e-38, -3.1631e-40, -1.5762e-39,\n         1.2004e-37, -1.1393e-39,  2.1720e-42,  2.7138e-38,  7.5175e-39,\n         3.9024e-38,  1.5365e-38,  6.2456e-40,  1.0093e-38, -7.0762e-40,\n        -2.1101e-39, -3.9092e-39,  2.1273e-38,  9.2257e-38,  9.6645e-38,\n         8.3292e-38, -3.9500e-40,  3.0191e-38,  6.7107e-38,  6.0685e-38,\n         1.0766e-38, -2.1579e-39, -1.1782e-40,  2.0773e-38,  2.1818e-38,\n         1.4521e-38, -1.5183e-39, -5.0302e-39, -3.0014e-39,  7.0045e-38,\n         9.2542e-38, -6.0592e-39, -7.2036e-39,  6.5780e-38,  4.7671e-38,\n        -2.7286e-39,  4.1443e-39,  7.8752e-40, -1.3785e-38,  8.0037e-39,\n         1.0101e-37, -7.7933e-39, -6.9854e-39,  7.8472e-38,  5.0149e-38,\n         5.7845e-38,  7.9108e-38, -2.8392e-39,  4.5869e-38,  3.5743e-38,\n        -4.7606e-40,  5.6164e-38,  5.9967e-38,  1.0147e-38, -2.6325e-39,\n         9.7965e-39,  3.2678e-38,  8.0726e-38, -3.0338e-39, -1.0246e-39,\n         1.8221e-38, -4.8737e-39,  1.5113e-38,  9.6830e-38,  6.0354e-39,\n         6.8767e-39, -7.6022e-41,  4.1976e-38,  7.9014e-38,  6.1436e-38,\n        -1.2662e-40,  5.4670e-38, -2.0171e-38,  8.2354e-40,  1.2280e-38,\n        -9.1711e-41, -1.0424e-38,  1.8435e-38, -1.1267e-39, -3.9285e-39,\n         6.4896e-38,  2.1954e-38, -6.2186e-40,  8.8330e-40,  2.9408e-40,\n         1.0256e-37,  6.9933e-41,  7.4100e-38,  3.9666e-38, -1.1918e-38,\n         5.2274e-38, -1.7019e-39,  2.2792e-38,  5.2906e-38,  2.5258e-38,\n         4.6992e-38, -9.0483e-41, -2.4247e-39,  3.8443e-40,  1.0527e-37,\n         1.3396e-37,  1.4571e-38,  5.0986e-38,  7.0069e-38,  1.5425e-38,\n         2.1080e-39, -7.2869e-40,  1.6262e-38,  5.3910e-39, -4.4847e-39,\n        -3.2859e-39,  2.1993e-38,  3.4330e-38,  8.3940e-38,  5.4600e-38,\n        -9.7702e-40, -2.9334e-39,  1.0614e-37,  7.7205e-38,  4.8645e-38,\n         1.0964e-38, -7.1441e-39,  4.1023e-38,  1.0504e-39,  4.9854e-39,\n        -8.1132e-40,  7.1685e-38,  7.0087e-39,  5.4973e-38, -2.3591e-38,\n         1.2611e-39,  2.3336e-38,  4.8329e-38,  3.2353e-40, -2.7719e-39,\n         5.8127e-38,  2.8589e-39, -3.4979e-39,  6.6796e-38,  4.3544e-38,\n         1.2836e-38,  7.1346e-38,  4.3151e-38,  4.3682e-38,  5.6613e-38,\n        -5.3251e-39,  2.2051e-39,  3.6345e-38,  6.2579e-38,  7.2292e-41,\n         3.9067e-38,  4.6231e-38,  2.0117e-38,  8.5576e-38, -4.7997e-39,\n         6.4412e-38, -1.5375e-39,  1.2302e-37,  4.8210e-38,  1.7260e-38,\n         3.6859e-38,  1.2019e-38,  2.6933e-38, -1.0038e-38,  9.5528e-40,\n         3.4640e-39, -4.0115e-39,  3.5160e-38,  3.7066e-38,  3.3766e-38,\n         7.5740e-38, -1.9087e-39, -1.1258e-39,  1.4418e-38, -1.4536e-39,\n         9.7980e-38, -2.2255e-39,  2.2847e-39,  5.9503e-39,  8.6110e-38,\n         7.9497e-38,  2.9084e-38,  6.4684e-38,  9.7043e-40, -1.5272e-39,\n        -1.4807e-39,  2.4303e-38,  7.3300e-39, -3.8254e-40,  3.5237e-38,\n        -3.2133e-39,  1.3105e-37], device='cuda:0')",
        "exp_avg_sq": "tensor([7.0776e-14, 4.1952e-15, 2.9846e-14, 3.0139e-17, 8.0649e-17, 4.0357e-19,\n        1.3603e-14, 2.5654e-17, 1.5677e-13, 4.9440e-16, 1.2194e-14, 4.8504e-15,\n        7.8330e-14, 2.6714e-15, 1.8957e-15, 5.8706e-15, 2.7268e-17, 3.6966e-14,\n        2.9058e-14, 1.1298e-14, 1.5320e-14, 1.7803e-15, 1.0877e-16, 1.4086e-14,\n        1.3024e-16, 2.2100e-17, 3.2558e-14, 3.8602e-15, 1.0777e-14, 5.1726e-15,\n        2.4506e-14, 1.2260e-17, 2.7829e-15, 1.0886e-15, 2.4077e-16, 2.5341e-17,\n        1.4438e-14, 1.2708e-16, 1.0550e-13, 1.9053e-15, 8.2764e-18, 7.5848e-17,\n        2.9393e-16, 1.3615e-15, 1.6359e-14, 2.8501e-14, 3.8589e-17, 6.3356e-15,\n        8.3092e-15, 1.0642e-14, 2.4354e-15, 1.8737e-15, 1.6269e-14, 6.4558e-16,\n        2.6369e-15, 5.3677e-14, 1.0344e-13, 1.7923e-13, 1.4100e-13, 4.3826e-14,\n        4.9116e-19, 7.0999e-14, 1.1628e-15, 1.0681e-14, 7.5517e-15, 7.3249e-15,\n        6.5383e-15, 6.2761e-15, 2.0827e-16, 3.0785e-15, 5.9939e-18, 1.3659e-14,\n        9.9060e-17, 1.6506e-16, 2.8774e-14, 1.8068e-16, 3.7567e-14, 2.7388e-14,\n        1.3518e-14, 1.2769e-14, 8.0061e-15, 1.2896e-16, 1.4812e-16, 4.5903e-14,\n        1.7539e-14, 1.5092e-14, 2.1407e-14, 2.0459e-14, 5.5681e-16, 1.4069e-15,\n        3.5317e-17, 1.6000e-14, 3.8502e-14, 1.0534e-15, 2.4944e-15, 2.3925e-14,\n        6.0423e-15, 3.8921e-14, 1.3396e-15, 1.5265e-14, 3.9381e-15, 1.1127e-15,\n        1.8267e-18, 8.0239e-15, 4.8795e-17, 1.6633e-15, 1.7014e-15, 5.1250e-15,\n        9.9830e-17, 3.0962e-15, 1.4486e-16, 2.7716e-18, 2.9486e-17, 1.7189e-14,\n        7.5582e-15, 6.1234e-16, 1.6362e-15, 2.4451e-14, 5.9153e-15, 1.4677e-14,\n        4.7785e-14, 2.6809e-15, 5.8501e-17, 2.2310e-15, 6.6307e-18, 5.9306e-19,\n        2.0792e-18, 1.2161e-16, 3.6396e-19, 9.1967e-16, 1.6507e-15, 1.6748e-16,\n        1.2347e-14, 2.1107e-15, 1.4738e-13, 2.2594e-16, 7.0151e-17, 2.0007e-15,\n        2.1074e-16, 2.2271e-15, 3.2371e-15, 1.2249e-14, 2.8209e-14, 3.6701e-16,\n        3.9061e-15, 8.4589e-16, 3.1227e-14, 3.9641e-16, 1.8381e-14, 2.8061e-14,\n        4.4994e-16, 8.0527e-16, 1.1571e-15, 1.4749e-13, 9.5416e-17, 1.7097e-14,\n        3.1344e-14, 3.5288e-14, 2.0992e-17, 3.1590e-14, 1.9407e-17, 2.8819e-16,\n        5.4130e-19, 4.9666e-15, 3.0101e-16, 4.4893e-15, 6.4528e-16, 6.2914e-18,\n        1.6769e-15, 3.0833e-14, 3.2944e-14, 3.5817e-15, 4.0960e-14, 1.0899e-18,\n        1.6689e-16, 8.0432e-15, 1.5259e-15, 3.6770e-15, 2.8383e-14, 1.8849e-14,\n        3.6689e-14, 1.2116e-14, 4.3248e-18, 4.9395e-15, 4.6780e-16, 1.1568e-15,\n        7.3907e-16, 1.2315e-16, 4.6843e-16, 3.2863e-14, 2.3113e-17, 9.8863e-17,\n        1.2645e-14, 4.7882e-19, 3.6391e-14, 7.0297e-14, 1.0793e-14, 1.5051e-15,\n        3.3043e-17, 1.6501e-17, 4.3081e-14, 1.3280e-14, 1.4639e-16, 1.6394e-15,\n        1.3702e-16, 4.8535e-14, 9.5385e-15, 1.0846e-16, 1.9045e-15, 8.3094e-18,\n        3.9183e-16, 8.6580e-14, 3.3016e-14, 2.6123e-16, 3.0511e-13, 2.4404e-16,\n        7.1505e-14, 1.4089e-14, 6.9678e-16, 1.9497e-17, 3.1143e-15, 1.5772e-14,\n        6.4255e-17, 5.4844e-16, 1.4271e-14, 3.8705e-17, 1.2329e-14, 1.0412e-14,\n        1.2831e-13, 2.2268e-14, 1.6064e-14, 9.7917e-17, 9.7574e-18, 1.6923e-13,\n        8.7298e-15, 5.8181e-18, 1.5741e-13, 2.3075e-16, 1.8746e-14, 3.5295e-15,\n        1.0731e-13, 5.6990e-15, 5.7491e-17, 2.0509e-17, 3.6105e-14, 1.1115e-17,\n        2.0614e-14, 2.9807e-15, 2.1381e-14, 4.5546e-15, 1.8595e-16, 8.5980e-15,\n        1.1192e-13, 3.7624e-16, 3.5782e-16, 5.5734e-17, 6.4642e-16, 3.5386e-15,\n        3.8987e-15, 1.1443e-14, 2.7255e-15, 9.5284e-17, 1.7343e-17, 4.8432e-14,\n        1.1409e-15, 1.1885e-18, 1.4745e-15, 6.5870e-17, 1.0926e-14, 1.7372e-15,\n        5.0764e-15, 1.1396e-18, 2.0623e-19, 1.8365e-15, 1.5988e-17, 8.0577e-17,\n        8.5117e-15, 1.0740e-17, 6.0573e-16, 2.9378e-15, 2.5343e-15, 1.0894e-15,\n        1.7183e-15, 2.1682e-15, 2.8824e-16, 2.0539e-16, 1.2301e-16, 2.1248e-15,\n        4.7815e-14, 6.8676e-17, 2.2647e-14, 1.1336e-13, 2.0393e-16, 4.1735e-19,\n        1.6138e-14, 4.3275e-18, 3.8573e-17, 1.9334e-15, 1.0248e-14, 1.4288e-15,\n        3.2318e-17, 8.9238e-17, 1.5520e-14, 7.4051e-14, 5.2842e-14, 1.3602e-16,\n        1.5399e-16, 4.6324e-15, 5.2524e-16, 2.0589e-13, 1.7869e-15, 8.3578e-15,\n        2.0401e-16, 7.3166e-17, 5.7453e-18, 5.1246e-17, 4.1003e-14, 3.5982e-15,\n        2.7898e-17, 1.3454e-17, 1.5439e-13, 2.1020e-17, 8.7952e-15, 5.6540e-15,\n        9.9234e-15, 3.2502e-15, 1.8408e-15, 3.2602e-15, 4.8300e-15, 1.4537e-16,\n        1.8218e-16, 4.4964e-16, 1.7780e-15, 3.9544e-15, 2.5112e-14, 1.7931e-14,\n        3.5292e-16, 3.6361e-15, 1.7703e-14, 5.5359e-15, 3.3125e-14, 2.9101e-16,\n        5.6387e-18, 3.4193e-15, 1.5884e-15, 2.8217e-15, 1.8139e-17, 2.1675e-18,\n        6.3299e-16, 2.7631e-14, 7.3102e-15, 3.3833e-16, 3.4792e-16, 2.8773e-15,\n        1.4417e-15, 7.7698e-16, 1.5205e-14, 6.7881e-17, 7.5411e-16, 8.7954e-14,\n        1.3810e-14, 1.1828e-15, 6.6745e-17, 2.2796e-14, 1.3969e-14, 7.3277e-15,\n        9.6147e-14, 1.2293e-17, 4.1456e-15, 1.4008e-15, 4.3000e-16, 2.4094e-14,\n        8.6783e-16, 1.1912e-15, 2.3697e-16, 6.1824e-15, 1.8168e-15, 3.7728e-14,\n        7.2625e-17, 1.3259e-16, 7.4987e-16, 1.7806e-16, 1.3956e-14, 5.9736e-14,\n        1.9453e-16, 2.4012e-14, 1.5910e-16, 7.2311e-16, 1.1981e-15, 8.4557e-15,\n        1.0947e-18, 1.6302e-14, 6.2389e-16, 6.0038e-19, 2.9457e-14, 2.4760e-17,\n        4.5340e-16, 4.6715e-15, 8.1472e-17, 6.0906e-16, 4.0456e-14, 2.5526e-14,\n        4.3136e-15, 9.2009e-17, 7.7002e-18, 8.4691e-14, 7.9762e-17, 2.3086e-14,\n        8.4430e-15, 1.3691e-15, 5.0020e-15, 1.8370e-18, 1.7426e-15, 1.7717e-14,\n        5.7510e-15, 1.2207e-14, 2.4249e-18, 3.9721e-17, 6.8939e-15, 6.0238e-14,\n        9.7862e-14, 1.1932e-14, 1.1581e-14, 1.9363e-14, 2.4507e-15, 2.4914e-17,\n        1.1198e-17, 2.1344e-14, 2.1631e-15, 3.4009e-17, 2.6585e-17, 1.0118e-14,\n        1.0512e-14, 2.2844e-14, 7.2010e-15, 6.1987e-17, 6.0903e-18, 3.5496e-14,\n        4.0604e-14, 3.4526e-15, 9.1239e-15, 2.3925e-16, 1.0027e-14, 2.6165e-15,\n        1.6326e-15, 1.8243e-16, 1.3546e-14, 9.2489e-17, 6.0356e-14, 6.3231e-14,\n        5.7431e-17, 3.8862e-15, 1.4180e-16, 2.3188e-17, 3.3413e-16, 9.1521e-15,\n        3.8973e-16, 3.3290e-15, 3.5165e-14, 9.1057e-16, 3.5056e-15, 1.9679e-14,\n        5.8591e-15, 2.8926e-14, 1.6975e-15, 2.3147e-17, 1.6990e-16, 2.4374e-15,\n        9.6145e-15, 2.1355e-17, 2.1005e-14, 4.9705e-15, 2.2118e-15, 1.8338e-14,\n        1.1735e-16, 3.3898e-14, 1.7773e-16, 5.9230e-14, 6.7648e-15, 6.6799e-17,\n        1.6141e-15, 1.0287e-14, 1.9328e-15, 6.8865e-16, 7.9244e-17, 7.7824e-16,\n        2.5828e-17, 1.7341e-15, 3.4424e-15, 1.4687e-15, 7.4537e-14, 3.5160e-18,\n        1.5978e-17, 9.5704e-16, 1.9290e-19, 1.3912e-13, 8.7867e-18, 2.2503e-17,\n        2.3990e-14, 4.4871e-14, 7.8953e-15, 1.0246e-15, 3.3329e-14, 1.7105e-15,\n        3.3382e-15, 6.6772e-18, 8.5322e-15, 1.0280e-16, 1.0487e-15, 5.6408e-15,\n        9.3161e-19, 7.9376e-14], device='cuda:0')"
      },
      "60": {
        "step": "tensor(11268.)",
        "exp_avg": "tensor([[-6.8573e-41,  3.7623e-39,  1.0757e-40,  ...,  5.9587e-40,\n          1.6411e-40,  3.2270e-39],\n        [ 2.1385e-41,  2.3700e-41,  1.2885e-41,  ...,  1.7124e-41,\n          5.8827e-41,  3.0205e-40],\n        [-5.3479e-41,  8.2829e-40, -1.4387e-40,  ..., -1.1701e-40,\n         -7.5807e-41, -5.4197e-39],\n        ...,\n        [-7.2711e-41, -1.1888e-39, -1.7960e-40,  ..., -2.1285e-40,\n         -1.0748e-40,  2.5146e-39],\n        [ 6.8397e-41, -7.9564e-40,  1.6092e-40,  ...,  6.6339e-41,\n          1.4502e-40, -1.2968e-39],\n        [ 1.5422e-40,  1.0231e-39,  1.9962e-40,  ...,  1.9881e-40,\n          7.1405e-41,  1.3386e-39]], device='cuda:0')",
        "exp_avg_sq": "tensor([[1.4782e-16, 4.5821e-17, 2.4976e-17,  ..., 1.4887e-16, 4.0435e-17,\n         1.9427e-16],\n        [6.9194e-19, 1.3556e-19, 9.2066e-20,  ..., 7.6196e-19, 1.3637e-19,\n         1.4574e-19],\n        [1.4402e-16, 3.7733e-17, 3.4900e-17,  ..., 1.8074e-16, 7.5175e-17,\n         2.4791e-16],\n        ...,\n        [1.8678e-16, 2.4025e-17, 3.5079e-18,  ..., 2.7689e-17, 8.3019e-18,\n         2.4524e-17],\n        [2.9098e-18, 7.8856e-18, 2.3437e-18,  ..., 8.0184e-18, 3.7704e-18,\n         1.7636e-17],\n        [1.1162e-17, 5.7698e-19, 2.8698e-18,  ..., 3.5083e-18, 1.4027e-18,\n         5.3336e-18]], device='cuda:0')"
      },
      "61": {
        "step": "tensor(11268.)",
        "exp_avg": "tensor([ 8.3140e-37, -3.8702e-38, -3.7346e-37, -6.8683e-37, -1.1281e-36,\n        -5.9318e-37,  3.5267e-37, -1.0686e-37,  2.3003e-37,  2.0016e-37,\n        -2.2012e-37, -4.1354e-37,  5.7555e-37, -9.3494e-38, -7.4777e-37,\n         5.0424e-38, -4.2114e-37, -1.2814e-36,  2.5118e-37, -1.0147e-37,\n         5.3365e-37, -1.4363e-37, -5.0363e-38, -1.2639e-36, -2.3142e-37,\n        -1.9063e-37,  3.7745e-37, -3.6245e-37,  2.5683e-38, -7.7579e-37,\n        -9.5228e-37,  1.3067e-37, -2.2868e-37, -6.1930e-37,  3.8171e-37,\n         1.4256e-37,  6.5807e-37,  5.1107e-38,  1.5557e-36, -2.2589e-37,\n        -4.9370e-38, -3.1980e-37, -3.0685e-37, -3.0456e-37,  1.5142e-36,\n         1.6792e-36,  7.5661e-38, -4.7646e-37,  1.7946e-36, -7.3170e-38,\n        -1.5128e-37, -3.8057e-37, -1.8281e-37, -1.2133e-36, -1.9887e-37,\n         1.3109e-36,  2.7982e-36,  1.6858e-36,  1.4131e-36,  9.5363e-37,\n        -6.2766e-38,  1.7743e-36,  8.2818e-37,  2.5106e-36, -4.2907e-37,\n        -5.6418e-37, -2.4168e-37,  7.0470e-37,  2.2730e-37, -1.0399e-36,\n         2.7419e-37,  3.3121e-37, -4.9421e-38, -1.1136e-36,  1.5264e-36,\n        -2.0663e-37, -2.4213e-36,  1.0780e-36, -2.6330e-36, -1.5527e-37,\n        -1.6421e-36,  3.1546e-38, -2.2060e-37,  4.3595e-37,  1.3276e-36,\n         1.8423e-36, -1.7607e-37,  3.9497e-37,  2.5325e-36, -5.1437e-37,\n        -8.2735e-37, -1.9817e-37,  4.1209e-37, -5.8288e-37, -1.1790e-37,\n         5.3959e-37,  5.9718e-37, -2.5570e-37, -7.3000e-37,  1.8557e-37,\n        -1.2289e-36,  1.6885e-36, -1.3155e-37, -1.6417e-36, -4.1964e-37,\n        -4.5661e-37,  7.0203e-37, -3.7178e-37,  2.8697e-36, -3.8009e-37,\n        -1.7332e-36,  2.1708e-37, -9.6002e-37, -1.6738e-36, -3.7034e-38,\n        -1.4576e-37,  7.1919e-37,  5.9796e-37,  4.4457e-37,  1.1072e-36,\n         7.0784e-37, -9.1374e-38, -5.8415e-38,  6.7250e-37,  1.3859e-37,\n        -6.8465e-38,  2.1339e-37,  8.6859e-38, -2.8103e-37, -5.7770e-37,\n        -7.9543e-38, -4.2010e-38, -1.4688e-37, -2.9571e-37,  5.4944e-37,\n        -5.0073e-37,  2.8772e-37,  2.3232e-37, -5.1945e-37, -7.6997e-37,\n        -3.8744e-37, -1.4512e-37,  5.4263e-37, -2.1115e-37, -4.0715e-38,\n         4.8295e-37,  1.3235e-36,  7.3887e-37, -1.0825e-36, -4.7156e-37,\n         1.0411e-37,  9.4643e-37, -5.1889e-37,  7.5453e-38,  4.2450e-37,\n        -7.6283e-38,  8.8626e-38,  8.8547e-38, -4.9374e-37,  1.0393e-37,\n        -4.3519e-37,  7.7142e-37, -6.2840e-37, -8.9656e-37,  3.0638e-37,\n         1.4285e-37, -1.8834e-36,  1.8468e-37,  7.6154e-38,  1.3718e-36,\n        -9.0107e-37, -2.8212e-36,  2.4640e-37, -4.2147e-37, -1.3644e-37,\n         2.0162e-38,  2.4820e-37, -4.5449e-38, -1.0374e-36,  2.6366e-37,\n        -2.8270e-37,  4.0857e-37, -1.3241e-37, -1.4959e-37,  3.2218e-37,\n        -3.5050e-36,  5.8477e-37,  1.0983e-36, -5.1003e-37,  4.7596e-37,\n         1.4772e-37, -3.8391e-37,  4.1043e-37,  1.2260e-37,  5.6407e-37,\n         1.7238e-36, -2.5424e-37, -7.5124e-37, -8.5961e-37, -1.6553e-36,\n         5.1555e-37,  1.4366e-37,  1.5363e-36, -2.6954e-37,  3.8798e-37,\n        -7.4582e-37, -1.9669e-37, -1.1313e-36, -2.2558e-37, -4.4405e-38,\n         8.7647e-37,  8.5101e-37,  3.2934e-37, -1.7424e-36,  2.1870e-36,\n        -1.1254e-36, -5.9694e-37,  4.9978e-39, -1.0054e-37, -1.6162e-37,\n        -1.7250e-37, -7.9521e-39, -6.2991e-37, -2.6871e-37,  2.2940e-37,\n        -9.1717e-37, -1.0636e-36,  6.3829e-38, -2.7643e-39,  3.2228e-38,\n        -1.9740e-37, -1.3802e-36, -2.2526e-37,  1.4242e-36,  6.6440e-37,\n        -4.7195e-37,  2.5464e-36, -4.9763e-38,  5.6698e-37, -3.0178e-37,\n         1.7264e-36, -1.8412e-37, -8.1388e-37, -8.2928e-37,  1.5924e-36,\n        -4.6841e-37,  6.0393e-37, -4.6532e-37,  4.7530e-37, -8.3155e-39,\n        -4.2363e-37, -7.0608e-37, -1.4977e-37, -1.8770e-36, -6.2132e-37,\n         8.9242e-38, -2.7137e-37,  1.4813e-37,  7.5953e-37, -2.6437e-37,\n         1.3981e-38, -9.8844e-37, -7.6836e-38, -6.9885e-37,  3.9677e-37,\n        -1.3520e-37,  3.3289e-36, -2.5001e-37, -2.2638e-37,  2.0509e-36,\n        -2.2264e-36,  9.2303e-37, -2.6237e-37,  1.1253e-36, -5.0737e-37,\n        -8.8577e-39,  3.7611e-38,  1.4295e-37, -1.9386e-37, -1.4329e-37,\n        -1.1707e-36,  7.5497e-37, -2.9946e-37,  1.5321e-36,  7.0419e-37,\n        -9.1133e-37, -1.2266e-36, -1.7186e-36,  2.2000e-36, -2.4988e-38,\n        -7.5265e-37,  7.9808e-37,  6.9745e-38, -2.0162e-38, -2.2868e-37,\n         2.4433e-37,  5.5231e-38,  8.0106e-37,  8.8616e-37, -1.9413e-37,\n        -1.5663e-37, -3.6613e-37,  4.9552e-37,  2.0880e-36,  1.5090e-38,\n        -1.8237e-37, -1.2422e-36, -1.1432e-37,  2.9134e-37, -2.9889e-37,\n         2.2394e-36, -1.3888e-36,  9.4044e-38, -1.8388e-36, -2.0090e-36,\n        -6.3498e-37, -4.4175e-37, -4.3701e-38, -1.4236e-38, -2.0159e-37,\n         7.0815e-37, -6.3189e-38, -1.9474e-36,  2.7798e-37, -1.5909e-36,\n         6.9091e-37,  8.1915e-38,  6.5929e-37, -2.5926e-37, -1.2809e-37,\n        -1.1081e-36, -1.6717e-37, -5.7965e-37,  3.7014e-37, -1.3391e-37,\n         2.4664e-37, -2.2976e-37,  7.4427e-38,  5.8756e-37,  9.8452e-37,\n        -1.4327e-36, -3.6477e-37, -1.1895e-36, -1.8090e-37, -4.3265e-37,\n        -1.0113e-37, -7.8050e-38, -1.5334e-37,  2.5791e-37, -1.8891e-37,\n         1.0497e-36,  3.1639e-37,  1.0814e-36,  7.2255e-38, -3.8219e-38,\n        -4.4235e-37, -6.3278e-37, -7.9255e-38,  1.3431e-36, -1.8660e-37,\n         3.1904e-37,  1.3067e-36, -3.0230e-38, -2.8074e-38, -3.7446e-38,\n        -2.9591e-37,  5.5687e-37, -7.8503e-37,  7.3672e-38, -1.1176e-36,\n        -2.2104e-37, -2.0798e-37, -3.5125e-37,  8.1623e-38, -5.3788e-37,\n        -1.6856e-37, -1.7130e-37, -1.2482e-37, -2.1917e-37, -2.3415e-37,\n         4.5850e-37,  2.8885e-37, -4.0475e-37,  3.4350e-38,  1.2999e-37,\n        -4.7037e-37,  5.3845e-38,  8.5909e-37, -2.4971e-37, -1.5166e-36,\n        -6.6211e-37, -2.4309e-37, -3.6101e-36, -1.8806e-37, -1.7704e-37,\n        -3.8625e-37,  4.6243e-37, -5.1221e-38, -1.3809e-36, -1.5768e-36,\n         2.7367e-36, -1.0856e-37, -4.1536e-37,  6.3658e-37, -6.5962e-37,\n         2.4005e-36, -1.3114e-36,  1.3369e-36, -5.0675e-37, -3.8097e-37,\n         9.5258e-37, -8.9310e-38,  3.1728e-37,  4.1419e-38,  8.9668e-39,\n        -1.2568e-37, -1.2674e-36, -6.7779e-37, -5.7065e-38,  6.5732e-37,\n         9.4517e-38, -4.1006e-37, -2.7769e-37, -5.4151e-37,  9.9010e-38,\n        -8.0514e-37,  2.1495e-37, -4.6631e-37, -1.7339e-37, -2.5699e-38,\n         1.5797e-39, -1.2869e-37, -3.0820e-37,  2.4253e-36,  7.4889e-37,\n         8.0092e-38, -2.4958e-37,  5.6599e-37, -5.3900e-38,  4.8885e-38,\n        -3.2050e-37,  4.9321e-37, -6.6123e-38, -6.8474e-37, -3.2009e-37,\n        -1.0140e-36,  3.0570e-36,  2.3583e-36, -7.1120e-37,  2.0443e-37,\n        -4.7902e-38, -2.6488e-37, -4.8749e-37, -2.1718e-37, -1.3778e-37,\n        -2.5012e-37, -1.1698e-37,  7.3706e-39,  1.2109e-36,  2.2900e-37,\n        -6.0904e-37,  1.0306e-36,  3.8030e-39,  1.9623e-37,  6.5240e-37,\n         1.7264e-37,  3.8219e-37,  1.2610e-37, -7.9168e-38, -8.9362e-37,\n        -5.8132e-38, -5.0075e-37,  4.0403e-37,  3.4691e-37, -7.8590e-38,\n         4.4811e-37, -2.3362e-37,  2.2363e-36,  1.7255e-37, -7.7321e-39,\n        -9.8555e-38, -2.5456e-37, -2.1764e-36,  1.4137e-36, -1.8523e-36,\n        -3.5071e-37, -1.8967e-37,  2.7673e-36,  1.5105e-36,  1.1952e-36,\n         3.0962e-36, -3.4554e-37,  4.5062e-37, -1.7712e-37,  1.4051e-37,\n         1.3967e-36,  7.2417e-37, -5.1193e-37, -2.7366e-36,  5.8606e-37,\n        -4.9493e-37,  1.6073e-36, -3.1818e-37, -5.3074e-37, -3.3337e-37,\n        -6.1992e-37, -2.2592e-37,  1.1420e-36, -1.3214e-36, -5.2400e-37,\n         4.0015e-37,  7.1429e-37], device='cuda:0')",
        "exp_avg_sq": "tensor([5.7205e-11, 2.4668e-13, 9.2228e-11, 8.4244e-11, 4.9405e-11, 9.9138e-12,\n        3.2100e-12, 2.9733e-13, 2.4706e-12, 1.8452e-12, 1.0872e-11, 4.2191e-12,\n        5.1633e-11, 6.3078e-13, 7.7498e-11, 8.1807e-14, 2.6266e-11, 8.1975e-11,\n        1.0802e-14, 7.9716e-12, 3.7726e-11, 1.1751e-10, 1.8035e-13, 1.0902e-10,\n        2.5953e-12, 3.7820e-12, 5.0037e-12, 4.2441e-11, 1.4196e-14, 4.1252e-13,\n        1.4088e-10, 9.9228e-16, 8.7526e-14, 3.4898e-10, 1.0232e-10, 4.9322e-13,\n        4.2853e-11, 3.1666e-12, 1.3635e-10, 3.8765e-13, 5.1310e-13, 6.7721e-11,\n        1.9248e-11, 5.3798e-12, 3.0231e-11, 2.6925e-11, 2.6733e-12, 2.8283e-11,\n        1.6223e-11, 4.1205e-13, 5.7811e-13, 1.3500e-11, 3.3360e-13, 8.6060e-12,\n        1.1812e-12, 1.0011e-10, 2.5929e-10, 6.7812e-10, 4.7660e-10, 4.6221e-11,\n        5.0031e-13, 4.9477e-11, 4.5620e-12, 3.0041e-10, 9.7047e-11, 3.3664e-12,\n        6.0112e-12, 1.2441e-10, 9.6067e-14, 3.7396e-10, 1.1752e-12, 1.4218e-10,\n        1.5067e-11, 5.5847e-11, 3.9759e-11, 1.0383e-12, 4.1977e-10, 5.6279e-11,\n        5.0425e-10, 7.0705e-14, 2.7060e-10, 2.0313e-14, 1.8375e-11, 3.7532e-12,\n        7.0332e-11, 1.7795e-10, 1.6912e-13, 7.3351e-11, 2.2326e-10, 1.5597e-10,\n        1.1683e-10, 1.3507e-11, 7.7964e-12, 1.9127e-10, 3.8296e-14, 1.6550e-12,\n        3.6693e-11, 1.3020e-10, 2.4525e-11, 1.3253e-10, 2.4190e-11, 3.7555e-11,\n        5.8355e-13, 5.8712e-11, 7.5346e-12, 4.3708e-12, 1.4054e-10, 5.3072e-13,\n        4.1850e-11, 2.8962e-11, 2.5644e-10, 1.3178e-14, 3.9128e-11, 5.3775e-10,\n        7.5416e-11, 3.4642e-13, 5.7944e-12, 9.8714e-11, 1.0522e-10, 7.3989e-11,\n        5.4142e-10, 4.9643e-13, 6.4782e-14, 1.4528e-11, 2.7757e-12, 6.3018e-13,\n        5.2093e-13, 6.6254e-11, 3.9862e-11, 3.9890e-11, 6.8825e-12, 5.2110e-12,\n        8.3847e-13, 1.8443e-12, 4.6362e-12, 4.5694e-11, 3.2487e-12, 2.1361e-12,\n        3.1749e-12, 3.4411e-11, 2.2893e-11, 4.0709e-11, 8.9196e-12, 2.1966e-12,\n        2.0294e-12, 3.0589e-12, 2.7502e-11, 4.4126e-11, 9.3857e-11, 5.7265e-11,\n        1.2298e-12, 1.4449e-10, 5.7811e-11, 3.7037e-14, 4.0334e-13, 9.1131e-12,\n        9.9881e-14, 7.1145e-11, 1.9813e-11, 1.7095e-13, 3.0696e-11, 2.9871e-11,\n        7.3120e-12, 4.4438e-11, 1.4657e-13, 5.3964e-14, 3.1727e-10, 1.7210e-11,\n        2.7599e-13, 6.9037e-11, 2.3451e-10, 1.1354e-10, 3.8025e-13, 2.5332e-10,\n        1.7118e-11, 6.4274e-13, 1.1495e-12, 1.0749e-12, 1.3410e-10, 3.3668e-11,\n        1.6336e-13, 2.6481e-11, 6.4026e-12, 2.2525e-13, 4.7559e-11, 6.8656e-11,\n        1.6684e-10, 1.8402e-11, 7.5543e-11, 6.1988e-14, 5.3221e-13, 2.8455e-11,\n        3.3352e-11, 7.1904e-12, 6.9531e-11, 1.6544e-11, 2.1839e-14, 4.7165e-11,\n        1.4750e-10, 1.4067e-10, 2.8734e-12, 1.2104e-10, 1.2098e-11, 2.1751e-11,\n        5.6714e-11, 2.1522e-10, 8.7082e-14, 2.7203e-11, 9.8921e-13, 2.7945e-12,\n        2.5802e-12, 1.6009e-10, 1.5710e-13, 4.7312e-11, 7.4762e-11, 2.1235e-10,\n        9.2342e-11, 1.2540e-13, 2.7224e-10, 1.0465e-12, 2.3728e-13, 1.7086e-13,\n        9.6487e-12, 1.7856e-12, 3.7141e-13, 1.0613e-12, 2.3332e-11, 1.0715e-13,\n        3.2935e-14, 1.0454e-11, 5.7923e-14, 1.7254e-10, 3.4828e-13, 3.6965e-11,\n        1.1861e-11, 6.6079e-12, 9.3804e-11, 8.9123e-14, 3.2711e-13, 7.6600e-12,\n        4.1873e-11, 2.0558e-12, 3.5608e-11, 7.5204e-11, 2.6658e-10, 1.1691e-11,\n        5.4069e-13, 1.0490e-11, 9.2288e-12, 2.2070e-11, 4.6754e-11, 9.5561e-11,\n        1.8118e-11, 4.5133e-12, 1.0358e-11, 6.3577e-13, 5.3453e-12, 2.4041e-13,\n        6.9804e-11, 1.5861e-13, 1.0079e-12, 1.2285e-10, 2.7915e-11, 1.5964e-10,\n        2.6709e-13, 9.8789e-13, 1.0997e-11, 1.6975e-12, 1.1517e-13, 1.2766e-10,\n        1.2717e-10, 1.1160e-11, 3.4206e-13, 1.5302e-10, 3.5737e-11, 2.8260e-13,\n        2.2826e-11, 7.1190e-13, 3.6572e-11, 1.1234e-14, 5.1398e-11, 6.1471e-12,\n        1.5788e-10, 1.3058e-10, 3.9451e-11, 2.2329e-13, 1.2741e-10, 3.0176e-11,\n        8.6278e-11, 4.4514e-11, 5.5300e-11, 2.3435e-10, 1.0122e-12, 1.8263e-13,\n        1.0303e-12, 3.6802e-13, 5.4049e-14, 1.3364e-11, 8.5967e-12, 1.8741e-12,\n        4.9973e-14, 8.1743e-12, 1.2510e-12, 6.3998e-11, 1.4482e-12, 1.5442e-13,\n        8.1207e-12, 8.4884e-14, 1.9197e-12, 3.4738e-13, 9.2252e-10, 5.5535e-11,\n        1.7896e-13, 8.6479e-11, 2.1867e-11, 1.1837e-11, 1.0920e-10, 1.7989e-11,\n        2.6230e-13, 4.5973e-13, 3.1877e-12, 1.2989e-14, 2.7948e-10, 1.0321e-10,\n        1.1893e-10, 1.1140e-11, 1.0575e-13, 5.1781e-13, 5.5123e-13, 2.8102e-12,\n        2.7955e-10, 4.9780e-13, 2.3393e-12, 1.3251e-12, 4.3979e-14, 1.5709e-13,\n        8.0652e-14, 1.2549e-12, 7.6814e-12, 4.2606e-12, 2.1383e-10, 1.3631e-11,\n        1.8852e-10, 9.1292e-13, 1.4505e-11, 3.7867e-11, 7.6303e-12, 2.1397e-13,\n        2.5138e-11, 9.3788e-14, 2.4284e-11, 2.2620e-13, 8.1311e-12, 1.4237e-10,\n        9.0418e-11, 6.0037e-11, 1.8959e-10, 2.7862e-13, 3.8977e-11, 4.4177e-10,\n        3.3105e-12, 1.8173e-11, 3.7495e-11, 3.7254e-13, 3.4644e-13, 3.5271e-14,\n        6.9043e-12, 4.3581e-12, 9.8387e-15, 1.9179e-12, 3.6607e-13, 7.4063e-12,\n        3.4732e-11, 4.9527e-13, 3.3029e-11, 1.1616e-11, 1.5197e-13, 1.7574e-13,\n        1.2093e-12, 3.9189e-13, 7.7578e-12, 6.5606e-12, 2.9051e-11, 5.4750e-13,\n        1.3534e-12, 1.8780e-12, 7.4658e-13, 1.1327e-11, 9.0747e-12, 2.9929e-12,\n        4.7405e-11, 3.7715e-13, 1.9277e-11, 3.4066e-13, 1.6896e-10, 2.7259e-11,\n        6.6616e-11, 2.7804e-12, 2.0648e-10, 5.5990e-11, 3.2884e-11, 1.6374e-12,\n        1.5045e-10, 4.2206e-12, 1.1051e-11, 2.3034e-10, 8.2719e-11, 4.9765e-11,\n        1.0941e-10, 4.6300e-10, 1.3344e-10, 8.1111e-13, 1.2731e-12, 1.0541e-13,\n        1.2092e-13, 2.0435e-12, 3.5539e-11, 2.0803e-10, 1.8328e-12, 3.0450e-12,\n        9.9274e-13, 5.5660e-11, 2.5020e-12, 4.5879e-11, 1.9301e-13, 1.1652e-12,\n        1.8888e-14, 1.4788e-10, 3.0241e-13, 3.6599e-11, 3.6342e-12, 2.8128e-11,\n        6.3630e-11, 3.8403e-11, 1.1531e-12, 1.8381e-12, 2.0300e-12, 7.9199e-11,\n        1.1639e-13, 9.5953e-14, 1.6490e-11, 4.7653e-11, 4.3583e-13, 1.9230e-10,\n        4.9737e-12, 2.7863e-11, 1.8971e-10, 1.5195e-10, 1.2622e-11, 6.1790e-12,\n        9.3459e-12, 8.2395e-12, 6.7769e-12, 7.4644e-11, 6.5160e-12, 1.2602e-11,\n        1.8907e-12, 4.6893e-12, 7.1406e-11, 3.1290e-12, 2.9621e-11, 1.7484e-12,\n        7.2992e-13, 1.8806e-12, 3.8011e-12, 3.2060e-12, 7.1773e-12, 3.6855e-12,\n        3.9359e-13, 4.3257e-12, 4.6950e-11, 3.5279e-12, 1.4202e-12, 3.2752e-12,\n        1.1799e-13, 2.3865e-11, 8.8961e-13, 2.8562e-10, 1.2879e-11, 2.0812e-13,\n        1.5685e-13, 5.7810e-13, 6.5458e-11, 3.1151e-10, 1.0725e-10, 1.7516e-11,\n        5.2628e-11, 5.1872e-11, 1.3352e-10, 1.1416e-11, 2.0922e-10, 5.1521e-12,\n        6.9364e-12, 9.0754e-14, 6.5940e-11, 2.5497e-10, 8.5133e-11, 1.9082e-11,\n        5.1171e-10, 1.5278e-10, 5.4127e-13, 2.8246e-10, 3.7229e-11, 5.3437e-11,\n        3.3550e-12, 4.1018e-12, 1.0987e-10, 1.8472e-11, 4.2383e-11, 2.0776e-11,\n        3.3702e-12, 1.9605e-12], device='cuda:0')"
      },
      "62": {
        "step": "tensor(11268.)",
        "exp_avg": "tensor([ 3.1637e-38,  1.1376e-38,  3.6504e-38,  2.6428e-38,  8.0497e-40,\n         1.3380e-38,  2.0630e-39,  1.8285e-39,  1.0997e-38, -1.9610e-39,\n         1.5682e-38, -8.4488e-40,  4.4512e-38,  1.5228e-39,  3.6865e-38,\n        -5.9483e-40,  1.2243e-39,  1.1390e-38,  1.1678e-38,  1.8316e-39,\n         3.2077e-38,  5.9394e-38,  2.7218e-39,  5.8252e-39, -1.4924e-39,\n         8.6416e-39, -4.5249e-39, -1.4106e-39,  4.6602e-39,  1.4703e-38,\n        -1.0775e-40,  8.0529e-39,  1.0974e-39,  7.2100e-38,  8.5133e-38,\n         4.4319e-39,  8.5931e-38, -5.6448e-40,  4.7457e-38,  1.0606e-39,\n         3.1436e-39,  2.8783e-38,  1.3484e-38, -9.5241e-40,  5.3023e-38,\n         3.6558e-38,  2.8279e-39,  8.9422e-39,  3.0396e-38,  3.9913e-39,\n         3.0292e-39, -2.7886e-40,  1.7106e-39,  4.2563e-39,  1.4182e-39,\n         6.6401e-38,  9.9579e-38,  1.2987e-37,  8.7522e-38,  7.2236e-38,\n         3.7340e-38,  5.2703e-38,  1.6446e-38,  7.6036e-38,  1.0457e-38,\n         5.0555e-41, -8.1556e-40,  3.2764e-38,  7.2175e-39,  8.3387e-39,\n         7.1343e-39,  5.5850e-38,  1.5943e-39,  2.0705e-38,  7.1981e-38,\n         1.8680e-39, -4.5856e-39,  2.1951e-38, -3.2928e-39,  1.6882e-39,\n        -1.5812e-40,  3.3430e-39,  8.3815e-39,  4.7955e-39,  8.9573e-38,\n         6.0192e-38,  4.6345e-39,  1.0815e-37,  1.0013e-37, -7.3404e-40,\n         5.4946e-38, -4.5476e-41,  8.8318e-39, -7.7111e-40,  3.0701e-39,\n         2.2601e-39,  3.2123e-38,  7.0285e-39, -5.8237e-40,  3.9050e-38,\n         1.5424e-38,  4.0904e-38,  1.3063e-39,  2.1771e-38, -1.3890e-39,\n         3.3668e-38,  5.4074e-38, -3.5743e-40,  5.4964e-38, -9.9788e-40,\n         9.3076e-41,  3.0831e-39,  1.9209e-40,  6.1089e-39, -5.7310e-40,\n         2.9432e-39,  2.7774e-38,  6.7905e-39,  3.4465e-38,  6.3933e-38,\n         3.9387e-38,  1.2195e-38,  2.4597e-39,  6.0623e-38, -8.2512e-40,\n         9.0086e-40,  2.4274e-39,  1.7575e-38, -1.5811e-39,  2.8463e-41,\n         1.4107e-38,  1.7167e-39,  2.1615e-39, -6.9324e-40,  1.3037e-38,\n         1.5993e-38,  1.0863e-38,  4.2321e-39, -6.2665e-40,  2.0004e-38,\n        -8.3436e-41, -2.0275e-39,  1.8223e-38,  2.7790e-39,  3.1535e-40,\n         3.8786e-39,  3.9242e-38,  3.7227e-38, -1.1023e-40,  4.6816e-38,\n         2.6456e-39,  1.0476e-37, -7.9689e-40,  1.5851e-39,  1.0850e-38,\n         2.0373e-39,  9.6542e-39,  6.6436e-38, -1.0972e-39,  5.7471e-39,\n         5.9815e-38,  2.1370e-38,  4.6463e-38,  2.0785e-39,  4.6295e-39,\n         3.2669e-39,  1.8319e-39,  6.0496e-39,  2.4278e-39,  1.1233e-37,\n        -9.3995e-41,  4.4052e-40,  9.9531e-39,  4.9637e-38,  6.1067e-39,\n         1.6536e-39,  8.5954e-40, -3.0830e-39, -1.4336e-40,  1.0144e-38,\n        -4.4952e-41,  5.3124e-38,  1.4028e-39,  2.5667e-39,  5.8063e-38,\n        -1.0145e-38,  1.0983e-37,  3.2647e-38, -9.9934e-40,  8.4899e-39,\n         1.3152e-39, -1.1697e-40,  6.9457e-38,  1.8887e-38,  6.4569e-38,\n         2.5601e-38,  4.8138e-40,  2.6816e-39,  5.3143e-41,  5.6437e-39,\n         2.5735e-38,  6.7726e-38,  2.5261e-38,  6.9097e-39,  7.2266e-38,\n         2.9026e-38, -4.5459e-40,  7.4117e-39,  3.2873e-40,  1.2331e-39,\n        -3.5591e-39,  6.2897e-38,  9.3148e-39,  7.0553e-39,  1.2619e-38,\n         7.4005e-39,  6.6105e-38,  4.1458e-39,  5.8615e-39,  6.7010e-40,\n        -2.0249e-39,  3.6467e-39, -7.7162e-40, -9.8679e-40,  8.4941e-39,\n         7.1973e-39,  6.5372e-38,  8.3128e-39, -2.3474e-39,  3.4024e-38,\n         1.4730e-39,  2.4786e-39,  6.7608e-40,  6.1187e-39,  3.6258e-38,\n        -8.9248e-40,  3.1192e-38,  5.6728e-39,  6.5160e-39, -2.2435e-39,\n         1.4065e-38,  4.8729e-40, -4.7894e-40,  4.6906e-39,  1.5091e-37,\n        -5.1700e-40,  8.8359e-39, -6.2625e-40, -1.1724e-39,  7.4131e-39,\n        -1.1896e-39,  4.8648e-38,  3.1892e-39,  1.1240e-38,  5.5722e-38,\n        -1.0770e-39,  8.7765e-39,  1.5135e-39,  1.0506e-37,  1.4411e-40,\n        -2.8415e-39,  5.3921e-38,  3.1660e-40,  1.9380e-38,  1.8670e-39,\n         2.1155e-39,  1.3173e-37, -1.5611e-39, -4.7943e-41,  8.7896e-38,\n         1.6497e-38,  1.3642e-38,  9.1170e-40,  1.5362e-38,  1.8337e-39,\n         1.4163e-39,  2.4762e-38,  4.4220e-39, -8.7031e-40,  1.8853e-39,\n         2.7179e-39, -4.7448e-39,  9.0170e-38,  1.0881e-37,  2.8308e-38,\n        -7.1435e-41,  7.9712e-39,  1.9239e-39,  1.4025e-37,  5.3138e-38,\n         3.6496e-39,  1.1644e-37,  1.1252e-39,  2.3402e-39,  3.2702e-40,\n         6.0103e-39,  5.2515e-39,  5.9257e-38,  4.0198e-38,  1.3422e-40,\n         2.6141e-39, -2.8309e-41,  1.6222e-39,  5.7318e-38,  8.3368e-40,\n         1.7489e-39,  1.0360e-38,  1.1997e-39,  7.1377e-39,  1.0807e-38,\n         1.2119e-37,  8.4333e-40,  3.7647e-39,  3.3068e-38,  2.5722e-38,\n         4.7826e-38,  7.6831e-38,  8.6848e-38,  8.6224e-40,  9.7735e-40,\n         9.9482e-39,  3.1476e-39, -5.9591e-40,  1.3350e-38,  3.1004e-39,\n         2.0992e-38,  1.9598e-39, -7.9384e-39,  6.1793e-40,  1.7909e-40,\n         1.5001e-38,  3.9507e-39, -7.5866e-40,  1.0815e-38,  5.6805e-39,\n        -2.1128e-39,  5.6818e-41,  4.4891e-39,  7.3372e-39,  1.1711e-38,\n         9.2124e-40, -1.5477e-39,  1.4512e-39,  1.5013e-39,  2.7309e-38,\n         7.1708e-39,  2.5989e-39,  3.2742e-40,  4.5947e-38, -1.1610e-39,\n         7.1622e-38,  4.7988e-39,  8.9515e-38,  1.0305e-37,  3.3151e-38,\n         2.7932e-38,  5.8168e-40,  2.9864e-39,  7.1267e-39,  2.7220e-39,\n         5.3353e-39,  8.0323e-39,  2.8518e-38,  6.8873e-39,  1.1076e-38,\n        -1.6046e-39,  8.3965e-40,  2.2543e-39,  4.5095e-39,  3.6766e-39,\n         1.9804e-40, -3.2607e-40, -8.8738e-40,  4.2681e-39, -5.7290e-40,\n         1.8115e-39,  7.2193e-39,  7.0938e-39,  1.4868e-39,  1.9362e-40,\n         8.3797e-39,  4.8015e-40,  2.0062e-38,  1.0649e-38, -6.4358e-40,\n        -1.0870e-40,  7.7601e-38,  6.3633e-38,  5.6638e-39,  2.0785e-38,\n         3.2052e-38,  9.8599e-39, -1.1357e-38, -1.5344e-40, -1.4786e-39,\n        -1.3552e-39,  7.1687e-38,  5.2651e-40,  1.0246e-39,  1.0740e-38,\n         1.0622e-37,  1.1575e-39,  8.7050e-40,  2.0543e-39, -4.5421e-40,\n         6.9303e-38,  6.5074e-38,  6.8188e-38,  3.4567e-38,  5.9234e-38,\n         1.3181e-37, -1.3090e-39,  6.7731e-39,  4.2972e-39,  4.0040e-39,\n         8.0109e-39,  2.0420e-38, -1.3429e-39,  2.2161e-39,  1.1261e-38,\n         1.1785e-38,  3.6853e-39, -5.6472e-41,  7.5283e-38,  5.6365e-39,\n         6.8205e-38,  5.5999e-39,  3.8631e-39,  9.5544e-40,  7.4348e-38,\n        -2.6550e-39, -1.8714e-39,  6.7306e-39,  5.0933e-38,  3.9446e-39,\n         5.5467e-39,  3.6033e-39,  6.2436e-38,  4.6184e-39,  4.4153e-39,\n        -2.0642e-40,  3.1649e-38,  1.3357e-39,  2.0219e-38,  5.6530e-41,\n         1.5104e-39,  8.1596e-38,  1.3229e-37,  9.4920e-41,  8.2961e-40,\n         5.5213e-40,  6.6796e-39, -8.0456e-40,  7.1649e-40,  2.5080e-40,\n         2.6349e-39, -1.9062e-39, -6.9387e-41,  6.7660e-38,  5.1630e-39,\n        -6.7158e-40,  9.2337e-39,  1.0690e-38, -2.8677e-40,  3.3712e-39,\n         5.5612e-39,  4.1947e-38,  6.7729e-39,  1.4737e-38,  3.5331e-39,\n         3.3018e-38,  4.9394e-38,  9.4092e-39,  8.4569e-39,  5.2771e-39,\n         7.4219e-38, -6.3742e-40,  1.4582e-37,  2.0916e-38,  2.3053e-39,\n         1.2171e-39, -5.4831e-40,  1.0977e-38,  1.4260e-37,  6.7045e-39,\n        -1.6445e-39,  2.2823e-38,  9.4221e-38,  7.3371e-38,  3.7310e-39,\n         1.5585e-37, -7.9047e-40, -2.9826e-40,  4.1242e-40,  3.1638e-38,\n         1.1286e-37,  6.0876e-38, -4.8542e-40, -5.8046e-39,  9.3133e-38,\n         2.5618e-38,  8.6934e-38, -1.1462e-39, -8.7881e-40,  1.2825e-40,\n         2.2035e-38,  1.2022e-38,  5.3879e-38,  2.0658e-39, -5.6369e-40,\n         5.8034e-39,  5.4711e-40], device='cuda:0')",
        "exp_avg_sq": "tensor([1.7490e-15, 2.9150e-16, 1.2942e-14, 4.9361e-14, 1.2915e-16, 3.3339e-15,\n        1.4196e-17, 1.0725e-19, 1.6662e-15, 1.3765e-17, 5.1245e-16, 3.6594e-18,\n        8.2376e-15, 1.1598e-16, 4.1628e-15, 2.1732e-18, 1.7029e-15, 4.5717e-15,\n        1.4521e-15, 7.2172e-16, 2.5098e-15, 2.7791e-14, 1.4705e-16, 2.7315e-14,\n        8.5373e-19, 9.5199e-16, 1.0983e-17, 1.7085e-16, 2.5736e-17, 7.4995e-15,\n        2.6279e-15, 3.9125e-17, 1.1008e-17, 1.7239e-13, 8.8446e-14, 1.6080e-16,\n        4.3470e-14, 8.9202e-16, 1.3918e-14, 1.9103e-16, 6.5748e-17, 1.6566e-14,\n        9.7743e-16, 8.0723e-19, 1.9436e-14, 9.8664e-16, 1.8285e-16, 7.9301e-17,\n        1.1668e-15, 2.2720e-19, 1.7830e-18, 2.6224e-19, 1.8623e-16, 4.4113e-16,\n        5.2535e-16, 1.5690e-14, 4.5108e-14, 3.3606e-13, 1.4827e-13, 4.7712e-14,\n        3.2689e-15, 4.4262e-15, 6.9863e-15, 3.0352e-14, 1.3419e-14, 8.2103e-18,\n        3.8679e-18, 7.7725e-15, 1.4709e-16, 6.3448e-14, 3.2100e-16, 2.8824e-14,\n        6.4440e-16, 2.7792e-14, 1.6610e-14, 7.7452e-18, 3.1391e-14, 6.5178e-15,\n        1.3250e-13, 5.5211e-17, 1.0370e-14, 4.9957e-18, 3.7304e-15, 6.9829e-16,\n        6.4046e-14, 2.2630e-14, 4.8907e-18, 8.6708e-14, 4.7112e-14, 2.7717e-15,\n        4.6739e-14, 1.6838e-17, 2.1781e-17, 3.7841e-15, 1.7344e-17, 4.0098e-18,\n        2.1287e-15, 1.7554e-14, 5.9283e-16, 1.0740e-14, 1.8265e-15, 5.9624e-15,\n        6.0698e-17, 8.7358e-15, 3.0596e-17, 7.6192e-15, 2.4009e-14, 1.8398e-18,\n        3.6698e-15, 5.9871e-17, 2.1659e-14, 1.7925e-17, 3.7747e-16, 9.5259e-14,\n        3.2892e-15, 2.5824e-16, 6.9197e-15, 1.2599e-14, 2.3273e-14, 5.0429e-14,\n        2.1002e-13, 3.3695e-16, 2.1478e-17, 1.5602e-14, 1.4948e-18, 7.6657e-20,\n        1.5934e-16, 1.4953e-14, 4.4514e-16, 1.1369e-15, 3.0117e-15, 6.1976e-16,\n        2.1671e-16, 6.3859e-18, 3.6655e-15, 1.3206e-14, 7.1433e-16, 4.8082e-17,\n        1.7589e-17, 1.5772e-14, 2.6817e-18, 9.4808e-16, 3.8532e-16, 1.2716e-15,\n        2.6174e-16, 1.7442e-16, 2.6892e-15, 1.9490e-14, 4.6663e-15, 2.5006e-14,\n        1.6540e-18, 1.1927e-13, 2.0049e-15, 1.9860e-16, 8.4538e-16, 2.1512e-17,\n        1.5130e-16, 1.9037e-14, 2.9985e-17, 1.7466e-16, 4.8270e-14, 8.7970e-16,\n        2.0446e-14, 2.6837e-15, 1.8988e-16, 4.3240e-17, 5.9364e-14, 1.3568e-15,\n        5.1434e-20, 5.9270e-14, 5.3406e-15, 9.8347e-15, 7.2113e-16, 4.4360e-14,\n        1.0631e-16, 7.0746e-17, 1.9653e-17, 1.0942e-19, 2.6519e-15, 1.8422e-16,\n        1.6966e-16, 8.5465e-15, 1.9045e-17, 1.0434e-17, 1.9548e-14, 2.5597e-15,\n        1.4750e-13, 1.8057e-15, 1.1146e-15, 5.1956e-16, 7.2626e-20, 8.6799e-16,\n        1.7623e-14, 5.3856e-16, 2.2193e-14, 3.0432e-16, 6.4895e-17, 4.1957e-16,\n        8.3164e-16, 1.1251e-14, 1.0177e-15, 4.4563e-14, 2.0285e-16, 2.3967e-17,\n        2.7686e-14, 2.1531e-14, 2.8405e-17, 1.0741e-15, 4.8531e-16, 1.8315e-17,\n        8.5683e-16, 3.9926e-14, 4.8347e-16, 4.6237e-16, 6.7013e-17, 4.5980e-14,\n        4.1746e-14, 3.6606e-18, 2.8900e-14, 1.3152e-17, 2.1704e-19, 1.2093e-17,\n        8.3298e-18, 1.2323e-18, 4.6447e-16, 2.5714e-16, 3.0789e-14, 1.0140e-16,\n        9.2474e-17, 2.7909e-15, 5.5980e-19, 1.1312e-14, 1.2017e-16, 6.9723e-17,\n        1.4868e-15, 3.1871e-18, 1.3078e-15, 5.1372e-17, 3.8975e-16, 1.0993e-16,\n        3.3863e-17, 1.1697e-17, 9.8107e-16, 2.3496e-15, 2.2931e-13, 7.1382e-19,\n        7.0060e-16, 3.2730e-17, 7.4272e-16, 9.2282e-17, 2.7464e-16, 2.7727e-14,\n        5.2507e-17, 3.3296e-15, 1.4680e-14, 4.1814e-18, 2.5867e-15, 5.2242e-18,\n        7.4574e-14, 5.3440e-19, 1.6350e-17, 4.4766e-14, 1.5775e-15, 2.1531e-14,\n        2.5640e-16, 3.8042e-16, 2.1950e-14, 1.6119e-18, 8.1380e-18, 2.8649e-14,\n        2.2222e-14, 1.5956e-15, 1.8986e-16, 1.3014e-14, 2.8681e-16, 4.2825e-17,\n        1.4993e-14, 4.4059e-16, 7.4196e-16, 1.7069e-17, 5.6324e-17, 2.5305e-16,\n        1.0084e-13, 9.0183e-14, 1.4717e-15, 5.9278e-17, 2.2990e-14, 5.4835e-16,\n        1.0878e-13, 1.5315e-14, 7.4399e-15, 1.8982e-13, 7.7117e-18, 1.8720e-17,\n        3.2422e-16, 2.4770e-16, 1.2548e-16, 1.7320e-14, 2.0143e-15, 3.9394e-16,\n        1.3156e-16, 1.2742e-18, 1.4042e-17, 4.8184e-15, 5.4918e-18, 1.6477e-17,\n        4.1168e-15, 5.6025e-17, 1.1738e-15, 3.3847e-16, 2.1909e-13, 1.1755e-16,\n        1.9416e-16, 3.0190e-14, 8.0109e-15, 1.1496e-14, 1.0861e-13, 2.9501e-14,\n        8.9585e-17, 1.9877e-19, 2.8583e-15, 2.0511e-17, 3.8805e-14, 1.9699e-14,\n        1.4346e-14, 3.4856e-15, 5.5725e-18, 6.9576e-17, 5.0935e-17, 2.1868e-19,\n        8.7740e-14, 2.8873e-16, 4.2714e-18, 4.8928e-17, 3.7163e-16, 1.6758e-17,\n        1.3011e-17, 2.5168e-16, 2.6860e-18, 1.7018e-18, 2.2269e-14, 1.0420e-16,\n        9.8788e-15, 4.2526e-19, 2.2185e-15, 5.7591e-18, 2.9322e-15, 4.1193e-20,\n        3.0890e-14, 5.8463e-17, 9.5987e-15, 2.5760e-16, 2.8624e-14, 1.4285e-13,\n        6.0403e-15, 2.9969e-14, 8.2890e-15, 2.0969e-20, 6.0762e-18, 3.4651e-14,\n        1.9358e-18, 2.8122e-17, 2.4526e-15, 8.7572e-16, 9.8069e-16, 7.9328e-18,\n        6.6123e-17, 8.1860e-16, 1.4300e-17, 8.6282e-16, 1.4596e-16, 4.0601e-18,\n        1.8099e-16, 1.4569e-16, 3.2017e-16, 3.7790e-17, 4.4335e-17, 8.9748e-17,\n        2.0675e-16, 2.2654e-16, 1.7268e-16, 2.9562e-17, 1.6659e-14, 3.5479e-16,\n        9.5336e-18, 8.1145e-16, 1.1973e-14, 1.1653e-14, 2.2506e-16, 1.5969e-14,\n        5.6026e-15, 3.8366e-16, 4.6869e-16, 1.2317e-17, 4.3880e-15, 1.0925e-16,\n        5.0131e-14, 2.2534e-16, 1.4270e-14, 2.4630e-14, 2.4290e-14, 5.0931e-16,\n        7.6833e-15, 7.5536e-16, 3.2278e-17, 3.2885e-14, 7.2213e-14, 4.5928e-14,\n        1.8421e-14, 1.2810e-13, 1.3565e-13, 9.9925e-19, 4.7561e-16, 2.2358e-17,\n        1.0086e-17, 2.3389e-17, 1.1065e-15, 1.0860e-14, 8.3836e-16, 3.2750e-15,\n        6.5739e-16, 6.2412e-15, 3.5223e-18, 4.6218e-14, 1.1729e-16, 2.1966e-14,\n        2.1307e-17, 1.3748e-14, 1.8283e-16, 1.9769e-14, 7.9706e-17, 1.4244e-17,\n        8.5571e-15, 1.9582e-15, 1.3098e-17, 2.0113e-16, 1.1425e-17, 4.6312e-14,\n        5.7208e-16, 1.1100e-17, 1.3468e-17, 2.5937e-15, 2.0750e-19, 6.9483e-14,\n        5.4803e-19, 6.4511e-17, 2.5780e-14, 1.0486e-13, 3.4991e-17, 7.8168e-16,\n        1.8228e-17, 1.9078e-16, 3.2395e-18, 3.6087e-15, 8.4028e-18, 5.2309e-18,\n        1.2910e-16, 9.1442e-16, 5.5001e-14, 6.2880e-18, 1.2199e-17, 4.1726e-19,\n        4.1619e-16, 1.7487e-17, 2.8868e-16, 2.9677e-17, 4.1530e-15, 3.2231e-18,\n        1.4446e-15, 3.4671e-16, 2.6784e-14, 1.5791e-14, 9.3714e-16, 2.9322e-18,\n        1.1545e-17, 2.6549e-14, 4.0968e-16, 2.2806e-13, 6.9400e-16, 3.6927e-19,\n        1.4206e-17, 3.0788e-16, 2.2140e-15, 2.2837e-13, 7.5975e-15, 9.9274e-17,\n        8.2963e-16, 3.9790e-14, 2.8534e-14, 1.1246e-16, 1.0587e-13, 1.4002e-17,\n        4.8755e-16, 6.3628e-17, 1.3045e-14, 1.8210e-13, 2.6775e-14, 5.8597e-17,\n        1.2253e-13, 7.5161e-14, 3.1743e-15, 5.4045e-14, 6.9986e-17, 1.1302e-15,\n        1.0094e-15, 8.8462e-15, 1.6517e-14, 4.1202e-15, 3.0013e-17, 1.4411e-18,\n        4.0592e-18, 1.8984e-17], device='cuda:0')"
      },
      "63": {
        "step": "tensor(11268.)",
        "exp_avg": "tensor([ 4.4653e-38, -8.3668e-39,  3.4432e-38,  2.7626e-38, -3.7831e-39,\n         2.2420e-38,  2.4786e-38, -1.6162e-39, -7.4820e-39,  1.7414e-38,\n         2.6651e-38,  1.8994e-39,  4.6265e-38, -1.0756e-39,  3.1252e-38,\n         1.6963e-38,  1.4709e-38,  1.4959e-38, -7.6976e-39,  1.8167e-38,\n         4.1823e-38,  4.4195e-38, -2.3581e-39,  1.0844e-38,  3.9594e-39,\n         2.2501e-38,  1.7229e-38,  3.8940e-39, -3.7823e-39,  2.1476e-38,\n         9.9923e-40, -5.8578e-39, -6.8827e-40,  4.3150e-38,  5.5527e-38,\n        -3.4948e-39,  5.8164e-38,  4.1796e-40,  5.8481e-38, -7.5147e-40,\n        -2.6882e-39,  3.1461e-38,  2.3386e-38,  2.0300e-39,  5.8410e-38,\n         5.6331e-38, -2.1777e-39,  2.0090e-38,  5.5928e-38, -3.5086e-39,\n        -2.7453e-39,  6.3505e-40, -9.5034e-40,  9.4655e-39, -7.6100e-40,\n         6.0239e-38,  8.3839e-38,  7.7637e-38,  6.6333e-38,  5.7974e-38,\n         3.7679e-38,  6.1954e-38, -8.4087e-39,  7.5777e-38,  2.1285e-38,\n         9.4081e-39,  2.8933e-39,  4.3992e-38, -5.1667e-39,  1.4415e-38,\n        -4.9390e-39,  4.7317e-38,  1.8601e-38,  2.1736e-38,  6.4221e-38,\n        -1.4216e-39, -1.2386e-38,  4.4432e-38, -4.5169e-39, -1.4617e-39,\n        -1.9953e-39, -2.6994e-39,  2.2188e-38, -3.3259e-39,  6.5678e-38,\n         6.4663e-38, -4.1425e-39,  6.0622e-38,  8.1004e-38,  6.9185e-39,\n         3.6656e-38,  1.3262e-40,  3.0394e-38,  5.3899e-39, -2.6591e-39,\n         2.7064e-38,  4.2708e-38,  2.1243e-38,  2.6473e-39,  4.0851e-38,\n         1.8030e-38,  5.7714e-38, -7.7020e-40,  1.8132e-38,  6.1519e-39,\n         3.2351e-38,  5.0108e-38,  1.1456e-39,  7.5714e-38,  2.4765e-39,\n        -4.1103e-40, -2.4607e-39,  6.5165e-39,  8.1184e-39,  1.6490e-38,\n        -2.3968e-39,  4.2406e-38,  3.1571e-38,  4.1691e-38,  5.7848e-38,\n         4.6410e-38, -9.0662e-39, -2.1658e-39,  5.3318e-38,  7.4104e-40,\n        -7.8140e-40, -1.8877e-39,  3.1062e-38,  8.3915e-39,  9.5944e-39,\n         2.7538e-38, -1.5799e-39, -1.7152e-39,  2.4349e-39, -7.6629e-39,\n         2.4102e-38, -7.1072e-39, -3.2306e-39,  8.8966e-39,  2.5091e-38,\n         3.1047e-40,  1.1592e-38,  3.6780e-38, -1.4169e-39, -1.0861e-40,\n        -2.6828e-39,  5.2880e-38,  4.5951e-38,  2.4515e-39,  3.7496e-38,\n        -2.2221e-39,  6.4654e-38,  8.2253e-39, -1.3883e-39, -6.7709e-39,\n        -1.8812e-39, -6.9862e-39,  4.7703e-38,  7.1446e-39, -4.4285e-39,\n         4.1661e-38,  4.0738e-38,  3.5772e-38,  8.9349e-39, -3.2847e-39,\n        -2.6177e-39,  3.2775e-39,  2.5208e-38, -2.1217e-39,  7.0898e-38,\n         8.0373e-40,  1.1305e-39, -6.7133e-39,  3.8718e-38,  2.1545e-38,\n        -1.3952e-39,  2.2114e-38,  1.0730e-38,  2.6190e-39,  2.9171e-38,\n         4.6924e-40,  4.7382e-38, -1.2699e-39, -2.2753e-39,  4.7996e-38,\n        -7.9899e-39,  6.3017e-38,  4.8512e-38,  6.6167e-39, -5.4678e-39,\n        -1.0574e-39,  1.2090e-38,  5.1946e-38,  3.2230e-38,  5.2085e-38,\n         5.3575e-38, -4.8161e-41,  1.1703e-38, -3.6827e-40,  8.5552e-39,\n         3.9709e-38,  4.8734e-38,  5.1314e-38,  2.1202e-38,  5.2111e-38,\n         2.8285e-38,  2.5891e-39,  1.3078e-38,  8.5644e-40,  1.8232e-38,\n         2.4124e-39,  5.4466e-38, -6.1544e-39,  8.9738e-39,  5.5249e-38,\n         1.3698e-38,  4.1522e-38, -3.5108e-39,  2.1750e-38,  1.2276e-40,\n         5.0843e-39, -3.1783e-39,  2.8003e-39,  2.4272e-39, -5.9333e-39,\n         1.3913e-38,  3.7769e-38, -6.2254e-39,  7.7242e-39,  3.7382e-38,\n        -5.1156e-40,  5.6518e-39, -3.5970e-40,  4.2672e-38,  4.4574e-38,\n         2.8149e-39,  6.5822e-38, -4.5753e-39, -4.1246e-39,  8.2048e-39,\n         4.9826e-38,  1.5126e-38,  2.9469e-39,  1.2566e-38,  7.9946e-38,\n         1.4253e-39, -5.3896e-39,  1.5054e-39,  8.5024e-40,  2.4288e-38,\n         3.2724e-39,  3.5310e-38, -2.8860e-39,  1.1007e-38,  3.8567e-38,\n         1.0252e-39,  2.1918e-38, -1.2454e-39,  6.3315e-38,  1.0858e-39,\n         1.3307e-38,  3.5206e-38,  1.6592e-38,  2.3891e-38, -1.4285e-39,\n        -1.6966e-39,  9.5587e-38,  4.5098e-39,  1.0973e-39,  7.2969e-38,\n         1.2621e-38,  3.9082e-38,  4.5203e-40,  4.2085e-38,  1.3115e-38,\n        -1.2764e-39,  3.3011e-38, -3.4305e-39,  1.2924e-38, -1.6697e-39,\n        -8.0173e-39,  3.5017e-39,  5.0396e-38,  7.1822e-38,  4.2306e-38,\n         3.7227e-39,  1.2198e-38,  3.3898e-39,  8.4877e-38,  4.3108e-38,\n         1.3832e-38,  6.6064e-38, -9.9427e-40, -2.1001e-39,  4.7706e-40,\n        -4.3798e-39, -4.2426e-39,  5.3104e-38,  4.8304e-38,  1.5072e-40,\n        -2.2475e-39,  5.0738e-40,  2.5505e-38,  6.6471e-38, -6.5453e-40,\n        -1.2357e-39,  1.5212e-38, -9.5845e-40, -5.0336e-39, -8.7432e-39,\n         8.1312e-38, -6.7603e-39, -3.0032e-39,  2.2148e-38,  1.8171e-38,\n         3.6265e-38,  4.5805e-38,  5.1701e-38, -8.0081e-40, -3.6544e-40,\n        -5.7500e-39, -2.8173e-39, -7.2117e-40,  3.0978e-38,  6.3923e-39,\n         3.9368e-38, -1.6925e-39,  6.9541e-39, -1.5843e-40, -1.5598e-40,\n         1.8627e-38, -3.2295e-39,  3.6045e-39,  3.0645e-38, -4.5859e-39,\n         1.8497e-38,  4.5650e-40, -3.5529e-39,  3.1602e-38,  3.9049e-38,\n         3.1779e-39,  6.0179e-39,  4.9704e-39, -1.0975e-39,  2.9269e-38,\n         2.2935e-38, -2.0291e-39,  1.5842e-38,  4.4419e-38,  4.8532e-39,\n         5.8772e-38, -3.4494e-39,  6.3937e-38,  5.6118e-38,  3.6468e-38,\n         3.0444e-38,  1.0214e-38, -2.5969e-39,  4.1691e-38,  1.7384e-38,\n         2.6711e-38,  4.1861e-38,  3.4424e-38, -5.3560e-39, -7.9296e-39,\n         9.7001e-39,  2.6833e-38,  1.0818e-38, -3.6209e-39,  9.4431e-39,\n         5.0686e-40,  1.4277e-39,  1.1174e-38, -3.4129e-39,  2.1270e-39,\n        -1.2798e-39, -5.9515e-39, -5.8976e-39, -2.6458e-40,  4.0062e-40,\n         3.0265e-38,  2.2785e-38,  2.7504e-38, -7.8616e-39,  1.8673e-38,\n         4.1659e-40,  5.0363e-38,  5.4807e-38,  2.0744e-38,  1.8442e-38,\n         3.0115e-38, -8.0569e-39, -2.5949e-38,  2.0786e-40,  1.2649e-38,\n         5.8053e-39,  5.2865e-38, -4.4525e-40,  3.0812e-39,  1.2985e-38,\n         8.4454e-38, -5.8655e-40,  1.2897e-38, -1.3696e-39,  4.5575e-39,\n         7.3139e-38,  3.6095e-38,  6.1335e-38,  3.2399e-38,  4.1685e-38,\n         7.0472e-38,  1.4534e-38, -4.7499e-39, -3.5342e-39, -3.2813e-39,\n         2.3232e-38,  2.0131e-38,  6.2873e-39, -1.6205e-39, -6.5131e-39,\n        -8.2083e-39,  1.7255e-38,  9.6861e-40,  4.4862e-38, -4.3939e-39,\n         4.0658e-38, -4.1588e-39,  1.5732e-38,  4.9802e-40,  4.8955e-38,\n         1.3461e-38,  4.6932e-39,  2.0542e-38,  6.9270e-38,  3.1722e-38,\n        -4.1989e-39, -3.1283e-39,  5.1729e-38, -3.7420e-39, -3.6599e-39,\n         5.8842e-40,  4.1186e-38, -1.1746e-39,  2.5164e-38,  6.7249e-40,\n         7.0914e-39,  8.3556e-38,  8.4888e-38,  6.6477e-39, -6.8390e-40,\n        -4.9964e-40,  2.0713e-38,  2.9787e-39,  1.5428e-38, -6.3517e-41,\n         1.7356e-38,  1.2203e-38, -3.5733e-43,  5.9588e-38, -3.6672e-39,\n         3.0360e-39,  3.8430e-38, -7.8145e-39,  2.0112e-38, -2.1759e-39,\n        -4.2131e-39,  4.3652e-38,  2.5416e-38, -1.0376e-38,  1.2027e-38,\n         3.6066e-38,  3.7823e-38, -6.0374e-39,  2.9170e-38, -4.2102e-39,\n         5.3321e-38,  2.1258e-39,  8.5949e-38,  3.3279e-38, -1.9669e-39,\n        -7.9554e-40,  1.7179e-39,  9.8263e-39,  7.6907e-38,  7.6664e-39,\n         7.3114e-39,  3.0717e-38,  8.2937e-38,  6.4099e-38,  3.7747e-38,\n         9.6775e-38,  1.0080e-38,  2.3690e-40, -1.9653e-41,  3.8128e-38,\n         7.1409e-38,  5.2929e-38,  1.6608e-39, -6.4140e-39,  5.8927e-38,\n         2.9657e-38,  6.7939e-38,  3.7985e-39,  5.4705e-39,  3.1172e-41,\n         2.6339e-38,  2.4478e-38,  5.5214e-38, -7.8477e-39,  1.9854e-39,\n         2.8213e-38,  2.7896e-38], device='cuda:0')",
        "exp_avg_sq": "tensor([1.3812e-14, 1.4136e-16, 2.5119e-14, 2.7307e-14, 3.9369e-15, 2.8455e-15,\n        1.6432e-15, 1.4135e-17, 7.5172e-16, 1.1084e-15, 4.1078e-15, 2.0748e-16,\n        2.3276e-14, 5.2735e-17, 2.2650e-15, 4.7640e-16, 5.4364e-15, 2.1315e-14,\n        5.8930e-16, 4.5576e-15, 1.7429e-14, 2.4640e-14, 6.8610e-17, 2.3780e-14,\n        9.6802e-17, 1.5995e-15, 3.0123e-15, 7.1784e-15, 5.9655e-18, 5.7156e-15,\n        2.0603e-14, 1.8777e-17, 4.3015e-18, 1.0452e-13, 5.4431e-14, 7.9990e-17,\n        3.0856e-14, 3.7485e-16, 4.8581e-14, 9.0441e-17, 3.1544e-17, 8.5496e-15,\n        1.0557e-15, 1.7993e-16, 1.7502e-14, 1.2637e-14, 7.1822e-17, 1.0341e-14,\n        1.0975e-14, 5.5545e-17, 1.4041e-16, 6.4792e-16, 9.7070e-17, 2.0299e-15,\n        2.3656e-16, 3.5190e-14, 8.2140e-14, 2.0222e-13, 1.4343e-13, 3.1533e-14,\n        5.7082e-15, 2.1310e-14, 2.7340e-15, 8.5658e-14, 1.3435e-14, 5.4708e-16,\n        7.6989e-16, 3.2431e-14, 7.9584e-17, 5.7279e-14, 1.2741e-16, 4.2767e-14,\n        2.0680e-15, 2.9442e-14, 2.5542e-14, 3.7766e-16, 6.7618e-14, 2.4396e-14,\n        7.8796e-14, 2.9199e-17, 3.3851e-14, 9.2283e-19, 5.8989e-15, 2.9752e-16,\n        4.2543e-14, 6.0081e-14, 1.9994e-17, 4.0808e-14, 7.2087e-14, 2.3773e-14,\n        3.9794e-14, 1.6516e-15, 2.9384e-15, 3.3128e-14, 5.5666e-18, 1.9848e-15,\n        4.1041e-15, 2.4534e-14, 3.6837e-15, 3.1960e-14, 1.8876e-15, 2.0588e-14,\n        2.8274e-17, 7.5897e-15, 7.7327e-16, 3.9156e-15, 5.1704e-14, 1.0010e-16,\n        2.0325e-14, 4.8502e-15, 3.4601e-14, 7.9486e-18, 1.2026e-15, 7.4121e-14,\n        1.7701e-14, 1.1743e-16, 5.7276e-15, 2.7582e-14, 1.2480e-14, 3.8827e-14,\n        1.1224e-13, 1.3042e-16, 6.6293e-18, 1.6685e-14, 1.0406e-16, 5.8915e-18,\n        7.1233e-17, 2.1731e-14, 6.2766e-15, 3.5507e-15, 3.0623e-15, 2.9214e-16,\n        8.9679e-17, 8.9988e-16, 1.4851e-15, 1.6482e-14, 2.6339e-16, 2.2323e-17,\n        5.5666e-16, 9.7934e-15, 2.7420e-15, 8.5473e-15, 7.3231e-15, 6.0637e-16,\n        1.1278e-16, 6.4543e-17, 1.5407e-14, 1.2249e-14, 1.4286e-14, 2.9556e-14,\n        1.5024e-16, 6.9230e-14, 1.0425e-14, 2.1542e-17, 3.8543e-16, 1.4429e-15,\n        7.6371e-17, 1.5019e-14, 4.5044e-16, 2.7015e-17, 2.8339e-14, 1.0964e-14,\n        1.4564e-14, 4.7724e-15, 9.8257e-17, 1.9487e-17, 4.3065e-14, 1.8741e-15,\n        6.7474e-20, 4.1482e-14, 3.7137e-14, 5.7238e-15, 3.2525e-16, 5.7543e-14,\n        3.2311e-15, 3.1988e-17, 2.2952e-15, 2.8289e-16, 1.3724e-14, 8.3460e-15,\n        4.3649e-17, 1.5523e-14, 8.2669e-16, 9.1273e-20, 2.4906e-14, 1.3971e-15,\n        7.1138e-14, 1.1667e-14, 8.6846e-15, 2.4327e-16, 1.0237e-17, 3.2893e-15,\n        2.1720e-14, 3.9025e-15, 3.1799e-14, 9.0233e-15, 3.1428e-17, 4.0685e-15,\n        2.0552e-14, 9.8357e-15, 5.6984e-15, 3.7282e-14, 6.7842e-15, 7.0222e-17,\n        2.8106e-14, 5.4293e-14, 1.0289e-16, 9.2622e-16, 2.6184e-16, 1.0800e-15,\n        4.1055e-16, 6.2873e-14, 2.4110e-16, 3.7685e-16, 2.2280e-14, 3.8707e-14,\n        3.4027e-14, 9.1173e-18, 5.6776e-14, 6.8034e-18, 8.1523e-16, 3.2527e-18,\n        9.5869e-16, 6.8214e-17, 2.1837e-16, 2.2109e-15, 1.3778e-14, 4.6529e-17,\n        7.2836e-17, 8.8996e-15, 1.3216e-16, 1.7115e-14, 5.9821e-17, 1.0905e-14,\n        7.2048e-15, 2.6298e-16, 3.1833e-14, 7.2895e-19, 2.0108e-16, 8.6363e-16,\n        1.4720e-14, 9.4957e-16, 4.3935e-15, 4.5424e-15, 9.9083e-14, 4.5114e-16,\n        3.4699e-16, 9.8100e-16, 2.7383e-16, 5.1506e-16, 6.9640e-15, 2.7731e-14,\n        3.2544e-15, 4.4948e-15, 1.2370e-14, 2.7575e-16, 3.5645e-15, 1.8978e-20,\n        4.3351e-14, 4.8496e-16, 1.2517e-15, 3.6370e-14, 5.0866e-15, 1.4497e-14,\n        5.8774e-17, 1.6319e-16, 1.6872e-14, 9.8595e-16, 4.2366e-17, 4.6135e-14,\n        1.1205e-14, 8.6094e-15, 9.7778e-17, 4.5895e-14, 1.0319e-15, 2.6670e-18,\n        1.1590e-14, 1.7335e-16, 3.7356e-15, 5.0562e-18, 4.6015e-15, 9.8265e-17,\n        5.2943e-14, 6.1255e-14, 1.4292e-14, 1.9765e-15, 1.7683e-14, 1.0143e-15,\n        5.2591e-14, 2.4545e-14, 1.2042e-14, 9.3692e-14, 3.4301e-17, 2.5467e-18,\n        1.6219e-16, 1.2605e-16, 6.8199e-17, 1.6311e-14, 8.8563e-15, 1.9809e-16,\n        6.7658e-17, 6.9132e-16, 1.0736e-15, 2.4883e-14, 2.1837e-16, 8.9406e-18,\n        2.9306e-15, 2.8198e-17, 5.6307e-16, 3.1913e-17, 2.1815e-13, 5.0630e-15,\n        2.1054e-17, 2.1269e-14, 4.3998e-15, 1.2730e-14, 5.5550e-14, 1.4518e-14,\n        3.6780e-17, 3.1454e-17, 1.3522e-15, 7.5334e-18, 4.1107e-14, 2.7803e-14,\n        1.2409e-14, 9.6033e-15, 1.2238e-18, 2.4546e-17, 2.4535e-17, 1.3004e-16,\n        5.4934e-14, 1.2326e-16, 2.1543e-16, 1.6747e-15, 1.7791e-16, 6.7102e-16,\n        5.7595e-18, 9.7437e-17, 5.1503e-15, 3.0166e-15, 3.1446e-14, 3.8211e-15,\n        2.2728e-14, 5.3568e-17, 2.1045e-15, 9.8107e-15, 1.0964e-15, 8.5042e-16,\n        1.7101e-14, 1.1446e-16, 1.0991e-14, 6.3841e-17, 1.6576e-14, 6.7633e-14,\n        2.7737e-14, 2.0025e-14, 3.1121e-14, 3.1890e-17, 1.3233e-14, 7.9920e-14,\n        1.6518e-15, 6.7831e-15, 1.1117e-14, 3.3282e-16, 4.7098e-16, 8.0658e-16,\n        2.7171e-15, 4.5888e-15, 3.6926e-18, 1.8478e-15, 3.6368e-17, 1.5687e-15,\n        1.3742e-15, 7.0904e-17, 5.9858e-15, 2.1357e-15, 6.6465e-18, 5.7530e-18,\n        1.0222e-16, 1.1296e-16, 5.1219e-16, 4.6644e-15, 1.3519e-14, 1.7281e-16,\n        7.3107e-16, 4.1980e-16, 8.9732e-15, 1.4509e-14, 9.4763e-16, 1.0908e-14,\n        5.4641e-15, 1.6056e-16, 3.7064e-15, 6.1531e-18, 3.4901e-14, 2.4663e-15,\n        3.9981e-14, 1.0991e-16, 2.9163e-14, 1.2516e-14, 2.4863e-14, 2.3659e-16,\n        2.4405e-14, 2.9700e-16, 2.7760e-16, 7.7744e-14, 3.6612e-14, 3.0902e-14,\n        2.5255e-14, 1.2630e-13, 6.1708e-14, 6.8521e-16, 2.4653e-16, 6.3941e-18,\n        1.6798e-19, 2.0922e-15, 6.3108e-16, 3.5218e-14, 3.5499e-16, 1.4009e-15,\n        3.0376e-16, 1.2120e-14, 6.2686e-16, 2.7605e-14, 4.7664e-17, 1.1512e-14,\n        2.1960e-18, 2.6051e-14, 9.4716e-17, 1.3485e-14, 8.4420e-16, 4.0756e-15,\n        1.0269e-14, 1.7045e-14, 1.1891e-15, 7.2485e-17, 4.9061e-16, 4.0657e-14,\n        2.6461e-16, 4.8217e-19, 3.0327e-15, 1.6449e-14, 2.3295e-17, 3.9652e-14,\n        4.5593e-16, 3.4472e-15, 6.5697e-14, 6.9592e-14, 2.7305e-15, 3.2652e-16,\n        1.2222e-15, 4.2338e-16, 1.9564e-16, 1.0477e-14, 1.3835e-15, 3.2177e-15,\n        1.1531e-15, 4.3832e-16, 3.4007e-14, 2.5166e-18, 2.0449e-15, 2.2204e-15,\n        1.5539e-16, 9.3587e-16, 1.1412e-16, 1.2165e-17, 9.3669e-15, 2.4464e-15,\n        6.3311e-16, 1.5718e-15, 1.8642e-14, 6.1199e-15, 4.4354e-16, 2.2926e-15,\n        5.7534e-18, 2.1744e-14, 2.1511e-16, 1.1247e-13, 6.2892e-15, 2.8619e-19,\n        3.9122e-17, 1.3959e-16, 1.0757e-15, 9.8387e-14, 4.4082e-15, 1.2334e-15,\n        8.1280e-16, 3.2407e-14, 4.7021e-14, 6.9078e-15, 7.5597e-14, 1.2113e-15,\n        1.9377e-16, 3.3285e-17, 6.4389e-15, 1.0209e-13, 4.0685e-14, 1.6309e-15,\n        7.0344e-14, 5.4555e-14, 4.7622e-15, 8.4126e-14, 7.2730e-15, 7.1075e-15,\n        5.1248e-16, 8.7985e-15, 1.5965e-14, 6.0044e-15, 3.4085e-15, 1.0892e-15,\n        2.4941e-15, 1.2590e-15], device='cuda:0')"
      },
      "64": {
        "step": "tensor(11268.)",
        "exp_avg": "tensor([[ 1.4055e-40,  4.0007e-39,  3.5527e-40,  ...,  1.3028e-40,\n          2.8431e-40, -4.0241e-39],\n        [ 4.5879e-40,  1.1664e-38,  1.2043e-39,  ...,  9.9953e-40,\n          2.7753e-40, -2.4745e-38],\n        [-2.9970e-40, -1.1858e-40, -7.2390e-41,  ...,  2.6444e-40,\n          1.7581e-41, -4.8541e-40],\n        ...,\n        [-3.1739e-40, -3.5825e-39, -6.8058e-40,  ..., -8.9667e-40,\n         -5.4532e-40,  5.0832e-39],\n        [ 8.0534e-41,  1.0864e-38,  2.9304e-40,  ...,  9.3050e-40,\n         -3.0394e-40, -2.0146e-38],\n        [-5.7966e-41, -1.2659e-39, -3.2537e-40,  ..., -6.2040e-41,\n         -2.3226e-40, -7.0305e-39]], device='cuda:0')",
        "exp_avg_sq": "tensor([[1.1021e-17, 1.9759e-18, 6.3399e-19,  ..., 7.2113e-18, 2.8060e-18,\n         6.8962e-18],\n        [1.2645e-16, 2.3686e-17, 6.1751e-18,  ..., 2.1969e-17, 1.6896e-17,\n         6.1816e-17],\n        [4.4788e-19, 2.0348e-18, 1.1309e-18,  ..., 8.3674e-19, 1.0117e-19,\n         3.3868e-17],\n        ...,\n        [4.8064e-16, 1.0134e-16, 1.3195e-17,  ..., 1.4406e-16, 3.2916e-17,\n         2.4602e-16],\n        [5.3255e-16, 1.1517e-16, 3.4107e-17,  ..., 2.6772e-16, 4.1614e-17,\n         1.8921e-16],\n        [1.7398e-16, 3.2826e-17, 3.2222e-17,  ..., 1.2085e-16, 2.2012e-17,\n         1.8727e-16]], device='cuda:0')"
      },
      "65": {
        "step": "tensor(11268.)",
        "exp_avg": "tensor([ 1.0483e-36,  3.7438e-36, -5.0526e-37,  5.1252e-37, -1.2760e-36,\n         6.9280e-37, -1.8966e-36, -4.8198e-37,  2.7078e-36,  4.7378e-36,\n        -3.3013e-36, -3.3931e-38,  2.3465e-36, -4.3724e-37, -2.0681e-36,\n         9.7485e-39,  1.1658e-37, -5.2005e-36,  2.4748e-36, -1.4275e-36,\n         3.0116e-36,  2.9050e-37, -1.2526e-36,  1.7173e-37, -9.6064e-37,\n        -1.4729e-36,  9.2060e-37,  4.7095e-37, -1.3153e-36, -3.1010e-37,\n         2.5174e-37,  4.1002e-36, -2.2470e-36,  5.5332e-37,  5.0076e-37,\n         1.1026e-36,  1.6630e-36, -8.7138e-38,  5.0015e-36, -3.3692e-37,\n         2.9271e-36, -1.4798e-36, -1.5075e-37, -3.2767e-36,  2.2018e-36,\n         4.0919e-37,  2.5345e-36,  8.6235e-37, -1.9357e-36,  2.9387e-37,\n         3.5389e-36, -2.1499e-36,  2.7067e-36, -1.2047e-36, -3.2732e-37,\n         1.7722e-36,  8.9857e-37,  1.2304e-36,  6.1975e-37,  1.4182e-36,\n         7.1728e-37,  7.0290e-37,  6.7465e-36,  3.1057e-36, -8.9349e-37,\n        -4.4944e-37, -1.5371e-36,  4.0436e-36,  9.4654e-37,  1.2986e-37,\n        -3.2129e-36,  1.6420e-37, -1.7592e-37, -5.8312e-37,  2.9611e-36,\n         1.0023e-36, -7.9176e-37, -7.3295e-37, -2.5578e-37, -2.0120e-37,\n         2.0582e-37, -7.1014e-37, -3.6494e-37,  3.9694e-37, -1.2998e-36,\n         3.9008e-36, -5.0827e-37,  1.0583e-36,  4.4876e-36, -3.1848e-36,\n         1.0531e-36,  2.3527e-37, -1.6980e-36, -3.5836e-36,  4.0235e-37,\n         1.7988e-36, -6.3630e-38, -7.2025e-38, -5.1700e-37,  2.8884e-36,\n        -5.5353e-37,  3.1358e-36,  2.9026e-36, -2.0386e-36,  8.2509e-38,\n        -3.8440e-37, -4.4371e-37, -3.3678e-36,  3.0728e-36, -1.2988e-36,\n         7.1536e-38, -1.9188e-36, -1.8253e-36, -2.2081e-36, -6.8319e-37,\n        -5.0307e-37,  6.4400e-37, -4.7676e-37, -7.9627e-37, -2.5837e-37,\n         3.9702e-36,  2.1016e-37, -3.0713e-36,  1.0607e-36,  4.0037e-37,\n        -1.4917e-37, -3.6536e-37,  5.2762e-37, -2.1216e-36, -4.6234e-37,\n        -1.2483e-37, -4.3605e-37, -4.7309e-37,  1.4428e-36,  1.7184e-36,\n        -1.1082e-36,  3.8629e-37, -2.7119e-36, -2.7516e-36, -2.6552e-37,\n        -8.7580e-37, -1.5781e-36, -8.9275e-38,  2.9090e-36, -1.7849e-36,\n        -2.1561e-36, -6.2753e-37,  4.3559e-37, -1.0275e-36,  2.9498e-36,\n        -1.0275e-36,  4.3018e-36, -1.0969e-36,  3.7445e-38, -3.6139e-36,\n        -1.2805e-36, -1.3824e-36, -3.8236e-37, -4.8461e-37, -4.3252e-37,\n        -5.7920e-37,  9.1989e-37, -1.4550e-37,  6.8046e-37,  1.2620e-36,\n         7.5198e-37,  1.1204e-37,  1.9335e-36, -1.7525e-36,  5.7016e-39,\n        -4.7627e-37, -1.5601e-36,  1.0793e-36, -5.3321e-36,  2.1538e-37,\n        -2.5687e-36,  1.3454e-36,  1.0894e-36,  4.2476e-37, -5.3045e-37,\n        -3.7178e-36, -4.3379e-38, -2.3393e-36, -5.6762e-37, -1.0387e-36,\n        -1.9671e-36,  1.2616e-36,  1.8662e-37, -3.2939e-36, -3.5211e-36,\n        -1.2883e-36, -2.7720e-37, -1.0901e-36, -2.2482e-36,  4.4220e-36,\n         2.0451e-36, -2.5185e-36, -4.6373e-37,  6.2140e-37,  1.5195e-37,\n         4.5876e-36, -3.7587e-37,  4.3520e-36,  9.7735e-38,  4.6847e-36,\n        -1.3033e-36,  2.1107e-36,  1.8814e-37,  2.2558e-36, -2.6183e-37,\n        -8.6139e-37,  3.8158e-36,  4.5421e-36, -1.1732e-37,  1.1037e-36,\n        -7.0194e-37, -1.8395e-36, -4.7991e-36,  5.2743e-37, -1.3593e-37,\n        -3.6516e-38,  2.1234e-37, -1.3864e-36, -6.6487e-37, -6.6150e-37,\n        -2.9151e-36, -1.5922e-36,  6.3783e-37,  1.4190e-36, -2.1796e-37,\n        -8.1519e-37, -2.2386e-36,  3.0279e-38, -1.8209e-37, -9.1882e-37,\n        -1.5704e-36,  1.1984e-36, -4.5324e-37,  5.9522e-37,  2.9100e-38,\n         7.2304e-36,  6.7175e-37, -7.9857e-37, -3.0623e-36,  5.4699e-36,\n        -6.5662e-37, -1.8566e-37, -5.9808e-37, -7.9186e-37,  1.9630e-37,\n        -2.2811e-37,  6.4994e-37, -1.4445e-36, -7.1753e-37,  4.1145e-37,\n        -1.2486e-36, -1.8266e-36, -1.4734e-36,  3.3547e-36,  1.0983e-36,\n        -1.9442e-36, -6.5852e-37, -8.7456e-37, -1.5097e-36,  1.1458e-36,\n        -2.7520e-36,  3.8921e-36,  4.7481e-36,  7.6550e-37,  4.0247e-37,\n        -3.4511e-37, -4.4800e-37, -1.4715e-37,  1.6825e-37, -3.8335e-36,\n        -1.8022e-36, -1.4342e-36, -2.9071e-36,  2.5470e-38, -7.3874e-37,\n        -3.5590e-36,  3.0581e-37,  5.3280e-36, -4.2198e-37, -9.2743e-37,\n         3.0357e-38,  2.5473e-37,  3.4645e-37,  5.2875e-37,  8.8895e-37,\n        -1.5741e-36,  4.6401e-36, -3.7572e-37, -1.9733e-37, -1.6860e-37,\n        -2.6362e-37, -4.6064e-37, -7.7151e-39,  1.0909e-38,  3.1962e-37,\n         3.5165e-39, -7.8292e-37,  5.1784e-36,  1.1185e-36, -1.2606e-36,\n         1.1825e-36, -1.6225e-36, -2.3220e-37,  6.6215e-37,  1.8635e-36,\n        -4.8330e-37, -3.9028e-36, -5.1869e-39, -2.9457e-36, -2.0600e-38,\n         2.7622e-37, -5.2429e-37, -1.0100e-36, -7.0166e-37, -4.3060e-37,\n         3.6800e-37,  2.2075e-38, -3.3903e-36,  1.7982e-37,  1.2890e-37,\n         7.2381e-38, -1.3420e-36,  1.0505e-36, -5.1696e-37,  3.5533e-38,\n        -5.1006e-37, -2.8675e-37, -3.9880e-37, -3.9943e-37,  9.2196e-38,\n        -2.9227e-37, -1.4188e-36, -6.8181e-37,  3.0529e-36,  6.9909e-37,\n        -2.0464e-36, -4.4698e-37,  7.1806e-37, -1.1956e-37, -8.7193e-37,\n        -1.6291e-36, -4.6999e-37,  2.0662e-36,  4.6013e-37,  2.2187e-36,\n         2.7695e-36,  9.5245e-37, -7.2041e-37,  1.6076e-36,  3.1918e-38,\n         4.1450e-38,  7.8356e-37, -2.4840e-36,  1.3539e-36, -4.1058e-37,\n        -8.6357e-37,  4.0077e-36,  2.8820e-38, -1.0815e-36,  5.7557e-37,\n        -2.6370e-37, -1.1841e-36, -9.9996e-37,  3.3779e-36, -2.3540e-36,\n         4.6089e-36, -3.8708e-37, -5.7664e-37, -5.4804e-37, -1.0806e-37,\n        -2.2076e-37, -5.7376e-37,  1.4764e-36, -3.6863e-37,  3.1353e-37,\n         1.2160e-36, -2.1785e-36, -5.6754e-37,  1.3179e-36,  9.7994e-37,\n        -5.8142e-37,  3.2301e-38, -2.5018e-37, -1.8602e-36,  8.1068e-37,\n         2.0272e-39,  1.4869e-36,  3.1049e-37, -1.8789e-36,  2.7280e-36,\n         9.1176e-38,  4.0177e-37, -3.2739e-37, -4.5965e-36, -1.0096e-36,\n         1.0193e-36, -1.4457e-36, -3.5030e-37, -3.8855e-36, -1.8397e-38,\n         4.8023e-36, -7.8222e-37,  1.7444e-36, -4.5349e-36, -8.5508e-38,\n         7.8698e-37, -4.8696e-37,  3.5022e-37, -6.7035e-37,  4.5274e-38,\n        -1.0717e-36,  1.4895e-37, -1.0625e-36, -4.7501e-37,  1.4922e-36,\n         1.5540e-36,  1.0126e-37, -4.3706e-37, -8.7560e-37, -1.4171e-37,\n        -1.5759e-36, -2.0111e-36, -2.2865e-37, -1.3786e-36,  9.5943e-39,\n        -5.0113e-37,  3.0543e-36, -1.3847e-36,  1.0215e-36,  2.6733e-36,\n         5.1224e-37,  2.6398e-36,  1.0487e-36, -5.9654e-37, -1.1646e-37,\n        -1.7216e-36, -7.8185e-37, -1.6261e-36, -3.3146e-37, -1.6056e-36,\n        -5.2855e-37, -1.3658e-37,  5.7236e-36,  7.8183e-37, -1.4943e-36,\n        -2.9621e-36, -1.0409e-36, -2.6072e-37, -8.0205e-37, -3.0786e-37,\n        -7.3755e-37, -2.1754e-37, -3.7368e-36, -4.3285e-37, -2.4755e-37,\n        -7.3218e-37,  9.2367e-37,  1.9941e-36,  6.2142e-37, -5.2288e-37,\n        -4.5394e-37,  1.6066e-37,  4.1700e-36,  1.9344e-36, -2.8594e-38,\n        -1.1433e-37, -4.9593e-38,  5.7675e-37, -1.4456e-37, -1.3726e-36,\n         8.9571e-37,  2.6734e-37,  7.1374e-37, -1.9118e-36,  9.4187e-37,\n        -1.2897e-37, -1.7181e-36, -1.8634e-36,  1.4653e-36, -6.9773e-37,\n        -1.5760e-37, -3.4782e-37, -2.1072e-37,  3.1466e-36,  2.9894e-36,\n         6.0432e-36,  6.0246e-37,  1.7896e-37,  1.3511e-37, -3.0186e-36,\n         1.1560e-36, -4.5886e-37, -1.7653e-36, -7.5864e-37, -1.0328e-36,\n        -5.6056e-37,  1.6029e-36, -1.1890e-36, -6.3801e-37, -5.8212e-37,\n        -4.6651e-37, -7.4281e-37, -3.7083e-37, -5.9381e-37, -2.9102e-36,\n         6.9839e-38, -1.5540e-36], device='cuda:0')",
        "exp_avg_sq": "tensor([2.6552e-12, 2.4680e-11, 2.2180e-12, 2.2071e-12, 3.0419e-11, 3.5147e-12,\n        6.9918e-11, 3.8261e-12, 1.8384e-10, 1.1780e-10, 7.3422e-11, 1.0988e-13,\n        1.5976e-10, 7.3602e-14, 7.6221e-11, 5.2979e-11, 3.1474e-12, 2.3865e-11,\n        2.0975e-10, 2.1847e-12, 2.4692e-11, 1.0079e-12, 2.8245e-13, 2.2306e-13,\n        2.3460e-11, 1.5825e-11, 7.7451e-12, 2.3976e-10, 2.3406e-11, 1.0327e-12,\n        7.9408e-12, 2.4732e-13, 2.4556e-11, 5.7516e-11, 5.8564e-11, 1.2056e-11,\n        5.1692e-11, 1.7336e-13, 2.9762e-10, 1.2341e-12, 1.8370e-11, 1.7222e-10,\n        5.2915e-13, 3.2310e-11, 1.0711e-10, 1.1898e-13, 1.1792e-10, 6.1767e-13,\n        2.0665e-11, 2.9338e-13, 5.6787e-11, 1.0252e-10, 6.1569e-11, 5.1814e-12,\n        3.2998e-13, 1.3577e-11, 9.9350e-13, 2.2128e-12, 1.6845e-12, 9.4265e-12,\n        7.0587e-13, 1.5993e-11, 1.0440e-09, 1.3421e-10, 8.8020e-11, 2.8791e-13,\n        3.2973e-11, 6.8141e-10, 5.2555e-12, 1.5573e-12, 1.0097e-12, 2.5002e-11,\n        4.9726e-14, 1.6134e-11, 5.9024e-11, 9.4037e-11, 1.6410e-12, 4.8006e-11,\n        3.9195e-11, 3.7245e-12, 8.8358e-13, 3.3399e-11, 7.3773e-11, 2.1643e-10,\n        7.8941e-11, 5.4503e-11, 2.2417e-12, 1.2793e-12, 1.7455e-10, 4.2191e-10,\n        2.8488e-12, 6.6649e-13, 1.0488e-10, 6.6251e-10, 4.0295e-12, 6.1092e-11,\n        1.7822e-13, 1.5659e-13, 5.3649e-11, 1.6900e-10, 4.0926e-11, 1.7292e-11,\n        8.4150e-12, 2.9635e-11, 1.1206e-11, 1.3655e-12, 1.2433e-13, 1.6357e-11,\n        1.1428e-11, 3.6137e-11, 5.7157e-12, 1.2433e-11, 1.8663e-11, 1.9145e-10,\n        2.3518e-10, 1.4827e-12, 1.8560e-11, 5.5830e-11, 2.2479e-11, 7.0775e-11,\n        4.4031e-10, 5.8306e-13, 2.1464e-11, 2.1277e-13, 2.9210e-13, 2.9738e-13,\n        1.3011e-12, 5.4697e-11, 1.1906e-10, 3.4433e-13, 1.0556e-12, 1.8772e-11,\n        4.6941e-13, 7.0913e-13, 6.9045e-10, 3.4032e-11, 6.5716e-11, 5.5172e-11,\n        3.1609e-11, 8.7929e-11, 6.4624e-13, 1.5270e-10, 2.2199e-13, 2.5888e-10,\n        3.0867e-11, 1.3607e-10, 3.2628e-12, 8.8973e-12, 1.1490e-10, 2.2227e-11,\n        1.2406e-11, 2.5288e-11, 3.2966e-11, 3.3948e-11, 4.7000e-11, 2.9719e-10,\n        2.1430e-11, 1.9990e-13, 4.8671e-12, 5.9690e-13, 5.3720e-12, 1.3712e-10,\n        8.3955e-14, 2.4442e-13, 1.5764e-12, 5.7896e-12, 3.5354e-12, 1.1756e-10,\n        2.4649e-11, 1.3994e-13, 3.2559e-12, 1.3236e-10, 2.4710e-13, 3.3318e-10,\n        3.8429e-12, 3.2524e-12, 1.4452e-11, 1.3778e-12, 5.8616e-13, 8.5065e-11,\n        4.6301e-10, 1.7429e-14, 2.3466e-10, 1.9619e-13, 1.3039e-10, 4.4039e-11,\n        9.3882e-13, 8.1525e-11, 2.9894e-10, 5.2575e-12, 7.7022e-11, 8.5073e-12,\n        2.7518e-13, 4.3732e-11, 2.7577e-10, 2.1399e-11, 8.6730e-14, 6.4437e-13,\n        7.6098e-13, 2.9101e-12, 1.2919e-10, 4.1543e-13, 3.4355e-11, 4.2785e-12,\n        5.6662e-12, 4.1139e-11, 2.2460e-10, 1.5383e-11, 2.6656e-10, 3.8917e-11,\n        6.1730e-13, 8.9837e-11, 8.1134e-11, 2.3984e-12, 1.0343e-09, 7.4452e-11,\n        7.5453e-11, 1.6063e-12, 1.8518e-10, 2.9072e-12, 1.6428e-11, 1.8822e-13,\n        1.2924e-11, 2.4559e-12, 2.7371e-11, 5.6914e-12, 3.6941e-11, 2.4345e-12,\n        2.2964e-12, 2.5836e-11, 1.5215e-11, 5.9050e-11, 8.1884e-13, 9.5289e-11,\n        7.7669e-13, 3.8068e-11, 2.0778e-11, 3.8233e-14, 8.5523e-14, 6.3501e-13,\n        2.4509e-10, 1.0294e-12, 5.0252e-11, 6.2670e-11, 2.7499e-10, 4.6158e-12,\n        3.4419e-11, 2.5402e-13, 2.4903e-12, 1.9102e-12, 6.0891e-12, 5.5835e-13,\n        1.0372e-09, 2.7342e-13, 1.2536e-12, 1.4495e-11, 3.5415e-13, 3.7733e-11,\n        3.5750e-11, 1.7472e-11, 5.7348e-11, 2.2783e-12, 2.9135e-11, 4.9705e-11,\n        6.3846e-12, 8.9275e-11, 3.0451e-11, 4.5590e-11, 7.2190e-14, 1.8451e-12,\n        6.3077e-13, 8.0098e-14, 4.6639e-11, 2.6877e-10, 7.4515e-11, 3.5700e-10,\n        8.4845e-11, 1.0458e-12, 3.0242e-12, 1.2205e-11, 1.1112e-10, 9.9342e-13,\n        2.4354e-10, 6.4609e-11, 4.1384e-13, 3.2380e-13, 3.1274e-12, 2.5890e-13,\n        4.2844e-13, 6.2875e-13, 5.3609e-11, 2.4347e-10, 4.9446e-12, 2.5808e-12,\n        3.8041e-12, 1.5312e-12, 3.2419e-14, 2.0228e-14, 5.4956e-12, 4.3577e-13,\n        1.8097e-12, 6.7763e-12, 1.6672e-10, 2.9897e-12, 5.2541e-12, 1.5449e-11,\n        2.0617e-11, 1.5926e-12, 1.5832e-12, 3.5636e-12, 1.3692e-11, 2.9976e-10,\n        1.1233e-13, 1.3467e-10, 2.5454e-12, 3.8669e-12, 1.0092e-10, 3.2223e-12,\n        7.9813e-12, 2.9384e-12, 2.5707e-10, 1.0899e-11, 3.8807e-10, 1.8320e-12,\n        1.5117e-12, 1.7855e-13, 8.7147e-13, 1.7134e-11, 2.0657e-13, 2.8851e-12,\n        1.8227e-12, 1.5448e-12, 1.4487e-13, 3.8548e-12, 1.1044e-12, 1.9487e-13,\n        3.6967e-11, 2.1323e-12, 5.0703e-11, 5.9367e-13, 2.1033e-10, 6.2733e-13,\n        5.5495e-13, 1.2578e-13, 9.4846e-12, 1.9034e-10, 1.4895e-12, 1.5539e-11,\n        1.2021e-13, 1.2271e-10, 1.7817e-11, 4.2699e-12, 3.6206e-11, 9.2257e-11,\n        1.7540e-12, 7.4355e-12, 2.2060e-10, 1.0398e-11, 1.1881e-11, 6.6493e-12,\n        1.8647e-12, 2.0818e-10, 1.9934e-10, 2.8841e-12, 5.6639e-11, 5.8010e-11,\n        6.5275e-10, 4.7239e-13, 3.8961e-11, 1.5304e-11, 1.3250e-10, 2.9101e-13,\n        5.9915e-12, 9.8083e-13, 8.9979e-12, 2.0701e-13, 9.2713e-13, 1.6763e-11,\n        2.3099e-12, 5.6868e-13, 6.1155e-12, 4.5695e-11, 3.9613e-11, 3.8617e-11,\n        1.2232e-12, 7.6485e-11, 1.8853e-12, 3.3454e-13, 1.3270e-11, 6.2128e-11,\n        2.0713e-12, 1.4888e-10, 4.4415e-12, 2.7408e-11, 3.6315e-10, 5.1832e-13,\n        1.1206e-13, 7.2538e-12, 3.6847e-10, 2.0639e-11, 7.3541e-13, 9.6222e-11,\n        8.0026e-12, 2.2729e-10, 3.6158e-11, 3.3634e-10, 3.3849e-11, 5.3382e-12,\n        1.4263e-10, 1.0123e-12, 3.5789e-13, 9.8658e-13, 3.0988e-11, 1.5063e-12,\n        2.7126e-12, 1.0315e-11, 4.1217e-11, 1.0989e-10, 3.7004e-13, 4.7730e-13,\n        2.3106e-12, 3.1865e-13, 9.4694e-13, 1.4608e-13, 8.1951e-11, 3.2209e-12,\n        7.9104e-11, 1.0653e-12, 2.3100e-11, 6.4931e-12, 3.1481e-12, 6.4479e-11,\n        5.2515e-11, 4.9988e-11, 1.7928e-11, 3.1475e-14, 9.7692e-11, 9.7568e-11,\n        1.0397e-11, 5.1818e-13, 1.2136e-10, 2.0995e-11, 5.9121e-11, 2.1115e-11,\n        7.5806e-11, 1.6902e-10, 2.2501e-10, 2.7460e-11, 1.4451e-12, 1.5564e-10,\n        3.0582e-10, 1.5905e-11, 3.6848e-12, 1.7599e-10, 3.1325e-12, 5.4982e-12,\n        1.3996e-12, 4.1023e-10, 4.1494e-11, 1.5982e-11, 8.2321e-11, 6.2793e-13,\n        2.6494e-10, 8.9830e-13, 1.5754e-13, 4.7815e-13, 2.2317e-12, 1.0885e-10,\n        1.8031e-10, 1.7507e-12, 4.9403e-11, 1.2220e-12, 1.1635e-10, 1.5037e-13,\n        2.8651e-11, 1.4346e-13, 3.8825e-13, 1.6773e-12, 2.8053e-12, 3.0288e-11,\n        1.0079e-11, 1.4727e-10, 7.2525e-11, 2.5728e-12, 1.9979e-13, 1.7486e-12,\n        2.0827e-12, 1.7693e-13, 2.8427e-11, 8.7708e-11, 5.2950e-10, 4.9302e-12,\n        1.2341e-12, 3.9632e-12, 2.0448e-11, 2.5137e-13, 1.3419e-14, 8.9460e-11,\n        4.6771e-11, 5.0046e-11, 4.1304e-12, 1.7831e-10, 2.2284e-10, 2.4496e-11,\n        3.7653e-12, 8.2324e-12, 1.4783e-11, 4.0752e-13, 1.8699e-11, 1.1499e-10,\n        9.9360e-11, 6.5051e-11], device='cuda:0')"
      },
      "66": {
        "step": "tensor(11268.)",
        "exp_avg": "tensor([ 8.0308e-39,  5.2759e-38, -2.7098e-40,  1.9711e-39, -9.0286e-40,\n         4.0585e-39,  1.6090e-37,  4.1859e-39,  4.3516e-38,  2.7270e-37,\n         9.1348e-38,  4.2446e-39,  1.8367e-37,  2.1392e-39,  5.4690e-38,\n         1.8159e-37,  3.5588e-39, -7.7896e-39,  1.9749e-37,  1.4132e-38,\n         2.4202e-37,  1.4112e-38, -3.8318e-40,  6.5855e-40,  4.6897e-39,\n         1.0156e-38, -6.2234e-39,  4.4065e-38,  1.0699e-37,  1.7928e-39,\n        -1.3553e-39,  7.8154e-38,  9.7066e-38,  3.5778e-38,  9.9193e-38,\n         5.2140e-38,  2.0387e-37, -1.8621e-39,  2.0134e-37,  1.5992e-39,\n         1.8966e-37,  1.9700e-38,  5.5373e-39, -3.5808e-40,  9.9520e-38,\n         1.5488e-38,  5.1315e-38,  1.3492e-38,  1.8469e-38,  4.0246e-39,\n         5.4034e-38,  7.9998e-39,  7.7092e-38, -1.8153e-39,  5.5140e-39,\n         2.2291e-38,  1.7836e-38,  2.8123e-38,  1.7578e-38,  6.8420e-40,\n         9.1530e-38,  1.2805e-38,  2.2603e-37,  7.9009e-38,  6.5551e-38,\n         1.0323e-39,  3.1544e-38,  2.5540e-37,  5.1207e-38,  2.7783e-39,\n         4.5808e-39,  1.3801e-38,  3.1693e-39,  2.1243e-38,  9.4357e-38,\n         3.0345e-38, -2.1349e-40,  8.1822e-38,  1.1248e-39,  2.8889e-39,\n        -2.9697e-39,  2.9398e-38,  1.3472e-38,  1.3157e-37,  8.0718e-38,\n         9.0104e-38,  4.1570e-40,  1.2866e-38,  2.3151e-37,  2.1514e-38,\n         1.1674e-38, -3.5998e-40,  1.4632e-37,  7.1152e-39,  1.1811e-38,\n         1.3283e-37,  7.6511e-39,  5.3956e-39, -1.1012e-39,  8.9230e-38,\n         5.3780e-38,  3.5853e-38,  7.5825e-38,  6.1139e-39,  7.0455e-38,\n         6.7168e-39, -3.2278e-39,  5.8758e-38,  2.4213e-38,  1.8922e-39,\n         1.1560e-39,  2.2383e-39,  1.9606e-39,  1.2708e-39,  2.1575e-38,\n         4.2777e-41,  4.1273e-38, -3.9471e-39, -1.0305e-39,  1.5899e-37,\n         1.4432e-37,  2.0057e-38,  2.2077e-39,  1.3216e-38, -2.3033e-40,\n         6.5893e-39,  1.3263e-39,  1.0024e-37,  5.9000e-40,  4.3394e-39,\n         2.4682e-39,  7.5358e-41, -1.8327e-39,  5.0054e-38,  2.4827e-37,\n        -1.8990e-39,  7.3664e-38,  7.6178e-39,  4.6886e-38,  1.3832e-38,\n        -4.0411e-40,  7.8316e-41,  9.2073e-39,  4.0763e-38,  3.0237e-39,\n         1.4829e-37, -3.8467e-39, -5.3146e-39, -1.0631e-39,  1.5395e-37,\n        -1.7000e-39,  3.9611e-38, -2.3958e-39,  1.6539e-39,  2.1767e-38,\n         1.3687e-38,  3.2722e-38,  2.0173e-39, -8.1430e-40,  1.9494e-38,\n         4.5206e-38,  1.9383e-37,  5.4986e-39, -3.3240e-39, -4.3588e-40,\n         8.2457e-39,  1.0612e-39,  1.0121e-37,  7.4217e-39,  7.0170e-39,\n         6.1751e-40,  3.5279e-38,  1.6129e-38,  4.4639e-38,  1.9084e-38,\n         3.3269e-39,  8.8993e-38,  3.8848e-38, -2.4729e-40,  4.1269e-38,\n         1.5277e-39,  3.5088e-39,  2.7831e-39, -1.4382e-39,  1.9583e-37,\n         2.4636e-39,  1.2026e-38,  9.1656e-38,  1.6734e-39,  1.0956e-38,\n         2.3532e-38,  1.1860e-38, -1.8923e-39,  6.4555e-38,  3.8309e-37,\n         3.2549e-38, -7.7506e-42, -4.8910e-41, -1.6765e-39, -1.0713e-39,\n         2.2659e-37,  2.1064e-40,  1.0242e-37, -4.9549e-39,  6.1567e-38,\n         1.2874e-38,  1.9950e-37,  2.4836e-38,  6.8069e-38,  9.6297e-39,\n         1.8516e-40,  7.3606e-38,  1.2825e-37,  3.1915e-39,  1.7337e-37,\n        -3.0835e-39,  2.7353e-38,  7.2510e-38,  3.8052e-39,  2.1964e-39,\n         3.2128e-38,  4.1760e-39, -1.4418e-39, -1.4147e-40,  1.6504e-37,\n         7.5320e-38,  4.1383e-38, -1.6698e-39,  1.3667e-38,  7.5134e-38,\n         1.1886e-37, -5.2543e-40,  2.2917e-39,  2.4296e-38, -1.1235e-39,\n        -2.7134e-40,  1.2502e-38,  4.1445e-39,  1.0389e-38,  1.6421e-39,\n         2.2111e-37,  1.1941e-38, -2.4601e-40, -3.2764e-40,  2.8345e-37,\n        -9.4842e-40,  6.9668e-38,  2.4032e-40,  6.3688e-41,  2.0545e-39,\n         7.0756e-39,  4.1221e-39,  1.0384e-38, -4.7333e-40,  1.4970e-39,\n         1.7007e-39,  8.2040e-38, -6.9591e-40,  5.4754e-38,  6.0020e-38,\n         6.7433e-38, -2.9201e-39, -1.4755e-39,  6.2824e-39,  2.5128e-39,\n         9.1956e-39,  2.8151e-37,  1.0879e-37,  1.2009e-38,  1.0682e-39,\n         2.2870e-39,  1.0702e-39, -1.4530e-39,  5.9707e-38,  3.1550e-38,\n         9.0576e-39,  1.9693e-38,  5.8582e-38,  2.4296e-39,  4.7196e-39,\n         4.4736e-39, -4.5999e-39,  2.1087e-37,  1.5860e-37, -1.6110e-39,\n         2.7687e-40,  1.4539e-40,  1.1257e-39,  2.4117e-38, -1.8162e-39,\n        -9.7665e-40,  3.4297e-37,  9.4566e-40,  3.6443e-39,  6.8762e-40,\n        -4.6820e-39,  6.1635e-40,  3.8096e-39, -6.2131e-39, -1.5230e-40,\n         1.1018e-38, -5.0393e-40,  2.6634e-37, -3.1121e-39,  8.7216e-40,\n         1.2481e-37,  3.8577e-38,  7.0045e-39,  1.5714e-38,  1.9346e-38,\n         3.2975e-39,  1.5204e-39,  9.1629e-39,  7.0109e-38, -5.7958e-39,\n         1.0352e-38,  7.2103e-38, -5.4770e-40, -1.3463e-39, -3.5459e-39,\n         3.7672e-38,  1.5786e-38,  4.6554e-40,  5.7739e-39, -1.7855e-39,\n         2.8415e-39,  8.8583e-39, -1.2351e-38,  5.8609e-40,  9.8417e-40,\n         1.0099e-40, -1.6696e-39,  1.9888e-39, -4.1783e-39,  3.1882e-39,\n         8.1647e-39, -6.6882e-40,  1.0651e-37,  5.3844e-38,  5.8342e-39,\n         2.1709e-39,  4.3935e-40, -5.6303e-39,  4.9179e-39,  1.3148e-38,\n         1.6118e-37,  2.1246e-41,  8.3013e-38,  4.3691e-39,  6.5473e-38,\n         1.0541e-37,  1.9844e-38,  2.2600e-37,  1.1997e-37,  5.8144e-39,\n        -5.7582e-39,  3.5177e-38,  5.5439e-38,  6.2658e-39,  8.7215e-40,\n        -7.4458e-40,  1.2763e-37,  1.9888e-37,  2.4163e-39,  3.7067e-38,\n         2.0692e-37,  1.4269e-37,  9.7875e-40,  2.5811e-37,  2.9384e-38,\n         1.0772e-37,  2.9033e-39, -2.1411e-40,  1.9662e-38,  3.0999e-39,\n         3.2835e-39,  6.5576e-39,  1.7948e-38,  2.0104e-39, -3.9900e-40,\n         3.0889e-38,  2.6086e-38,  2.6054e-38,  5.4368e-38,  1.5251e-38,\n         1.0355e-40,  4.9462e-38,  1.1512e-39,  3.9204e-40,  4.4706e-38,\n         1.1085e-38,  2.3707e-37, -1.6660e-39,  1.7131e-39,  1.3351e-37,\n         3.6011e-39,  5.9086e-39, -7.2685e-40, -5.8909e-39,  2.7797e-40,\n         1.7882e-39,  1.7749e-39,  3.7179e-39,  8.3334e-38,  6.0831e-38,\n         3.1688e-37,  8.4965e-38,  1.9538e-38,  2.6828e-38, -5.0484e-39,\n         1.3638e-38, -7.9740e-40,  8.3290e-38, -6.3179e-40,  2.3540e-38,\n         9.2396e-38,  2.9702e-38, -2.1406e-39,  1.7526e-39,  1.5528e-38,\n         1.2257e-38,  5.8906e-39,  2.5584e-40,  4.2801e-39,  7.5781e-38,\n         6.9459e-38,  5.3154e-38,  6.4036e-39, -1.8273e-39,  2.9807e-39,\n         5.6911e-38,  9.9393e-38,  3.8353e-39,  1.5944e-37,  8.6385e-38,\n         7.1350e-39,  2.8845e-38,  7.4572e-38, -1.2834e-39,  7.3851e-40,\n         3.7929e-39,  4.8825e-38, -5.2727e-40, -2.1102e-39,  4.1648e-38,\n         7.6838e-38,  2.3474e-37,  6.6960e-38, -6.6670e-39, -5.9085e-40,\n         2.1198e-39, -2.6662e-39,  1.1615e-39,  2.8133e-38,  2.8258e-39,\n        -8.7979e-40,  4.6609e-39, -8.3288e-40,  1.2353e-37,  1.7879e-38,\n         3.0353e-39,  3.3320e-39,  2.6413e-37,  2.0636e-38,  5.7059e-40,\n         2.8250e-39,  1.1595e-38,  2.7291e-37,  1.8823e-37,  4.6739e-39,\n        -9.3719e-40,  1.7638e-39,  1.6963e-37,  6.1009e-39,  5.9036e-38,\n         2.4985e-38,  5.8304e-39,  9.2454e-39,  1.6615e-39,  5.4090e-38,\n         2.4464e-38,  7.5726e-39,  7.4708e-38,  1.5422e-38, -2.9159e-40,\n         8.4678e-40, -4.1665e-39,  6.1406e-39,  8.3012e-38,  2.3695e-37,\n         4.0859e-37, -5.2248e-39, -5.8596e-40,  6.9602e-40,  1.0649e-39,\n         2.1849e-38,  4.3039e-40, -3.4703e-40, -1.1211e-39,  5.5258e-38,\n        -2.5784e-39,  9.7761e-38,  4.4761e-38,  8.9001e-41,  2.4231e-40,\n         2.8431e-40,  1.1206e-40, -1.6359e-40, -2.0315e-40,  6.8307e-39,\n         1.7878e-37,  2.4878e-38], device='cuda:0')",
        "exp_avg_sq": "tensor([9.9722e-16, 8.1790e-17, 2.1381e-19, 3.9607e-18, 1.9092e-17, 8.8915e-17,\n        6.2837e-14, 9.0247e-16, 1.3250e-15, 7.7000e-14, 4.8408e-14, 9.3837e-17,\n        7.7862e-14, 3.8838e-17, 5.9632e-15, 5.8050e-14, 2.9169e-16, 9.9182e-17,\n        1.1158e-13, 3.4141e-15, 4.1624e-14, 5.1916e-16, 1.1910e-18, 1.6253e-17,\n        1.1474e-16, 2.4783e-15, 9.0850e-18, 4.8436e-14, 1.7506e-14, 3.5653e-17,\n        3.0904e-18, 1.3394e-15, 1.4710e-14, 2.3484e-16, 2.1113e-14, 6.7432e-16,\n        5.7361e-14, 5.2255e-17, 9.9661e-14, 1.9401e-16, 1.8733e-14, 2.6744e-14,\n        3.1932e-18, 4.2184e-17, 5.5662e-14, 3.8260e-16, 1.7523e-14, 8.2772e-16,\n        4.6566e-15, 3.4579e-18, 1.1226e-14, 3.4347e-15, 2.7964e-14, 1.5347e-18,\n        1.0149e-16, 8.0900e-18, 1.2477e-15, 2.7082e-15, 1.2952e-15, 5.9360e-17,\n        3.9844e-15, 4.3632e-18, 2.0411e-13, 6.3176e-15, 3.0199e-14, 2.9017e-17,\n        1.0612e-14, 2.0137e-13, 1.0369e-15, 1.9483e-17, 1.4838e-15, 9.4053e-17,\n        1.6524e-17, 1.0290e-15, 1.0487e-14, 1.6746e-14, 6.1969e-16, 4.3372e-14,\n        6.9452e-17, 2.1484e-18, 5.9805e-17, 1.0966e-15, 9.8686e-15, 4.6617e-14,\n        3.8535e-14, 5.2346e-15, 2.6691e-17, 6.5270e-18, 6.2312e-14, 8.1816e-14,\n        1.4698e-15, 7.0809e-18, 6.1054e-14, 1.1514e-13, 3.7100e-16, 4.0006e-14,\n        1.3804e-16, 5.3612e-18, 1.4103e-15, 1.0720e-14, 4.3906e-15, 3.4067e-16,\n        1.3867e-15, 3.9228e-16, 6.2966e-15, 1.2864e-16, 1.1164e-17, 1.2810e-14,\n        1.6153e-18, 2.6105e-15, 5.8065e-16, 2.6623e-17, 7.1095e-18, 5.1690e-15,\n        3.5196e-14, 9.2137e-19, 8.3749e-15, 7.3454e-16, 2.5401e-17, 6.8698e-14,\n        2.1365e-13, 3.3436e-16, 3.2587e-16, 5.8042e-16, 1.8873e-17, 1.7204e-18,\n        2.1211e-18, 5.8994e-14, 2.6053e-15, 1.6471e-17, 1.0658e-18, 4.3582e-18,\n        1.5518e-16, 7.3215e-16, 3.3089e-13, 6.8585e-16, 6.6753e-15, 4.8744e-16,\n        6.0933e-15, 1.0905e-14, 2.0891e-16, 8.5036e-15, 1.5488e-16, 3.3434e-14,\n        1.0072e-15, 9.7089e-14, 3.7301e-17, 3.0229e-16, 4.8687e-16, 1.9697e-14,\n        4.0158e-17, 1.8140e-16, 4.3842e-16, 5.1694e-17, 8.9790e-15, 3.0981e-14,\n        6.3576e-15, 4.5297e-17, 2.9674e-19, 6.0584e-16, 2.0937e-15, 1.0146e-13,\n        7.8698e-17, 6.9649e-18, 2.7599e-18, 2.2652e-16, 2.3867e-18, 3.3289e-14,\n        1.6099e-16, 4.3042e-17, 6.6034e-16, 5.2100e-15, 9.9613e-16, 1.0592e-13,\n        4.7452e-16, 9.5142e-16, 8.3183e-15, 9.8360e-17, 5.3666e-17, 2.8651e-15,\n        5.9508e-14, 1.1079e-16, 1.4803e-14, 4.5659e-18, 1.5828e-13, 1.8545e-16,\n        9.4197e-16, 2.8175e-14, 2.4117e-14, 1.7917e-15, 9.1549e-15, 9.2507e-16,\n        2.8977e-18, 1.3741e-14, 2.8109e-13, 7.5238e-17, 3.6553e-16, 5.5552e-17,\n        5.3594e-17, 2.7076e-20, 6.2393e-14, 1.3180e-17, 3.3859e-15, 2.5432e-18,\n        1.1910e-16, 4.2593e-17, 8.9810e-14, 4.6333e-16, 8.7538e-14, 1.7722e-17,\n        1.3316e-17, 2.9333e-15, 1.2981e-14, 2.6967e-19, 2.7728e-13, 3.8869e-16,\n        4.0501e-15, 2.6343e-14, 1.1443e-14, 4.7629e-19, 7.5414e-15, 7.3364e-18,\n        3.3857e-18, 4.2074e-19, 4.8967e-14, 1.9084e-14, 4.5546e-15, 1.3884e-18,\n        1.8878e-15, 7.1004e-15, 2.0135e-14, 2.6818e-16, 1.1138e-19, 1.6393e-16,\n        8.8959e-19, 3.1785e-16, 1.8930e-17, 2.7408e-17, 1.2631e-16, 3.9505e-19,\n        4.7158e-14, 4.9310e-16, 1.6072e-16, 4.3057e-16, 1.5268e-13, 3.2681e-20,\n        9.9830e-15, 1.0473e-16, 4.6471e-16, 9.0956e-17, 6.9164e-16, 2.2896e-16,\n        1.3323e-13, 5.7673e-17, 7.6463e-17, 2.6073e-17, 2.0655e-14, 8.9596e-17,\n        1.1991e-15, 1.4780e-15, 3.5324e-14, 8.4424e-18, 7.3545e-17, 8.3155e-16,\n        1.0596e-15, 1.8007e-14, 4.6612e-14, 2.7152e-14, 2.1742e-16, 4.0973e-18,\n        8.9315e-17, 1.0182e-16, 2.5592e-15, 7.7709e-14, 3.2082e-16, 3.6164e-14,\n        1.8687e-14, 1.3965e-14, 1.3955e-18, 1.1885e-15, 8.2139e-16, 2.6279e-18,\n        5.3739e-14, 8.1341e-14, 6.3544e-18, 3.8192e-18, 1.3160e-18, 1.5792e-18,\n        7.6158e-16, 2.2482e-18, 2.0429e-15, 2.1363e-13, 5.9098e-16, 1.5263e-18,\n        3.6617e-18, 9.7888e-18, 6.0068e-18, 7.0952e-17, 4.9964e-17, 5.7748e-17,\n        4.2791e-16, 3.7652e-19, 7.3886e-14, 3.0802e-18, 1.4165e-15, 7.3933e-15,\n        7.4654e-15, 1.7721e-16, 9.7555e-16, 4.3900e-15, 3.7002e-17, 8.1664e-15,\n        3.7023e-17, 3.2987e-14, 5.1553e-19, 2.2694e-16, 2.4075e-14, 9.0392e-18,\n        3.1201e-18, 3.3187e-19, 5.9480e-15, 9.9403e-16, 3.6494e-14, 3.6285e-18,\n        1.0537e-19, 3.3543e-17, 2.3818e-16, 2.5214e-15, 7.3792e-17, 2.1770e-18,\n        5.9305e-16, 2.6236e-18, 2.6167e-17, 2.5532e-17, 3.3865e-16, 1.3168e-17,\n        2.1644e-16, 2.4766e-14, 4.7077e-15, 4.7175e-16, 1.2047e-14, 2.4377e-16,\n        3.2548e-17, 1.3852e-17, 3.6546e-16, 1.2406e-13, 6.5254e-16, 2.7108e-15,\n        2.5271e-17, 2.8169e-15, 4.1767e-15, 1.8179e-15, 1.1566e-13, 2.7277e-14,\n        1.3139e-16, 4.4915e-17, 2.4564e-14, 9.5541e-15, 3.8097e-17, 2.1581e-15,\n        3.3185e-19, 2.7118e-14, 1.2652e-13, 1.0418e-16, 4.4955e-15, 7.5945e-14,\n        2.2951e-13, 3.4079e-18, 3.9514e-14, 4.7666e-15, 3.8554e-14, 7.0490e-17,\n        3.6909e-19, 1.4467e-16, 1.2748e-17, 6.2744e-17, 1.5413e-18, 2.3085e-16,\n        2.8890e-16, 1.3971e-18, 2.8922e-16, 1.3600e-14, 1.5858e-14, 1.7330e-15,\n        9.8595e-16, 1.3617e-17, 1.7553e-15, 2.4483e-19, 1.9014e-17, 2.0531e-14,\n        9.8538e-17, 2.2897e-13, 2.8673e-17, 4.8482e-17, 1.7397e-13, 3.6570e-18,\n        1.7472e-16, 2.1098e-18, 3.0427e-14, 1.2411e-17, 1.6320e-18, 3.9486e-15,\n        1.9086e-17, 8.8150e-14, 6.9907e-16, 2.5352e-13, 1.3714e-14, 3.6008e-16,\n        2.3976e-14, 6.3713e-17, 6.0461e-16, 2.8349e-19, 5.5379e-15, 4.2473e-16,\n        2.8751e-17, 1.9122e-14, 2.9472e-15, 1.0809e-15, 1.2006e-16, 1.2636e-15,\n        2.2395e-15, 1.7883e-18, 2.4766e-16, 3.0202e-18, 1.0847e-14, 7.3661e-15,\n        1.0161e-14, 4.8423e-19, 6.6723e-16, 6.7814e-18, 9.1344e-15, 1.4644e-14,\n        5.6434e-15, 3.9804e-14, 1.8620e-15, 2.0526e-16, 1.3395e-14, 4.1714e-14,\n        6.1914e-17, 9.6010e-20, 8.7952e-15, 2.4807e-15, 1.7003e-15, 9.3621e-18,\n        1.1391e-14, 2.0686e-14, 2.7935e-13, 4.5006e-16, 4.1171e-16, 1.0562e-15,\n        2.0426e-14, 9.2094e-17, 7.0687e-19, 2.9865e-14, 9.5423e-18, 4.2305e-18,\n        6.1940e-17, 1.5512e-14, 7.3752e-14, 3.1593e-16, 7.7582e-16, 2.9304e-19,\n        2.3472e-13, 8.8574e-16, 1.2285e-17, 3.8589e-18, 3.9313e-16, 7.3193e-14,\n        1.6169e-13, 1.4522e-16, 1.9403e-15, 2.2650e-19, 4.4501e-14, 2.3195e-17,\n        6.3632e-15, 7.5363e-16, 3.7314e-17, 8.8433e-18, 3.3086e-18, 9.7440e-16,\n        9.1700e-17, 1.8429e-14, 5.4831e-16, 2.1785e-15, 4.3444e-17, 2.7652e-17,\n        4.3163e-18, 7.2924e-16, 2.1339e-15, 1.0033e-13, 3.7927e-13, 5.7074e-18,\n        8.8790e-18, 2.0193e-16, 3.3215e-16, 1.7573e-15, 3.3033e-18, 1.1128e-15,\n        5.6639e-17, 6.2756e-15, 3.2765e-17, 1.6157e-14, 6.7279e-14, 1.6357e-18,\n        4.2779e-16, 1.0273e-18, 8.0872e-18, 2.6727e-19, 6.0666e-18, 3.1327e-15,\n        7.8424e-14, 1.7962e-15], device='cuda:0')"
      },
      "67": {
        "step": "tensor(11268.)",
        "exp_avg": "tensor([-5.5011e-39,  1.0928e-37,  2.5647e-38, -1.7023e-39,  2.9815e-39,\n        -3.0261e-39,  8.6080e-38,  3.4033e-38,  9.3614e-38,  1.6912e-37,\n         5.6246e-38, -3.7292e-39,  1.2837e-37, -5.5515e-40,  5.1962e-38,\n         1.0627e-37, -2.5938e-39, -2.4055e-38,  1.3258e-37,  3.3951e-38,\n         1.4621e-37, -1.0598e-38,  1.5983e-38, -5.9809e-40,  2.9268e-38,\n         2.9734e-38,  4.2104e-38,  6.8695e-38,  7.5486e-38, -1.3369e-39,\n         1.2259e-39,  1.2136e-37,  6.6033e-38,  6.5891e-38,  8.9240e-38,\n         7.9212e-38,  1.2612e-37,  1.2929e-39,  1.5944e-37, -7.4027e-40,\n         1.3536e-37,  3.7382e-38, -4.8443e-39, -2.3934e-39,  1.0618e-37,\n        -1.1194e-38,  9.4416e-38, -8.9331e-39,  3.2460e-38, -3.4044e-39,\n         1.0671e-37,  2.1436e-38,  1.0538e-37,  6.9432e-39, -4.9106e-39,\n         7.4118e-38, -1.1631e-38, -1.6063e-38, -1.1934e-38,  5.7030e-38,\n         8.8815e-38,  5.5650e-38,  1.8259e-37,  1.0968e-37,  6.4997e-38,\n        -2.1391e-40,  4.4864e-38,  1.5831e-37,  7.7086e-38, -2.3973e-39,\n         9.8442e-39,  4.9717e-38, -2.6261e-39,  4.6717e-38,  1.1298e-37,\n         6.8297e-38,  5.9414e-40,  7.2645e-38, -8.4144e-40, -2.5267e-39,\n         2.7830e-39,  5.0520e-38,  4.2569e-38,  9.7130e-38,  6.7476e-38,\n         1.2309e-37,  2.9445e-38,  6.0092e-38,  1.5909e-37,  2.5387e-38,\n        -7.8598e-39,  3.2039e-40,  8.3475e-38,  1.0319e-38, -8.9169e-39,\n         1.1252e-37, -6.3451e-39, -4.9526e-39,  2.6669e-38,  1.1028e-37,\n         6.3658e-38,  9.6323e-38,  1.0674e-37,  2.0233e-38,  7.7293e-38,\n        -4.8485e-39,  1.7750e-38,  4.4061e-38,  9.0701e-38,  2.1507e-38,\n        -7.6284e-40,  1.3693e-38, -5.9063e-39,  1.0988e-38,  4.5116e-38,\n         1.3761e-39,  6.9819e-38,  1.6440e-38,  3.0875e-39,  9.9934e-38,\n         1.3685e-37, -1.4402e-38,  7.6736e-39, -8.6920e-39,  2.0895e-40,\n        -5.8759e-39, -1.0036e-39,  9.0445e-38, -2.7592e-39, -2.8058e-39,\n        -2.1620e-39, -1.1050e-40,  4.8773e-39,  8.1795e-38,  1.3594e-37,\n         1.0249e-38,  7.9786e-38,  1.6350e-38,  4.2927e-38,  4.4340e-38,\n         1.2082e-39,  1.3505e-38, -7.8128e-39,  9.5171e-38,  1.5792e-38,\n         8.1140e-38,  1.5269e-38,  3.4826e-38,  3.1486e-39,  1.2828e-37,\n         5.9343e-39,  1.1163e-37,  9.7675e-39, -1.4812e-39,  2.3638e-38,\n         3.3825e-38,  4.6118e-38, -1.1798e-39,  3.2114e-39, -1.5277e-38,\n         5.9632e-38,  1.1754e-37, -4.8695e-39,  2.8958e-39,  5.2311e-38,\n        -5.7709e-39, -9.4940e-40,  1.0164e-37,  2.3082e-38, -5.8559e-39,\n         3.8218e-40,  4.6203e-38, -1.0242e-38,  2.7660e-38, -1.3553e-38,\n         1.0734e-38,  9.4518e-38,  7.4567e-38,  2.3174e-40,  5.7950e-38,\n         3.9136e-39, -3.1183e-39,  1.2089e-38,  4.8153e-39,  1.0122e-37,\n         1.2574e-38, -7.7804e-39,  8.4339e-38,  4.7930e-39,  1.5375e-38,\n         4.0639e-38,  4.3240e-38,  1.1242e-38,  5.4732e-38,  1.8370e-37,\n         8.1306e-38,  4.6655e-39,  9.9285e-41,  1.3921e-39,  9.6207e-40,\n         1.5930e-37,  3.6866e-39,  1.2990e-37,  2.8360e-38,  1.2303e-37,\n         3.3659e-38,  1.2943e-37,  5.6343e-38,  9.7685e-38,  4.1451e-38,\n        -1.2571e-39,  1.1619e-37,  1.3852e-37, -2.7897e-39,  1.1475e-37,\n         9.8480e-39,  3.9087e-38,  4.0906e-38,  4.4806e-38, -1.9349e-39,\n         5.8954e-38, -3.5902e-39,  6.9382e-39,  5.2112e-40,  9.7022e-38,\n         5.3022e-38,  4.8693e-38,  4.1894e-38, -8.3679e-39,  7.4912e-38,\n         8.3314e-38,  1.7486e-39, -2.0307e-39,  5.2531e-38,  3.3164e-39,\n         5.2898e-40,  6.1330e-38, -2.1933e-39, -7.6028e-39, -1.4205e-39,\n         1.8779e-37, -8.4088e-39,  6.0597e-40, -6.2916e-39,  1.7822e-37,\n         3.4471e-39,  7.3061e-38,  1.6742e-40, -2.1728e-39, -1.6510e-39,\n        -5.6899e-39, -3.1820e-39,  3.0099e-38,  1.7231e-39,  4.3346e-38,\n        -4.7695e-39,  6.3942e-38,  9.5205e-39,  1.0527e-37,  8.1882e-38,\n         5.7970e-38,  1.8514e-38,  4.1115e-39,  2.5425e-38, -1.7560e-39,\n         1.8350e-38,  1.6245e-37,  1.3665e-37, -8.3490e-39,  4.2066e-38,\n        -1.7289e-39,  2.0583e-40,  3.1333e-38,  7.4019e-38,  2.8577e-38,\n         2.4842e-38,  3.6809e-38,  4.6859e-38, -2.1968e-39,  3.1036e-38,\n        -2.4734e-38,  4.1248e-39,  1.6429e-37,  9.7017e-38,  6.9471e-39,\n        -2.4137e-40, -1.2051e-40, -8.7521e-40, -1.5857e-38,  4.6507e-38,\n         6.4991e-39,  1.7948e-37, -1.3886e-40, -3.3005e-39, -6.1274e-40,\n         2.1849e-38,  1.9520e-39, -3.3156e-39,  2.1461e-38,  1.4435e-40,\n        -8.9111e-39,  1.0862e-39,  1.7289e-37,  4.7903e-38, -4.8283e-39,\n         1.0231e-37,  4.6457e-38, -5.2278e-39, -1.0887e-38, -1.0553e-38,\n         3.2082e-38, -2.2508e-38, -7.3696e-39,  5.0845e-38,  1.9384e-38,\n        -8.0216e-39,  7.0524e-38,  1.8678e-38,  3.5655e-39,  9.9025e-39,\n         6.5507e-38,  4.7503e-38,  1.7181e-39, -4.8610e-39,  1.6826e-39,\n        -2.3918e-39,  2.9464e-38,  1.0010e-38,  7.4686e-40, -8.2129e-40,\n         1.2080e-39,  7.2145e-39, -1.2375e-39,  2.1335e-38, -2.4895e-39,\n        -7.3444e-39,  3.0169e-39,  8.0474e-38,  1.0213e-37, -4.3136e-39,\n         1.1857e-38,  2.5372e-39,  4.9057e-39, -4.4487e-39,  3.8226e-38,\n         8.8491e-38,  1.1040e-39,  9.9524e-38, -3.4898e-39,  9.4329e-38,\n         1.1314e-37, -1.2379e-38,  1.0995e-37,  1.0583e-37, -5.0233e-39,\n         2.3729e-38,  6.7624e-38,  4.8566e-38,  5.8376e-38,  6.1461e-40,\n         2.2591e-39,  1.3257e-37,  1.1047e-37,  2.3330e-38,  6.6802e-38,\n         1.1006e-37,  8.6698e-38, -2.1889e-39,  1.5281e-37,  3.6545e-38,\n         1.3436e-37, -1.0963e-39,  1.3252e-39,  4.6071e-38, -2.8443e-39,\n        -2.8430e-39, -4.7921e-39,  6.8036e-38, -7.1284e-40,  3.8418e-40,\n         7.0455e-38,  3.5996e-38,  4.9788e-38,  8.2087e-38, -9.9875e-39,\n         7.7063e-40,  6.7451e-38, -9.0148e-40,  9.3322e-39,  7.3598e-38,\n        -8.8100e-39,  1.3319e-37,  1.3320e-39, -5.2346e-39,  1.2039e-37,\n        -3.1320e-39, -4.7495e-39,  6.1099e-40, -1.3806e-38, -1.2260e-39,\n         5.0771e-38,  1.8126e-38, -3.4816e-39,  4.9792e-38,  7.0942e-38,\n         1.7767e-37,  7.3255e-38,  7.1968e-38,  2.2231e-38,  2.1987e-38,\n        -9.3547e-39,  2.9190e-39,  8.2369e-38,  1.9060e-39,  5.3226e-38,\n         7.3753e-38,  5.7362e-38,  8.3341e-39, -4.7160e-40, -9.3231e-39,\n        -7.3944e-39, -5.1006e-39,  2.8867e-40,  2.8610e-38,  7.5811e-38,\n         6.1022e-38,  5.1021e-38, -5.4224e-39,  8.0436e-39,  3.8582e-38,\n         6.5239e-38,  1.1521e-37,  2.3513e-38,  1.1028e-37,  1.0725e-37,\n        -5.5501e-39,  8.6477e-38,  8.6697e-38,  2.5595e-38, -6.1798e-40,\n         1.8583e-38,  5.9746e-38,  1.1237e-38,  5.3056e-39,  4.8722e-38,\n         7.2727e-38,  1.1805e-37,  1.3728e-37,  5.7667e-39,  3.2424e-39,\n         5.1500e-39,  1.2703e-38, -1.0443e-39,  4.8307e-38, -2.3494e-39,\n         2.2053e-38, -3.4772e-39, -1.3172e-38,  8.8387e-38,  4.7901e-38,\n         2.9804e-38,  5.1214e-38,  1.4125e-37, -1.3752e-38,  5.5140e-40,\n        -3.4412e-40, -9.0779e-39,  1.6330e-37,  1.2576e-37, -3.6474e-39,\n         3.2810e-38, -1.6019e-39,  1.0867e-37, -5.3720e-39,  5.8492e-38,\n        -1.5398e-38, -4.8642e-39,  5.3030e-38,  1.1885e-38,  7.7917e-38,\n         5.3200e-38,  2.3250e-38,  6.0269e-38, -9.1316e-39,  1.4441e-39,\n        -5.8777e-40,  1.6423e-38, -4.3281e-39,  1.1153e-37,  1.4575e-37,\n         2.0350e-37,  3.7837e-38,  5.3791e-40, -5.0094e-40,  4.2812e-39,\n        -1.3140e-38,  1.8226e-39,  3.4748e-39,  3.4127e-39,  6.0430e-38,\n         6.8980e-39,  9.7386e-38,  5.4250e-38,  7.8016e-40, -3.8953e-40,\n         1.2210e-39, -8.7707e-42,  4.0190e-39, -5.9034e-40,  1.4513e-38,\n         1.0707e-37,  3.9345e-38], device='cuda:0')",
        "exp_avg_sq": "tensor([4.6779e-16, 9.0684e-15, 1.0032e-15, 3.9459e-16, 1.1714e-15, 3.3418e-17,\n        3.2543e-14, 2.4835e-15, 4.5039e-14, 5.3259e-14, 3.0999e-14, 2.7225e-17,\n        6.3594e-14, 1.7835e-17, 5.0265e-15, 3.3113e-14, 1.1854e-16, 4.9755e-15,\n        8.5051e-14, 5.4890e-15, 2.6638e-14, 2.5366e-16, 1.3865e-15, 7.3675e-18,\n        1.7886e-15, 3.1715e-15, 4.1361e-15, 3.3489e-14, 1.7935e-14, 1.6358e-17,\n        9.5435e-16, 4.8621e-15, 7.2872e-15, 1.6787e-14, 3.2778e-14, 6.3612e-15,\n        3.7719e-14, 1.1129e-17, 9.9462e-14, 8.1055e-17, 1.4705e-14, 1.9046e-14,\n        5.5579e-19, 1.6397e-16, 3.0494e-14, 1.1888e-16, 1.6254e-14, 3.8197e-16,\n        1.4007e-14, 4.8874e-17, 1.8153e-14, 5.3297e-15, 2.0932e-14, 1.8128e-16,\n        3.1317e-17, 5.3699e-15, 6.0772e-16, 1.2666e-15, 5.2047e-16, 5.9690e-15,\n        6.4464e-15, 6.8268e-15, 2.6180e-13, 4.0091e-14, 1.1287e-14, 1.4448e-17,\n        6.3008e-15, 1.5170e-13, 4.4514e-15, 1.7787e-16, 3.8162e-15, 6.8750e-15,\n        7.4062e-18, 1.1190e-14, 2.8739e-14, 1.8212e-14, 2.5167e-16, 3.0919e-14,\n        5.7744e-15, 5.7930e-16, 2.7552e-17, 2.3108e-15, 1.3776e-14, 4.1176e-14,\n        4.1824e-14, 2.4627e-14, 3.2772e-15, 1.6023e-15, 6.2570e-14, 4.4865e-14,\n        5.9297e-16, 2.9434e-17, 2.9954e-14, 9.4198e-14, 1.4042e-16, 3.7229e-14,\n        4.2790e-17, 5.4534e-18, 5.7162e-15, 3.9234e-14, 9.1903e-15, 1.0379e-14,\n        4.5509e-15, 2.9581e-15, 3.4770e-15, 5.4123e-17, 2.5153e-16, 8.0614e-15,\n        5.8263e-15, 5.3646e-15, 2.6117e-16, 2.8434e-15, 6.5578e-16, 1.5662e-14,\n        4.2178e-14, 8.4216e-16, 7.1328e-15, 1.1610e-14, 2.4551e-15, 4.5390e-14,\n        8.8846e-14, 1.3247e-16, 3.3799e-15, 2.8117e-16, 2.5279e-18, 1.1070e-19,\n        3.3365e-16, 3.1053e-14, 1.4278e-14, 7.7697e-18, 1.7760e-18, 1.1423e-15,\n        7.9089e-17, 4.7784e-15, 1.6761e-13, 7.9990e-15, 1.8169e-14, 7.9181e-16,\n        6.2710e-15, 1.3846e-14, 9.0302e-17, 2.4562e-14, 6.8993e-17, 6.0538e-14,\n        2.4562e-15, 3.6520e-14, 2.5303e-15, 2.0280e-15, 1.6806e-14, 2.2180e-14,\n        1.0567e-15, 1.1900e-14, 5.4456e-15, 5.4016e-15, 2.3503e-14, 3.8397e-14,\n        1.6065e-14, 2.7097e-17, 3.8345e-17, 2.5233e-16, 8.6053e-15, 5.2100e-14,\n        2.7280e-17, 8.5864e-18, 1.1792e-15, 7.4608e-17, 3.3759e-16, 1.3581e-14,\n        2.8332e-16, 7.7901e-19, 3.0778e-16, 3.5644e-15, 4.7121e-16, 6.7926e-14,\n        1.9048e-16, 2.2181e-15, 1.5696e-14, 2.7966e-15, 1.1419e-17, 1.8018e-14,\n        5.7821e-14, 4.8814e-17, 2.8785e-14, 3.4138e-16, 6.6726e-14, 2.0224e-15,\n        4.4821e-16, 3.9342e-14, 3.4407e-14, 8.3153e-15, 7.3797e-15, 1.8208e-15,\n        3.3336e-16, 1.4359e-14, 1.0207e-13, 9.2587e-15, 2.9658e-15, 2.3333e-17,\n        6.6548e-17, 5.5972e-17, 5.5154e-14, 7.8817e-17, 1.7010e-14, 3.7832e-16,\n        3.8208e-15, 8.8189e-15, 6.3539e-14, 1.0266e-15, 5.1318e-14, 1.0832e-14,\n        1.3397e-16, 3.2131e-14, 3.5902e-14, 5.4114e-17, 2.6313e-13, 1.1746e-14,\n        2.2831e-14, 1.2893e-14, 3.6909e-14, 8.4235e-17, 1.3332e-14, 9.6611e-18,\n        8.1468e-16, 1.2526e-16, 2.5526e-14, 9.2073e-15, 1.1116e-14, 1.0034e-15,\n        7.6489e-16, 1.5500e-14, 1.8479e-14, 3.0180e-15, 8.1252e-17, 2.4583e-14,\n        2.1910e-16, 2.3036e-15, 7.9856e-15, 4.5382e-17, 3.9380e-17, 1.3799e-17,\n        7.7421e-14, 2.2198e-16, 6.7991e-15, 3.0934e-15, 9.8066e-14, 2.0376e-17,\n        2.0816e-14, 3.9807e-17, 2.1268e-16, 3.9560e-17, 2.6016e-16, 9.9095e-17,\n        1.5887e-13, 2.8046e-17, 1.2828e-15, 2.7389e-15, 1.0734e-14, 1.0514e-15,\n        1.6315e-14, 9.8607e-15, 3.5367e-14, 7.8700e-16, 4.1991e-15, 1.6145e-15,\n        4.5580e-16, 2.2045e-14, 2.8932e-14, 2.2068e-14, 9.8769e-17, 1.2491e-15,\n        3.2221e-17, 5.2521e-17, 9.1776e-15, 7.7873e-14, 1.6553e-16, 4.5479e-14,\n        1.9268e-14, 1.1402e-14, 1.5743e-16, 2.2206e-15, 8.7408e-15, 9.7229e-18,\n        6.9371e-14, 4.5037e-14, 2.1001e-16, 7.6431e-18, 2.7914e-16, 2.3664e-20,\n        3.3668e-16, 7.9795e-16, 9.1698e-15, 9.6700e-14, 2.7640e-16, 1.2697e-16,\n        3.9228e-16, 6.0107e-16, 3.3475e-16, 3.4927e-17, 2.3411e-15, 1.2350e-17,\n        1.6043e-16, 2.5151e-16, 6.0059e-14, 1.3101e-15, 6.8761e-16, 9.6917e-15,\n        4.6798e-15, 7.4818e-17, 4.5479e-16, 1.8514e-15, 3.7371e-15, 3.4675e-14,\n        6.7449e-18, 2.4575e-14, 1.6248e-16, 7.9487e-17, 4.2788e-14, 5.4622e-16,\n        3.5468e-16, 1.0364e-16, 6.4481e-14, 1.7628e-15, 5.0916e-14, 3.1512e-16,\n        4.4887e-17, 1.9264e-18, 2.3842e-15, 1.0330e-15, 3.2663e-17, 1.0142e-16,\n        2.8706e-16, 9.6628e-16, 8.0525e-18, 4.3254e-16, 1.4025e-16, 8.9790e-18,\n        2.8179e-15, 1.3759e-14, 2.3850e-14, 2.1613e-16, 2.7139e-14, 1.1741e-16,\n        7.5402e-18, 3.9569e-18, 1.8727e-15, 4.6185e-14, 2.5281e-16, 1.1028e-14,\n        1.2811e-19, 3.1679e-14, 8.3582e-15, 7.7198e-16, 3.9168e-14, 4.1867e-14,\n        5.7541e-17, 2.3098e-15, 2.4170e-14, 1.0491e-14, 4.8699e-15, 1.0103e-15,\n        2.2202e-16, 6.2864e-14, 5.8903e-14, 3.9716e-15, 2.4323e-14, 3.4417e-14,\n        1.6544e-13, 1.0771e-15, 1.9428e-14, 2.6883e-15, 3.2427e-14, 1.2774e-17,\n        1.0603e-16, 2.5563e-15, 1.1812e-15, 3.1369e-18, 5.4281e-16, 9.1517e-15,\n        1.3570e-16, 8.7145e-17, 1.6305e-15, 2.3616e-14, 1.4991e-14, 1.8173e-14,\n        4.5162e-16, 1.2214e-14, 4.5710e-15, 9.0444e-17, 4.4635e-16, 1.4610e-14,\n        4.5882e-17, 8.1039e-14, 1.2292e-17, 1.4032e-15, 6.8170e-14, 1.4576e-18,\n        6.6917e-17, 3.3139e-16, 4.3497e-14, 2.4989e-15, 1.0551e-15, 1.4910e-14,\n        1.1206e-15, 4.7345e-14, 4.3591e-16, 1.2562e-13, 1.5346e-14, 6.1970e-15,\n        2.6603e-14, 5.3397e-16, 2.3496e-16, 2.0583e-16, 7.1839e-15, 2.0971e-16,\n        2.5685e-15, 6.2899e-15, 3.1920e-15, 1.6474e-14, 3.7335e-17, 5.8773e-16,\n        1.0820e-15, 3.3469e-17, 1.1647e-16, 8.4673e-16, 1.7487e-14, 7.2729e-15,\n        1.2640e-14, 2.0417e-16, 4.2314e-15, 2.2410e-15, 6.2928e-15, 7.6675e-15,\n        8.3408e-15, 3.3989e-14, 1.2286e-14, 7.5621e-17, 2.2037e-14, 3.7464e-14,\n        6.0211e-15, 2.3172e-17, 1.5919e-14, 8.3817e-15, 7.8977e-15, 4.2552e-15,\n        4.8204e-15, 3.0005e-14, 1.0325e-13, 1.3184e-14, 2.1846e-16, 1.8900e-14,\n        3.2983e-14, 9.8834e-16, 1.1329e-16, 2.3666e-14, 5.3384e-16, 1.1795e-15,\n        2.6571e-17, 4.7785e-14, 3.4165e-14, 1.2107e-15, 4.6926e-15, 1.0225e-15,\n        1.0633e-13, 3.8718e-16, 4.0128e-18, 2.0017e-18, 1.4545e-16, 4.5892e-14,\n        7.7649e-14, 6.4793e-17, 1.0894e-14, 5.9353e-17, 3.5703e-14, 1.3838e-17,\n        1.0884e-14, 3.6578e-16, 7.7219e-17, 1.4825e-15, 1.3348e-15, 2.2165e-15,\n        4.7528e-15, 2.0499e-14, 2.3886e-16, 8.4562e-16, 1.6481e-17, 1.3210e-17,\n        3.9666e-16, 3.2012e-16, 1.4499e-14, 5.3161e-14, 1.5814e-13, 1.3643e-15,\n        2.9180e-17, 8.0826e-17, 5.2384e-16, 7.9947e-16, 1.2907e-16, 1.0174e-14,\n        3.9872e-15, 2.0955e-14, 9.6846e-16, 5.2430e-14, 3.6681e-14, 3.1355e-15,\n        2.0449e-16, 1.6165e-15, 1.7369e-15, 1.2919e-16, 9.3197e-16, 4.1179e-15,\n        4.9445e-14, 2.1637e-14], device='cuda:0')"
      },
      "68": {
        "step": "tensor(11268.)",
        "exp_avg": "tensor([[ 6.3550e-41,  4.4437e-40,  1.8296e-40,  ...,  1.2257e-40,\n          9.9233e-41, -1.5926e-39],\n        [ 7.4101e-42,  6.5321e-39,  5.5344e-40,  ...,  3.0382e-40,\n          3.0973e-40, -7.9176e-39],\n        [-1.0231e-41,  2.3257e-39, -3.7493e-40,  ..., -2.9720e-40,\n         -3.6646e-40,  1.0039e-38],\n        ...,\n        [-6.2951e-41, -2.9344e-39, -2.3950e-40,  ..., -2.3235e-40,\n         -1.4908e-40,  3.5335e-39],\n        [ 3.2218e-40,  7.7511e-39,  6.1618e-40,  ...,  8.3739e-40,\n          4.3200e-40, -1.4301e-38],\n        [ 1.1612e-40,  1.4679e-39,  8.4626e-41,  ..., -5.6039e-40,\n          1.3322e-40, -3.4241e-40]], device='cuda:0')",
        "exp_avg_sq": "tensor([[4.5235e-18, 1.3771e-18, 1.4257e-18,  ..., 8.2952e-18, 2.2511e-18,\n         4.1676e-18],\n        [3.2227e-16, 4.2961e-17, 4.8663e-17,  ..., 4.2681e-16, 4.6939e-17,\n         2.7964e-16],\n        [6.7760e-17, 2.1506e-17, 1.5457e-17,  ..., 1.6281e-16, 6.0088e-17,\n         1.3511e-16],\n        ...,\n        [2.7670e-17, 2.4075e-17, 1.8867e-18,  ..., 3.1543e-17, 1.1006e-17,\n         4.1855e-17],\n        [2.9817e-16, 3.4848e-17, 8.7613e-18,  ..., 4.1146e-17, 2.1467e-17,\n         9.3937e-17],\n        [3.2339e-18, 1.3323e-18, 1.4749e-18,  ..., 4.1996e-18, 3.9412e-18,\n         1.5699e-18]], device='cuda:0')"
      },
      "69": {
        "step": "tensor(11268.)",
        "exp_avg": "tensor([ 4.3834e-37,  2.1041e-36, -1.8550e-37,  1.7445e-37, -2.2165e-36,\n        -1.4620e-36,  2.4654e-37,  1.6777e-37,  2.3607e-36,  1.2217e-36,\n        -2.3900e-37, -4.1787e-36,  4.2742e-37,  2.0491e-37, -1.1103e-37,\n         7.1207e-37, -3.6438e-37,  7.5975e-37,  2.9035e-36, -9.2117e-37,\n        -1.6778e-37,  5.7312e-37,  3.0299e-37, -1.5914e-36, -1.6781e-36,\n        -7.3062e-38, -1.6242e-37, -8.3229e-37, -1.2692e-37,  1.3432e-37,\n        -8.2749e-39, -1.1385e-37,  1.8049e-37,  1.4444e-36, -2.1828e-37,\n        -1.3262e-37, -1.5618e-37, -8.7218e-37,  1.2777e-37, -2.5346e-36,\n         1.0722e-37, -3.1583e-37, -2.6485e-36,  2.3377e-38, -3.2287e-37,\n         1.8051e-36, -3.2897e-37, -1.8005e-36,  1.8467e-36, -2.1994e-37,\n         3.0116e-37, -1.8481e-36, -3.9737e-37,  1.7292e-37,  2.0061e-36,\n         4.5995e-37,  2.4395e-36,  4.8546e-37,  2.0320e-37,  7.8223e-38,\n         5.6211e-37,  3.7785e-36,  3.9116e-36,  1.1475e-36, -1.6066e-36,\n        -3.4927e-37, -1.1338e-36, -1.5332e-36, -1.5023e-37, -4.9017e-37,\n         2.6818e-36,  3.6348e-37,  1.5959e-36, -6.7819e-37,  1.2515e-36,\n         1.5140e-36, -7.4536e-37,  2.6413e-36, -2.8284e-36, -1.4653e-37,\n         1.4685e-37, -3.1318e-38, -7.6852e-38,  5.1945e-37, -3.0588e-37,\n         2.6714e-36,  7.7727e-37,  2.5439e-36,  2.7216e-37, -2.4792e-37,\n        -1.4651e-36, -8.2058e-37, -2.6695e-37, -1.3832e-37, -1.5290e-36,\n         1.4256e-36, -1.1731e-36,  1.0404e-36, -9.6314e-37, -1.3378e-36,\n        -1.2600e-36,  2.7117e-36, -2.3622e-37,  1.3206e-38, -4.1865e-37,\n        -2.7459e-37, -4.9441e-37, -3.0558e-36,  4.0481e-36,  3.8408e-37,\n        -4.9298e-37, -1.0787e-38, -1.6386e-36, -1.7306e-36,  9.4483e-37,\n         5.2805e-37, -1.5626e-37,  2.0852e-37, -2.4765e-37,  7.4001e-37,\n         1.3108e-36, -3.2071e-37,  1.1015e-37,  2.1829e-36,  7.5488e-37,\n         3.4945e-37,  7.6733e-38, -1.5068e-37, -6.5265e-37, -1.7940e-36,\n        -3.3157e-37, -2.7451e-37, -4.6113e-37, -1.4932e-37,  1.8135e-37,\n        -1.2955e-37,  7.1949e-37, -8.1189e-37, -1.2113e-36, -2.3127e-36,\n        -2.6555e-36, -2.5269e-37, -1.4902e-36, -2.0189e-37, -5.1098e-37,\n        -2.5489e-36, -1.7839e-36, -3.7648e-37, -7.6978e-37,  4.4996e-37,\n        -1.4780e-36,  1.2774e-36,  1.6960e-36,  1.7679e-38,  2.7030e-36,\n        -2.6155e-37, -2.4040e-37,  1.1273e-37,  2.0481e-37,  1.4357e-36,\n         1.3496e-37, -3.1096e-37, -2.6840e-38, -8.5198e-37, -4.9542e-37,\n        -3.4913e-37, -1.0755e-36,  8.1110e-38, -1.0516e-36,  6.7899e-37,\n        -2.6094e-36, -3.9057e-37, -9.4773e-37,  8.2102e-38, -1.1899e-36,\n        -4.8021e-37, -2.2420e-37, -7.6186e-38,  1.4395e-37, -2.7842e-36,\n        -2.6896e-38,  1.6120e-36, -2.6210e-37,  2.7430e-37,  4.8402e-37,\n         2.8632e-37,  1.9157e-36,  1.8155e-37, -3.4126e-37,  3.6226e-37,\n        -1.0497e-37,  1.4797e-36,  5.1913e-37, -1.7566e-37, -6.0711e-37,\n         5.9638e-37, -1.9267e-37, -4.5581e-37, -3.4355e-38, -2.8367e-36,\n         1.1637e-36, -2.1456e-36,  4.1449e-37,  6.2948e-37, -2.2297e-36,\n        -2.7101e-37,  1.8931e-37, -1.0533e-36, -2.8731e-37,  1.3898e-36,\n         9.7171e-37,  1.9064e-37,  6.8760e-37, -3.1545e-36,  6.4341e-36,\n        -2.0768e-36,  2.1695e-37,  9.8587e-37, -1.9219e-36,  1.2852e-37,\n         2.2222e-38, -2.5604e-37,  5.4119e-37, -5.0358e-37,  8.5722e-38,\n         5.1695e-37,  7.8495e-38, -5.8976e-37,  2.3885e-36, -5.8871e-38,\n        -4.1280e-37, -2.0522e-36, -1.4401e-36,  5.4223e-36, -1.6732e-37,\n         1.7385e-37,  5.2294e-36,  6.4429e-37,  3.9756e-37,  1.4224e-37,\n         5.9774e-37, -2.7813e-37, -3.1613e-37, -1.3530e-36,  2.5407e-36,\n        -3.0163e-37, -3.1142e-37, -4.3754e-37, -4.7478e-37, -1.3354e-37,\n         1.0957e-37, -3.7806e-37, -1.6603e-36, -3.5230e-37, -5.4012e-37,\n        -6.7818e-37, -1.4638e-37,  2.1252e-37, -1.0450e-36,  5.1560e-38,\n        -3.1652e-37,  9.4655e-37,  1.0776e-37, -2.0174e-37,  8.0493e-37,\n        -9.6038e-39,  3.9692e-36,  1.1207e-36,  1.4088e-37, -3.4572e-37,\n        -4.7469e-38,  1.2919e-36,  2.5362e-37,  4.0509e-36, -2.1618e-37,\n        -2.2507e-36, -8.7439e-37,  1.3285e-37, -1.2904e-36,  1.1967e-37,\n        -2.2301e-36, -2.7176e-36, -2.4029e-37,  1.3891e-36,  2.3215e-36,\n         6.7386e-38, -3.2359e-36, -3.5392e-37, -2.3729e-37,  4.4400e-37,\n        -1.2999e-37,  2.1728e-36, -3.1980e-37, -1.3782e-37, -5.0417e-37,\n         9.7169e-38,  1.6104e-37,  1.4470e-36, -2.1484e-37, -9.1966e-37,\n        -1.4336e-37, -3.7681e-37,  5.0208e-38, -1.4123e-37, -5.2213e-37,\n        -9.1815e-38, -3.8355e-37,  2.5846e-37, -1.5403e-36,  4.3838e-36,\n         1.2327e-36, -2.4163e-36, -4.6635e-37, -3.6549e-37, -1.7694e-37,\n        -1.2923e-37,  7.1728e-38,  2.7052e-37, -7.9424e-37, -4.9208e-37,\n         4.6631e-36, -2.9175e-38, -5.3636e-37, -1.9371e-37, -6.7147e-37,\n         5.3602e-37,  8.7014e-38,  8.9650e-37, -1.3619e-36, -1.1623e-36,\n        -8.7811e-38,  1.2730e-36, -6.3889e-37, -5.1880e-37,  3.4355e-36,\n        -2.3875e-37, -2.9542e-37, -3.1223e-37,  1.6302e-36,  1.9651e-36,\n        -4.7202e-37, -7.9308e-39, -4.5837e-37, -2.5953e-37, -1.9609e-36,\n         5.9285e-37, -7.4073e-37, -2.5636e-36, -3.1273e-37, -9.5619e-37,\n        -1.2790e-37, -1.4908e-36,  2.1815e-36,  1.8625e-36, -1.1488e-37,\n         2.7165e-38, -3.3543e-37,  3.1623e-37,  6.6856e-37, -1.6292e-37,\n         8.8617e-37,  1.9175e-36, -1.0333e-36,  2.1389e-37,  3.7668e-36,\n        -2.1296e-36,  1.8912e-36, -2.6558e-37, -2.0910e-37,  8.0650e-37,\n        -1.3082e-36, -2.2813e-37, -8.3650e-37, -2.7479e-37, -2.5423e-38,\n        -2.4547e-37,  5.9429e-37, -2.5640e-37,  2.7540e-37, -1.2443e-37,\n        -2.5732e-37,  1.9049e-36, -2.6576e-37,  4.7558e-36, -5.8465e-37,\n        -2.8223e-36, -2.4414e-37,  1.0442e-36,  2.6843e-37, -8.2490e-37,\n         6.6221e-38, -1.2526e-37, -3.0465e-36, -3.1692e-38, -2.4483e-37,\n        -2.3532e-36,  5.4923e-37, -1.8565e-36, -1.4802e-37,  6.3961e-37,\n         2.1818e-36, -1.0843e-36, -3.2670e-37,  2.4158e-37, -9.7888e-37,\n         7.9472e-37,  1.3027e-36, -7.7828e-38,  5.5546e-37, -1.5717e-37,\n         1.2824e-36, -2.6470e-37,  3.2905e-37, -3.3695e-37,  2.0605e-37,\n        -4.3904e-37, -1.3829e-36, -1.0806e-36, -2.1745e-37,  1.9902e-36,\n         2.9137e-36, -1.9030e-36, -1.9715e-37, -2.6391e-37, -1.3486e-36,\n         3.8001e-37,  2.7862e-37, -1.8158e-36,  3.9930e-38,  2.3024e-37,\n         1.3612e-36, -3.5763e-37, -2.3588e-37,  1.4131e-36,  1.8234e-36,\n         5.4752e-37, -1.2565e-37,  1.7696e-36, -3.9439e-38,  1.3611e-36,\n        -1.4502e-36,  4.6225e-37, -7.9151e-37,  5.0193e-38, -1.4351e-37,\n         7.6798e-37,  2.2830e-36, -6.3103e-37,  6.1922e-37,  6.0807e-37,\n        -5.0979e-37, -2.9039e-37, -1.0527e-37, -2.2282e-37,  8.6129e-38,\n        -6.5811e-37, -3.5578e-37, -3.2809e-37,  4.2179e-37,  2.8593e-37,\n        -1.1775e-36, -2.3302e-37,  3.2965e-36, -8.4489e-37, -3.6530e-37,\n         1.2442e-37,  2.0066e-37, -2.4078e-36, -4.9159e-37, -1.4175e-36,\n         8.8762e-38, -1.3438e-37,  5.7222e-37,  1.5808e-36,  6.3092e-38,\n         2.1525e-36, -2.5025e-37,  1.0412e-36,  6.5362e-37, -2.4488e-36,\n         2.9354e-37, -7.0239e-37, -1.2446e-36, -3.1160e-37, -2.4634e-36,\n        -7.4381e-38, -1.0118e-37,  5.9520e-38,  1.3586e-36, -2.5420e-37,\n        -4.4523e-37, -2.9803e-36, -3.2978e-36, -7.9015e-37, -2.7724e-37,\n         3.3778e-38, -1.8120e-36, -1.0341e-36,  5.4579e-37,  2.8196e-37,\n        -2.1228e-37, -1.4693e-37,  7.7579e-37,  3.2722e-37, -6.8601e-37,\n        -2.9717e-37, -2.6930e-38, -1.2912e-36,  2.7365e-37, -6.3487e-37,\n         2.5678e-36,  2.1366e-37], device='cuda:0')",
        "exp_avg_sq": "tensor([2.8053e-12, 1.2622e-10, 4.6663e-11, 2.6599e-11, 4.7415e-11, 8.1427e-12,\n        7.9210e-14, 4.7882e-12, 3.4524e-11, 1.3077e-10, 1.8542e-13, 7.5021e-11,\n        1.8495e-11, 8.8840e-13, 6.9097e-12, 1.3245e-13, 3.8944e-11, 8.9580e-11,\n        1.1171e-10, 7.1761e-11, 5.5656e-11, 6.0432e-13, 1.1343e-11, 1.7345e-10,\n        7.7554e-11, 1.4916e-13, 4.5123e-13, 4.6259e-11, 2.3621e-12, 2.3198e-13,\n        1.0884e-13, 5.4407e-12, 1.0871e-11, 4.2624e-10, 3.6789e-11, 5.8888e-13,\n        9.5600e-14, 4.3688e-11, 2.1131e-14, 1.2167e-10, 1.0428e-12, 2.3741e-12,\n        1.5128e-11, 3.8528e-13, 8.0503e-12, 4.5822e-11, 9.0545e-12, 1.8834e-10,\n        3.4060e-11, 1.4976e-10, 5.7870e-12, 9.4980e-11, 6.2033e-12, 9.2325e-14,\n        7.5344e-11, 1.0324e-11, 3.0253e-10, 3.7394e-12, 6.9856e-13, 2.4446e-13,\n        5.7489e-12, 3.3050e-10, 3.1003e-10, 5.2359e-12, 2.8336e-10, 2.5903e-12,\n        1.0799e-10, 2.1187e-10, 7.0958e-13, 3.2077e-10, 4.3166e-11, 2.1923e-12,\n        3.4056e-11, 1.0515e-10, 2.5743e-11, 4.5493e-11, 6.2296e-13, 6.5355e-11,\n        2.6373e-10, 3.0641e-13, 2.9744e-13, 2.1714e-12, 2.7794e-13, 3.9813e-12,\n        2.4628e-13, 2.3506e-10, 5.1952e-13, 3.5175e-11, 1.2331e-10, 4.2392e-11,\n        1.7968e-10, 8.7827e-11, 8.7174e-11, 5.0711e-10, 4.5965e-11, 1.1831e-11,\n        3.7323e-11, 2.0095e-10, 8.5252e-12, 2.3923e-10, 5.2319e-12, 4.9262e-11,\n        3.1385e-13, 7.0560e-14, 1.2087e-11, 4.5631e-15, 1.2460e-10, 1.9374e-11,\n        2.5063e-10, 7.0733e-11, 5.7715e-11, 3.1090e-11, 1.1065e-10, 2.2045e-10,\n        2.8293e-10, 1.1101e-11, 4.9891e-13, 1.2759e-10, 1.0599e-13, 3.6269e-11,\n        3.0212e-10, 9.3399e-14, 4.6389e-11, 9.8241e-12, 4.1583e-12, 1.3263e-12,\n        7.5862e-12, 2.6938e-13, 1.6700e-10, 7.6676e-11, 5.5934e-14, 1.3598e-11,\n        1.2929e-12, 8.6730e-13, 1.2214e-12, 1.6662e-11, 2.3967e-12, 2.8837e-11,\n        9.4221e-12, 1.4817e-11, 1.9296e-10, 7.0294e-12, 2.8940e-11, 1.4757e-12,\n        4.3052e-12, 2.2722e-11, 4.4848e-11, 4.4617e-11, 5.1581e-11, 5.0955e-12,\n        1.1403e-10, 1.2877e-10, 7.0990e-11, 2.1688e-11, 3.9962e-12, 3.2681e-12,\n        9.1529e-14, 1.6213e-12, 7.1504e-15, 9.0553e-12, 1.1366e-14, 7.5923e-13,\n        7.2182e-13, 3.1512e-11, 3.3163e-11, 4.8528e-12, 2.2396e-10, 1.9793e-12,\n        2.5565e-11, 8.2315e-14, 6.4783e-10, 8.1972e-11, 1.0356e-10, 3.8277e-13,\n        1.9488e-10, 2.6338e-12, 7.9177e-13, 4.4710e-12, 7.1328e-13, 3.9446e-10,\n        4.3014e-13, 6.6707e-12, 2.1376e-12, 3.6077e-11, 1.6373e-13, 4.4134e-12,\n        5.0316e-10, 2.1859e-13, 1.2168e-11, 1.0720e-12, 1.8445e-12, 1.6950e-11,\n        5.9597e-11, 1.8058e-13, 2.4343e-11, 3.2493e-13, 3.9246e-12, 9.2351e-11,\n        9.8187e-12, 8.9502e-11, 2.3420e-12, 8.2109e-11, 5.5524e-12, 8.2689e-12,\n        4.6489e-11, 3.2675e-13, 1.5058e-10, 8.8133e-12, 1.2300e-10, 2.4469e-10,\n        1.9409e-12, 4.7468e-13, 4.8629e-11, 1.7364e-10, 1.5060e-09, 1.5060e-10,\n        8.7437e-11, 6.0549e-12, 2.8926e-10, 2.2293e-11, 2.9059e-13, 4.8137e-13,\n        4.6187e-12, 9.6842e-11, 4.9026e-11, 9.8116e-13, 1.3118e-13, 4.2711e-12,\n        4.5021e-10, 2.4055e-13, 2.9562e-11, 2.1280e-10, 1.4105e-10, 6.3172e-10,\n        2.9442e-12, 6.3178e-14, 5.8482e-10, 1.4752e-12, 1.6495e-12, 1.2914e-12,\n        6.1435e-11, 3.3833e-12, 3.3990e-12, 8.8923e-11, 9.5750e-11, 5.0445e-12,\n        1.1631e-11, 9.0480e-12, 4.2679e-14, 6.5038e-13, 5.4461e-13, 9.7236e-13,\n        9.3795e-10, 1.9711e-13, 1.0409e-12, 7.2136e-12, 4.6841e-13, 8.8454e-11,\n        6.0236e-11, 4.7542e-13, 2.9770e-13, 2.7905e-11, 4.3767e-13, 5.0152e-12,\n        2.9323e-10, 3.2676e-13, 2.4663e-11, 1.9121e-11, 6.4716e-13, 2.6067e-10,\n        2.9920e-12, 3.9191e-11, 2.9050e-13, 2.7608e-10, 4.2164e-12, 5.8131e-10,\n        1.0287e-10, 2.0288e-13, 9.0394e-11, 7.3272e-13, 6.9960e-11, 5.8893e-11,\n        2.0065e-12, 6.7601e-11, 8.1389e-11, 3.3141e-12, 3.1497e-10, 2.3304e-12,\n        1.5257e-13, 3.9335e-12, 6.5432e-11, 1.1669e-10, 2.4794e-12, 3.3031e-12,\n        2.7053e-11, 9.7820e-13, 1.6664e-12, 2.2308e-11, 3.3629e-13, 8.4347e-11,\n        2.9429e-13, 5.2435e-14, 4.2625e-14, 1.6471e-13, 1.4419e-10, 2.8643e-12,\n        5.5102e-12, 7.2157e-12, 2.6606e-11, 5.8508e-10, 9.5945e-12, 1.1703e-10,\n        3.8390e-13, 1.1725e-11, 6.1657e-13, 3.5727e-12, 1.3923e-10, 6.1332e-14,\n        8.0265e-12, 1.0327e-11, 2.2303e-10, 2.9762e-12, 8.4232e-11, 1.1550e-12,\n        5.1520e-11, 7.5099e-11, 2.5041e-14, 1.9781e-12, 7.4452e-11, 7.3123e-11,\n        2.2634e-10, 5.3216e-11, 3.1706e-12, 9.7125e-12, 1.1894e-11, 3.6360e-13,\n        1.6010e-12, 2.3215e-12, 7.1500e-11, 9.9388e-11, 3.7101e-11, 3.1124e-13,\n        1.9775e-13, 2.5617e-12, 9.1832e-12, 3.1131e-12, 4.6259e-10, 1.1163e-11,\n        7.2863e-13, 3.5815e-10, 1.3335e-12, 2.4968e-10, 1.3492e-11, 2.7967e-10,\n        4.0182e-13, 1.0354e-12, 2.8669e-12, 5.8123e-13, 2.5037e-13, 1.2118e-11,\n        1.2210e-11, 2.7176e-11, 4.7586e-11, 1.1713e-12, 4.0151e-11, 1.7609e-11,\n        2.8715e-10, 7.0807e-12, 7.5428e-13, 2.1819e-11, 1.0389e-10, 4.0214e-13,\n        1.8279e-11, 4.8919e-13, 4.6621e-12, 7.9157e-12, 5.2880e-12, 3.1053e-13,\n        7.3733e-14, 3.8121e-11, 4.1133e-12, 5.2377e-11, 2.3001e-12, 8.9567e-11,\n        1.9678e-12, 4.9195e-10, 1.2625e-12, 3.2754e-11, 5.4761e-12, 1.1138e-11,\n        1.2448e-12, 4.2122e-13, 1.1337e-11, 1.5900e-12, 1.8007e-10, 7.0099e-11,\n        6.2319e-11, 1.0222e-10, 3.2963e-12, 1.0641e-10, 1.0974e-10, 2.0877e-10,\n        4.3476e-12, 1.1160e-13, 1.1529e-11, 1.7923e-13, 1.4144e-11, 7.7689e-13,\n        3.2641e-12, 6.2096e-10, 1.5109e-10, 1.2744e-12, 7.5879e-13, 2.5072e-12,\n        2.2929e-11, 6.8015e-13, 5.9260e-12, 1.7259e-10, 3.5032e-13, 1.1753e-10,\n        2.0216e-10, 1.3767e-11, 1.6529e-12, 4.0446e-14, 1.9756e-10, 4.1606e-14,\n        1.5512e-12, 1.3135e-10, 3.9392e-13, 9.2260e-13, 6.2652e-12, 8.4449e-13,\n        8.4383e-11, 4.9024e-11, 2.4288e-11, 6.3371e-12, 4.4579e-13, 7.1907e-11,\n        1.4180e-12, 1.8612e-13, 2.6070e-10, 2.8633e-11, 1.0394e-10, 2.1617e-10,\n        5.3224e-12, 3.5313e-12, 6.7651e-12, 1.3577e-10, 2.9302e-12, 3.0922e-13,\n        2.9307e-12, 4.2355e-12, 9.4453e-13, 1.7510e-12, 6.8579e-13, 2.8190e-11,\n        4.5408e-12, 3.6418e-12, 2.1055e-13, 5.3508e-12, 1.0009e-10, 1.4596e-13,\n        1.7093e-11, 3.3370e-10, 8.8921e-13, 4.6722e-11, 2.9533e-12, 1.0573e-11,\n        3.9499e-13, 8.1998e-12, 4.4478e-13, 7.4979e-12, 1.3068e-12, 8.8330e-11,\n        7.9111e-13, 1.9330e-11, 4.4072e-13, 1.0136e-10, 7.1555e-11, 9.0159e-11,\n        7.6273e-13, 2.8028e-10, 3.5206e-11, 2.0370e-13, 7.0734e-11, 1.0103e-12,\n        1.1762e-11, 6.6262e-12, 1.6969e-10, 1.6512e-13, 3.8274e-10, 2.4040e-10,\n        1.5074e-10, 7.0353e-12, 1.0925e-11, 1.9034e-10, 6.7527e-11, 3.9155e-11,\n        1.7471e-12, 3.8941e-13, 5.5027e-13, 5.3039e-14, 9.5026e-11, 3.9970e-13,\n        7.0451e-13, 1.4736e-11, 2.2153e-13, 2.1607e-11, 4.1809e-12, 2.3047e-11,\n        4.2430e-11, 3.6836e-12], device='cuda:0')"
      },
      "70": {
        "step": "tensor(11268.)",
        "exp_avg": "tensor([ 8.3270e-39,  1.8070e-37,  5.2016e-38, -4.0043e-39,  1.3304e-39,\n         3.8293e-38,  5.7121e-39,  5.5233e-40,  8.5861e-39,  7.5788e-38,\n         4.0047e-39, -1.0398e-38,  5.2317e-38,  2.8463e-40,  3.0945e-40,\n         1.2396e-38, -1.8560e-39,  6.4649e-38,  1.1008e-37,  2.4845e-38,\n         1.1698e-37,  7.3044e-39, -3.9177e-39,  4.1015e-39,  3.9693e-38,\n         4.3170e-39,  1.1043e-38, -1.2541e-39,  3.5680e-38,  4.1418e-39,\n         1.1060e-40,  3.6664e-39,  7.2298e-38,  1.6382e-37,  6.2799e-38,\n         1.5462e-39,  4.5236e-39,  6.8642e-40, -4.2373e-39,  6.0301e-39,\n         1.4979e-39, -2.5304e-40,  2.5171e-39,  5.2495e-39, -6.9641e-40,\n         4.9114e-38, -7.9548e-41,  3.9993e-38,  9.7952e-38,  3.3006e-38,\n         8.4260e-40, -7.9644e-41, -1.5341e-39,  3.4097e-39,  2.1036e-38,\n         1.4836e-38,  9.0209e-38,  1.7149e-38,  1.2733e-38,  8.3160e-39,\n         3.6774e-39,  2.6737e-37,  7.0662e-38,  1.3530e-38,  1.5237e-38,\n        -2.2838e-39,  3.4512e-39,  1.8239e-38,  1.0795e-39,  1.9702e-38,\n         2.2141e-37,  1.1865e-38,  3.5303e-38,  3.8317e-38,  1.0428e-37,\n         3.9917e-38,  8.9538e-40,  4.2864e-38, -9.4871e-40,  3.9653e-39,\n        -1.8965e-39,  3.0363e-39,  5.9848e-39,  8.0472e-39,  7.3857e-39,\n         1.1310e-37, -1.3526e-39,  1.3438e-37,  1.5632e-37,  4.7159e-40,\n         1.0237e-37, -9.1560e-40,  5.0850e-38,  2.1109e-38,  4.9432e-38,\n         2.5248e-38,  4.7274e-39,  1.8213e-38,  6.6251e-40,  4.2088e-38,\n        -2.5340e-40,  9.3095e-38,  1.2464e-39,  2.1982e-39, -1.4181e-40,\n         4.5126e-40,  1.1282e-37,  4.4021e-38,  1.6434e-37,  2.8674e-38,\n        -4.3785e-40,  5.8107e-38,  1.4576e-38,  4.7608e-39,  5.5798e-38,\n         3.2453e-38,  6.1548e-39,  1.6165e-38,  2.1697e-39,  3.7185e-38,\n         3.0682e-38,  9.8713e-39,  8.0489e-38,  1.1681e-37, -5.5631e-39,\n         2.5217e-39,  5.1976e-39,  5.3328e-39,  1.7522e-38,  1.7630e-39,\n        -1.4593e-40, -3.5764e-40, -8.9711e-42,  4.3068e-39, -3.0989e-39,\n        -1.2179e-39,  3.5001e-40,  1.0711e-39,  5.9500e-39,  1.0928e-38,\n         1.5259e-39,  6.4203e-40,  2.9339e-38,  1.0629e-39,  8.8360e-41,\n         3.6947e-39,  7.5744e-38, -2.9195e-39, -9.8806e-40,  1.6389e-38,\n         5.8590e-39,  7.9611e-38,  3.6636e-38,  1.6630e-39,  1.2608e-38,\n         5.2530e-41,  6.9840e-39,  7.3287e-39,  1.8404e-39,  2.3618e-38,\n         1.0834e-38, -2.0509e-39,  3.1813e-39,  2.0951e-38,  6.5608e-38,\n         1.1837e-38,  1.7780e-39,  2.6209e-39,  5.3177e-39,  1.4755e-38,\n        -1.6331e-40,  1.5471e-38,  8.2877e-38,  1.1083e-38,  6.7967e-38,\n        -1.7579e-39,  1.6784e-39,  3.0658e-39, -4.1962e-40,  3.4210e-38,\n         3.6012e-39,  1.0023e-37,  3.4067e-40, -1.0691e-39,  1.4332e-38,\n         2.6390e-39,  2.0133e-37,  7.1555e-39,  1.7555e-40,  2.5525e-39,\n         8.3135e-40,  9.3083e-39,  1.5484e-37,  5.1978e-39,  2.0973e-38,\n         1.5023e-38,  1.5550e-39,  7.8723e-38, -5.5131e-40,  2.2377e-39,\n         1.6532e-37,  4.0591e-38, -4.1200e-39,  2.4885e-39,  8.1520e-38,\n         5.5819e-40,  8.9446e-38, -3.8716e-40,  1.1894e-38,  8.0806e-38,\n        -1.0583e-39,  1.2381e-38,  4.7523e-38, -3.9590e-39,  2.6945e-37,\n         9.6827e-40,  6.7050e-38,  2.0196e-38,  1.0953e-38,  2.5025e-38,\n         6.4087e-39,  2.4264e-39,  7.9457e-40,  2.1096e-38,  6.5825e-38,\n         9.9453e-39,  1.0563e-38, -1.7366e-39,  1.2754e-37,  9.9441e-39,\n         1.4477e-37,  9.1683e-39,  7.8405e-39,  1.9149e-37, -6.7565e-40,\n         6.3119e-39,  2.2276e-37,  7.6984e-39,  1.2049e-38,  6.9061e-38,\n         3.0467e-38,  6.9194e-40, -2.5181e-40,  8.6222e-40,  1.4042e-37,\n         1.8613e-40,  1.4521e-38,  9.0615e-39,  1.6572e-40,  2.9549e-39,\n         3.5200e-40, -1.5487e-39,  1.5362e-38, -1.7816e-40, -1.4564e-39,\n        -6.2505e-40,  2.5756e-39,  2.4536e-38,  1.3086e-37,  2.8234e-39,\n        -3.6926e-40,  3.1822e-38,  3.9217e-39,  9.3751e-40,  7.8063e-38,\n         4.5815e-39,  1.6572e-37,  1.4457e-38,  5.9523e-39,  9.0948e-38,\n         2.1677e-39,  4.7146e-38,  1.0794e-39,  8.4699e-38,  1.6172e-39,\n         1.1138e-38,  3.5064e-38,  5.2101e-39,  3.1805e-40,  6.9804e-39,\n         4.2083e-39,  2.5268e-39, -2.4748e-40,  9.5085e-38,  7.8645e-38,\n         4.8107e-39,  7.3714e-39, -1.8987e-39,  8.2825e-39,  8.7167e-39,\n        -2.5286e-39,  7.0340e-38,  1.7495e-41,  1.3386e-39, -1.7382e-39,\n         9.0273e-39,  1.9182e-38,  8.4723e-38,  5.9812e-39, -8.0114e-40,\n         2.7866e-39, -4.5956e-40,  8.9945e-39,  1.1460e-38,  1.0100e-39,\n         3.3792e-39, -6.8037e-40,  2.9239e-38,  2.4786e-39,  2.0830e-37,\n         1.0128e-38, -4.2894e-40, -1.3463e-39, -2.8424e-39,  7.4279e-40,\n         1.7825e-38,  1.0652e-37,  1.2125e-38, -1.0721e-39, -1.5645e-39,\n         6.9765e-38, -2.4254e-39, -5.7723e-40,  2.5020e-39,  3.8044e-38,\n         1.3861e-37,  2.0366e-39, -1.1522e-38,  2.6317e-38, -2.6010e-40,\n         2.3160e-38,  1.0579e-38, -9.6108e-41,  7.1941e-38,  6.3990e-38,\n         2.9445e-39,  2.0971e-40, -2.8261e-39,  1.3443e-37,  1.2248e-37,\n        -5.1409e-40,  2.0353e-39, -1.9342e-39,  1.7618e-39,  4.6571e-39,\n         7.8247e-39,  1.5789e-38,  1.5575e-38,  1.8690e-39,  1.1320e-37,\n         2.5532e-39,  6.9639e-38,  1.2728e-37,  2.0095e-37, -2.0511e-39,\n         5.3069e-39,  2.2018e-40,  4.1961e-39, -5.4919e-39,  6.4666e-40,\n         3.1146e-38,  1.7089e-38,  2.8818e-38, -4.6122e-39,  2.8491e-38,\n         7.7235e-38,  8.0346e-38, -3.2593e-39,  2.4177e-39,  5.9569e-38,\n         3.8247e-38,  4.1088e-40, -1.3170e-39,  1.1560e-39, -6.2205e-41,\n        -9.0695e-40, -1.6805e-39,  1.1243e-38, -2.0748e-40,  1.6623e-38,\n         8.1165e-39,  1.1244e-37,  1.2796e-39,  1.2852e-37, -2.0918e-39,\n         1.7929e-39,  7.9949e-40,  7.0840e-38, -2.6081e-39,  8.9153e-41,\n         3.5928e-39,  8.3126e-39,  3.5009e-39,  6.6434e-41, -1.3126e-39,\n         2.1227e-38,  1.8164e-37,  3.1374e-38,  1.1148e-40,  4.4832e-38,\n         1.4581e-37,  3.6397e-38,  2.9909e-40,  4.1432e-39,  3.4426e-39,\n         1.4012e-38,  4.5182e-38,  2.7324e-39, -1.6333e-40,  1.1396e-37,\n         1.2965e-37,  7.2988e-40,  4.3824e-39, -3.5004e-40,  8.5294e-38,\n         5.2463e-40,  3.1017e-39, -7.2922e-41,  1.6566e-39,  3.6953e-38,\n         1.1309e-37,  1.3413e-38,  5.0634e-39, -1.9406e-39,  8.0324e-38,\n         9.2741e-39,  4.3481e-39, -8.6644e-41,  6.6943e-39,  4.4481e-39,\n         5.5451e-38,  6.4947e-41,  2.0987e-39,  1.8775e-37,  1.1359e-37,\n         4.8392e-38,  1.0986e-38,  1.2934e-37, -2.6562e-39,  1.5164e-38,\n         2.5099e-38,  4.6155e-38,  2.3069e-38,  3.7736e-38,  3.0767e-40,\n         1.3680e-39,  5.0276e-39,  1.3474e-37,  7.8920e-39, -4.4396e-39,\n        -4.5389e-40,  4.1487e-40,  1.1714e-40,  1.0764e-39,  2.8635e-39,\n         1.4790e-38, -6.0826e-40,  6.1645e-40,  1.0298e-38,  3.7203e-39,\n        -2.1418e-40,  5.6954e-39,  2.7572e-38,  1.0874e-37, -1.8037e-39,\n         1.2161e-38,  6.0164e-38,  3.9421e-38,  1.1480e-38,  7.3788e-41,\n         4.2535e-39,  2.1713e-38,  7.9756e-39,  2.1461e-37,  1.0741e-38,\n         9.3639e-38,  2.3292e-39,  1.2883e-37,  1.1362e-37,  2.4950e-39,\n         4.9035e-39,  2.7270e-38,  4.8285e-39,  5.0003e-39,  3.7290e-39,\n         2.1537e-39,  1.0050e-38, -4.8405e-39,  1.0730e-37,  3.3349e-39,\n         1.5602e-37,  6.5773e-38,  1.5169e-38, -6.7214e-40, -1.7618e-39,\n         6.2053e-38,  2.9850e-38, -4.8718e-40, -4.4011e-39,  1.1285e-38,\n         3.9331e-39, -1.0346e-39,  7.0292e-39, -8.0014e-40,  9.1575e-41,\n        -1.2600e-39,  4.6642e-39,  9.1387e-40, -2.2510e-39, -8.8161e-40,\n         8.6391e-38,  1.3215e-39], device='cuda:0')",
        "exp_avg_sq": "tensor([9.6554e-16, 1.0396e-13, 8.7057e-15, 4.5583e-16, 6.6170e-17, 1.0417e-14,\n        9.0357e-17, 2.0301e-16, 6.7913e-17, 2.2423e-14, 2.3116e-17, 3.3651e-15,\n        3.1315e-15, 1.6789e-19, 3.4487e-18, 4.8780e-16, 2.8189e-17, 1.1646e-14,\n        2.9777e-14, 2.5058e-14, 7.5704e-14, 2.8650e-16, 1.9507e-16, 2.6202e-14,\n        1.5259e-14, 1.7641e-17, 1.1253e-17, 5.0713e-16, 9.6075e-15, 3.8829e-18,\n        9.8831e-17, 5.3290e-16, 7.3647e-15, 1.9405e-13, 2.7102e-14, 8.4304e-19,\n        2.0613e-16, 1.4390e-16, 1.5875e-17, 3.9184e-14, 1.1301e-16, 6.7627e-16,\n        6.6697e-16, 5.6001e-19, 8.8199e-19, 2.7322e-15, 8.5472e-18, 3.3983e-14,\n        1.5056e-14, 3.8374e-14, 6.1244e-16, 1.2954e-15, 6.6782e-19, 1.1097e-17,\n        9.2056e-15, 1.1290e-17, 2.8830e-14, 3.2154e-15, 7.2343e-16, 1.2400e-16,\n        8.4178e-16, 3.0212e-13, 6.9029e-15, 4.3099e-15, 4.9395e-14, 2.6463e-19,\n        8.0945e-15, 9.2899e-15, 5.6299e-19, 5.0607e-14, 7.4502e-14, 7.0864e-16,\n        9.3014e-15, 6.1107e-14, 2.3729e-14, 1.4981e-14, 4.7609e-16, 5.5548e-15,\n        1.0687e-14, 2.3719e-18, 8.6261e-17, 2.3711e-17, 1.7279e-17, 9.6720e-16,\n        5.2934e-17, 4.6651e-14, 9.5772e-18, 2.2289e-14, 1.3504e-13, 2.2306e-17,\n        1.2919e-13, 7.6510e-16, 1.3381e-14, 8.9163e-14, 2.2897e-15, 2.3388e-16,\n        3.4589e-16, 2.2631e-14, 9.3832e-18, 4.0680e-14, 7.1208e-19, 2.1627e-14,\n        9.8749e-18, 2.3416e-17, 2.7263e-18, 5.5517e-18, 1.0377e-13, 2.5660e-14,\n        6.4713e-14, 1.5281e-14, 6.6487e-17, 7.3004e-15, 7.8074e-15, 1.0838e-14,\n        9.5529e-14, 7.6387e-15, 1.9785e-19, 2.2960e-14, 5.7748e-17, 7.7871e-15,\n        5.7248e-14, 4.0743e-17, 1.0563e-14, 1.9974e-14, 4.7258e-18, 1.9448e-16,\n        1.4429e-15, 1.4488e-18, 2.0945e-14, 3.2713e-15, 7.4548e-19, 1.2805e-17,\n        4.9295e-16, 6.2770e-19, 4.3506e-17, 5.5964e-16, 4.8042e-18, 7.0992e-17,\n        2.5388e-16, 1.0694e-14, 2.0491e-14, 1.9214e-17, 2.0901e-14, 7.6332e-16,\n        2.9609e-19, 7.4022e-16, 5.1896e-14, 2.5830e-16, 1.8447e-16, 1.8076e-18,\n        1.2018e-14, 3.1586e-14, 1.7163e-14, 5.6628e-17, 1.2173e-18, 6.4726e-16,\n        3.7532e-16, 2.2840e-16, 1.5466e-18, 7.4141e-17, 3.2369e-16, 1.0483e-17,\n        5.2343e-19, 3.8354e-15, 1.0896e-14, 1.5674e-16, 8.9402e-15, 1.5682e-18,\n        3.6261e-16, 8.8554e-16, 8.8347e-14, 1.9363e-15, 5.9282e-14, 2.9968e-16,\n        5.2801e-14, 5.4225e-18, 4.2075e-18, 1.7618e-16, 3.7044e-17, 9.5218e-14,\n        3.3135e-17, 6.2755e-15, 4.5298e-16, 1.2556e-15, 6.5063e-16, 8.9891e-17,\n        3.4841e-13, 9.2955e-17, 7.6270e-18, 2.4834e-18, 1.8593e-16, 1.0974e-15,\n        9.7108e-14, 2.7058e-17, 6.2907e-16, 6.1005e-16, 2.2363e-18, 1.5979e-14,\n        1.9292e-15, 2.4231e-15, 3.2940e-14, 2.5016e-14, 1.5177e-16, 1.3202e-17,\n        5.1808e-14, 3.5420e-17, 4.1259e-14, 2.2261e-19, 1.8494e-14, 3.1637e-14,\n        8.6144e-16, 5.4463e-16, 8.9383e-15, 1.3706e-14, 5.3520e-13, 1.6633e-14,\n        1.3588e-14, 7.2350e-16, 7.1656e-14, 1.6726e-16, 5.7512e-18, 3.1399e-16,\n        2.4697e-16, 2.8311e-15, 1.5033e-14, 4.5536e-18, 1.1788e-16, 1.1775e-18,\n        9.4562e-14, 7.5361e-17, 5.6860e-14, 2.0555e-14, 2.3647e-14, 1.4058e-13,\n        3.3902e-18, 9.4645e-19, 2.2749e-13, 1.1700e-18, 9.3870e-16, 1.9714e-14,\n        1.3672e-15, 8.2667e-18, 1.6626e-18, 1.6640e-15, 5.4996e-14, 3.3791e-19,\n        9.4106e-17, 2.8319e-15, 7.3848e-17, 4.3919e-18, 7.2709e-17, 7.4023e-19,\n        1.9292e-13, 7.4975e-17, 2.2952e-19, 5.0941e-18, 6.3238e-19, 2.0089e-15,\n        1.2791e-13, 7.2029e-19, 8.4379e-18, 4.5700e-16, 1.0696e-17, 8.8422e-16,\n        4.0017e-14, 2.7732e-17, 3.5902e-14, 2.8960e-15, 2.2379e-16, 9.4140e-14,\n        1.5080e-16, 2.2254e-14, 4.2102e-18, 8.2064e-14, 8.0818e-19, 1.2885e-13,\n        3.9897e-14, 6.5619e-17, 2.6101e-15, 8.4185e-16, 1.0259e-16, 1.6355e-15,\n        1.3454e-17, 2.4923e-14, 6.2787e-15, 1.6415e-16, 9.0321e-14, 1.2664e-18,\n        1.7116e-18, 3.5221e-18, 1.1058e-15, 9.8919e-15, 6.1766e-16, 1.2609e-16,\n        6.1318e-16, 3.5201e-16, 2.3840e-16, 2.9706e-14, 2.9408e-18, 4.3077e-16,\n        3.0094e-18, 3.3885e-17, 1.9009e-16, 1.0895e-16, 5.7293e-15, 8.3413e-17,\n        6.5035e-20, 7.2915e-15, 2.2554e-17, 2.5759e-13, 5.6358e-15, 2.4972e-15,\n        3.3444e-18, 6.6726e-19, 5.8687e-19, 2.9991e-16, 6.9825e-14, 1.1497e-16,\n        1.6119e-17, 6.4314e-19, 4.1163e-15, 1.0281e-16, 3.8804e-17, 6.1167e-16,\n        1.3989e-14, 1.0191e-13, 2.5221e-18, 3.2989e-17, 9.6130e-15, 4.0053e-16,\n        3.8599e-14, 3.4268e-15, 1.2024e-21, 1.1711e-14, 4.8287e-15, 4.0157e-16,\n        1.9782e-16, 4.2938e-17, 8.1092e-14, 4.8653e-14, 5.6052e-18, 1.1348e-17,\n        1.6821e-16, 2.1551e-19, 4.3385e-16, 1.4798e-15, 7.8295e-14, 3.8131e-15,\n        2.4876e-16, 1.6276e-13, 2.3809e-17, 8.5506e-14, 1.7529e-14, 2.6596e-13,\n        1.6975e-18, 1.7013e-18, 7.7602e-16, 2.8933e-16, 3.0241e-17, 3.8933e-15,\n        2.5656e-16, 2.4553e-17, 4.1649e-15, 2.7152e-17, 1.0403e-15, 5.2065e-14,\n        3.6546e-14, 1.0956e-16, 1.3520e-18, 1.4385e-14, 5.2485e-14, 2.0918e-16,\n        4.8296e-18, 6.0455e-19, 1.0975e-15, 7.0136e-18, 1.1988e-17, 6.7954e-17,\n        4.8359e-17, 8.7027e-15, 1.4472e-16, 5.3150e-14, 7.0661e-16, 1.8814e-14,\n        6.5183e-19, 1.1653e-13, 4.7475e-19, 6.9410e-15, 6.8101e-18, 7.7614e-16,\n        1.7492e-18, 1.5327e-16, 6.9738e-20, 2.1313e-17, 4.7156e-15, 4.0543e-16,\n        1.4042e-13, 4.1792e-15, 5.7099e-16, 2.8783e-14, 6.1490e-14, 7.5525e-14,\n        1.3703e-15, 8.5262e-17, 9.8196e-17, 1.6126e-15, 1.1978e-15, 3.5072e-18,\n        1.3220e-17, 2.2835e-13, 7.5720e-14, 5.6602e-19, 1.9773e-16, 8.4347e-16,\n        1.4246e-14, 2.0366e-18, 2.5668e-16, 8.5409e-15, 1.3856e-16, 1.6317e-15,\n        3.8675e-14, 1.1043e-14, 1.9259e-18, 2.6737e-19, 8.3741e-14, 1.5261e-16,\n        5.0283e-16, 6.0547e-15, 1.1522e-18, 2.2006e-16, 5.6990e-15, 2.3673e-16,\n        4.2923e-15, 9.2086e-14, 1.8932e-14, 2.1670e-15, 5.2776e-18, 9.6786e-14,\n        1.0094e-17, 1.1163e-15, 7.5834e-14, 2.3151e-15, 1.6939e-14, 6.3473e-14,\n        9.3848e-18, 6.0034e-16, 2.7212e-17, 1.8008e-13, 1.0189e-15, 5.6277e-17,\n        7.4106e-16, 2.2395e-18, 1.5779e-19, 2.1165e-16, 2.2736e-18, 7.6706e-16,\n        3.3169e-19, 7.5983e-16, 4.3122e-16, 4.6097e-18, 8.3682e-16, 1.6485e-18,\n        8.6090e-17, 1.6912e-13, 1.5896e-18, 3.1726e-16, 6.6690e-15, 1.1137e-14,\n        1.5092e-16, 4.8714e-19, 1.4573e-17, 3.1517e-16, 7.6940e-16, 1.1123e-13,\n        3.7164e-16, 8.4107e-15, 1.9936e-16, 9.3226e-14, 3.0690e-14, 3.7283e-15,\n        2.6600e-16, 7.1427e-14, 2.9816e-16, 2.6282e-17, 9.8279e-16, 5.7090e-20,\n        5.2198e-17, 1.7125e-18, 4.3980e-14, 1.7037e-18, 3.3658e-13, 1.1519e-13,\n        1.3303e-14, 1.9424e-16, 4.7886e-19, 3.2373e-14, 1.9060e-14, 3.4160e-16,\n        2.5865e-17, 2.7298e-16, 1.1714e-19, 7.9237e-17, 6.8780e-15, 4.4882e-18,\n        3.6867e-16, 4.2063e-16, 1.6037e-17, 3.5974e-17, 2.4980e-16, 1.6064e-18,\n        7.1349e-15, 1.5298e-17], device='cuda:0')"
      },
      "71": {
        "step": "tensor(11268.)",
        "exp_avg": "tensor([-6.0303e-39,  1.0562e-37,  5.1428e-38,  2.1148e-38, -1.1566e-38,\n         3.5926e-38, -4.3081e-39,  2.7942e-38,  6.3561e-38,  7.3381e-38,\n        -2.2595e-39, -1.4260e-38,  5.7653e-38, -2.4318e-40,  2.3598e-38,\n        -7.9384e-39,  4.4172e-39,  6.4699e-38,  9.9691e-38,  3.3139e-38,\n         7.0465e-38, -5.0169e-39,  2.2860e-38,  1.2070e-38,  3.4043e-38,\n        -3.8574e-39, -9.0762e-39,  4.4398e-39,  4.5274e-38, -3.5573e-39,\n        -9.0143e-41,  2.6566e-38,  6.1321e-38,  9.5539e-38,  5.5673e-38,\n        -1.0574e-39, -3.8958e-39, -1.6249e-39,  1.6707e-38,  8.1381e-39,\n        -1.2581e-39,  1.9386e-39,  3.8382e-39, -4.4986e-39,  2.4560e-39,\n         7.1327e-38,  5.5438e-41,  3.3027e-38,  8.5523e-38,  4.3348e-38,\n         2.8444e-38, -3.7839e-40,  4.0392e-39, -2.8623e-39,  6.3040e-38,\n         4.2131e-38,  8.9875e-38, -1.0928e-38, -9.1014e-39, -6.8129e-39,\n         3.7409e-38,  1.3663e-37,  1.0274e-37, -7.5660e-39,  2.0519e-38,\n         5.9518e-39,  1.4865e-38,  2.4262e-38,  2.3639e-38,  3.5334e-38,\n         1.1819e-37, -8.1439e-39,  6.4681e-38,  4.3157e-38,  8.1371e-38,\n         6.4819e-38, -2.5188e-39,  7.9498e-38, -1.5831e-38, -3.6290e-39,\n         1.7066e-39, -2.5824e-39, -5.3685e-39, -5.6947e-39, -6.8211e-39,\n         9.8479e-38,  3.4776e-38,  1.0132e-37,  8.3371e-38,  4.9101e-42,\n         5.6325e-38,  5.2862e-39,  5.0179e-38,  3.9794e-38,  3.9734e-38,\n         5.9064e-38,  1.6376e-38,  5.0552e-38, -1.8862e-39,  3.8439e-38,\n         1.3170e-39,  9.3768e-38, -8.3662e-40, -1.8906e-39,  3.7658e-40,\n        -1.7737e-40,  6.6697e-38,  2.7088e-38,  1.2332e-37,  4.7819e-38,\n         1.4112e-39,  5.5057e-38,  2.1941e-38,  1.2375e-38,  6.3580e-38,\n         5.1308e-38, -5.5037e-39,  4.0358e-38, -1.8206e-39,  5.5501e-38,\n         5.9805e-38, -8.4763e-39,  6.3848e-38,  9.3765e-38,  4.3153e-39,\n        -2.0528e-39,  3.0319e-38, -4.7218e-39,  3.0357e-38,  7.5588e-39,\n         4.7591e-40,  1.9601e-40,  5.4605e-40, -3.6243e-39,  1.9018e-38,\n         2.0229e-38,  3.6075e-38,  1.4764e-38,  1.7271e-38,  1.4071e-38,\n         2.7095e-39, -3.1639e-40,  3.1200e-38,  1.5605e-39, -2.0843e-40,\n         5.5694e-39,  4.6313e-38,  1.1000e-38,  4.5298e-39,  4.3064e-38,\n         1.3909e-38,  7.4927e-38,  6.6191e-38, -1.3644e-39,  7.0748e-38,\n         3.9130e-40, -5.7957e-39, -5.9240e-39, -1.5715e-39,  5.8250e-38,\n        -8.2682e-39,  1.5428e-38, -2.9162e-39,  3.2815e-38,  5.3042e-38,\n         3.0692e-38,  1.2558e-38, -2.2588e-39,  1.7595e-38, -9.3441e-39,\n         3.1574e-40,  3.2752e-38,  5.4590e-38, -8.4680e-39,  4.8758e-38,\n         6.4295e-39, -8.6543e-40,  2.7073e-38,  4.0163e-40,  2.4514e-38,\n        -3.1658e-39,  8.3722e-38,  1.0212e-39,  2.7816e-38, -9.4550e-39,\n        -2.0585e-39,  1.0759e-37, -5.6736e-39,  8.5090e-40,  3.3794e-38,\n        -2.5803e-40,  5.1468e-38,  8.5314e-38, -4.4866e-39,  3.3801e-38,\n        -9.5752e-39, -1.1163e-39,  5.7834e-38,  3.5237e-40,  3.8716e-39,\n         9.3465e-38,  3.1607e-38,  2.5740e-38,  3.5349e-38,  4.4859e-38,\n        -5.3462e-40,  6.7130e-38,  1.1377e-39,  3.0848e-38,  7.6113e-38,\n         7.5444e-40, -9.0513e-39,  5.8986e-38, -8.4132e-39,  1.6501e-37,\n         4.9645e-39,  6.0376e-38,  5.0631e-38,  1.6505e-38,  4.2978e-38,\n        -5.3539e-39, -7.9908e-40, -5.8790e-40,  3.5590e-38,  5.9436e-38,\n         3.8659e-38, -8.1388e-39,  5.5920e-39,  9.8074e-38, -8.1274e-39,\n         7.5144e-38,  1.4163e-38,  1.7372e-38,  1.4337e-37,  2.0583e-38,\n        -5.2083e-39,  1.4616e-37,  4.0059e-38, -8.3501e-39,  6.1576e-38,\n         5.1134e-38,  2.1883e-38,  1.5703e-39,  7.8486e-39,  1.0152e-37,\n         2.3578e-40,  3.2676e-38,  2.7178e-38, -7.1039e-40, -2.5521e-39,\n        -3.3272e-40,  4.4989e-39,  2.1904e-38,  1.1496e-39,  4.1124e-39,\n         2.7181e-39, -2.0068e-39,  4.2864e-38,  6.6610e-38, -2.4874e-39,\n         1.6627e-39,  5.5398e-38, -3.4013e-39, -6.2297e-40,  6.9593e-38,\n        -4.0426e-39,  1.2220e-37,  4.9908e-38, -4.8157e-39,  6.2183e-38,\n        -1.6998e-39,  6.4911e-38, -8.7137e-40,  1.0747e-37, -1.1662e-39,\n         1.3733e-38,  3.8716e-38, -4.2769e-39,  7.0820e-39,  3.2034e-38,\n        -1.7382e-38,  4.1638e-39,  1.8495e-38,  8.0294e-38,  8.5444e-38,\n         3.1105e-38,  7.9061e-39,  5.3892e-39, -7.0326e-39,  3.8340e-38,\n         1.6983e-38,  8.2075e-38,  9.0433e-40, -1.0566e-39,  1.2301e-38,\n        -7.1295e-39,  4.1593e-38,  7.8201e-38, -5.4330e-39,  4.9585e-39,\n        -2.5906e-39,  1.6878e-39, -7.1465e-39, -9.3275e-39,  1.8730e-38,\n        -2.7589e-39,  1.8998e-39,  4.6685e-38,  8.8299e-39,  1.3458e-37,\n        -5.7168e-39, -6.5507e-39,  5.1250e-39,  1.2822e-38, -5.9050e-40,\n         3.7125e-38,  7.0007e-38, -8.7674e-39,  5.5506e-39,  1.1983e-38,\n         1.1215e-37,  1.9680e-38,  1.4497e-39, -1.6883e-39,  4.2298e-38,\n         8.2181e-38, -1.9036e-39,  1.0244e-38,  3.0419e-38,  5.8431e-39,\n         4.0187e-38,  5.0289e-38,  1.2713e-40,  5.5402e-38,  9.5424e-38,\n        -2.1528e-39,  1.7470e-40,  1.0889e-38,  9.1292e-38,  9.2394e-38,\n         1.2352e-39, -1.9554e-39,  4.6855e-39, -1.1967e-39,  1.0550e-38,\n        -5.4501e-39,  2.8517e-38,  1.5567e-38, -7.9771e-40,  6.2841e-38,\n         2.6243e-38,  4.6174e-38,  9.5676e-38,  1.0734e-37,  5.8632e-39,\n        -4.4940e-39,  6.4838e-40, -3.3623e-39,  2.5631e-38, -2.6898e-40,\n         5.4273e-38,  6.1586e-38,  3.4717e-38,  1.9337e-38,  8.9573e-38,\n         4.4843e-38,  8.1738e-38,  1.3201e-38, -1.9180e-39,  6.3655e-38,\n         3.6998e-38,  3.5766e-40,  5.7438e-39,  8.4437e-41,  3.2812e-40,\n         2.9914e-39,  3.1208e-38, -9.4037e-39,  1.9133e-40,  3.7116e-38,\n         2.9439e-38,  8.9834e-38,  6.2523e-40,  1.2470e-37,  8.4160e-39,\n         2.9420e-39,  3.6001e-41,  7.0157e-38,  2.4817e-38,  1.3980e-38,\n        -2.8822e-39, -6.8480e-39, -2.4772e-38, -3.3109e-40,  1.8209e-38,\n         2.0765e-38,  9.0517e-38,  2.8800e-38,  7.6868e-41,  5.8367e-38,\n         9.9981e-38,  3.6789e-38,  1.0107e-40, -3.4460e-39,  1.6275e-38,\n        -8.5945e-39,  6.4624e-38, -2.2765e-39,  3.2570e-38,  6.9908e-38,\n         8.7357e-38,  6.6716e-41, -3.4751e-39,  1.8791e-39,  6.5880e-38,\n         6.3961e-40,  1.2612e-38,  8.1151e-39, -1.5455e-39,  6.9929e-38,\n         1.0044e-37,  1.9580e-38, -4.4140e-39,  7.6505e-39,  5.0843e-38,\n        -6.7494e-39, -3.5950e-39,  1.2645e-39, -5.3565e-39, -3.4217e-39,\n         6.8638e-38,  1.3127e-39,  2.3768e-38,  1.0074e-37,  8.8782e-38,\n         5.7344e-38, -8.9531e-39,  9.1805e-38,  8.0704e-39,  5.3076e-38,\n         2.8653e-38,  5.6856e-38,  3.3739e-38,  4.9357e-38, -8.6958e-41,\n        -1.0169e-39,  6.0337e-38,  7.1255e-38, -5.3509e-39,  3.8169e-39,\n         6.1316e-40, -8.1676e-41,  2.2522e-41, -1.9164e-40, -2.5302e-39,\n         2.9934e-38,  2.2100e-39, -8.0330e-40, -7.1745e-39, -2.8545e-39,\n         4.6635e-39, -4.7821e-39,  8.3182e-38,  6.2662e-38,  5.4866e-39,\n         3.6431e-38,  5.7964e-38,  2.8433e-38, -9.8668e-39, -1.6967e-39,\n        -3.6248e-39,  3.9023e-38, -5.6184e-39,  1.0695e-37, -8.3507e-39,\n         8.7855e-38, -2.0086e-39,  8.4614e-38,  7.7096e-38,  4.5942e-39,\n        -3.8933e-39,  3.6571e-38,  1.6724e-38, -4.0652e-39,  6.6715e-39,\n        -1.8744e-39,  3.2566e-38,  1.2354e-38,  8.2961e-38, -2.8274e-39,\n         7.7097e-38,  3.5270e-38,  1.3013e-38,  1.1081e-38,  4.3519e-39,\n         5.7077e-38,  2.8651e-38,  5.1867e-39,  3.7749e-39, -8.1513e-39,\n        -3.6422e-39,  7.7275e-39,  4.1713e-38,  6.9240e-40, -5.5164e-40,\n         1.6692e-38, -4.2156e-39,  9.3929e-39,  1.8043e-39,  2.5552e-39,\n         9.0591e-38,  2.9676e-38], device='cuda:0')",
        "exp_avg_sq": "tensor([4.3243e-16, 5.1241e-14, 1.5812e-14, 7.1995e-15, 2.8202e-15, 4.6632e-15,\n        3.6026e-17, 9.1110e-16, 1.0878e-14, 4.7574e-14, 1.2255e-17, 9.1925e-15,\n        1.2526e-14, 6.8235e-17, 1.4554e-15, 2.4834e-16, 5.5832e-15, 2.6878e-14,\n        4.9912e-14, 1.5626e-14, 4.0271e-14, 1.1868e-16, 4.6877e-15, 3.1471e-14,\n        1.4657e-14, 6.8208e-18, 5.8978e-17, 5.8056e-15, 8.1860e-15, 3.3095e-19,\n        2.4386e-18, 1.8002e-15, 6.9095e-15, 1.2087e-13, 2.7607e-14, 2.0696e-17,\n        1.0112e-16, 7.1196e-15, 1.8714e-16, 2.4816e-14, 5.1245e-17, 3.2777e-16,\n        4.8932e-15, 4.3654e-19, 1.6580e-15, 1.9771e-14, 8.4025e-16, 5.9297e-14,\n        2.4652e-14, 2.7038e-14, 3.9879e-15, 6.3607e-15, 1.2975e-15, 6.8261e-19,\n        1.8683e-14, 4.2868e-15, 8.9407e-14, 1.5159e-15, 1.1143e-16, 3.6442e-17,\n        2.7964e-15, 1.1906e-13, 8.7775e-14, 1.8861e-15, 3.4676e-14, 6.4506e-17,\n        1.6073e-14, 5.1200e-14, 1.0731e-15, 4.1867e-14, 3.4026e-14, 3.0384e-16,\n        8.0066e-15, 3.6494e-14, 2.4504e-14, 1.2187e-14, 1.7381e-16, 2.7237e-14,\n        3.7949e-14, 1.4469e-17, 1.9827e-18, 1.2161e-17, 3.3876e-18, 4.1268e-16,\n        2.1750e-18, 7.9489e-14, 1.1473e-15, 2.4744e-14, 6.0194e-14, 5.9538e-15,\n        5.6073e-14, 9.8509e-15, 2.8647e-14, 7.9266e-14, 9.1966e-16, 8.4122e-15,\n        4.9561e-15, 3.4520e-14, 5.7878e-16, 5.4222e-14, 6.1139e-16, 3.0457e-14,\n        4.6365e-18, 9.2983e-18, 8.2148e-16, 2.3811e-18, 6.2026e-14, 1.5716e-14,\n        8.4890e-14, 1.3713e-14, 6.1987e-15, 9.1082e-15, 5.8212e-15, 1.9786e-14,\n        5.6647e-14, 1.1285e-14, 2.0388e-17, 2.8831e-14, 2.2268e-17, 2.1462e-14,\n        6.4639e-14, 2.1540e-17, 1.1415e-14, 1.7417e-14, 1.7905e-18, 8.3592e-17,\n        3.3598e-15, 1.0001e-16, 2.4051e-14, 5.7641e-15, 3.0493e-19, 9.2219e-16,\n        2.4202e-16, 3.3835e-17, 7.3886e-16, 6.2658e-15, 8.5315e-16, 3.8300e-16,\n        6.1424e-16, 6.9294e-15, 1.7768e-14, 1.0664e-15, 2.2215e-14, 3.7452e-16,\n        2.9732e-16, 2.3458e-15, 3.5686e-14, 6.1952e-15, 7.0591e-15, 4.2748e-15,\n        1.3414e-14, 5.4942e-14, 1.5646e-14, 3.5310e-15, 3.0305e-15, 2.8605e-16,\n        1.7415e-16, 1.1524e-16, 4.2787e-19, 6.2685e-15, 1.6104e-16, 6.0879e-16,\n        7.2457e-17, 2.4439e-15, 1.7052e-14, 3.1845e-15, 2.7786e-14, 1.5151e-16,\n        6.8878e-16, 4.0741e-16, 8.9148e-14, 3.9832e-15, 4.9272e-14, 5.9798e-17,\n        4.6339e-14, 3.5896e-16, 3.5206e-16, 1.1033e-15, 1.7428e-17, 9.7888e-14,\n        3.7019e-17, 9.8090e-15, 2.2153e-16, 5.4791e-15, 3.2004e-16, 3.7787e-17,\n        1.6146e-13, 3.1715e-17, 9.8107e-16, 1.6539e-15, 8.3498e-17, 2.1613e-15,\n        4.2185e-14, 8.9684e-18, 1.1992e-14, 3.0191e-16, 6.1286e-16, 8.9633e-15,\n        8.7151e-16, 2.0735e-15, 1.3560e-14, 2.3017e-14, 1.4821e-15, 1.6957e-16,\n        2.6195e-14, 4.4604e-17, 4.6548e-14, 2.1937e-17, 2.6336e-14, 6.8185e-14,\n        4.2186e-16, 2.8618e-16, 2.6392e-14, 1.4589e-14, 3.9000e-13, 2.4100e-14,\n        2.8681e-14, 7.1593e-15, 5.1331e-14, 2.0748e-16, 2.0156e-18, 1.4417e-16,\n        1.0906e-16, 3.1096e-15, 2.6218e-14, 1.6473e-15, 4.4501e-17, 1.2967e-16,\n        1.2493e-13, 1.9925e-17, 2.7025e-14, 1.6695e-14, 2.2829e-14, 1.6074e-13,\n        7.9877e-16, 1.9205e-19, 1.8012e-13, 2.5535e-15, 3.5965e-16, 8.6700e-15,\n        2.2945e-14, 1.5396e-15, 5.1230e-16, 5.9081e-15, 4.8938e-14, 2.4265e-17,\n        6.4757e-15, 3.9753e-15, 2.1170e-17, 5.3500e-19, 8.5374e-18, 1.1642e-16,\n        1.4681e-13, 3.7421e-17, 3.1393e-16, 2.5171e-15, 8.3282e-17, 3.4107e-15,\n        4.9370e-14, 1.5674e-17, 5.8323e-16, 8.2509e-15, 5.4384e-18, 3.9284e-16,\n        6.7507e-14, 2.5158e-18, 2.6563e-14, 5.9080e-15, 1.0989e-16, 8.3609e-14,\n        7.1811e-17, 1.3992e-14, 1.0585e-17, 8.2131e-14, 1.3607e-16, 7.9799e-14,\n        1.7122e-14, 2.5063e-17, 8.2576e-15, 3.5038e-15, 5.8554e-15, 3.9371e-15,\n        1.1112e-15, 3.7430e-14, 2.5881e-14, 4.7495e-15, 3.4819e-14, 7.8412e-17,\n        7.5987e-17, 3.2856e-15, 1.1096e-14, 4.3712e-14, 3.0480e-16, 5.5172e-17,\n        4.2202e-15, 1.5945e-16, 4.2722e-15, 1.6553e-14, 7.1015e-18, 7.6465e-15,\n        9.1530e-18, 1.5023e-17, 9.2919e-17, 2.0497e-17, 2.3337e-14, 3.5951e-17,\n        3.2023e-16, 9.7291e-15, 7.3907e-15, 1.7542e-13, 2.0707e-15, 9.0151e-15,\n        2.2550e-16, 9.2124e-17, 5.9620e-19, 4.8564e-15, 6.2584e-14, 6.1737e-17,\n        9.8424e-16, 9.4745e-17, 6.1530e-14, 1.2177e-15, 1.2191e-14, 2.7548e-16,\n        5.8970e-15, 5.1196e-14, 6.6942e-19, 1.1595e-16, 5.3751e-15, 5.0576e-15,\n        4.1114e-14, 1.8773e-14, 7.0292e-17, 1.1523e-14, 1.1914e-14, 1.8136e-16,\n        8.8660e-17, 1.3760e-15, 4.5513e-14, 4.4415e-14, 4.8891e-15, 9.4740e-19,\n        6.0695e-17, 1.1480e-16, 3.8820e-15, 7.4552e-16, 8.2451e-14, 1.0403e-14,\n        1.1243e-16, 9.4727e-14, 1.1755e-15, 5.8529e-14, 1.8385e-14, 1.1277e-13,\n        1.9012e-16, 1.0361e-16, 3.7572e-16, 1.4349e-16, 3.9219e-16, 1.7620e-15,\n        5.4476e-15, 9.5576e-15, 1.0430e-14, 1.2448e-15, 1.6826e-14, 2.6446e-14,\n        8.0334e-14, 3.6335e-15, 1.3117e-17, 7.1835e-15, 2.4032e-14, 1.0040e-16,\n        2.8713e-16, 8.2653e-17, 4.4042e-16, 1.5685e-15, 4.0519e-15, 2.8024e-18,\n        5.6112e-18, 1.4602e-14, 1.6465e-15, 3.7192e-14, 3.1343e-16, 4.1100e-14,\n        5.1643e-16, 7.6136e-14, 3.5510e-17, 1.9981e-14, 1.7582e-16, 3.7000e-15,\n        1.1701e-17, 4.3841e-17, 3.1591e-16, 1.0039e-17, 3.2632e-14, 2.3372e-16,\n        4.7779e-14, 2.2025e-15, 2.4695e-16, 1.9902e-14, 5.3222e-14, 3.3056e-14,\n        6.6940e-16, 1.4504e-17, 4.3682e-16, 7.0134e-16, 8.8545e-15, 2.8542e-16,\n        1.3943e-15, 1.6957e-13, 6.2348e-14, 2.3270e-17, 9.3350e-17, 3.9965e-16,\n        1.7580e-14, 1.7808e-16, 1.0936e-15, 2.4375e-14, 2.6737e-17, 3.7021e-14,\n        7.0824e-14, 1.0526e-14, 2.6583e-16, 3.2035e-16, 5.1359e-14, 6.6465e-17,\n        2.4313e-16, 2.0604e-14, 7.4096e-17, 1.0759e-16, 4.2431e-15, 1.2071e-16,\n        1.2627e-14, 4.0117e-14, 2.1751e-14, 9.2210e-15, 4.8949e-17, 4.6786e-14,\n        1.0536e-15, 3.4349e-15, 3.9457e-14, 1.2654e-14, 1.1497e-14, 3.9173e-14,\n        4.7042e-16, 2.6741e-16, 3.9054e-15, 7.4201e-14, 4.9879e-16, 8.4626e-18,\n        3.6876e-16, 1.8448e-16, 3.9660e-20, 9.8434e-17, 7.2950e-17, 6.9848e-15,\n        5.5417e-16, 4.1360e-16, 2.1536e-16, 1.5990e-18, 7.7487e-15, 7.4394e-17,\n        9.6499e-15, 1.0045e-13, 1.6420e-16, 5.5261e-16, 1.0129e-14, 5.5499e-15,\n        1.3002e-17, 7.2279e-17, 2.4260e-17, 5.2873e-15, 3.9121e-16, 4.7309e-14,\n        1.6294e-16, 1.6285e-14, 3.7626e-17, 5.7146e-14, 2.7195e-14, 3.8926e-15,\n        1.3291e-16, 4.0385e-14, 1.4338e-15, 1.8740e-17, 9.8761e-16, 1.0037e-19,\n        2.6537e-15, 2.7284e-15, 5.7777e-14, 2.5028e-16, 1.3058e-13, 5.2232e-14,\n        5.8684e-15, 1.3937e-15, 1.0421e-15, 6.8823e-14, 3.5501e-14, 2.7145e-15,\n        1.4801e-16, 1.0699e-16, 3.4409e-17, 1.2258e-16, 2.0190e-14, 3.4412e-18,\n        1.7304e-16, 3.7421e-15, 9.4482e-19, 2.6551e-16, 1.1633e-16, 1.0342e-15,\n        2.2061e-14, 2.2758e-15], device='cuda:0')"
      },
      "72": {
        "step": "tensor(11268.)",
        "exp_avg": "tensor([[ 3.9620e-40,  2.6890e-40,  5.7648e-41,  ...,  3.0595e-41,\n          5.4144e-40, -8.7412e-39],\n        [ 1.0886e-40,  1.4941e-39,  3.2311e-40,  ...,  4.3265e-40,\n          2.2622e-40, -1.7144e-39],\n        [ 3.5936e-41, -4.3930e-39,  4.0935e-40,  ...,  2.9465e-40,\n          2.0876e-40,  6.3892e-39],\n        ...,\n        [ 2.1870e-41, -5.1660e-41, -1.0723e-40,  ..., -1.3480e-40,\n         -1.7558e-41,  5.3013e-40],\n        [-3.6125e-42, -7.2166e-40,  3.1057e-41,  ...,  3.6923e-41,\n         -2.2770e-41,  5.4797e-40],\n        [ 4.3834e-40,  4.4106e-39,  3.2919e-40,  ...,  6.4417e-40,\n          4.0309e-40, -6.9104e-39]], device='cuda:0')",
        "exp_avg_sq": "tensor([[8.9864e-16, 1.3248e-16, 1.4164e-16,  ..., 8.3300e-16, 1.9022e-16,\n         1.1340e-15],\n        [4.3866e-18, 7.1487e-19, 3.1071e-19,  ..., 3.3519e-18, 3.5868e-19,\n         9.6296e-18],\n        [3.9031e-17, 3.2395e-18, 9.8554e-18,  ..., 4.4820e-17, 1.0420e-17,\n         6.6124e-17],\n        ...,\n        [6.3591e-18, 1.0022e-17, 1.3294e-18,  ..., 7.5699e-18, 3.4290e-18,\n         3.8484e-18],\n        [1.1171e-18, 2.1719e-19, 5.1945e-20,  ..., 9.2021e-19, 1.1483e-19,\n         3.9933e-19],\n        [1.2620e-15, 1.1732e-16, 9.0464e-17,  ..., 5.1056e-16, 2.1473e-16,\n         5.2539e-16]], device='cuda:0')"
      },
      "73": {
        "step": "tensor(11268.)",
        "exp_avg": "tensor([ 9.7167e-37,  6.8790e-37,  6.8129e-37,  1.1988e-36, -3.0927e-36,\n        -4.0076e-37,  6.4780e-37, -4.9484e-38,  2.7961e-36,  7.1402e-37,\n        -4.6676e-37, -3.3070e-36,  3.6753e-37, -2.4888e-37, -3.4056e-37,\n         1.4419e-36,  3.1950e-36, -2.0577e-36,  1.2373e-36, -1.3555e-37,\n         2.5989e-37, -1.4090e-36, -2.3198e-37, -1.7464e-36, -3.4540e-37,\n        -3.8713e-37,  2.5109e-36,  5.5265e-37, -2.3566e-37, -8.0799e-38,\n        -2.8196e-36, -2.8622e-37, -4.4229e-37, -2.6825e-37,  5.3327e-37,\n        -6.1640e-37,  9.4205e-37, -3.5482e-38,  7.7598e-37, -1.3422e-37,\n        -6.0399e-38, -1.5418e-37, -2.8333e-37,  2.0883e-37,  2.8055e-37,\n        -1.2515e-36, -3.8938e-37,  7.2906e-37,  8.8019e-37, -3.2128e-37,\n        -1.3899e-37, -2.4398e-37, -2.9454e-37, -3.1615e-36, -2.9392e-37,\n         2.6829e-36,  4.2614e-36,  2.0774e-36,  4.4078e-36, -1.8158e-37,\n        -2.3922e-38, -8.1205e-37,  6.6422e-36,  2.1069e-36, -4.5323e-37,\n         1.3655e-36, -2.1568e-37,  5.7318e-37,  6.8545e-37, -2.2555e-36,\n         9.6550e-37, -1.1064e-37, -8.7606e-37,  7.1344e-37,  4.0231e-37,\n         8.9046e-38,  1.8109e-37, -2.3677e-37, -2.9155e-36, -3.2470e-37,\n        -1.2904e-36, -6.6999e-37, -2.9770e-37, -1.0120e-36,  6.1835e-37,\n         9.0295e-37,  1.7430e-36,  1.5196e-36,  1.1878e-36, -1.8234e-37,\n         6.0707e-37, -2.8575e-36, -1.8186e-37, -1.0782e-36, -1.9424e-36,\n         6.1003e-37, -4.1603e-37, -3.2667e-37, -4.2018e-37,  4.8784e-37,\n        -3.7091e-37, -4.3729e-38,  2.7394e-38, -1.2058e-36, -1.1408e-36,\n        -1.3228e-37,  3.1008e-36, -3.7900e-39,  8.0540e-37, -7.5349e-37,\n        -2.2249e-36, -4.4389e-37, -1.7357e-36,  3.6678e-37, -1.9184e-37,\n         4.1600e-38, -8.0689e-37, -3.0066e-37, -1.3578e-36,  1.0794e-36,\n        -6.2130e-37, -7.3521e-38, -1.3296e-36, -1.4143e-37, -3.1837e-36,\n        -8.6216e-38, -7.0140e-37,  1.6529e-36, -1.5666e-36, -4.0540e-37,\n        -2.6884e-37, -2.1440e-36, -1.7119e-37, -2.9855e-37,  8.2077e-37,\n         8.7970e-37,  1.6727e-36,  6.1733e-37,  6.8866e-38, -2.2445e-37,\n        -9.0535e-37, -3.6550e-38, -1.2662e-37,  9.9884e-37, -1.6818e-37,\n         2.7004e-37,  5.1765e-37, -9.6787e-38, -5.6719e-37, -3.2522e-37,\n        -2.0713e-37,  7.1006e-37,  4.1923e-37, -1.2291e-36,  3.7847e-37,\n        -2.1240e-37,  5.5262e-37,  4.3109e-37,  1.2456e-37, -3.2274e-37,\n        -1.4855e-37,  5.2730e-37, -2.6405e-38, -1.3154e-36,  2.4828e-37,\n        -7.8259e-37, -6.0752e-38, -3.7730e-37, -6.5570e-38,  1.6871e-36,\n        -4.1890e-37,  5.2461e-37,  2.5226e-36,  6.4184e-37, -6.6883e-38,\n        -3.8168e-37, -9.5610e-38, -3.1172e-38, -1.7989e-37, -1.6494e-37,\n        -3.6822e-37,  4.6771e-38, -1.7095e-36, -5.0155e-37,  9.2129e-38,\n        -3.1331e-37,  3.7872e-36, -3.3931e-36, -2.1143e-36, -1.8294e-37,\n        -5.6673e-37, -1.7072e-37, -3.0551e-37, -6.6536e-37,  4.1066e-37,\n         2.4707e-36, -7.9931e-39, -6.7002e-37, -8.2265e-37,  3.8519e-39,\n         5.3765e-38, -9.0250e-37,  3.7534e-37, -5.4376e-37,  4.0891e-38,\n         6.7785e-37,  1.1571e-36, -1.8419e-37, -2.3851e-37, -1.4200e-36,\n        -3.0767e-36,  3.0010e-36,  3.0855e-37,  2.2716e-38,  4.3011e-36,\n        -1.1528e-36,  3.0008e-37, -6.3400e-37, -8.3063e-37,  1.2397e-38,\n        -1.1610e-37, -1.7676e-36, -6.7369e-37, -2.7768e-36,  1.5276e-36,\n         3.2406e-37,  3.6995e-37,  1.1542e-37,  1.4244e-36,  1.6032e-37,\n        -1.7241e-37, -3.2117e-37, -2.6805e-37,  1.0101e-36,  6.2390e-37,\n        -7.8675e-37,  3.2423e-36, -1.6067e-37,  4.0563e-37, -2.6924e-37,\n         3.8112e-36, -7.2561e-38, -2.2917e-36, -8.4124e-37,  1.4364e-36,\n        -4.8684e-37,  2.3339e-36, -5.1003e-37,  6.0313e-38, -4.9628e-37,\n        -1.1201e-36,  6.2575e-37, -4.2539e-37, -3.2135e-37,  8.9403e-38,\n         3.1344e-37, -4.2497e-37, -1.4479e-36,  1.5512e-37, -2.9587e-37,\n         4.9257e-37, -5.6847e-37, -1.3894e-36, -2.1571e-36, -1.2970e-36,\n        -6.9037e-38, -9.3240e-37, -2.4110e-38, -1.6300e-37, -2.6684e-37,\n        -9.7486e-37, -4.6405e-38, -1.8262e-37,  3.2536e-37, -7.4791e-37,\n        -1.9219e-37, -2.6156e-37, -1.0001e-36, -6.5774e-37, -3.4107e-37,\n        -7.8080e-37, -5.2756e-37,  2.6969e-37,  2.6007e-36,  6.5217e-37,\n        -2.0286e-37, -2.3522e-37,  4.8835e-38,  4.0258e-36,  7.8675e-38,\n        -7.3903e-37,  7.1007e-37, -1.1276e-36, -5.3791e-38, -2.5869e-36,\n        -9.4833e-37, -1.7657e-37,  1.4642e-36,  2.3392e-36,  3.0788e-37,\n        -3.4682e-37, -1.0087e-36,  3.7037e-37,  1.5718e-36, -2.8962e-37,\n        -3.1665e-37, -2.2831e-37, -1.0275e-37, -7.4070e-37, -4.2568e-37,\n        -1.2238e-36,  4.2390e-37, -5.1978e-37, -5.2326e-37, -5.5976e-37,\n        -1.6115e-37,  1.0897e-36,  3.4854e-37, -7.0224e-37, -9.4756e-37,\n         8.6099e-37,  8.2957e-37,  1.0659e-37,  8.5288e-37, -6.0971e-38,\n         9.7670e-37, -3.2397e-37,  4.5142e-37, -1.8567e-37, -2.2660e-37,\n        -5.4268e-37,  5.7759e-37,  2.1435e-37, -2.9508e-37,  1.6806e-36,\n         2.5462e-36, -4.2948e-37, -1.8908e-37,  3.3710e-36, -8.8576e-37,\n        -2.7957e-37, -2.3964e-37, -2.7053e-36, -1.0358e-36, -9.6533e-37,\n         3.5247e-37, -3.4998e-37, -3.4711e-37,  3.4904e-37,  3.9775e-37,\n         2.8896e-36, -4.7075e-38, -3.4356e-37,  2.5492e-36,  2.9133e-37,\n        -4.2312e-37, -2.8800e-37,  6.8772e-37,  4.0532e-36, -1.1494e-36,\n         8.4154e-37,  3.4668e-36, -1.5682e-36,  1.4587e-36,  1.4303e-36,\n         2.0981e-37,  2.9485e-37, -4.2665e-38, -9.1948e-39, -2.2927e-37,\n        -1.2666e-36, -2.8665e-37,  2.3640e-37, -3.8768e-37, -2.6451e-37,\n         8.3437e-38,  1.8757e-36,  9.5069e-37, -5.4139e-37, -1.0706e-36,\n         1.0267e-37,  1.5594e-36, -6.1177e-37, -3.4134e-38, -2.3382e-36,\n        -1.0291e-37,  1.2380e-36,  2.9241e-36,  3.5315e-37, -5.8766e-38,\n        -3.7802e-36,  3.1102e-38,  5.7128e-37, -4.6338e-38, -2.4327e-37,\n        -1.1670e-36, -1.6764e-38, -2.8865e-37,  1.4385e-37,  1.0635e-36,\n        -2.5628e-37, -1.2300e-36, -4.5335e-37, -1.1044e-36, -8.3858e-37,\n         1.8246e-37,  4.0925e-37, -2.3622e-37, -8.5112e-37,  2.5922e-36,\n        -3.2585e-38, -5.3625e-39, -2.5794e-37, -2.2323e-37,  4.1589e-38,\n         6.8015e-38, -4.1745e-37, -2.3867e-37, -2.8770e-37,  7.4079e-37,\n         2.4758e-36, -3.3814e-37,  5.3743e-37,  8.6557e-37,  4.2488e-37,\n         2.0428e-37, -6.2797e-38, -7.2232e-38, -1.0334e-37,  2.0937e-36,\n        -2.7482e-37, -1.6611e-37,  2.5078e-37,  1.8496e-37, -7.3570e-38,\n        -7.4375e-37,  3.6730e-38,  2.5209e-36, -3.0083e-37, -1.7402e-36,\n         2.1089e-38,  2.7744e-37,  6.1168e-38, -1.0972e-37, -2.6966e-37,\n         6.5153e-37,  5.4087e-37, -9.1579e-37, -1.5988e-36, -5.0137e-37,\n        -4.1843e-36, -5.4581e-37, -4.7996e-37, -8.2393e-38,  1.8143e-38,\n        -5.6721e-38, -2.6958e-37, -4.8923e-36,  5.9253e-37, -1.9625e-37,\n        -7.0872e-37,  3.5417e-37,  2.1298e-36,  8.7603e-37,  4.6570e-37,\n        -3.9838e-37, -2.3119e-37,  1.7952e-37,  3.0789e-36, -4.3524e-37,\n        -2.7968e-37, -4.0479e-37, -4.3597e-37,  8.8416e-37,  5.4272e-37,\n         7.4114e-37, -1.3557e-36,  8.4171e-39, -6.1998e-37, -2.9341e-37,\n         2.4137e-36, -8.3660e-37, -3.2682e-37,  7.3270e-38, -1.6938e-36,\n        -1.7370e-36,  1.6457e-37,  5.1857e-37,  2.7129e-36, -2.7664e-37,\n         6.9357e-37, -1.1896e-36, -3.2099e-36, -3.1811e-37, -3.0026e-37,\n         2.8113e-37,  1.4451e-37, -1.1103e-37, -1.6329e-36,  7.0442e-37,\n        -2.2055e-37,  9.4927e-37, -6.1155e-37, -4.3685e-37, -8.9882e-37,\n         1.5154e-39, -3.5222e-37, -1.4470e-36,  5.2350e-37, -1.6156e-37,\n        -1.4754e-37,  2.3239e-36], device='cuda:0')",
        "exp_avg_sq": "tensor([2.7880e-10, 1.7457e-12, 1.6857e-11, 4.1415e-11, 1.2815e-10, 6.2190e-13,\n        2.1573e-11, 2.3480e-13, 6.3594e-10, 2.2956e-12, 6.9038e-12, 1.1398e-10,\n        4.0636e-13, 1.9957e-12, 7.0830e-13, 3.6487e-13, 3.4546e-11, 2.9060e-10,\n        7.5633e-12, 1.4356e-12, 1.7903e-12, 6.9458e-11, 3.4503e-13, 2.6596e-10,\n        5.0002e-11, 3.2239e-15, 7.3054e-11, 3.4180e-10, 3.5671e-13, 1.4483e-12,\n        2.5135e-10, 1.9704e-12, 1.2135e-11, 5.7968e-14, 1.0556e-10, 3.6548e-11,\n        1.0962e-11, 3.3473e-12, 4.4795e-12, 5.7906e-13, 4.5912e-13, 9.6941e-13,\n        1.4333e-12, 1.3977e-11, 1.4746e-11, 1.1873e-10, 2.3371e-12, 1.2753e-12,\n        5.7970e-11, 1.3311e-12, 4.9645e-12, 6.7728e-12, 5.7447e-13, 1.9886e-11,\n        7.2274e-12, 1.3695e-10, 4.5072e-10, 7.2512e-10, 5.3583e-10, 8.8161e-15,\n        2.7240e-13, 1.1378e-10, 1.0266e-09, 1.7633e-11, 9.6737e-13, 1.0416e-11,\n        5.3622e-12, 1.2662e-10, 1.6673e-11, 3.4341e-10, 4.0071e-12, 9.8812e-14,\n        4.2244e-11, 6.7056e-12, 3.7002e-11, 3.7081e-11, 1.1830e-11, 1.9951e-12,\n        3.8138e-10, 1.3102e-11, 6.9595e-11, 4.4875e-11, 1.6525e-13, 2.0073e-11,\n        5.6382e-11, 1.9636e-10, 2.1174e-11, 8.1161e-12, 1.5066e-10, 2.0513e-11,\n        6.8867e-11, 1.8869e-10, 1.7752e-11, 5.1911e-10, 2.8736e-11, 2.0023e-11,\n        6.1232e-12, 2.0426e-11, 9.9066e-11, 2.7511e-12, 1.4496e-12, 4.9981e-11,\n        3.4016e-13, 2.4710e-11, 4.0596e-12, 1.1388e-13, 3.7408e-10, 3.4726e-12,\n        8.7631e-12, 1.6825e-10, 4.4802e-10, 2.7314e-11, 9.2108e-11, 8.0408e-12,\n        7.9215e-11, 7.4107e-12, 2.9899e-12, 4.4038e-13, 1.0615e-10, 8.1505e-12,\n        2.6747e-10, 1.0110e-14, 7.6441e-12, 8.9282e-11, 5.1630e-11, 4.2377e-13,\n        1.0457e-12, 6.4631e-11, 1.2515e-10, 1.5525e-11, 1.4434e-12, 2.4970e-10,\n        2.2477e-10, 2.1510e-12, 3.3109e-12, 8.4378e-11, 6.7788e-11, 6.3487e-11,\n        2.2300e-12, 1.3232e-12, 1.0617e-10, 1.9247e-10, 1.4647e-13, 2.5226e-10,\n        2.5256e-13, 2.1927e-14, 7.8699e-14, 5.6346e-13, 1.6074e-10, 3.4030e-14,\n        1.1569e-12, 7.7809e-12, 1.2591e-10, 4.7577e-10, 2.7222e-13, 9.0957e-12,\n        3.6131e-11, 2.7716e-12, 1.6090e-12, 1.0393e-10, 5.6160e-15, 1.1018e-12,\n        3.6610e-15, 1.6399e-11, 1.9331e-12, 1.4318e-11, 8.3513e-12, 1.1090e-13,\n        6.5276e-13, 2.1406e-11, 4.8606e-11, 3.2624e-12, 1.7415e-11, 2.2956e-10,\n        2.5410e-13, 1.5127e-12, 1.2465e-11, 4.1585e-13, 9.7484e-12, 1.7151e-13,\n        3.9814e-12, 1.9763e-13, 2.1763e-10, 3.4135e-12, 5.0213e-13, 9.0242e-12,\n        1.4743e-10, 1.3850e-11, 2.0265e-10, 9.2612e-13, 1.1482e-13, 7.8217e-12,\n        2.2772e-11, 4.7862e-11, 1.3608e-10, 8.8441e-11, 3.9058e-13, 9.8939e-12,\n        7.4299e-11, 1.9121e-12, 4.4192e-12, 4.3400e-11, 2.5061e-13, 1.1603e-12,\n        8.2276e-14, 1.0951e-10, 3.4419e-11, 7.0254e-12, 1.1768e-11, 3.1230e-10,\n        1.4048e-11, 3.5258e-10, 2.2668e-14, 2.4772e-12, 4.8151e-10, 1.4367e-10,\n        2.9998e-12, 9.2070e-12, 4.8222e-10, 4.3386e-13, 1.5788e-11, 6.7235e-11,\n        3.6544e-12, 9.2149e-11, 8.6482e-11, 6.8827e-14, 1.9486e-12, 1.2497e-12,\n        3.1320e-10, 7.9807e-12, 8.5186e-16, 7.8229e-12, 1.2580e-11, 3.1282e-12,\n        2.8928e-12, 1.1592e-11, 3.1821e-10, 1.9485e-11, 5.2151e-11, 1.2674e-13,\n        6.0080e-10, 1.5466e-11, 2.3532e-10, 1.9199e-10, 5.9219e-12, 1.5236e-11,\n        1.3328e-10, 5.6007e-12, 6.2924e-12, 4.9211e-12, 2.2281e-10, 2.8052e-12,\n        6.6263e-12, 1.4679e-12, 1.1040e-13, 2.3383e-13, 5.7431e-13, 7.4140e-11,\n        4.5585e-14, 2.6588e-13, 5.2414e-12, 6.0194e-12, 1.3913e-10, 2.0559e-10,\n        3.8185e-11, 6.8343e-11, 2.6694e-11, 1.9939e-13, 6.4917e-12, 9.9078e-12,\n        2.7660e-11, 3.5259e-13, 1.8445e-12, 2.7372e-11, 6.1634e-11, 1.0434e-11,\n        7.3669e-11, 1.5288e-11, 1.9605e-10, 2.5818e-13, 8.3983e-13, 9.5375e-12,\n        2.2914e-12, 4.8772e-11, 1.0829e-10, 2.4606e-11, 2.3852e-12, 2.6062e-13,\n        1.1664e-10, 2.4524e-13, 3.0300e-11, 1.3241e-12, 1.0893e-10, 1.7080e-12,\n        1.0533e-10, 1.2339e-10, 1.1043e-12, 3.0525e-11, 7.3361e-13, 1.9749e-12,\n        1.2504e-11, 1.1323e-10, 1.9005e-13, 3.0674e-10, 2.5684e-11, 1.5820e-13,\n        1.0721e-12, 4.8039e-13, 2.0275e-11, 1.5329e-12, 1.0973e-09, 1.4515e-12,\n        6.9864e-11, 9.2319e-12, 6.8439e-12, 3.1334e-13, 1.5024e-12, 1.7244e-12,\n        1.5760e-11, 1.0707e-11, 1.8242e-11, 2.7968e-13, 1.1832e-12, 2.7624e-11,\n        2.9753e-12, 5.1439e-11, 2.4155e-12, 1.2390e-11, 1.2163e-11, 1.7743e-12,\n        2.7646e-11, 1.1048e-10, 8.3079e-13, 1.3902e-13, 4.1533e-11, 5.3843e-11,\n        1.3516e-12, 3.0520e-13, 4.5757e-11, 1.1459e-10, 2.9267e-13, 1.0880e-12,\n        2.0966e-10, 4.5734e-12, 1.5322e-11, 1.3022e-13, 3.4657e-12, 6.6265e-12,\n        1.9190e-11, 1.4292e-12, 4.2261e-11, 1.5600e-10, 1.0980e-10, 4.0375e-11,\n        2.2855e-12, 4.0820e-11, 2.4842e-11, 5.9708e-12, 1.7958e-10, 7.4308e-10,\n        9.0606e-12, 2.4258e-10, 9.5998e-11, 5.4042e-12, 1.6878e-10, 6.3539e-12,\n        6.5037e-10, 2.3560e-11, 2.0668e-14, 1.8495e-12, 1.2083e-10, 1.6579e-11,\n        7.2706e-12, 4.8487e-13, 7.7275e-12, 2.8814e-12, 5.0670e-11, 2.1854e-12,\n        8.4637e-11, 5.9274e-11, 1.7140e-12, 5.3261e-11, 1.7220e-11, 7.5010e-11,\n        1.7656e-10, 8.3582e-12, 5.7053e-11, 2.6022e-11, 1.8532e-11, 8.3916e-13,\n        7.4095e-11, 1.2745e-10, 1.4368e-12, 1.3340e-12, 7.7913e-11, 1.7784e-11,\n        3.1983e-14, 2.2985e-12, 3.6174e-13, 2.0248e-10, 8.1060e-11, 3.1627e-10,\n        2.0287e-11, 2.4586e-11, 3.7625e-11, 8.7617e-13, 1.6707e-13, 1.7811e-13,\n        2.8988e-12, 1.1102e-10, 5.3517e-14, 7.1990e-12, 2.7267e-12, 2.0313e-10,\n        1.1141e-12, 1.5863e-12, 1.2142e-12, 4.1507e-12, 2.7771e-12, 1.9948e-12,\n        2.9532e-11, 9.3826e-12, 3.6546e-11, 3.8985e-13, 2.3390e-12, 7.0763e-13,\n        2.1151e-12, 5.2423e-12, 3.7607e-13, 5.1460e-11, 3.0324e-12, 1.9807e-10,\n        7.3179e-11, 1.5233e-10, 3.1674e-14, 5.7447e-13, 1.4066e-10, 1.0578e-10,\n        4.7761e-13, 8.7690e-12, 1.4776e-12, 1.2033e-12, 4.7515e-12, 4.7204e-12,\n        9.3650e-13, 3.2449e-12, 3.3318e-10, 1.2191e-10, 3.9100e-10, 1.1326e-12,\n        5.6217e-10, 4.0216e-12, 5.0197e-11, 1.6864e-10, 1.2651e-12, 3.1854e-12,\n        2.7944e-11, 3.1890e-10, 1.3793e-11, 1.9484e-11, 6.5819e-11, 4.9046e-12,\n        1.7938e-10, 1.6571e-12, 1.1784e-11, 3.8390e-12, 7.0343e-14, 1.2337e-12,\n        2.7681e-10, 9.3361e-13, 1.6557e-11, 5.6194e-13, 5.0949e-12, 1.1417e-11,\n        3.3813e-12, 1.3162e-11, 1.8651e-10, 4.7984e-13, 5.1859e-11, 2.6448e-11,\n        2.4332e-11, 2.3067e-10, 4.8578e-16, 6.8195e-12, 8.3717e-11, 6.3489e-11,\n        4.8295e-12, 4.9805e-11, 4.2960e-12, 7.9146e-14, 1.5210e-12, 7.5480e-11,\n        1.0565e-10, 1.4826e-13, 1.4029e-12, 2.9438e-13, 2.9642e-14, 1.2368e-12,\n        2.2656e-10, 2.8944e-10, 4.8541e-12, 5.1456e-10, 1.2436e-10, 5.3602e-14,\n        3.8211e-11, 5.7609e-13, 7.3577e-11, 2.3915e-11, 1.0051e-11, 4.0667e-12,\n        2.9857e-13, 3.1987e-10], device='cuda:0')"
      },
      "74": {
        "step": "tensor(11268.)",
        "exp_avg": "tensor([ 1.2638e-37,  8.3167e-39,  2.0513e-38,  1.0635e-37, -2.7329e-39,\n        -7.4309e-40,  5.7396e-38,  2.0110e-39,  1.5826e-37,  1.2489e-39,\n         4.1124e-39,  2.1800e-38,  1.2925e-38,  7.3449e-40,  1.1310e-40,\n         3.6732e-39,  1.1012e-37,  7.8469e-38,  5.6332e-39,  2.3890e-39,\n         1.1648e-38,  1.3091e-38,  2.0944e-39,  2.5123e-39,  6.6805e-38,\n         4.4665e-41,  6.3740e-38,  4.9492e-38,  1.3797e-39,  5.6542e-39,\n         3.6752e-40,  1.4428e-39, -1.7878e-39,  4.4491e-40,  1.6002e-37,\n         6.0037e-38,  1.0133e-38,  5.4562e-40, -8.5026e-40,  6.5312e-40,\n         2.3880e-39,  2.9703e-39,  3.9906e-39,  7.4352e-39,  6.2035e-39,\n         1.0488e-37, -8.7225e-41,  1.4189e-38,  1.3856e-37,  4.6955e-39,\n        -3.1725e-39,  7.8663e-40, -7.2165e-41,  8.6916e-39,  9.9080e-40,\n         1.7614e-37,  1.8311e-37,  1.4699e-37,  2.4089e-37,  3.0299e-39,\n         6.3146e-39,  7.5326e-38,  2.8821e-37,  1.3215e-38, -3.1184e-40,\n         5.7634e-38,  2.3667e-39,  4.7827e-38,  5.5301e-38,  4.2516e-40,\n         2.0404e-38,  3.5166e-39, -9.7255e-40,  1.3950e-38,  8.5024e-38,\n         5.1334e-38, -1.1937e-39, -3.1717e-39, -3.0263e-39,  3.8473e-40,\n         4.5363e-40,  7.4333e-38,  9.2057e-41, -1.1944e-40,  2.7084e-38,\n         1.4939e-37,  6.3673e-39,  4.9980e-38,  1.3563e-37,  2.8667e-40,\n         3.8570e-38, -2.8162e-39,  1.4631e-38, -2.6666e-40,  2.8718e-38,\n         6.0873e-38, -2.1104e-39,  6.4174e-40, -2.2863e-39,  6.4710e-39,\n        -2.3275e-39,  5.7104e-38,  3.3225e-39,  5.8556e-39,  2.9168e-38,\n         1.4627e-39,  2.1038e-37,  4.8858e-40,  1.9291e-38,  8.3272e-39,\n         1.4331e-38,  5.3306e-38,  2.4115e-38, -2.5452e-39, -9.2190e-40,\n         9.0211e-39,  9.2783e-38,  1.0269e-39,  2.0390e-38,  6.6260e-39,\n         9.3995e-40, -2.5236e-39,  7.5915e-40,  1.0864e-37, -4.1389e-39,\n         4.1449e-39, -1.5180e-39,  6.9434e-38,  1.0260e-38, -7.5645e-40,\n         1.8291e-38,  2.5632e-38,  3.1054e-38,  3.9309e-41,  1.5408e-38,\n         1.4770e-38,  1.0817e-37,  1.0680e-37, -2.6311e-39,  4.3606e-39,\n        -8.1183e-40,  5.5325e-38,  6.2812e-39,  4.4315e-38,  1.6753e-39,\n         5.8627e-39,  1.7400e-38,  8.5295e-39,  1.1303e-39, -1.3641e-39,\n         6.0833e-40,  7.8972e-39,  2.2181e-38,  1.7758e-40,  1.4209e-38,\n         5.6913e-40,  7.1041e-38,  1.1211e-38,  2.6981e-39,  1.0161e-37,\n         4.5995e-39,  7.2499e-39,  4.5703e-39,  3.5246e-38,  9.2564e-39,\n         7.4341e-38,  1.5988e-40, -2.7266e-40,  2.3504e-39,  3.3697e-38,\n        -1.7197e-40, -6.0244e-40,  4.4914e-38,  4.1831e-38,  2.2023e-39,\n        -2.3463e-39,  1.9950e-39,  4.5177e-39,  1.3101e-39, -1.9153e-39,\n        -4.0968e-40,  1.5222e-38,  1.9390e-38,  8.6743e-39,  1.8821e-39,\n         2.0459e-40,  1.1620e-37,  5.7689e-38,  6.3534e-39, -2.5524e-39,\n        -2.1553e-40,  1.1052e-38,  1.0656e-37,  5.8077e-38,  8.4681e-38,\n         5.7708e-38,  7.4916e-39, -1.0823e-39, -1.3703e-40,  6.4317e-40,\n         1.5591e-38,  3.8881e-38,  1.0065e-38, -1.8500e-39,  1.0473e-38,\n         4.7809e-38,  2.4626e-38,  3.9308e-40,  1.1681e-39,  6.2528e-38,\n        -1.0940e-39,  1.0765e-37, -2.7703e-39,  2.2777e-39,  7.3029e-38,\n         3.4164e-38,  1.2057e-39,  5.6247e-38,  2.4939e-38,  3.7691e-39,\n        -4.0953e-39,  1.9125e-38, -2.2875e-40,  1.6608e-38,  8.3960e-38,\n         8.2312e-39,  5.4877e-39,  8.4370e-39,  5.2551e-38,  1.8681e-38,\n         2.9291e-39,  1.0708e-40,  1.1868e-40, -4.0873e-39,  1.0708e-37,\n         3.3657e-39,  8.7397e-38,  5.0380e-38,  7.7156e-38,  1.8373e-39,\n         1.6534e-37,  7.4694e-39,  2.0410e-39,  1.4598e-38,  2.4716e-39,\n        -1.3540e-39,  1.4256e-37, -1.6209e-39, -1.2724e-39, -1.8224e-39,\n         1.4379e-38,  7.5581e-39, -1.1859e-39, -7.8200e-40,  7.4833e-40,\n        -1.7373e-39, -7.9789e-40,  1.5820e-38, -4.3164e-39,  1.3460e-39,\n        -4.8439e-39, -9.4219e-40,  1.3683e-38,  4.3511e-40,  5.7996e-40,\n         2.4305e-39,  1.0971e-37,  5.0553e-39,  2.0952e-38,  1.1979e-38,\n        -7.1477e-40,  2.3980e-39,  2.2169e-39, -3.5118e-39,  2.7647e-38,\n         2.1854e-39,  3.5523e-38,  4.8868e-38,  4.3408e-38,  2.6825e-40,\n         1.0310e-39,  1.7901e-40, -2.6104e-39,  6.2464e-38,  1.0455e-37,\n         1.5905e-38,  1.2353e-39,  2.4449e-39,  1.1666e-37,  1.0045e-38,\n        -7.7860e-40,  1.3426e-38,  1.6299e-39,  1.0740e-39,  1.4642e-39,\n         6.7748e-38, -3.6062e-39,  1.4254e-37,  8.0899e-39,  3.6427e-40,\n        -1.9609e-39,  3.3796e-38,  8.1517e-39,  1.5317e-37, -7.3379e-40,\n        -4.9726e-40,  9.9051e-41,  7.5551e-40,  5.0511e-39,  1.9092e-38,\n         1.1549e-37,  8.6771e-41,  3.9087e-38,  1.4383e-40, -1.0449e-39,\n         5.0115e-39,  2.5860e-39,  3.8375e-39, -1.2006e-39,  2.1681e-39,\n         4.5602e-39, -1.7117e-39,  2.4458e-40,  2.0613e-39,  1.0232e-39,\n         2.3476e-38,  4.7134e-40, -6.1919e-39,  1.0722e-39,  2.5001e-40,\n         2.6393e-40,  9.8763e-39,  4.9160e-39,  1.1214e-38,  1.1630e-37,\n         4.5969e-38, -3.2779e-40,  3.4327e-39,  7.1982e-38,  1.2063e-37,\n         1.3775e-40,  1.1485e-39,  1.6969e-38,  4.4018e-38,  2.8010e-38,\n         4.9697e-39, -4.2257e-40,  1.1176e-37,  3.6259e-38,  1.2713e-38,\n         1.5643e-37,  4.4141e-38,  9.1108e-38,  5.0469e-38,  1.2026e-38,\n         1.0573e-39,  2.7471e-40,  5.4245e-38,  7.2561e-38,  2.5587e-38,\n         3.6657e-38,  1.3352e-37,  1.0414e-37,  9.4415e-39,  1.2929e-37,\n         9.4513e-39,  1.3933e-37,  1.9428e-39,  4.1373e-39,  1.5497e-39,\n        -3.8013e-40, -6.8499e-40,  5.6117e-39, -3.6104e-40,  1.9574e-39,\n         2.5121e-39,  5.4021e-38, -1.7410e-39,  2.6489e-38,  2.6457e-38,\n         4.2556e-39,  1.4695e-37, -8.2957e-40,  1.0236e-37,  9.3739e-38,\n         2.3147e-39,  3.6199e-38,  1.3488e-37,  5.7302e-38,  2.1701e-39,\n        -7.2611e-39,  9.9029e-38, -4.2435e-39, -4.7462e-41, -2.5744e-39,\n         1.3540e-39, -3.5683e-39,  3.6664e-40,  1.6211e-39,  2.5843e-38,\n         1.2349e-37,  5.7126e-39,  4.6019e-42,  3.4364e-39, -1.0368e-40,\n         1.5442e-38,  8.8677e-39,  9.1890e-39, -1.0246e-39,  2.9033e-38,\n        -2.0145e-39,  6.5058e-38, -2.7884e-40,  2.7732e-38,  1.6570e-39,\n         1.0653e-38,  6.5718e-41,  1.7840e-39,  1.4779e-40,  1.7026e-38,\n         2.1402e-38, -6.7775e-41,  1.3791e-39,  8.5113e-39,  5.8186e-39,\n        -2.7417e-39,  3.0163e-39,  4.7756e-39,  3.3337e-39,  8.8074e-38,\n         1.0209e-39,  2.3656e-38,  1.2499e-38,  1.2971e-37,  8.0692e-39,\n        -9.9013e-40,  1.4932e-39,  8.4546e-38,  7.5570e-39,  5.5203e-38,\n         6.9015e-40,  8.0230e-39,  1.0707e-38,  3.9823e-39,  1.5771e-40,\n         3.2408e-40,  1.7608e-37,  1.2547e-37,  9.0742e-38,  1.6373e-40,\n        -1.1364e-38, -1.4238e-39,  2.3216e-38,  4.2914e-38,  3.5347e-39,\n        -2.5588e-39,  3.1055e-38, -1.2472e-38, -2.0286e-40, -1.9366e-39,\n         6.8178e-40,  2.6453e-38,  1.3974e-37,  1.0550e-38,  1.7901e-38,\n        -2.4095e-39, -2.7591e-39,  9.1837e-39,  1.8106e-37, -9.7886e-40,\n        -1.3900e-39, -2.3751e-39,  7.8884e-40,  1.5484e-38,  4.0048e-38,\n         1.9861e-38,  1.6809e-39,  2.0852e-38,  9.0064e-38,  8.3328e-39,\n         6.5326e-38,  1.0653e-38,  2.4317e-40,  2.6367e-39,  5.2226e-39,\n         2.6872e-38,  3.8781e-39,  1.0450e-37,  5.2811e-38,  3.4881e-39,\n         1.0658e-38,  1.9294e-38, -3.5937e-39, -3.6247e-41, -2.7575e-40,\n        -3.8416e-39,  1.0494e-38,  2.2291e-39,  9.9962e-39,  1.4146e-37,\n        -2.1235e-39,  1.7381e-37,  4.1605e-40, -4.2874e-40,  2.8294e-41,\n         4.7078e-39,  4.2345e-38,  6.5540e-38, -3.0876e-39,  1.1737e-39,\n        -3.1618e-39,  2.0646e-37], device='cuda:0')",
        "exp_avg_sq": "tensor([8.7865e-14, 1.0268e-17, 7.2736e-16, 5.1848e-14, 3.1349e-15, 2.1076e-18,\n        3.2497e-15, 6.8127e-17, 1.7146e-13, 1.7827e-17, 1.2916e-17, 2.3389e-14,\n        4.2551e-16, 8.7535e-19, 3.6837e-17, 9.9416e-18, 3.0857e-14, 1.5049e-13,\n        6.6244e-18, 2.1590e-18, 1.4106e-16, 8.8760e-16, 1.5624e-16, 2.9715e-14,\n        4.2972e-15, 5.4043e-17, 7.2234e-15, 9.4855e-14, 5.0127e-17, 1.9600e-17,\n        2.9186e-14, 2.5823e-19, 2.1014e-18, 9.9085e-17, 1.6845e-13, 1.3316e-14,\n        8.0115e-16, 8.6445e-18, 6.4986e-18, 2.4251e-16, 9.7363e-19, 4.2653e-19,\n        1.4384e-19, 7.9965e-17, 1.1696e-15, 1.3850e-13, 5.4199e-19, 1.5127e-15,\n        8.4471e-14, 4.4741e-16, 1.2136e-17, 3.0654e-18, 1.9737e-16, 6.0194e-15,\n        1.8820e-17, 6.7226e-14, 1.5501e-13, 2.3613e-13, 3.6242e-13, 5.2336e-19,\n        1.6324e-18, 3.8804e-14, 5.6024e-13, 1.5181e-17, 2.9954e-16, 3.5773e-15,\n        5.2135e-18, 7.3833e-15, 4.8601e-15, 3.4095e-14, 2.0248e-16, 1.2315e-16,\n        1.2093e-15, 7.2925e-16, 3.0183e-14, 2.9951e-14, 2.2256e-17, 6.5093e-18,\n        4.2859e-14, 5.9775e-20, 7.4526e-17, 1.4110e-14, 1.2412e-16, 4.2443e-18,\n        1.2472e-14, 1.3915e-13, 1.4964e-15, 1.3745e-15, 7.7770e-14, 1.5348e-18,\n        3.0493e-15, 1.1348e-14, 1.7210e-16, 2.2790e-14, 4.8400e-15, 1.2252e-14,\n        2.9230e-18, 2.8694e-18, 1.1183e-15, 5.5737e-16, 4.6122e-19, 1.8770e-14,\n        4.2786e-18, 2.7939e-16, 6.7290e-15, 5.8227e-17, 2.5509e-13, 1.9228e-17,\n        4.5383e-17, 1.6293e-14, 9.0177e-14, 4.8840e-15, 1.0405e-14, 6.5496e-16,\n        1.5108e-15, 1.2621e-15, 3.0341e-14, 2.4683e-16, 2.0555e-14, 8.3273e-17,\n        1.9993e-14, 1.2183e-16, 1.1116e-18, 8.4981e-14, 1.6249e-15, 4.3594e-19,\n        3.1549e-19, 3.6459e-14, 2.6299e-14, 4.2745e-18, 2.2951e-15, 3.0240e-14,\n        7.8418e-14, 1.0232e-18, 2.3676e-15, 1.2843e-14, 1.6212e-14, 7.0669e-15,\n        1.2340e-18, 3.2493e-18, 1.6663e-15, 9.7189e-14, 7.1323e-17, 7.1560e-14,\n        2.6348e-17, 4.5994e-17, 7.1889e-16, 3.5771e-19, 7.1941e-15, 1.7144e-18,\n        5.8331e-17, 7.7085e-19, 1.9345e-14, 1.8879e-14, 8.0724e-16, 1.0266e-17,\n        1.5302e-14, 8.5322e-16, 6.6640e-17, 9.5644e-14, 3.5567e-16, 7.2793e-16,\n        3.0198e-16, 7.3889e-15, 4.8049e-16, 1.6618e-14, 1.1216e-17, 3.2624e-17,\n        8.4877e-19, 3.6831e-16, 1.4594e-17, 1.7510e-16, 7.2054e-16, 1.6998e-14,\n        2.8174e-17, 1.5920e-18, 1.2619e-15, 1.6148e-16, 1.2134e-17, 4.2266e-17,\n        1.0321e-15, 2.9723e-16, 4.3254e-14, 1.5690e-15, 4.5963e-18, 1.4691e-18,\n        2.2016e-14, 6.0493e-14, 2.2169e-14, 9.8377e-18, 3.1149e-17, 2.8545e-16,\n        3.0194e-14, 1.2407e-14, 3.2147e-14, 3.6785e-15, 3.6862e-19, 2.7249e-18,\n        7.0891e-17, 5.2145e-17, 1.4764e-16, 5.2158e-15, 6.6244e-16, 1.7753e-18,\n        2.1642e-16, 8.1797e-15, 1.4575e-16, 1.9955e-18, 2.0337e-17, 8.8253e-14,\n        3.5472e-18, 6.1212e-14, 1.3129e-18, 8.3076e-19, 1.8036e-14, 5.9075e-14,\n        1.7029e-17, 1.3156e-14, 1.3933e-13, 2.9426e-20, 1.7029e-16, 2.8725e-14,\n        4.7196e-19, 6.4496e-15, 2.0559e-14, 1.5252e-16, 2.6736e-16, 2.8877e-16,\n        1.7203e-14, 1.5187e-16, 1.2898e-16, 3.0864e-18, 2.7623e-19, 4.4829e-18,\n        1.4041e-14, 5.5868e-17, 3.5446e-14, 8.9219e-15, 1.5509e-14, 8.8919e-18,\n        1.4369e-13, 4.8057e-18, 3.7498e-14, 1.7032e-14, 6.8319e-18, 3.9311e-18,\n        7.7201e-14, 1.7543e-17, 6.3827e-19, 5.2651e-18, 3.4355e-14, 6.5458e-16,\n        2.0491e-15, 1.8469e-19, 7.8434e-18, 4.5414e-18, 8.8496e-20, 1.7023e-15,\n        8.0956e-19, 1.7577e-19, 1.2871e-17, 2.0800e-18, 3.1855e-14, 1.5408e-14,\n        1.7706e-17, 5.1342e-15, 4.6175e-14, 1.5894e-17, 5.6205e-16, 4.6043e-18,\n        2.7143e-17, 3.1686e-16, 4.1441e-18, 3.7424e-17, 2.1351e-15, 1.2369e-17,\n        2.8362e-14, 1.3469e-14, 4.7779e-14, 4.0777e-17, 2.7478e-16, 1.6932e-18,\n        4.4029e-19, 5.5489e-15, 4.0849e-14, 6.6951e-15, 2.2844e-16, 6.9999e-19,\n        2.0697e-14, 2.9837e-16, 1.9492e-16, 1.5311e-15, 2.8158e-16, 2.6911e-17,\n        1.5639e-14, 3.7393e-14, 9.2060e-17, 4.0007e-14, 5.4540e-18, 2.0819e-18,\n        2.7076e-16, 1.9028e-14, 2.1640e-16, 1.6309e-13, 2.8191e-18, 1.6266e-17,\n        1.6877e-16, 1.3098e-18, 1.2055e-17, 1.2080e-15, 5.9568e-13, 6.5979e-18,\n        8.0596e-15, 4.1746e-19, 1.9319e-19, 5.0507e-20, 1.4896e-17, 2.3336e-16,\n        2.1485e-18, 8.3954e-17, 1.3222e-17, 2.0097e-17, 1.0279e-16, 1.7764e-15,\n        2.3094e-16, 1.3262e-14, 2.9126e-19, 2.1210e-15, 9.5170e-18, 1.5328e-16,\n        1.3567e-18, 1.2015e-14, 2.4614e-16, 4.2718e-18, 3.1440e-14, 1.4597e-14,\n        4.7792e-19, 1.1143e-18, 5.0377e-15, 1.3341e-13, 1.9429e-16, 2.9587e-16,\n        2.2192e-14, 5.2927e-15, 2.0642e-15, 2.2412e-16, 1.1862e-15, 2.7536e-14,\n        1.1843e-14, 9.7142e-16, 2.8364e-14, 1.1214e-14, 6.4451e-14, 1.2999e-15,\n        8.6521e-16, 3.0194e-15, 2.4835e-17, 1.9969e-15, 5.0143e-15, 2.1336e-13,\n        8.7073e-16, 4.6705e-14, 6.4118e-14, 5.2037e-18, 1.0560e-13, 8.7804e-18,\n        3.1879e-13, 2.0173e-15, 7.4179e-18, 3.6111e-19, 4.7903e-15, 1.8634e-19,\n        7.1980e-17, 2.6497e-18, 9.8231e-18, 3.8009e-18, 5.6762e-15, 1.7186e-17,\n        1.2369e-14, 3.0425e-14, 2.2033e-17, 6.9317e-14, 7.1557e-17, 6.6909e-14,\n        1.7106e-13, 2.0161e-17, 1.1426e-14, 2.9985e-14, 3.4956e-15, 1.7556e-18,\n        1.3242e-15, 1.2579e-13, 1.0720e-19, 2.1157e-19, 5.3414e-16, 1.2799e-18,\n        8.7030e-18, 2.3626e-16, 5.2915e-17, 3.2622e-14, 1.1375e-13, 3.8576e-14,\n        1.9946e-17, 1.3339e-16, 8.5425e-17, 8.4156e-16, 3.4840e-16, 2.5782e-16,\n        8.6509e-18, 9.2289e-17, 2.7574e-17, 6.0839e-15, 1.3511e-18, 4.8721e-14,\n        9.8470e-17, 8.6467e-18, 2.7243e-16, 7.9523e-16, 9.3506e-16, 2.1486e-15,\n        3.5755e-18, 1.4144e-19, 2.4590e-15, 5.2889e-16, 4.9356e-16, 1.5858e-16,\n        5.9246e-17, 8.9199e-18, 2.6924e-19, 6.8294e-15, 3.7549e-19, 2.9146e-14,\n        1.0280e-14, 1.2100e-13, 1.0001e-16, 2.9457e-19, 7.4907e-15, 4.5305e-14,\n        8.8489e-16, 1.3765e-14, 1.6546e-16, 4.9702e-16, 2.3554e-15, 6.8029e-18,\n        3.8664e-19, 7.6910e-16, 3.2559e-13, 1.7424e-13, 1.8971e-13, 3.6605e-16,\n        7.0284e-14, 1.2201e-17, 1.6481e-15, 4.0243e-14, 1.9118e-18, 8.3001e-18,\n        1.3784e-14, 2.1540e-14, 3.1147e-16, 1.3556e-16, 4.1764e-16, 3.1183e-16,\n        9.4117e-14, 1.4719e-15, 2.4883e-16, 1.6572e-18, 7.0373e-18, 3.8464e-16,\n        2.1833e-13, 9.8452e-19, 9.8368e-18, 1.3158e-17, 1.5966e-18, 6.5635e-17,\n        1.1436e-15, 5.0639e-17, 1.8799e-14, 1.2329e-15, 3.7161e-14, 3.2221e-18,\n        2.9873e-15, 3.2044e-14, 7.9661e-18, 2.7837e-17, 2.1032e-15, 9.4548e-15,\n        3.8219e-17, 4.8690e-14, 3.8354e-16, 3.4272e-17, 1.4169e-15, 3.6138e-15,\n        3.2675e-15, 3.7993e-17, 3.9386e-20, 2.4828e-17, 8.4186e-16, 6.3161e-17,\n        1.9709e-14, 1.5575e-13, 1.3655e-16, 3.2378e-13, 6.8036e-15, 9.9215e-18,\n        5.1575e-17, 1.3317e-18, 1.7487e-14, 1.5438e-14, 8.2224e-16, 8.0225e-19,\n        2.2519e-18, 2.2934e-13], device='cuda:0')"
      },
      "75": {
        "step": "tensor(11268.)",
        "exp_avg": "tensor([ 8.0742e-38,  3.9427e-38,  4.5964e-38,  7.8248e-38, -1.4945e-38,\n         2.3882e-39,  5.9837e-38, -1.5840e-39,  1.0508e-37,  3.4906e-38,\n         2.1514e-38,  1.5703e-38, -8.8217e-39, -3.2862e-40,  1.1555e-39,\n         4.6702e-38,  9.9162e-38,  4.3947e-38,  4.4953e-38, -2.2099e-39,\n         3.6954e-38,  2.1130e-38, -1.5629e-39,  8.1192e-39,  5.3612e-38,\n         5.9137e-40,  8.1742e-38,  5.5522e-38, -5.2259e-40, -4.3525e-39,\n         1.3324e-40, -6.2365e-40,  1.1884e-38,  1.3935e-39,  8.4314e-38,\n         4.8992e-38,  4.3985e-38, -5.1300e-40,  3.3023e-38,  1.6701e-41,\n        -2.0147e-39, -2.6831e-39, -3.1931e-39,  3.2719e-38,  3.2290e-38,\n         5.6654e-38,  5.3954e-40, -8.6861e-39,  8.3888e-38, -3.2814e-39,\n         1.1588e-38, -4.8102e-40,  1.6104e-39,  8.7464e-39,  1.1239e-40,\n         1.0721e-37,  1.2453e-37,  9.5845e-38,  1.3543e-37, -2.3687e-39,\n        -5.0722e-39,  5.2168e-38,  1.6607e-37,  6.0209e-38,  9.4907e-40,\n         6.7193e-38, -2.1496e-39,  5.5990e-38,  5.9602e-38,  9.5749e-40,\n         4.9441e-38, -2.9335e-39,  5.6411e-39,  4.3028e-38,  6.5953e-38,\n         5.2491e-38,  1.0992e-39,  1.1693e-38, -7.5743e-39,  1.3656e-40,\n        -2.2296e-39,  5.2778e-38,  1.1761e-39,  8.5467e-39,  4.8502e-38,\n         8.5003e-38,  5.0925e-38,  6.6362e-38,  8.4861e-38, -1.4116e-40,\n         5.2435e-38, -7.5315e-39,  3.3502e-38,  8.4127e-39,  2.6870e-38,\n         6.0435e-38,  5.9340e-39, -1.3160e-40,  7.7237e-39, -4.6509e-39,\n         9.1191e-39,  5.3551e-38, -2.9311e-39,  1.6396e-38,  3.2311e-38,\n        -1.1031e-39,  1.1723e-37,  2.3737e-38,  4.6951e-38,  2.2462e-38,\n         1.6003e-38,  4.8043e-38,  2.5619e-38,  1.9920e-39,  1.5656e-38,\n         3.1769e-38,  5.7028e-38, -1.9611e-40,  2.6076e-38,  4.3507e-38,\n         1.6718e-38,  8.1863e-39,  7.9512e-39,  6.6413e-38, -8.6591e-39,\n        -3.6664e-39,  7.8984e-39,  7.3497e-38,  1.7550e-38,  1.7777e-39,\n         3.4491e-38,  2.3261e-38,  4.2271e-38,  1.2586e-39, -9.0868e-39,\n         4.5253e-38,  8.3423e-38,  7.2247e-38,  2.1144e-38, -4.0591e-39,\n         3.6438e-39,  5.2262e-38, -5.4284e-39,  5.8647e-38, -1.5209e-39,\n        -4.4571e-39, -1.0798e-38, -6.9248e-39,  1.6821e-38,  4.4478e-39,\n        -4.0981e-40,  3.9404e-38,  4.3775e-38,  5.4266e-39, -9.4882e-39,\n        -4.8049e-40,  6.3037e-38, -7.6206e-39, -2.2306e-39,  6.3501e-38,\n        -3.7209e-39, -4.9783e-39, -3.9748e-39,  3.3291e-38, -6.7929e-39,\n         5.2008e-38, -1.4243e-40,  8.6689e-40, -1.9396e-39,  6.2534e-38,\n        -1.6442e-40,  4.2163e-40,  7.6347e-38,  5.4612e-38, -1.7791e-39,\n         9.5188e-39,  2.4870e-38, -3.7916e-39, -1.1671e-39,  7.2433e-39,\n         1.3450e-39, -1.0908e-38,  2.3189e-38,  2.5665e-38,  2.7074e-38,\n         1.1521e-40,  1.0774e-37,  2.9817e-38,  1.1338e-38,  7.5898e-39,\n         7.3456e-40,  3.1243e-38,  6.4945e-38,  4.8091e-38,  6.5428e-38,\n         7.9089e-38, -6.0994e-39,  3.6067e-39,  1.6282e-40, -5.6218e-40,\n         3.6707e-38,  3.8945e-38, -7.0229e-39,  6.5272e-39, -7.9733e-39,\n         5.6816e-38,  5.3290e-38, -2.3480e-40, -8.8027e-40,  4.3928e-38,\n        -1.7865e-38,  9.7470e-38,  2.1090e-38, -2.0325e-39,  1.0446e-37,\n         3.4263e-38,  2.8664e-38,  4.7817e-38,  3.3230e-38, -3.3011e-39,\n         1.1986e-38,  2.2791e-38,  5.6172e-40,  1.5235e-38,  7.6325e-38,\n        -5.9930e-39, -4.1521e-39, -6.4897e-39,  6.6208e-38,  3.9365e-38,\n        -2.1488e-39,  2.5708e-40,  7.5207e-40,  3.3727e-38,  7.2976e-38,\n         1.8398e-38,  9.5380e-38,  5.0351e-38,  6.3113e-38, -5.6728e-40,\n         1.1659e-37,  2.9474e-38,  2.9647e-39,  2.6692e-38,  4.5364e-38,\n         3.3524e-39,  9.7631e-38,  5.5119e-39,  1.1899e-39,  8.0642e-39,\n         2.3639e-38, -5.2019e-39,  2.9169e-39,  2.2120e-39, -5.8669e-40,\n         1.5235e-39,  3.0908e-39,  2.2321e-38,  1.6068e-38, -3.5422e-40,\n         2.3896e-38,  1.3280e-38,  2.2030e-38,  2.7576e-39,  8.0233e-39,\n         2.5377e-38,  6.0584e-38, -4.2175e-39,  3.7559e-38,  3.0289e-38,\n         4.2199e-39, -1.6007e-39, -1.8439e-39,  2.1503e-38,  3.5644e-38,\n        -2.1175e-39,  4.3909e-38,  4.1545e-38,  4.2745e-38,  1.0793e-39,\n        -3.7082e-39, -5.2419e-40,  2.2668e-38,  8.2185e-38,  7.2695e-38,\n         3.3750e-38, -5.4711e-40, -2.1548e-39,  1.1036e-37, -7.5706e-39,\n         2.5259e-39, -8.2026e-39, -4.4309e-39, -8.4693e-40,  2.4115e-39,\n         4.9667e-38,  1.2373e-38,  8.9082e-38,  6.0855e-38, -3.2802e-40,\n         1.0867e-38,  3.5573e-38, -5.8024e-39,  9.2135e-38,  2.7299e-39,\n         2.1437e-39,  4.7240e-40, -6.4612e-40,  1.9524e-38, -1.4499e-38,\n         5.9672e-38, -8.1100e-41,  4.2262e-38,  1.6397e-38,  3.3905e-39,\n        -4.4192e-39,  4.0640e-38, -2.9715e-39,  3.7756e-39,  1.4783e-38,\n         3.9352e-38,  3.4039e-38, -2.1874e-40,  3.7537e-38, -7.2426e-40,\n         5.0592e-38, -2.8376e-41,  4.7973e-39, -8.3709e-40, -1.4054e-40,\n        -2.5797e-40,  3.9242e-38, -3.9465e-39, -9.2467e-39,  8.5142e-38,\n         7.6750e-38,  1.1034e-39, -3.1005e-39,  9.3455e-38,  6.3343e-38,\n         4.3753e-41, -1.5759e-40,  1.5776e-38,  3.9711e-38,  3.3845e-38,\n        -3.7894e-39,  1.8196e-39,  6.6075e-38,  4.9516e-38, -8.6481e-39,\n         1.0541e-37,  4.8470e-38,  6.0607e-38,  7.8033e-38, -8.4517e-39,\n         1.9885e-38, -1.1872e-40,  5.9082e-38,  1.0152e-37,  2.9860e-38,\n         5.4846e-38,  1.0750e-37,  5.4332e-38,  5.0414e-38,  8.5928e-38,\n         3.3684e-38,  7.7330e-38,  2.4961e-38, -3.6610e-39, -1.3447e-39,\n         2.3252e-39,  2.4007e-39, -4.2345e-39,  1.5557e-39, -1.7016e-39,\n        -2.1760e-39,  7.1722e-38,  3.5463e-38,  3.8294e-38,  3.1825e-38,\n        -3.3075e-39,  9.0847e-38,  2.4235e-39,  6.6423e-38,  4.6150e-38,\n        -2.2083e-39,  5.9361e-38,  1.0211e-37,  5.6627e-38, -1.8531e-39,\n        -1.0441e-38,  6.5925e-38,  3.4705e-39,  2.8774e-41,  9.9260e-39,\n         1.0702e-38,  1.1226e-38, -9.1150e-41, -1.4285e-39,  5.3312e-38,\n         6.9463e-38,  1.6457e-38,  6.2330e-40,  1.4178e-38,  1.1368e-39,\n        -1.0663e-38, -6.3018e-39, -7.7060e-39,  6.3557e-39,  7.1969e-38,\n         9.0337e-39,  5.6099e-38,  1.9561e-38,  4.0503e-38, -1.4419e-39,\n         3.3680e-38,  4.0759e-40, -9.8362e-40,  3.9430e-40, -1.0044e-38,\n         6.7996e-38,  9.8680e-40,  3.1765e-38, -5.3208e-39, -4.2654e-39,\n         2.2760e-38, -2.5777e-39, -4.1999e-39, -3.0300e-39,  8.3014e-38,\n        -3.4742e-41,  3.8318e-38,  3.6222e-38,  7.4706e-38, -6.6088e-39,\n         7.0827e-39,  2.5626e-38,  8.7789e-38, -6.4151e-39,  3.8731e-38,\n        -4.3913e-40, -6.0379e-39,  3.4005e-38, -3.6054e-39,  3.8712e-40,\n        -2.4475e-40,  8.7263e-38,  6.4549e-38,  5.0432e-38, -3.8930e-40,\n        -2.0753e-38,  1.2609e-38,  3.4624e-38,  4.6965e-38, -3.1027e-39,\n         1.8963e-38,  4.1805e-38, -3.5759e-38,  3.2145e-38,  1.7664e-38,\n         1.5391e-38,  4.5382e-38,  9.4822e-38, -6.4814e-39,  4.2234e-38,\n         9.7737e-39,  1.0044e-38, -6.8997e-39,  1.1230e-37,  2.4154e-39,\n         4.9793e-39,  1.2237e-38,  1.7957e-38,  4.6139e-38,  5.2471e-38,\n         4.6137e-38,  1.0649e-38, -1.4265e-38,  5.7841e-38,  2.8409e-38,\n         8.0661e-38,  2.4493e-38,  4.9158e-40,  2.6703e-38,  1.2309e-38,\n         2.7376e-38, -3.1264e-39,  7.1416e-38,  8.0826e-38, -2.6472e-39,\n        -6.8691e-39,  2.7386e-38, -6.5904e-39,  1.3568e-39,  1.8486e-39,\n         1.7284e-38, -7.8576e-39, -1.3704e-39,  1.7071e-38,  8.1708e-38,\n         1.3752e-38,  9.0194e-38,  1.5156e-38,  1.0370e-39, -3.0037e-40,\n        -4.0403e-39,  4.5212e-38,  4.4211e-38,  2.4906e-39, -9.8288e-40,\n         1.0927e-38,  1.0881e-37], device='cuda:0')",
        "exp_avg_sq": "tensor([7.0096e-14, 1.5136e-15, 3.5084e-15, 2.8660e-14, 9.0111e-15, 1.7803e-16,\n        1.0404e-14, 3.5171e-17, 1.6059e-13, 1.5406e-15, 2.7244e-15, 1.5002e-14,\n        1.7487e-16, 2.6955e-16, 1.7534e-17, 8.6262e-16, 1.1990e-14, 7.7304e-14,\n        4.6397e-15, 2.1710e-16, 3.8663e-15, 1.3494e-14, 6.6614e-17, 4.7155e-14,\n        2.2789e-15, 2.6960e-17, 3.1325e-14, 5.1931e-14, 2.4589e-17, 8.4596e-18,\n        2.6361e-14, 2.4138e-16, 2.7732e-17, 5.5780e-17, 6.3905e-14, 1.5904e-14,\n        8.3250e-15, 5.3320e-16, 2.5173e-15, 1.1320e-16, 1.4256e-18, 3.0140e-17,\n        4.8511e-18, 2.1570e-16, 8.2231e-15, 5.9785e-14, 2.6209e-16, 7.1529e-16,\n        4.1984e-14, 2.0464e-16, 2.0957e-15, 2.6625e-16, 1.0159e-16, 7.1343e-15,\n        1.4735e-15, 4.8267e-14, 1.3436e-13, 1.9914e-13, 1.6649e-13, 1.5164e-16,\n        5.8825e-18, 4.5534e-14, 2.8912e-13, 6.3825e-15, 1.3504e-16, 2.4688e-15,\n        6.6588e-16, 2.9273e-14, 1.2095e-14, 4.7284e-14, 3.2641e-15, 5.2050e-18,\n        4.9836e-15, 6.2584e-15, 2.8508e-14, 1.3369e-14, 1.5014e-15, 1.9080e-15,\n        5.7567e-14, 2.2210e-15, 6.0433e-15, 6.3351e-15, 5.7440e-17, 3.2949e-15,\n        2.1760e-14, 7.8830e-14, 9.1360e-15, 8.0380e-15, 6.1264e-14, 2.5156e-15,\n        2.1917e-14, 2.0189e-14, 4.9649e-15, 8.5892e-14, 8.7270e-15, 1.8589e-14,\n        2.8827e-16, 3.4849e-15, 1.4385e-14, 2.6525e-16, 2.7969e-16, 2.9280e-14,\n        9.8105e-18, 2.8117e-15, 4.4198e-15, 2.7781e-17, 1.2804e-13, 5.5933e-16,\n        5.9494e-15, 2.1105e-14, 4.9149e-14, 6.2792e-15, 5.3366e-15, 2.7729e-16,\n        1.4682e-14, 7.0372e-15, 1.5414e-14, 1.0339e-16, 1.2182e-14, 5.9221e-15,\n        5.1183e-14, 6.7273e-17, 1.8105e-15, 4.6788e-14, 2.1029e-15, 3.3100e-19,\n        4.5253e-16, 2.9757e-14, 1.9544e-14, 1.2691e-15, 3.3755e-15, 1.4597e-14,\n        4.3133e-14, 6.1434e-16, 1.1555e-15, 2.2811e-14, 1.9473e-14, 2.6578e-15,\n        4.9462e-16, 2.1320e-16, 1.4152e-14, 3.6094e-14, 1.6443e-17, 6.0078e-14,\n        2.8798e-18, 1.3232e-17, 3.5427e-16, 5.4950e-17, 2.2733e-14, 2.2409e-16,\n        2.6427e-17, 5.1928e-15, 1.8466e-14, 7.9458e-14, 3.7456e-16, 1.1225e-15,\n        2.4450e-14, 3.9789e-16, 3.2479e-17, 5.2667e-14, 1.7259e-16, 3.0848e-16,\n        1.5385e-16, 3.4459e-15, 1.8615e-16, 8.3501e-15, 7.9681e-16, 1.2287e-17,\n        1.5875e-17, 1.0795e-14, 6.3987e-15, 8.1421e-17, 9.9318e-15, 4.8459e-14,\n        2.8109e-18, 4.3294e-16, 5.8572e-15, 8.1071e-17, 5.9741e-16, 6.3204e-17,\n        4.9657e-16, 1.4799e-16, 2.1633e-14, 4.8524e-15, 9.4044e-16, 2.6782e-16,\n        4.9766e-14, 2.4154e-14, 2.0946e-14, 1.0852e-15, 1.5817e-17, 7.1548e-16,\n        2.0419e-14, 1.6192e-14, 4.8403e-14, 2.9906e-14, 4.3099e-17, 3.9916e-16,\n        8.3299e-15, 2.6490e-17, 4.5445e-15, 1.0106e-14, 2.8378e-16, 2.2838e-16,\n        1.0083e-16, 2.8554e-14, 9.9227e-15, 4.9841e-16, 2.3842e-15, 8.0590e-14,\n        3.5875e-15, 1.0757e-13, 1.4240e-16, 1.0912e-16, 1.2242e-13, 2.6387e-14,\n        1.3890e-15, 1.4647e-14, 8.7610e-14, 1.4929e-19, 4.2127e-15, 1.5538e-14,\n        2.3101e-16, 5.4382e-15, 3.5149e-14, 6.9386e-17, 1.1534e-16, 1.2815e-16,\n        8.1678e-14, 5.3521e-15, 6.2258e-17, 2.9980e-16, 1.8382e-15, 1.2700e-15,\n        8.7661e-15, 3.4421e-16, 9.5028e-14, 1.7243e-14, 2.3812e-14, 4.3695e-18,\n        1.6450e-13, 4.7252e-15, 3.5979e-14, 1.7205e-14, 3.0419e-15, 8.4155e-16,\n        5.7733e-14, 3.9431e-16, 3.8657e-16, 5.7446e-16, 2.9062e-14, 2.6659e-16,\n        9.3301e-16, 2.9798e-16, 3.0148e-18, 9.6856e-18, 4.5247e-16, 1.5620e-15,\n        1.9611e-16, 8.9351e-17, 3.0258e-15, 6.6392e-16, 2.2979e-14, 2.1256e-14,\n        7.2722e-15, 1.4651e-14, 2.5733e-14, 2.9843e-18, 4.7045e-15, 4.4286e-15,\n        1.1340e-15, 1.4223e-16, 3.9938e-16, 8.5528e-15, 3.5290e-15, 1.3987e-15,\n        1.7231e-14, 1.7361e-14, 2.0047e-14, 2.0978e-17, 1.0724e-16, 3.5778e-16,\n        1.0937e-15, 2.3805e-14, 3.9330e-14, 7.1175e-15, 1.0000e-16, 1.2344e-19,\n        4.5658e-14, 1.1041e-16, 4.8490e-15, 7.4635e-16, 1.3129e-14, 1.2976e-17,\n        1.3215e-14, 3.4333e-14, 1.8169e-15, 2.7808e-14, 1.2546e-15, 7.2758e-17,\n        1.6825e-15, 9.4392e-15, 5.2494e-17, 9.8125e-14, 3.6459e-15, 8.6135e-18,\n        7.8506e-17, 1.3176e-16, 5.1135e-15, 3.8044e-16, 2.5634e-13, 4.7602e-17,\n        2.1314e-14, 1.3028e-15, 1.3056e-16, 2.6518e-17, 1.4711e-15, 9.9503e-17,\n        7.8404e-16, 1.5319e-15, 6.1182e-15, 1.1131e-15, 5.2355e-17, 1.0860e-14,\n        1.0301e-16, 1.4489e-14, 2.8490e-16, 9.4050e-16, 1.3571e-15, 7.7146e-17,\n        4.8999e-15, 3.3150e-14, 1.1778e-16, 2.0004e-18, 2.9462e-14, 2.2995e-14,\n        6.8636e-17, 1.1783e-17, 2.3164e-14, 5.6011e-14, 3.0965e-17, 1.3778e-16,\n        1.0627e-14, 6.3170e-15, 1.4361e-15, 3.6277e-17, 6.1737e-16, 1.4222e-14,\n        1.5603e-14, 5.1926e-16, 2.1233e-14, 3.6614e-14, 4.9740e-14, 1.7815e-14,\n        3.5718e-16, 9.2226e-15, 3.5139e-15, 7.2592e-15, 5.2272e-14, 1.1556e-13,\n        6.2364e-15, 7.4261e-14, 2.5098e-14, 3.9715e-15, 7.4136e-14, 3.9886e-15,\n        1.6850e-13, 8.5219e-15, 1.9417e-18, 2.2009e-17, 2.2549e-14, 3.2633e-15,\n        2.5124e-17, 1.9963e-16, 1.2561e-15, 5.5058e-16, 2.4505e-14, 1.9809e-15,\n        7.5764e-15, 1.9581e-14, 9.4806e-18, 3.9433e-14, 2.7775e-15, 4.5376e-14,\n        6.2126e-14, 1.3823e-15, 8.2227e-15, 2.5019e-14, 2.7409e-15, 1.1789e-16,\n        1.1772e-15, 6.0371e-14, 1.4743e-17, 2.4223e-17, 1.5157e-14, 1.6110e-17,\n        1.5057e-16, 1.2007e-16, 3.0979e-18, 3.1986e-14, 5.1525e-14, 5.1432e-14,\n        3.1345e-15, 3.8850e-15, 3.5804e-15, 3.9467e-16, 1.6984e-16, 1.2830e-16,\n        6.8850e-16, 2.8452e-14, 1.0817e-16, 6.9281e-15, 8.1979e-16, 3.3029e-14,\n        4.2934e-17, 2.2728e-15, 1.3914e-16, 3.6696e-16, 4.6368e-16, 1.0480e-15,\n        1.0683e-14, 1.7841e-15, 8.4247e-15, 2.8226e-16, 2.4045e-16, 1.5610e-15,\n        2.9077e-17, 9.3646e-16, 4.7030e-17, 1.6358e-14, 4.7477e-16, 2.2424e-14,\n        1.2166e-14, 6.4177e-14, 4.0529e-17, 6.3434e-16, 2.9727e-14, 4.6981e-14,\n        3.9238e-16, 7.4567e-15, 7.1529e-17, 2.0430e-16, 4.9037e-15, 9.0330e-16,\n        2.2150e-17, 3.8200e-16, 1.2793e-13, 6.7271e-14, 7.9809e-14, 1.6658e-16,\n        7.3825e-14, 3.2560e-16, 1.2315e-15, 2.1557e-14, 2.1903e-16, 6.9718e-16,\n        7.5409e-15, 3.0830e-14, 7.4259e-15, 6.2769e-16, 4.1342e-15, 5.4587e-15,\n        7.4415e-14, 6.8470e-16, 2.2945e-15, 3.1732e-16, 1.1338e-15, 1.6352e-16,\n        1.1068e-13, 1.0706e-16, 4.0026e-15, 9.0886e-16, 1.7500e-15, 5.1977e-15,\n        4.4792e-15, 6.1800e-15, 3.6048e-14, 5.8288e-16, 2.4310e-14, 4.1135e-18,\n        1.0184e-14, 3.3869e-14, 2.0890e-18, 2.8412e-15, 3.3638e-15, 3.6357e-15,\n        1.5468e-17, 3.5122e-14, 5.2120e-15, 1.9189e-17, 5.3032e-16, 1.4257e-14,\n        7.0607e-15, 1.6540e-17, 7.3179e-17, 3.1578e-16, 3.9173e-16, 2.8699e-17,\n        1.8706e-14, 9.3643e-14, 1.5195e-15, 1.4842e-13, 2.1002e-14, 4.2074e-18,\n        2.8659e-15, 1.0417e-16, 8.5811e-15, 8.7103e-15, 3.2774e-16, 1.1868e-16,\n        2.5419e-16, 1.0239e-13], device='cuda:0')"
      },
      "76": {
        "step": "tensor(11268.)",
        "exp_avg": "tensor([[ 1.1576e-38,  3.5089e-38,  1.0218e-38,  ...,  6.1739e-39,\n          1.2687e-38,  2.1135e-38],\n        [ 1.0486e-38,  2.6551e-39,  8.1210e-39,  ...,  5.9969e-39,\n          1.8183e-39,  1.5416e-38],\n        [ 1.2618e-38,  3.3055e-39, -1.2715e-39,  ...,  2.9707e-39,\n          1.5904e-39,  6.3622e-39],\n        ...,\n        [-8.4365e-38, -5.7873e-38, -5.9229e-38,  ..., -2.1116e-38,\n         -3.3179e-38, -9.8168e-38],\n        [-1.9121e-37, -1.2993e-37, -1.2843e-37,  ..., -4.5619e-38,\n         -7.0741e-38, -2.0305e-37],\n        [-7.0863e-38, -5.3600e-38, -4.9282e-38,  ..., -1.6480e-38,\n         -2.5795e-38, -7.9400e-38]], device='cuda:0')",
        "exp_avg_sq": "tensor([[1.3047e-15, 3.5658e-15, 3.2753e-17,  ..., 3.2421e-16, 4.0554e-16,\n         7.7453e-16],\n        [4.8639e-15, 6.2193e-16, 1.1374e-15,  ..., 4.6817e-15, 3.3601e-15,\n         6.1111e-15],\n        [1.6939e-16, 2.1047e-15, 6.5155e-16,  ..., 1.0478e-15, 1.2865e-15,\n         4.4542e-16],\n        ...,\n        [1.4960e-13, 1.5558e-13, 1.0871e-13,  ..., 1.2818e-13, 4.9310e-14,\n         2.0609e-13],\n        [5.0901e-13, 5.6484e-13, 3.1361e-13,  ..., 3.8826e-13, 1.6444e-13,\n         6.7328e-13],\n        [1.2459e-12, 9.5813e-13, 7.7809e-13,  ..., 7.9842e-13, 2.5647e-13,\n         1.6270e-12]], device='cuda:0')"
      },
      "77": {
        "step": "tensor(11268.)",
        "exp_avg": "tensor([ 4.4994e-38,  1.8788e-38,  1.7028e-38,  ..., -1.1096e-37,\n        -2.4830e-37, -9.7838e-38], device='cuda:0')",
        "exp_avg_sq": "tensor([3.9147e-15, 2.3972e-15, 1.2475e-14,  ..., 1.7804e-12, 6.5102e-12,\n        1.0490e-11], device='cuda:0')"
      },
      "78": {
        "step": "tensor(11268.)",
        "exp_avg": "tensor([[-1.0137e-37,  8.4623e-38, -2.7644e-37,  ...,  2.0287e-37,\n          2.3229e-37,  1.7978e-37],\n        [ 1.4747e-37, -1.2127e-37,  4.0638e-37,  ..., -3.1484e-37,\n         -3.2443e-37, -2.5609e-37],\n        [-1.9392e-38,  8.3829e-39, -3.0665e-38,  ...,  8.9807e-39,\n          4.5111e-38,  2.4748e-38],\n        ...,\n        [-1.3320e-37,  1.0757e-37, -3.5432e-37,  ...,  2.5053e-37,\n          3.0617e-37,  2.2795e-37],\n        [ 9.4387e-38, -5.6904e-38,  2.3082e-37,  ..., -1.6266e-37,\n         -1.8874e-37, -1.4078e-37],\n        [-1.7728e-38,  3.8989e-39, -3.2109e-38,  ...,  1.6087e-38,\n          3.1951e-38,  2.2502e-38]], device='cuda:0')",
        "exp_avg_sq": "tensor([[3.2701e-13, 1.0460e-12, 1.2211e-13,  ..., 5.3603e-14, 2.4000e-14,\n         2.1623e-13],\n        [1.3163e-12, 3.6053e-12, 1.0769e-12,  ..., 6.0127e-13, 2.3879e-13,\n         6.6051e-13],\n        [3.1250e-14, 5.0088e-14, 2.6626e-14,  ..., 8.4782e-15, 2.5403e-14,\n         6.0510e-15],\n        ...,\n        [1.7239e-13, 6.2386e-13, 3.0198e-13,  ..., 2.1667e-13, 5.6043e-14,\n         1.1186e-13],\n        [3.2821e-13, 1.1063e-12, 3.5227e-13,  ..., 1.9318e-13, 9.9735e-14,\n         1.8671e-13],\n        [6.8965e-14, 3.9926e-13, 4.4305e-14,  ..., 1.9908e-14, 4.9137e-15,\n         6.8734e-14]], device='cuda:0')"
      },
      "79": {
        "step": "tensor(11268.)",
        "exp_avg": "tensor([-2.6321e-37,  3.8083e-37, -3.0141e-38,  2.9271e-37,  2.2262e-37,\n        -1.7921e-37,  3.1918e-37, -2.6110e-37,  2.1527e-37, -1.9510e-39,\n         2.7603e-37,  2.8856e-37,  3.0732e-37,  2.7641e-37,  3.4909e-37,\n         1.0794e-37, -2.2477e-37,  1.2464e-37,  2.7291e-37,  4.1622e-38,\n         2.0276e-37, -1.8678e-37, -1.5725e-37, -2.8002e-37, -1.8800e-37,\n         3.3233e-37, -2.6239e-37,  2.6886e-37,  1.5140e-37, -8.8919e-38,\n         2.3978e-37,  3.8869e-37, -1.8457e-37, -2.5316e-37,  1.7459e-37,\n        -3.8648e-37,  1.9234e-37,  2.9653e-37, -1.8307e-37, -2.3719e-37,\n         1.5989e-37, -2.4079e-37,  2.3278e-37, -2.5090e-37,  2.1638e-37,\n        -3.2252e-37,  2.7312e-37, -1.7154e-37,  1.6601e-37,  5.4762e-38,\n        -1.8269e-37,  1.3005e-37, -2.3928e-37,  2.9268e-37,  6.5752e-38,\n        -1.4503e-37,  3.2099e-37,  2.7756e-37, -4.1507e-38, -2.7362e-37,\n         2.8282e-37, -1.7719e-37,  1.8859e-37, -3.7664e-37,  4.4125e-40,\n         1.2458e-37, -3.2238e-38, -2.2043e-37, -3.6862e-37, -3.1378e-37,\n         2.4130e-37, -1.2415e-37,  2.7902e-37,  2.0971e-37, -3.8896e-37,\n        -3.4889e-38,  3.0307e-37,  3.0024e-37, -2.2135e-37,  1.6960e-37,\n        -1.8150e-37, -2.0084e-37, -3.1529e-38, -2.2470e-37,  2.6302e-37,\n         2.9088e-37,  2.4580e-37,  2.2019e-37, -2.5449e-38,  2.9694e-37,\n         1.2268e-37, -1.1334e-37, -3.3673e-37, -2.7484e-37, -1.6998e-37,\n         2.1646e-37, -2.9052e-37, -1.3106e-38,  2.1386e-37, -1.9890e-37,\n         8.9316e-38, -2.1618e-37,  3.4659e-37, -2.5971e-37,  2.3421e-37,\n         2.1110e-37,  4.2259e-38,  2.5241e-37,  7.1137e-38,  2.1442e-37,\n         2.6792e-37, -2.7468e-37,  2.0384e-37, -1.8865e-37, -2.6325e-37,\n        -1.8031e-37,  2.3669e-37,  2.7688e-37, -8.1240e-38, -4.8831e-38,\n         2.6584e-38, -1.8406e-37, -2.5706e-37,  3.0593e-37, -1.5980e-37,\n         2.9478e-37,  2.9752e-37, -3.3072e-37,  1.1129e-37,  2.1453e-37,\n        -5.3121e-38,  2.4889e-37, -3.2799e-37, -2.4545e-37, -1.9889e-37,\n        -3.0003e-37, -1.9879e-37, -1.8943e-37,  2.2975e-37, -2.5907e-37,\n         2.3676e-37,  2.9839e-37,  3.7756e-38, -2.9878e-37, -2.0804e-37,\n         3.0400e-37, -2.6167e-37, -2.6929e-37,  1.2177e-37, -1.9907e-37,\n        -2.4199e-37,  2.5555e-37,  4.3076e-38,  1.8263e-37, -2.5702e-37,\n         2.5661e-37, -2.4857e-37,  9.9476e-38,  3.0129e-37,  3.0256e-37,\n         1.4898e-38,  2.6322e-37, -1.3319e-37, -1.9241e-37,  2.5429e-37,\n        -2.8953e-37,  2.6623e-37, -1.4463e-37, -3.6586e-38,  2.5827e-37,\n         2.0427e-37,  1.5411e-37, -2.0967e-37, -3.3176e-37,  3.4829e-37,\n        -1.9672e-37, -3.1761e-37, -2.2158e-37, -2.6994e-37,  2.4433e-37,\n         6.1856e-38,  1.7047e-37, -2.5198e-37,  2.6205e-37, -1.4663e-37,\n         2.4166e-37,  5.9094e-38, -9.0848e-38,  1.3274e-37, -1.9090e-37,\n         2.5220e-37,  2.4644e-37, -2.3481e-37,  1.6620e-37, -2.0346e-37,\n         2.1686e-37, -2.3918e-37, -2.6433e-37,  2.2119e-37, -1.8418e-37,\n         2.9791e-37,  1.1739e-37,  3.9903e-38, -3.9794e-37, -3.6534e-37,\n        -2.9063e-37,  1.5979e-37,  2.2015e-37,  2.1930e-37, -3.1876e-37,\n        -6.8129e-38,  1.3725e-37,  2.0284e-37,  2.7246e-37,  1.5083e-37,\n         1.1443e-38,  2.2910e-37, -3.1665e-37,  1.7853e-37, -3.3473e-37,\n        -2.0880e-37,  2.9445e-37, -3.1066e-37, -1.8439e-38, -1.9829e-37,\n         2.3746e-37, -2.4474e-37, -3.1420e-38, -1.4397e-37,  5.0848e-38,\n        -2.7780e-37,  2.8264e-37,  3.2096e-37,  2.3862e-37,  1.7818e-37,\n        -2.5047e-37, -2.8644e-37, -2.9563e-37, -8.8431e-38,  2.2495e-37,\n         1.6090e-37, -2.3335e-37,  4.0312e-37, -3.1495e-38,  2.5100e-37,\n         1.7112e-37,  1.0492e-37, -2.8028e-37, -2.4051e-37,  3.2954e-37,\n         2.2077e-37,  2.1971e-37, -2.2970e-37, -8.8366e-38, -2.4886e-37,\n        -1.3058e-37, -2.9436e-37, -2.4931e-37,  2.1693e-37, -3.4115e-37,\n         3.0765e-37, -1.8846e-37,  3.3314e-37,  2.4176e-37,  4.1460e-37,\n         1.7519e-37, -2.6126e-37,  4.3628e-38, -1.5949e-37, -2.9422e-37,\n        -2.4249e-37, -1.1730e-38,  3.2173e-37, -3.0802e-37, -3.5805e-37,\n        -3.4441e-37, -2.2551e-37,  9.5921e-38,  2.2337e-37, -3.4952e-38,\n         2.4668e-37, -1.4553e-38, -3.8951e-37,  3.4555e-38, -1.3763e-37,\n        -2.0910e-37,  2.4549e-37, -1.5738e-37, -1.3586e-37,  2.1630e-37,\n        -2.9617e-37, -2.6852e-37, -1.3456e-38, -3.4368e-37,  2.8935e-37,\n        -2.2373e-37,  1.8720e-37, -2.4813e-37, -1.5643e-37,  2.1943e-37,\n        -5.3093e-38, -2.6620e-37,  2.4335e-37,  2.5593e-37, -8.8570e-38,\n         3.5021e-37,  1.5719e-37, -2.5119e-37,  3.5227e-37,  3.4392e-37,\n        -2.0790e-37,  2.7011e-37, -2.5019e-37,  1.4396e-37, -1.8096e-37,\n         1.6849e-37,  3.1440e-37,  1.9517e-37,  2.4081e-37,  3.3058e-37,\n        -3.1271e-37,  2.6759e-37, -2.5040e-38,  2.6424e-37,  2.0239e-37,\n         1.1032e-37,  1.9604e-37, -1.8347e-37,  2.2724e-37, -2.4485e-37,\n        -2.9709e-37,  2.3258e-37,  3.1548e-37, -1.4110e-37,  3.6673e-37,\n         2.9767e-37, -3.0141e-37, -1.3142e-37, -2.6689e-37, -2.3955e-37,\n         3.1927e-37, -1.7909e-37, -1.4714e-38, -1.7525e-37,  2.4588e-37,\n         1.4224e-37,  2.9946e-37, -2.2313e-37, -5.7989e-39, -3.5159e-37,\n         2.7225e-37, -2.0538e-37, -3.5792e-37,  6.0428e-38, -3.2878e-37,\n        -1.9168e-37, -1.0335e-37, -2.6534e-37,  3.1561e-37, -4.5821e-37,\n         3.1674e-37, -2.8292e-37, -1.1555e-37,  2.9088e-37,  2.9818e-37,\n        -2.4270e-37,  3.5082e-37,  2.7531e-37,  6.4367e-38, -2.0384e-37,\n         3.7561e-37, -2.9131e-37,  1.8421e-37, -2.2226e-37,  3.0723e-37,\n         2.5107e-37,  2.6579e-37, -6.9652e-38, -2.8914e-37, -2.0049e-37,\n         4.2370e-38, -2.0183e-37, -2.9946e-37, -2.1226e-37, -1.9680e-37,\n        -2.0525e-37, -8.7495e-38, -2.6288e-37,  2.4706e-37, -2.1162e-37,\n         1.9602e-37,  2.3188e-37, -2.7350e-37,  1.3787e-37, -5.1297e-38,\n         1.6521e-37, -2.1562e-37,  2.1618e-37,  3.6695e-37, -1.9988e-37,\n         3.5667e-38,  1.5313e-37, -3.9630e-37,  2.9867e-37,  2.2449e-37,\n         2.5922e-37,  2.8580e-37,  4.1748e-38,  2.1509e-37, -2.7583e-37,\n         2.8490e-37,  2.6552e-37,  2.5206e-37, -2.5436e-37,  1.5894e-37,\n        -3.1834e-37,  2.6340e-37, -1.9739e-37,  2.2900e-37, -2.6695e-37,\n         2.4032e-37, -1.4990e-37,  2.5777e-37, -2.5055e-37, -6.4882e-39,\n        -2.8171e-37,  2.9364e-37,  2.7645e-37, -3.5677e-37,  6.6556e-38,\n        -1.5008e-37,  1.5075e-37,  2.9349e-37,  2.3297e-37,  1.9231e-37,\n         2.4738e-37,  2.7827e-37, -2.3263e-37, -2.8145e-37, -1.8693e-37,\n         2.0882e-37,  2.8277e-37, -2.6191e-37, -3.4376e-37, -2.6250e-37,\n         1.4815e-37, -1.4557e-37,  9.2072e-38,  2.3267e-37, -1.7737e-37,\n        -2.3783e-37,  2.3769e-37, -1.4967e-37, -2.2552e-37, -4.4909e-38,\n        -1.7469e-37, -3.0562e-37, -1.6710e-37, -1.0194e-37,  2.7848e-38,\n        -2.2819e-37, -3.9923e-39, -2.4881e-38,  2.3328e-37, -9.7280e-38,\n        -2.4434e-37,  2.1698e-37,  2.7465e-37, -1.7777e-37,  3.3820e-37,\n         2.4489e-37, -1.9880e-37,  2.0645e-37,  2.3871e-37, -3.2892e-37,\n         1.4659e-37,  2.7377e-37, -1.6392e-37, -2.0002e-37,  3.0541e-37,\n         2.7418e-37, -2.8768e-37,  1.8319e-37,  2.4004e-37,  3.1318e-37,\n        -2.7187e-37, -1.2697e-38, -3.1752e-37,  3.0955e-37, -2.3877e-37,\n        -2.6913e-37,  1.7840e-37, -2.3497e-37,  9.9906e-38, -3.0823e-37,\n        -2.9907e-38,  1.3910e-37, -9.2321e-38,  2.4072e-37, -1.8678e-37,\n        -2.5445e-37,  2.6734e-37,  3.1618e-37,  2.2358e-37,  1.2557e-37,\n        -2.5450e-37,  2.0443e-37, -8.1562e-39,  1.6178e-37, -3.3990e-37,\n         2.0786e-37, -2.6223e-38], device='cuda:0')",
        "exp_avg_sq": "tensor([1.1675e-11, 5.0122e-11, 1.1441e-12, 1.3132e-12, 2.3980e-11, 1.8434e-12,\n        5.2891e-12, 2.7357e-11, 1.2546e-12, 1.0790e-12, 9.6548e-12, 1.6269e-12,\n        1.0791e-12, 8.2262e-12, 1.4435e-12, 3.2000e-11, 1.1939e-11, 1.6618e-12,\n        3.8863e-11, 2.0152e-11, 5.2145e-12, 1.6504e-12, 4.8574e-12, 3.7251e-12,\n        1.3698e-11, 2.2774e-12, 1.3451e-11, 6.5875e-12, 6.1979e-12, 1.2200e-12,\n        1.7996e-11, 1.6014e-12, 3.2720e-12, 1.1886e-12, 6.4287e-12, 1.4371e-12,\n        1.8584e-12, 5.7428e-12, 2.3908e-12, 1.2204e-12, 8.2743e-12, 6.7556e-12,\n        1.3924e-11, 2.0008e-12, 1.1910e-12, 6.4998e-12, 9.6357e-13, 8.3881e-12,\n        2.0039e-11, 1.9248e-11, 1.3822e-11, 6.6856e-12, 7.5630e-12, 2.1518e-11,\n        2.2206e-12, 2.4154e-12, 1.2876e-11, 3.8617e-12, 4.2186e-13, 1.2547e-11,\n        4.5216e-12, 4.7906e-12, 1.4984e-12, 2.4882e-12, 1.1555e-11, 8.2955e-12,\n        1.6681e-12, 1.9610e-11, 1.1600e-11, 1.7905e-12, 5.7281e-12, 3.5064e-11,\n        2.6652e-11, 8.8139e-13, 2.4946e-12, 1.8326e-11, 7.5874e-12, 4.2516e-12,\n        2.9918e-12, 1.4647e-11, 5.1010e-12, 1.0165e-11, 3.8006e-11, 2.6989e-12,\n        2.3632e-12, 3.8471e-12, 6.9488e-12, 6.3400e-12, 6.1480e-12, 1.9555e-11,\n        7.5958e-11, 6.2477e-12, 5.3332e-12, 9.9638e-12, 3.4628e-12, 9.7449e-12,\n        2.8353e-11, 1.5020e-11, 1.2366e-12, 8.7788e-13, 7.0453e-11, 3.9408e-12,\n        6.0532e-12, 4.8632e-12, 3.9073e-12, 1.0097e-11, 1.0699e-12, 2.4120e-11,\n        2.1158e-11, 9.7173e-13, 5.5060e-11, 3.0483e-12, 1.0859e-11, 1.1109e-11,\n        6.0656e-12, 4.1056e-11, 1.5801e-12, 6.5814e-12, 6.0811e-12, 1.9620e-12,\n        1.8237e-11, 1.1818e-11, 3.3956e-11, 2.7461e-12, 1.5174e-12, 1.3879e-12,\n        1.1635e-11, 2.1388e-11, 1.1697e-11, 1.0419e-12, 2.2931e-12, 1.0052e-12,\n        6.6725e-12, 8.2272e-12, 8.2463e-12, 1.7240e-12, 1.5338e-13, 3.6541e-12,\n        8.5134e-12, 2.4707e-12, 1.5340e-11, 3.4868e-11, 5.9986e-12, 2.6628e-11,\n        2.6903e-11, 2.6501e-12, 1.4699e-12, 1.5714e-11, 1.0291e-12, 1.0695e-12,\n        1.3295e-11, 5.5130e-12, 9.1821e-12, 1.2767e-12, 2.7670e-11, 2.9855e-12,\n        8.6966e-12, 3.4956e-12, 9.1286e-12, 3.4118e-12, 1.4727e-11, 5.2553e-12,\n        1.8074e-11, 1.1284e-12, 2.2839e-12, 4.3293e-12, 2.6100e-11, 1.0794e-12,\n        1.0435e-12, 9.0387e-12, 1.8965e-11, 7.1881e-12, 1.3340e-12, 3.6826e-11,\n        1.0769e-12, 1.1755e-11, 6.3759e-12, 1.3530e-12, 2.0936e-12, 2.1072e-12,\n        2.9908e-12, 1.3208e-11, 2.3686e-11, 3.3296e-11, 1.5794e-11, 1.4589e-11,\n        7.7424e-12, 4.0372e-12, 2.1847e-12, 2.8505e-11, 5.0880e-12, 7.9454e-12,\n        1.3824e-11, 1.3658e-11, 7.5242e-12, 1.2806e-12, 3.5855e-12, 6.8047e-13,\n        6.6876e-11, 5.1604e-12, 1.2260e-11, 2.5732e-12, 3.2225e-11, 1.2281e-11,\n        1.2484e-12, 1.4041e-11, 1.0583e-12, 5.4701e-11, 1.7782e-12, 2.6664e-12,\n        3.0500e-11, 8.6891e-12, 1.2374e-12, 4.2312e-12, 1.0813e-12, 3.2176e-11,\n        9.3446e-12, 1.3426e-12, 1.9545e-11, 1.5918e-12, 1.1819e-12, 9.1483e-12,\n        1.9840e-12, 2.5901e-12, 7.0625e-12, 6.1055e-11, 1.6884e-12, 1.2224e-11,\n        2.1962e-12, 2.7838e-12, 7.5839e-12, 2.3677e-12, 5.9735e-11, 2.8733e-12,\n        4.8532e-12, 5.1714e-12, 8.8095e-12, 1.0226e-12, 6.4830e-12, 2.8337e-12,\n        9.7443e-12, 1.0702e-12, 4.2911e-11, 1.0442e-11, 1.9947e-11, 3.2450e-11,\n        1.4353e-12, 3.4531e-12, 3.0924e-12, 1.2952e-12, 5.2566e-12, 2.9146e-12,\n        6.1181e-11, 4.9636e-12, 5.9916e-12, 1.0586e-12, 9.7376e-12, 1.3228e-11,\n        1.0597e-10, 1.6042e-11, 6.9028e-12, 7.5671e-12, 1.5998e-11, 1.2867e-12,\n        2.8562e-12, 2.1279e-11, 1.7036e-11, 1.4638e-11, 1.4924e-11, 5.7490e-12,\n        2.4436e-11, 4.7054e-12, 1.9085e-11, 1.7373e-12, 9.3718e-12, 1.0119e-11,\n        2.3884e-11, 1.4589e-11, 9.4283e-12, 1.0939e-12, 1.9309e-12, 5.2179e-12,\n        1.3881e-11, 1.2357e-11, 9.5826e-13, 1.0404e-11, 2.8668e-12, 3.2995e-13,\n        1.0792e-12, 1.5224e-11, 1.6542e-12, 2.7065e-12, 2.8698e-11, 7.3566e-12,\n        1.0154e-11, 1.3145e-11, 3.4782e-11, 1.7575e-11, 1.9908e-12, 2.1812e-12,\n        1.8861e-11, 8.8969e-12, 1.3073e-11, 1.4193e-11, 1.0788e-12, 1.2016e-10,\n        7.7801e-13, 3.0218e-11, 1.7137e-12, 7.4503e-12, 7.0181e-12, 2.8274e-11,\n        3.0859e-11, 6.1624e-12, 2.8077e-11, 2.5911e-12, 1.4910e-12, 1.3539e-11,\n        1.1943e-11, 7.0290e-12, 1.1154e-10, 8.8242e-12, 1.3748e-13, 3.6692e-11,\n        1.6733e-12, 3.0557e-11, 1.3497e-12, 2.2268e-12, 1.4555e-11, 2.2196e-11,\n        2.2908e-11, 1.3541e-12, 2.7720e-11, 1.2944e-12, 1.1215e-10, 7.7488e-12,\n        3.7744e-12, 1.6624e-12, 3.0463e-12, 6.8732e-12, 6.9618e-12, 3.0381e-12,\n        3.6056e-12, 3.2907e-12, 1.2677e-11, 1.6556e-11, 9.6409e-12, 2.3949e-12,\n        9.4131e-12, 1.7510e-11, 2.1961e-11, 4.1157e-11, 4.5318e-11, 7.2331e-12,\n        1.8900e-11, 1.5496e-11, 1.5436e-11, 7.2867e-12, 1.8534e-12, 3.3970e-12,\n        2.6592e-12, 2.7454e-12, 2.9010e-12, 6.1733e-11, 2.2283e-11, 2.6497e-12,\n        1.4877e-11, 3.4819e-11, 2.1321e-11, 2.9937e-12, 1.2424e-11, 1.5284e-11,\n        1.8567e-12, 2.0166e-12, 4.0988e-12, 1.9968e-11, 4.2338e-12, 1.0356e-12,\n        1.1175e-12, 2.8375e-12, 3.2140e-11, 2.7133e-11, 8.5151e-13, 3.2484e-11,\n        3.6339e-12, 1.3862e-12, 1.0685e-12, 2.3114e-12, 4.8725e-12, 1.9344e-12,\n        1.7020e-12, 1.2065e-12, 1.4342e-12, 8.3399e-12, 1.5780e-11, 6.4466e-12,\n        5.6659e-12, 1.6605e-11, 4.8729e-11, 2.2106e-11, 2.2550e-11, 6.1265e-11,\n        1.6210e-11, 1.6199e-12, 1.3083e-11, 8.9638e-12, 4.3452e-12, 1.1320e-12,\n        6.0372e-11, 1.5975e-11, 3.6231e-12, 2.8447e-12, 6.7449e-11, 4.4400e-12,\n        3.0903e-12, 5.4358e-11, 8.9322e-12, 1.1778e-12, 9.7523e-12, 2.2276e-11,\n        1.1733e-12, 1.0501e-13, 3.8969e-12, 9.8004e-12, 3.3724e-11, 7.4506e-12,\n        8.0449e-12, 1.2271e-12, 1.2440e-11, 4.8026e-11, 7.4239e-12, 1.8355e-12,\n        2.0384e-11, 5.7968e-11, 1.2306e-12, 1.2798e-12, 9.9451e-13, 5.4968e-12,\n        1.0038e-11, 1.2230e-11, 2.3628e-12, 2.2537e-12, 2.1747e-12, 1.5553e-11,\n        5.1520e-11, 2.7652e-11, 6.2249e-12, 2.1239e-12, 2.2030e-11, 1.3911e-11,\n        3.4733e-11, 5.7758e-12, 9.4835e-13, 1.0104e-11, 1.1757e-12, 4.1015e-12,\n        3.9046e-12, 2.4164e-11, 1.1290e-11, 2.0128e-12, 3.9001e-12, 5.2738e-12,\n        1.5954e-11, 1.5887e-12, 1.7340e-11, 1.0223e-12, 4.2960e-11, 3.2261e-12,\n        5.8156e-12, 3.1244e-11, 2.5654e-12, 1.0802e-12, 8.5733e-12, 1.1234e-12,\n        3.6047e-12, 1.6020e-12, 2.7236e-12, 2.4998e-11, 2.6777e-11, 1.8221e-11,\n        3.3499e-12, 8.4230e-11, 1.7041e-12, 9.1307e-12, 2.2938e-12, 8.8043e-12,\n        2.4382e-12, 5.1889e-12, 6.2600e-12, 3.8010e-13, 1.1495e-12, 1.2488e-11,\n        2.5828e-12, 2.0201e-11, 5.0403e-12, 2.0492e-11, 1.1043e-12, 1.0195e-11,\n        1.0655e-12, 2.3772e-12, 8.7914e-12, 5.2873e-12, 3.9322e-12, 1.1119e-12,\n        5.5508e-12, 5.8180e-12, 2.5129e-11, 5.1812e-12, 1.4832e-12, 1.0954e-11,\n        1.6764e-11, 4.5140e-12], device='cuda:0')"
      },
      "80": {
        "step": "tensor(11268.)",
        "exp_avg": "tensor([[ 5.6052e-45, -5.6052e-45,  5.6052e-45,  ...,  5.6052e-45,\n         -5.6052e-45,  5.6052e-45],\n        [-7.0824e-38, -2.0201e-36,  9.5687e-37,  ...,  2.8413e-36,\n         -1.5480e-36, -1.1157e-36],\n        [-1.8598e-38, -3.7291e-37,  1.7793e-37,  ...,  5.4312e-37,\n         -2.8774e-37, -2.1883e-37],\n        ...,\n        [ 1.5885e-39, -1.1716e-37,  4.8167e-38,  ...,  1.6460e-37,\n         -8.9969e-38, -6.0849e-38],\n        [ 1.1643e-38,  1.4763e-37, -7.5103e-38,  ..., -2.1657e-37,\n          1.1513e-37,  9.0531e-38],\n        [ 1.3898e-38,  1.9792e-37, -9.9563e-38,  ..., -2.8844e-37,\n          1.5300e-37,  1.1945e-37]], device='cuda:0')",
        "exp_avg_sq": "tensor([[7.9932e-11, 1.4311e-11, 2.7122e-12,  ..., 3.2402e-10, 3.1335e-11,\n         4.5080e-11],\n        [1.1756e-12, 2.0261e-13, 3.8685e-14,  ..., 4.5282e-12, 4.5188e-13,\n         6.6964e-13],\n        [9.1399e-13, 1.6787e-13, 3.1114e-14,  ..., 3.8683e-12, 3.5947e-13,\n         5.1043e-13],\n        ...,\n        [9.2765e-13, 1.6857e-13, 3.1535e-14,  ..., 3.8384e-12, 3.6707e-13,\n         5.2111e-13],\n        [8.6704e-13, 1.6177e-13, 2.9794e-14,  ..., 3.7505e-12, 3.4486e-13,\n         4.8120e-13],\n        [9.1406e-13, 1.5986e-13, 3.1004e-14,  ..., 3.5714e-12, 3.5693e-13,\n         5.1914e-13]], device='cuda:0')"
      },
      "81": {
        "step": "tensor(11268.)",
        "exp_avg": "tensor([-5.6052e-45, -1.2599e-36, -2.3892e-37, -1.6297e-37, -1.1143e-37,\n        -2.0629e-37, -3.4774e-37, -6.5847e-38,  1.0171e-37,  1.3274e-37],\n       device='cuda:0')",
        "exp_avg_sq": "tensor([4.8925e-09, 7.2751e-11, 5.5600e-11, 5.2776e-11, 6.2007e-11, 5.8327e-11,\n        8.0375e-11, 5.6666e-11, 5.2725e-11, 5.6065e-11], device='cuda:0')"
      },
      "82": {
        "step": "tensor(11268.)",
        "exp_avg": "tensor([[ 5.6052e-45, -5.6052e-45,  5.6052e-45,  ...,  5.6052e-45,\n         -5.6052e-45,  5.6052e-45],\n        [-7.1317e-38, -2.0341e-36,  9.6354e-37,  ...,  2.8611e-36,\n         -1.5587e-36, -1.1235e-36],\n        [-1.8727e-38, -3.7551e-37,  1.7917e-37,  ...,  5.4690e-37,\n         -2.8975e-37, -2.2036e-37],\n        ...,\n        [ 1.5995e-39, -1.1798e-37,  4.8503e-38,  ...,  1.6575e-37,\n         -9.0596e-38, -6.1273e-38],\n        [ 1.1724e-38,  1.4865e-37, -7.5626e-38,  ..., -2.1808e-37,\n          1.1593e-37,  9.1161e-38],\n        [ 1.3995e-38,  1.9930e-37, -1.0026e-37,  ..., -2.9045e-37,\n          1.5407e-37,  1.2028e-37]], device='cuda:0')",
        "exp_avg_sq": "tensor([[7.9999e-11, 1.4330e-11, 2.7146e-12,  ..., 3.2452e-10, 3.1364e-11,\n         4.5110e-11],\n        [1.1765e-12, 2.0287e-13, 3.8718e-14,  ..., 4.5350e-12, 4.5226e-13,\n         6.7006e-13],\n        [9.1481e-13, 1.6810e-13, 3.1144e-14,  ..., 3.8744e-12, 3.5982e-13,\n         5.1080e-13],\n        ...,\n        [9.2845e-13, 1.6879e-13, 3.1564e-14,  ..., 3.8444e-12, 3.6742e-13,\n         5.2148e-13],\n        [8.6785e-13, 1.6200e-13, 2.9823e-14,  ..., 3.7565e-12, 3.4521e-13,\n         4.8157e-13],\n        [9.1478e-13, 1.6007e-13, 3.1030e-14,  ..., 3.5767e-12, 3.5725e-13,\n         5.1947e-13]], device='cuda:0')"
      },
      "83": {
        "step": "tensor(11268.)",
        "exp_avg": "tensor([-5.6052e-45, -1.2686e-36, -2.4058e-37, -1.6411e-37, -1.1221e-37,\n        -2.0773e-37, -3.5017e-37, -6.6306e-38,  1.0242e-37,  1.3367e-37],\n       device='cuda:0')",
        "exp_avg_sq": "tensor([4.8963e-09, 7.2803e-11, 5.5646e-11, 5.2819e-11, 6.2046e-11, 5.8371e-11,\n        8.0445e-11, 5.6711e-11, 5.2770e-11, 5.6106e-11], device='cuda:0')"
      },
      "84": {
        "step": "tensor(11268.)",
        "exp_avg": "tensor([[ 5.6052e-45, -5.6052e-45,  5.6052e-45,  ...,  5.6052e-45,\n         -5.6052e-45,  5.6052e-45],\n        [-7.1423e-38, -2.0371e-36,  9.6496e-37,  ...,  2.8653e-36,\n         -1.5610e-36, -1.1251e-36],\n        [-1.8755e-38, -3.7606e-37,  1.7944e-37,  ...,  5.4771e-37,\n         -2.9017e-37, -2.2068e-37],\n        ...,\n        [ 1.6019e-39, -1.1815e-37,  4.8574e-38,  ...,  1.6599e-37,\n         -9.0730e-38, -6.1364e-38],\n        [ 1.1742e-38,  1.4887e-37, -7.5738e-38,  ..., -2.1840e-37,\n          1.1611e-37,  9.1296e-38],\n        [ 1.4016e-38,  1.9960e-37, -1.0040e-37,  ..., -2.9088e-37,\n          1.5430e-37,  1.2046e-37]], device='cuda:0')",
        "exp_avg_sq": "tensor([[7.9999e-11, 1.4330e-11, 2.7146e-12,  ..., 3.2452e-10, 3.1364e-11,\n         4.5110e-11],\n        [1.1765e-12, 2.0287e-13, 3.8718e-14,  ..., 4.5350e-12, 4.5226e-13,\n         6.7006e-13],\n        [9.1481e-13, 1.6810e-13, 3.1144e-14,  ..., 3.8744e-12, 3.5982e-13,\n         5.1080e-13],\n        ...,\n        [9.2846e-13, 1.6879e-13, 3.1564e-14,  ..., 3.8444e-12, 3.6742e-13,\n         5.2148e-13],\n        [8.6785e-13, 1.6200e-13, 2.9823e-14,  ..., 3.7565e-12, 3.4521e-13,\n         4.8157e-13],\n        [9.1478e-13, 1.6007e-13, 3.1030e-14,  ..., 3.5767e-12, 3.5725e-13,\n         5.1947e-13]], device='cuda:0')"
      },
      "85": {
        "step": "tensor(11268.)",
        "exp_avg": "tensor([-5.6052e-45, -1.2705e-36, -2.4093e-37, -1.6435e-37, -1.1238e-37,\n        -2.0803e-37, -3.5068e-37, -6.6404e-38,  1.0257e-37,  1.3386e-37],\n       device='cuda:0')",
        "exp_avg_sq": "tensor([4.8963e-09, 7.2803e-11, 5.5646e-11, 5.2819e-11, 6.2046e-11, 5.8371e-11,\n        8.0445e-11, 5.6711e-11, 5.2770e-11, 5.6106e-11], device='cuda:0')"
      },
      "8": {
        "step": "tensor(10016.)",
        "exp_avg": "tensor([[ 5.7050e-08, -5.1685e-07,  1.6811e-06,  ..., -3.7226e-07,\n         -2.3871e-06,  1.3917e-06],\n        [-1.4297e-06, -1.2933e-06,  3.3386e-07,  ...,  1.1636e-07,\n         -6.4691e-09, -2.2877e-06],\n        [ 1.0100e-08, -2.1009e-06,  6.1287e-07,  ..., -3.3782e-07,\n          8.0755e-08,  4.0675e-08],\n        ...,\n        [-3.9232e-07, -2.8939e-07,  2.4743e-06,  ...,  5.9863e-08,\n         -1.2198e-06, -9.4787e-08],\n        [ 8.7591e-08, -2.8485e-06, -2.2011e-07,  ...,  3.8251e-07,\n         -6.0895e-07,  7.7556e-08],\n        [ 1.6157e-06, -6.1933e-07,  2.1022e-06,  ...,  4.5819e-07,\n          7.7928e-07,  2.6391e-07]], device='cuda:0')",
        "exp_avg_sq": "tensor([[1.3517e-11, 1.3955e-10, 4.2245e-11,  ..., 1.9300e-11, 1.5673e-10,\n         6.4332e-10],\n        [1.8164e-10, 2.4463e-11, 1.3559e-11,  ..., 1.0003e-11, 1.9250e-11,\n         1.3946e-10],\n        [7.7739e-13, 3.8199e-11, 9.1008e-12,  ..., 2.7321e-11, 1.9353e-12,\n         2.8163e-11],\n        ...,\n        [7.4574e-11, 6.4268e-11, 7.0429e-11,  ..., 5.6551e-12, 1.7052e-11,\n         3.5875e-11],\n        [4.9513e-12, 3.1334e-10, 3.4765e-12,  ..., 7.2852e-11, 1.1422e-11,\n         5.6882e-12],\n        [5.7027e-12, 4.2127e-11, 7.1064e-11,  ..., 1.1275e-11, 4.6165e-11,\n         5.8064e-11]], device='cuda:0')"
      },
      "9": {
        "step": "tensor(10016.)",
        "exp_avg": "tensor([ 1.1268e-04, -1.7006e-04, -3.7811e-04,  ...,  1.2157e-04,\n         2.5862e-05,  1.2286e-04], device='cuda:0')",
        "exp_avg_sq": "tensor([1.1939e-06, 1.5134e-06, 1.1786e-06,  ..., 1.0489e-06, 1.4385e-06,\n        2.1311e-06], device='cuda:0')"
      },
      "10": {
        "step": "tensor(10016.)",
        "exp_avg": "tensor([[ 1.3993e-06, -1.9184e-07,  1.8302e-06,  ..., -1.1271e-06,\n         -1.0958e-06, -4.3329e-07],\n        [-6.6412e-07,  2.3267e-06,  2.4629e-06,  ...,  1.2537e-06,\n          1.4436e-08,  3.4350e-07],\n        [ 4.6213e-09, -2.7099e-06, -1.8532e-06,  ..., -5.8464e-07,\n         -1.7645e-06, -4.1976e-07],\n        ...,\n        [-1.6098e-06, -2.2373e-07, -7.8352e-07,  ..., -2.5300e-07,\n          1.9053e-06, -3.4868e-06],\n        [-2.8411e-07, -2.1170e-06, -1.6711e-06,  ..., -6.0980e-07,\n          2.7728e-06, -1.2337e-06],\n        [ 9.1927e-07, -2.7363e-07, -2.8442e-06,  ...,  1.1821e-07,\n         -1.1793e-06, -3.2573e-07]], device='cuda:0')",
        "exp_avg_sq": "tensor([[1.8572e-11, 1.9635e-11, 1.7665e-11,  ..., 1.9751e-11, 3.7404e-11,\n         1.8815e-11],\n        [2.9104e-11, 4.2918e-11, 4.2583e-11,  ..., 3.7495e-11, 8.3576e-11,\n         3.1042e-11],\n        [3.6263e-11, 4.8757e-11, 6.8699e-11,  ..., 4.2702e-11, 6.0661e-11,\n         3.9455e-11],\n        ...,\n        [4.1266e-11, 4.4547e-11, 3.0892e-11,  ..., 3.5818e-11, 6.9796e-11,\n         4.1600e-11],\n        [5.9437e-11, 4.5215e-11, 5.0682e-11,  ..., 4.2509e-11, 1.0957e-10,\n         4.4842e-11],\n        [4.6853e-11, 4.1352e-11, 2.9335e-11,  ..., 3.4958e-11, 5.2776e-11,\n         4.6762e-11]], device='cuda:0')"
      },
      "11": {
        "step": "tensor(8764.)",
        "exp_avg": "tensor([[ 1.1960e-06,  4.8689e-08,  3.3830e-07,  ..., -1.8561e-07,\n         -4.8200e-06,  4.3799e-06],\n        [-7.1018e-06, -1.2513e-06, -3.7534e-07,  ...,  3.9371e-08,\n         -2.1806e-07, -4.8822e-09],\n        [-6.1516e-07, -4.3401e-07,  2.1065e-06,  ...,  7.4217e-07,\n          8.2772e-07,  9.7394e-08],\n        ...,\n        [ 1.2986e-06, -2.1679e-08,  6.4351e-07,  ...,  3.1842e-07,\n          2.1852e-07,  1.2099e-07],\n        [-1.2453e-06,  1.7357e-06, -1.4360e-07,  ..., -1.9772e-07,\n          3.9123e-07, -1.3123e-07],\n        [-3.6757e-07,  4.2039e-06,  3.7585e-07,  ...,  9.9166e-09,\n          1.1410e-06,  1.0187e-07]], device='cuda:0')",
        "exp_avg_sq": "tensor([[2.9564e-10, 2.3513e-11, 4.5745e-11,  ..., 8.9984e-12, 1.0203e-10,\n         1.8383e-10],\n        [2.4419e-11, 4.5158e-11, 6.2232e-11,  ..., 7.9474e-12, 1.3270e-11,\n         1.7439e-11],\n        [3.1277e-11, 1.2143e-11, 4.2765e-11,  ..., 1.3530e-10, 3.7011e-11,\n         5.0663e-11],\n        ...,\n        [1.3064e-12, 5.3427e-11, 6.8894e-12,  ..., 3.6728e-10, 7.3827e-12,\n         4.9152e-11],\n        [3.3066e-11, 6.5016e-11, 5.5852e-12,  ..., 6.9002e-12, 7.5802e-11,\n         3.8336e-12],\n        [6.1672e-12, 1.1459e-11, 1.2797e-10,  ..., 1.3299e-11, 4.7892e-11,\n         2.8678e-11]], device='cuda:0')"
      },
      "12": {
        "step": "tensor(8764.)",
        "exp_avg": "tensor([-3.5265e-04, -1.0758e-03, -3.5683e-05,  ...,  1.4777e-03,\n         1.6098e-04, -9.9712e-05], device='cuda:0')",
        "exp_avg_sq": "tensor([9.5481e-07, 1.4150e-06, 1.6084e-06,  ..., 1.2814e-06, 1.1326e-06,\n        8.8751e-07], device='cuda:0')"
      },
      "13": {
        "step": "tensor(8764.)",
        "exp_avg": "tensor([[-1.8908e-07,  9.8710e-07, -1.3207e-06,  ...,  1.7753e-06,\n          3.1482e-07,  7.6963e-07],\n        [-2.1734e-08,  6.4252e-07, -1.2457e-06,  ...,  6.7382e-07,\n         -1.5405e-06, -3.5947e-06],\n        [-8.6494e-07, -5.0943e-06,  1.0616e-06,  ..., -3.0242e-06,\n          1.5758e-06,  3.6576e-06],\n        ...,\n        [-5.0559e-07,  1.9893e-06,  9.6371e-08,  ..., -8.5843e-07,\n         -8.8860e-08, -2.1805e-06],\n        [-9.2237e-08,  6.5942e-07,  4.5598e-08,  ..., -3.4188e-06,\n         -3.5485e-06, -6.9475e-06],\n        [-1.5712e-06, -2.0907e-06,  4.8085e-07,  ..., -2.0015e-06,\n          3.4000e-07, -1.4756e-06]], device='cuda:0')",
        "exp_avg_sq": "tensor([[8.5186e-12, 1.0914e-11, 1.2175e-11,  ..., 2.1242e-11, 8.9989e-12,\n         8.9610e-12],\n        [2.0128e-11, 2.1503e-11, 3.2996e-11,  ..., 1.7734e-11, 2.0164e-11,\n         1.7386e-11],\n        [2.0752e-11, 2.6884e-11, 4.0814e-11,  ..., 4.0228e-11, 1.7165e-11,\n         1.7185e-11],\n        ...,\n        [2.0284e-11, 2.8665e-11, 2.7518e-11,  ..., 8.1900e-11, 2.2509e-11,\n         2.6044e-11],\n        [3.6423e-11, 1.9604e-11, 1.9486e-11,  ..., 6.3043e-11, 2.3930e-11,\n         4.6554e-11],\n        [2.3333e-11, 1.9578e-11, 2.1798e-11,  ..., 4.1505e-11, 2.5272e-11,\n         2.0514e-11]], device='cuda:0')"
      },
      "14": {
        "step": "tensor(7512.)",
        "exp_avg": "tensor([[-2.6486e-07,  7.6264e-07,  5.9651e-07,  ...,  3.5759e-07,\n          1.8258e-07, -1.0782e-07],\n        [-1.5140e-06, -1.9808e-08, -4.7973e-07,  ..., -1.1095e-08,\n         -8.3522e-07,  2.7755e-06],\n        [-2.3315e-07,  9.4581e-07,  2.0395e-06,  ...,  3.4064e-07,\n          2.5774e-07, -5.3986e-07],\n        ...,\n        [ 2.6702e-07,  2.6460e-06, -1.0933e-06,  ...,  1.2117e-06,\n         -6.2369e-07,  1.4578e-07],\n        [-1.1644e-06,  2.6904e-06,  1.0797e-06,  ..., -4.7912e-07,\n          5.8880e-07,  6.8877e-06],\n        [ 1.6223e-06,  1.9716e-06, -2.9099e-07,  ...,  8.6806e-08,\n         -1.1540e-07,  2.0154e-07]], device='cuda:0')",
        "exp_avg_sq": "tensor([[3.0970e-11, 3.2394e-11, 3.0175e-11,  ..., 1.7476e-11, 2.5266e-12,\n         4.5874e-12],\n        [4.7345e-11, 1.4324e-11, 7.2490e-12,  ..., 1.0044e-11, 1.2242e-11,\n         3.6448e-10],\n        [1.3549e-11, 4.5579e-11, 1.0192e-10,  ..., 1.2672e-11, 2.1776e-11,\n         9.4020e-11],\n        ...,\n        [1.3698e-11, 5.2590e-11, 4.7714e-11,  ..., 1.8872e-11, 2.6789e-11,\n         2.0764e-11],\n        [1.1558e-10, 3.4136e-11, 2.5325e-11,  ..., 6.3694e-12, 3.4435e-11,\n         1.3350e-10],\n        [1.4320e-10, 3.7838e-11, 2.5429e-11,  ..., 1.2798e-11, 6.9165e-11,\n         1.2430e-11]], device='cuda:0')"
      },
      "15": {
        "step": "tensor(7512.)",
        "exp_avg": "tensor([ 6.4186e-04, -1.3813e-04,  3.8554e-04,  ...,  8.3271e-05,\n         1.6040e-04, -3.1227e-04], device='cuda:0')",
        "exp_avg_sq": "tensor([2.1189e-06, 1.0022e-06, 9.5622e-07,  ..., 1.2634e-06, 1.1172e-06,\n        1.0429e-06], device='cuda:0')"
      },
      "16": {
        "step": "tensor(7512.)",
        "exp_avg": "tensor([[-1.1308e-06,  4.1987e-07, -7.2713e-07,  ...,  4.1051e-08,\n         -6.9818e-07,  4.6106e-07],\n        [ 1.3566e-06,  4.8331e-07, -8.0298e-07,  ..., -1.3705e-06,\n         -2.5061e-08,  2.4138e-07],\n        [-1.7547e-07,  6.0705e-07, -2.9897e-06,  ..., -8.1847e-08,\n         -1.0605e-07,  1.4517e-06],\n        ...,\n        [ 3.0162e-06,  1.3201e-06, -1.7535e-06,  ...,  1.0880e-06,\n          4.3863e-07,  5.0840e-07],\n        [-9.8453e-08,  3.0499e-07,  2.6118e-06,  ...,  1.2212e-07,\n         -1.1189e-06,  1.1227e-07],\n        [ 1.7745e-06,  1.0085e-06, -1.7517e-07,  ..., -6.2873e-08,\n         -7.1714e-07,  1.4144e-06]], device='cuda:0')",
        "exp_avg_sq": "tensor([[1.5145e-11, 7.7786e-12, 1.0542e-11,  ..., 6.7407e-12, 7.4589e-12,\n         8.6142e-12],\n        [1.8922e-11, 2.1663e-11, 1.4044e-11,  ..., 1.3589e-11, 1.1971e-11,\n         1.2119e-11],\n        [3.2659e-11, 2.0271e-11, 1.5750e-11,  ..., 1.7639e-11, 1.4690e-11,\n         1.7052e-11],\n        ...,\n        [3.3830e-11, 1.5603e-11, 2.3556e-11,  ..., 3.0987e-11, 1.2468e-11,\n         1.6311e-11],\n        [2.7884e-11, 1.6500e-11, 1.7814e-11,  ..., 2.2964e-11, 1.3546e-11,\n         1.6197e-11],\n        [1.8077e-11, 1.7469e-11, 1.5655e-11,  ..., 1.5150e-11, 1.6900e-11,\n         2.4985e-11]], device='cuda:0')"
      },
      "17": {
        "step": "tensor(6260.)",
        "exp_avg": "tensor([[-1.7330e-07, -3.1501e-07,  2.1203e-07,  ..., -1.3401e-07,\n         -1.5521e-06,  1.6324e-07],\n        [-9.4369e-07,  1.0181e-07,  6.0870e-08,  ...,  4.1206e-07,\n          1.2208e-06,  3.7920e-07],\n        [ 2.4497e-07,  3.2532e-06, -7.8083e-08,  ...,  3.3264e-08,\n          3.1814e-07,  4.3007e-07],\n        ...,\n        [-7.3579e-07, -2.8914e-07,  8.8775e-07,  ...,  3.6738e-07,\n         -3.6663e-08, -1.0775e-06],\n        [-6.1262e-07,  1.5515e-07,  1.6460e-06,  ...,  2.6810e-08,\n          1.4007e-06,  1.9532e-07],\n        [-8.1926e-07, -5.9779e-06, -8.8301e-07,  ..., -1.4908e-07,\n          1.6751e-06, -1.9060e-05]], device='cuda:0')",
        "exp_avg_sq": "tensor([[2.4884e-12, 1.4739e-11, 4.2727e-11,  ..., 2.6939e-11, 4.9045e-11,\n         3.4645e-11],\n        [2.7811e-10, 1.2395e-10, 7.9183e-11,  ..., 1.9934e-11, 3.1646e-11,\n         8.7867e-12],\n        [2.9639e-11, 2.7255e-10, 9.2071e-12,  ..., 1.2095e-11, 8.5230e-11,\n         5.1708e-11],\n        ...,\n        [8.0357e-11, 9.0016e-12, 6.5202e-11,  ..., 1.1478e-11, 7.5948e-12,\n         9.6122e-12],\n        [4.0841e-12, 3.5721e-11, 2.9992e-11,  ..., 1.0409e-11, 2.6228e-11,\n         1.1984e-10],\n        [1.3618e-11, 6.5598e-11, 1.1568e-10,  ..., 2.4455e-11, 3.2604e-10,\n         1.7298e-09]], device='cuda:0')"
      },
      "18": {
        "step": "tensor(6260.)",
        "exp_avg": "tensor([-1.1375e-04,  2.8648e-04,  1.8582e-04,  ...,  3.1660e-04,\n         5.0146e-05, -1.0496e-03], device='cuda:0')",
        "exp_avg_sq": "tensor([2.5128e-06, 1.4636e-06, 1.7333e-06,  ..., 1.1624e-06, 1.4712e-06,\n        2.3371e-06], device='cuda:0')"
      },
      "19": {
        "step": "tensor(6260.)",
        "exp_avg": "tensor([[ 1.1009e-06, -5.1273e-09,  1.0374e-07,  ..., -8.9355e-08,\n          2.4032e-07,  6.9289e-07],\n        [-1.5021e-06,  6.8152e-07,  6.5331e-07,  ...,  1.6676e-06,\n          6.6485e-07, -1.2881e-06],\n        [ 1.2996e-06,  1.5315e-06,  1.0814e-06,  ...,  2.6965e-07,\n         -1.7389e-08,  1.3087e-06],\n        ...,\n        [ 1.9104e-06, -4.8307e-07,  1.6462e-07,  ...,  1.0482e-06,\n          4.5014e-07,  2.1247e-06],\n        [-1.5727e-06, -8.0319e-07,  5.5558e-07,  ..., -8.6363e-07,\n          3.3640e-07, -6.1640e-07],\n        [-6.3016e-07, -2.0584e-07,  4.9503e-07,  ...,  9.7925e-08,\n          5.7949e-08,  1.3641e-06]], device='cuda:0')",
        "exp_avg_sq": "tensor([[6.5615e-12, 8.3190e-12, 6.4349e-12,  ..., 1.0229e-11, 4.6112e-12,\n         9.2764e-12],\n        [1.6656e-11, 1.3312e-11, 1.1915e-11,  ..., 9.3713e-12, 1.1798e-11,\n         2.1640e-11],\n        [4.3382e-11, 1.5194e-11, 2.0452e-11,  ..., 2.0931e-11, 1.9668e-11,\n         1.9982e-11],\n        ...,\n        [1.8454e-11, 1.4205e-11, 1.6185e-11,  ..., 1.2540e-11, 1.0684e-11,\n         2.0355e-11],\n        [3.3855e-11, 1.8805e-11, 1.7765e-11,  ..., 1.5283e-11, 1.2346e-11,\n         2.1364e-11],\n        [1.9370e-11, 1.5033e-11, 1.5132e-11,  ..., 1.5773e-11, 1.3393e-11,\n         2.1980e-11]], device='cuda:0')"
      },
      "20": {
        "step": "tensor(5008.)",
        "exp_avg": "tensor([[-8.2295e-07, -7.2032e-06,  7.1120e-07,  ...,  4.1082e-08,\n         -1.1965e-06,  3.1624e-06],\n        [-8.0443e-07,  5.2746e-07, -1.2269e-06,  ...,  3.9362e-07,\n         -3.2844e-06,  1.4410e-06],\n        [ 2.2667e-06, -1.2247e-06, -3.3254e-07,  ...,  1.8890e-06,\n         -5.4874e-07,  1.8567e-06],\n        ...,\n        [ 1.1173e-07,  4.0338e-07, -4.6203e-08,  ...,  3.6269e-07,\n         -3.2862e-07, -5.9698e-07],\n        [-5.2174e-07,  6.3277e-06, -6.8786e-07,  ...,  2.2828e-07,\n          8.9240e-07,  1.1883e-06],\n        [-4.3780e-07, -1.0966e-06,  5.7598e-07,  ...,  4.1342e-07,\n          1.6444e-08, -6.0602e-07]], device='cuda:0')",
        "exp_avg_sq": "tensor([[4.0928e-11, 3.8925e-11, 2.1527e-11,  ..., 1.6600e-11, 9.0558e-11,\n         3.4779e-11],\n        [1.5830e-09, 3.4727e-10, 8.3803e-11,  ..., 1.8061e-11, 2.5868e-10,\n         8.9061e-11],\n        [2.4825e-10, 9.6072e-11, 2.3111e-11,  ..., 4.6567e-11, 2.0907e-11,\n         1.8833e-10],\n        ...,\n        [1.4596e-11, 1.1622e-10, 3.0407e-11,  ..., 3.3292e-10, 5.2848e-10,\n         1.5236e-10],\n        [2.6042e-11, 6.4785e-11, 4.7233e-11,  ..., 2.0630e-11, 3.6177e-11,\n         1.4679e-09],\n        [1.0771e-11, 5.3287e-10, 3.2154e-11,  ..., 3.9186e-11, 4.9260e-11,\n         4.7586e-10]], device='cuda:0')"
      },
      "21": {
        "step": "tensor(5008.)",
        "exp_avg": "tensor([-4.6871e-04,  4.3166e-04,  3.9127e-05,  ..., -2.4907e-05,\n        -3.9612e-05,  3.6801e-04], device='cuda:0')",
        "exp_avg_sq": "tensor([1.6118e-06, 2.2941e-06, 3.4733e-06,  ..., 2.1300e-06, 1.6202e-06,\n        2.6506e-06], device='cuda:0')"
      },
      "22": {
        "step": "tensor(5008.)",
        "exp_avg": "tensor([[-7.3708e-07,  2.0341e-06, -5.3268e-07,  ...,  3.3727e-07,\n         -1.3975e-06,  3.8736e-07],\n        [-1.9185e-07, -4.7559e-06,  5.6406e-07,  ..., -5.7749e-07,\n         -2.0657e-06, -1.0139e-06],\n        [-6.8206e-07,  7.0447e-07,  2.4725e-07,  ...,  1.4039e-06,\n          1.7952e-06, -1.5379e-07],\n        ...,\n        [ 1.6052e-06, -7.0365e-07,  3.3640e-07,  ...,  7.3617e-08,\n          6.1993e-07, -8.3355e-07],\n        [ 1.1799e-07, -1.4564e-06, -9.6870e-08,  ..., -4.6235e-07,\n         -3.1537e-06,  1.5498e-07],\n        [ 1.3223e-06,  4.1186e-07, -3.2634e-07,  ..., -1.7054e-06,\n          4.9534e-07, -1.1708e-06]], device='cuda:0')",
        "exp_avg_sq": "tensor([[1.5133e-11, 1.5640e-11, 1.2609e-11,  ..., 1.1138e-11, 1.3320e-11,\n         1.2314e-11],\n        [1.4546e-11, 2.0823e-11, 1.9579e-11,  ..., 2.2209e-11, 1.9414e-11,\n         2.2285e-11],\n        [1.9346e-11, 2.2941e-11, 2.2349e-11,  ..., 2.1769e-11, 2.0254e-11,\n         3.9127e-11],\n        ...,\n        [1.6474e-11, 2.4684e-11, 2.7243e-11,  ..., 3.1310e-11, 2.3089e-11,\n         1.9151e-11],\n        [4.5990e-11, 3.0819e-11, 2.5504e-11,  ..., 3.4623e-11, 3.1781e-11,\n         2.0217e-11],\n        [2.4685e-11, 2.9307e-11, 2.2814e-11,  ..., 2.5241e-11, 2.6638e-11,\n         2.2540e-11]], device='cuda:0')"
      },
      "23": {
        "step": "tensor(3756.)",
        "exp_avg": "tensor([[-6.3150e-07,  3.3835e-06, -2.6972e-06,  ..., -2.3801e-06,\n         -3.6375e-06, -2.9015e-06],\n        [ 4.2548e-07,  1.8303e-06, -4.7875e-07,  ...,  7.8698e-06,\n         -4.8906e-07,  3.6943e-06],\n        [-3.9494e-06, -1.0209e-06,  3.6311e-06,  ...,  1.5526e-06,\n         -2.3883e-06, -2.0865e-06],\n        ...,\n        [-1.1375e-06,  6.7797e-06, -5.4201e-07,  ..., -7.9800e-07,\n          2.9587e-07,  2.7488e-07],\n        [ 7.7991e-07, -1.4230e-06, -3.4261e-07,  ..., -9.2657e-07,\n         -4.8141e-07,  7.2538e-07],\n        [-3.7056e-06, -2.4140e-06, -1.9163e-08,  ..., -1.1881e-06,\n         -6.4942e-07, -2.3501e-06]], device='cuda:0')",
        "exp_avg_sq": "tensor([[1.4234e-10, 1.7864e-10, 1.2458e-10,  ..., 3.9501e-11, 1.6845e-10,\n         3.0323e-10],\n        [8.7527e-11, 1.4939e-10, 4.0876e-11,  ..., 7.5912e-11, 3.8273e-11,\n         3.8224e-10],\n        [3.8642e-10, 2.1895e-10, 3.0750e-10,  ..., 5.6409e-11, 2.3532e-10,\n         1.2174e-10],\n        ...,\n        [1.6686e-10, 1.6279e-10, 8.9855e-11,  ..., 4.5212e-11, 4.7450e-11,\n         1.2962e-10],\n        [3.0198e-11, 6.5378e-11, 9.7956e-11,  ..., 5.1876e-11, 9.3102e-12,\n         1.4278e-10],\n        [1.2607e-10, 2.4506e-10, 1.4504e-11,  ..., 6.5568e-11, 7.4933e-11,\n         1.5035e-09]], device='cuda:0')"
      },
      "24": {
        "step": "tensor(3756.)",
        "exp_avg": "tensor([ 2.3134e-04, -8.8972e-05,  1.9765e-04,  ...,  6.2072e-04,\n         5.5567e-05, -3.3042e-04], device='cuda:0')",
        "exp_avg_sq": "tensor([3.8390e-06, 3.1507e-06, 6.3202e-06,  ..., 4.1144e-06, 3.4263e-06,\n        3.3025e-06], device='cuda:0')"
      },
      "25": {
        "step": "tensor(3756.)",
        "exp_avg": "tensor([[-5.2149e-07,  9.5402e-07,  4.3525e-07,  ..., -2.1959e-07,\n         -1.7159e-06, -5.4805e-07],\n        [ 3.7949e-07,  4.5399e-06,  1.9552e-06,  ..., -1.3559e-06,\n         -3.1429e-07, -2.5819e-06],\n        [-1.7227e-06,  5.0289e-07,  5.5874e-07,  ...,  4.1645e-07,\n          1.4241e-06, -2.3229e-07],\n        ...,\n        [ 1.7732e-06,  6.2241e-07,  6.5765e-07,  ...,  2.9272e-07,\n          2.5248e-06,  2.8494e-06],\n        [ 6.9161e-07,  6.5170e-07, -3.5903e-07,  ..., -5.7219e-07,\n          1.2496e-06, -1.0320e-06],\n        [ 2.0382e-06, -2.3020e-06, -1.1167e-06,  ..., -4.9296e-07,\n         -9.0276e-07,  1.7637e-07]], device='cuda:0')",
        "exp_avg_sq": "tensor([[2.7893e-11, 2.4078e-11, 2.8512e-11,  ..., 2.4465e-11, 2.3088e-11,\n         2.1927e-11],\n        [4.6955e-11, 4.4291e-11, 4.1413e-11,  ..., 4.4010e-11, 4.0537e-11,\n         4.1859e-11],\n        [7.1498e-11, 4.6716e-11, 5.1987e-11,  ..., 4.2478e-11, 5.3130e-11,\n         3.2828e-11],\n        ...,\n        [7.3611e-11, 4.1172e-11, 6.5415e-11,  ..., 5.1766e-11, 5.1223e-11,\n         3.5036e-11],\n        [7.7461e-11, 4.4375e-11, 6.5059e-11,  ..., 5.2796e-11, 4.7511e-11,\n         3.3096e-11],\n        [5.5564e-11, 4.1861e-11, 5.5235e-11,  ..., 4.5613e-11, 4.3515e-11,\n         4.4222e-11]], device='cuda:0')"
      },
      "26": {
        "step": "tensor(2504.)",
        "exp_avg": "tensor([[-1.6973e-06,  5.8527e-06, -5.4748e-06,  ...,  7.7705e-07,\n         -7.3080e-08, -8.8511e-07],\n        [-8.5396e-07,  1.1453e-05, -2.9107e-06,  ..., -9.0712e-07,\n          4.2041e-06, -1.2851e-05],\n        [-1.3529e-06,  6.1060e-06, -8.0345e-07,  ...,  1.1233e-06,\n          3.2708e-06, -6.3620e-06],\n        ...,\n        [ 1.0960e-06, -9.6453e-07, -2.2089e-06,  ..., -1.6540e-06,\n         -2.5888e-06,  5.1279e-06],\n        [-2.3110e-06, -1.0761e-06, -1.3288e-06,  ...,  2.2022e-07,\n         -3.8565e-07,  2.3899e-06],\n        [ 2.6624e-06,  5.1549e-07,  3.1861e-06,  ...,  5.3918e-07,\n          4.7691e-06, -2.7600e-06]], device='cuda:0')",
        "exp_avg_sq": "tensor([[1.0434e-10, 2.2538e-10, 3.7965e-10,  ..., 1.4912e-10, 1.0817e-10,\n         1.4740e-10],\n        [1.7399e-10, 1.1346e-10, 1.4545e-10,  ..., 1.5087e-10, 2.1916e-10,\n         2.3275e-10],\n        [5.6028e-11, 1.7519e-10, 3.1984e-10,  ..., 1.0024e-10, 3.6255e-10,\n         4.3789e-10],\n        ...,\n        [6.5941e-11, 3.0015e-10, 7.9302e-11,  ..., 9.5854e-11, 7.6468e-11,\n         3.6712e-10],\n        [5.3141e-11, 1.5230e-10, 2.2487e-10,  ..., 1.9951e-10, 5.5189e-11,\n         3.0762e-10],\n        [1.8130e-10, 1.9939e-10, 1.0468e-10,  ..., 5.1182e-11, 5.9189e-10,\n         4.8965e-10]], device='cuda:0')"
      },
      "27": {
        "step": "tensor(2504.)",
        "exp_avg": "tensor([-0.0009, -0.0003,  0.0002,  ...,  0.0003, -0.0002, -0.0002],\n       device='cuda:0')",
        "exp_avg_sq": "tensor([4.5050e-06, 5.7529e-06, 3.4394e-06,  ..., 3.4832e-06, 4.1110e-06,\n        4.6935e-06], device='cuda:0')"
      },
      "28": {
        "step": "tensor(2504.)",
        "exp_avg": "tensor([[-6.1742e-07,  7.9504e-07, -1.1355e-06,  ...,  1.3052e-06,\n         -2.5103e-07,  2.2820e-06],\n        [-3.5861e-06, -8.4926e-07,  2.9967e-06,  ...,  7.5647e-07,\n          3.1124e-06,  1.0118e-06],\n        [-1.9675e-06, -2.6654e-06, -1.7444e-06,  ...,  1.2241e-06,\n         -9.3232e-07, -5.6143e-06],\n        ...,\n        [ 3.5786e-06,  1.6723e-06,  7.6532e-07,  ..., -1.6175e-06,\n          2.3511e-06,  5.4803e-06],\n        [ 2.0428e-06,  1.9126e-06,  2.1259e-07,  ...,  3.5767e-06,\n         -1.0494e-07,  1.2516e-06],\n        [ 1.9037e-06, -6.3253e-07, -1.3919e-06,  ...,  6.7897e-07,\n          1.3036e-07, -3.7711e-06]], device='cuda:0')",
        "exp_avg_sq": "tensor([[4.6255e-11, 6.5529e-11, 4.4341e-11,  ..., 3.6636e-11, 4.2161e-11,\n         4.8270e-11],\n        [1.0113e-10, 8.2593e-11, 6.8456e-11,  ..., 7.6041e-11, 8.0455e-11,\n         8.9699e-11],\n        [1.0803e-10, 7.8008e-11, 7.6404e-11,  ..., 8.3780e-11, 8.4854e-11,\n         9.7344e-11],\n        ...,\n        [1.1011e-10, 8.2386e-11, 8.3526e-11,  ..., 7.3255e-11, 7.4846e-11,\n         8.4580e-11],\n        [9.8648e-11, 8.1055e-11, 7.7127e-11,  ..., 8.0316e-11, 7.6780e-11,\n         1.0066e-10],\n        [8.3123e-11, 9.5305e-11, 8.2457e-11,  ..., 7.5674e-11, 8.2651e-11,\n         1.0157e-10]], device='cuda:0')"
      },
      "29": {
        "step": "tensor(1252.)",
        "exp_avg": "tensor([[-2.6052e-06,  2.5219e-06,  7.2488e-07,  ...,  1.5918e-06,\n         -1.9013e-06,  3.4838e-06],\n        [-1.3042e-06, -6.0475e-06, -4.7019e-06,  ..., -5.5700e-07,\n         -1.0882e-06, -1.6787e-05],\n        [ 3.9001e-06,  9.4369e-06, -5.2936e-06,  ..., -1.4043e-06,\n         -3.5155e-06, -1.2354e-06],\n        ...,\n        [-3.1653e-06, -5.3490e-06,  3.8696e-06,  ..., -1.6639e-06,\n          1.9087e-07, -5.8424e-06],\n        [ 3.2597e-07, -5.4739e-06,  1.5007e-06,  ..., -1.5213e-08,\n          2.2616e-07,  5.9561e-07],\n        [ 2.6093e-06,  2.2761e-06,  2.5087e-06,  ...,  1.8993e-06,\n          1.9907e-06,  3.7199e-06]], device='cuda:0')",
        "exp_avg_sq": "tensor([[2.6168e-10, 3.9755e-10, 2.2285e-10,  ..., 1.5927e-10, 2.0798e-10,\n         4.5759e-10],\n        [3.5770e-10, 1.5860e-10, 1.1347e-10,  ..., 9.4876e-11, 1.5508e-10,\n         1.5948e-09],\n        [2.0544e-10, 2.6259e-10, 2.0819e-10,  ..., 1.6622e-10, 2.5033e-10,\n         8.7416e-11],\n        ...,\n        [3.5001e-11, 1.3137e-10, 2.6936e-10,  ..., 1.9653e-10, 3.9608e-10,\n         1.3868e-09],\n        [2.0421e-10, 3.5765e-10, 1.7765e-10,  ..., 1.5751e-10, 3.6307e-10,\n         6.1023e-10],\n        [2.0386e-10, 1.9421e-10, 2.4710e-10,  ..., 9.5157e-11, 8.6606e-11,\n         1.9460e-10]], device='cuda:0')"
      },
      "30": {
        "step": "tensor(1252.)",
        "exp_avg": "tensor([-0.0003, -0.0003, -0.0002,  ..., -0.0006,  0.0001, -0.0010],\n       device='cuda:0')",
        "exp_avg_sq": "tensor([6.4430e-06, 4.7279e-06, 4.7335e-06,  ..., 4.7203e-06, 6.3075e-06,\n        4.6359e-06], device='cuda:0')"
      },
      "31": {
        "step": "tensor(1252.)",
        "exp_avg": "tensor([[ 7.8205e-06,  1.6603e-06,  5.2407e-06,  ...,  4.8243e-06,\n          3.3522e-06,  2.6506e-06],\n        [ 2.4747e-08, -2.3531e-06, -4.1132e-06,  ...,  6.5673e-06,\n          1.4903e-06,  4.5747e-07],\n        [-1.9430e-06, -2.6426e-07,  4.1306e-06,  ..., -8.1902e-06,\n         -4.5977e-06, -5.2081e-06],\n        ...,\n        [ 1.4329e-06,  3.8159e-06,  2.7730e-06,  ...,  3.5250e-06,\n         -2.7279e-06,  2.3960e-06],\n        [ 7.8748e-06,  9.5467e-06,  4.6303e-06,  ...,  9.2087e-06,\n          1.0976e-05,  8.8552e-06],\n        [-8.4960e-07,  2.5192e-06,  4.6628e-06,  ..., -1.0892e-07,\n          1.9562e-06,  3.9289e-06]], device='cuda:0')",
        "exp_avg_sq": "tensor([[1.3862e-10, 9.5204e-11, 9.6698e-11,  ..., 1.0085e-10, 1.5824e-10,\n         1.0035e-10],\n        [2.2804e-10, 1.8775e-10, 1.6859e-10,  ..., 1.8060e-10, 2.8368e-10,\n         1.8392e-10],\n        [2.5401e-10, 1.7991e-10, 1.5312e-10,  ..., 1.7606e-10, 2.8376e-10,\n         1.8567e-10],\n        ...,\n        [2.3966e-10, 1.7100e-10, 1.5492e-10,  ..., 1.8217e-10, 2.9133e-10,\n         2.0380e-10],\n        [2.6597e-10, 2.1175e-10, 1.7693e-10,  ..., 1.9214e-10, 3.5947e-10,\n         2.0408e-10],\n        [2.4719e-10, 1.7294e-10, 1.9230e-10,  ..., 1.7751e-10, 3.2463e-10,\n         2.1664e-10]], device='cuda:0')"
      }
    },
    "param_groups": [
      {
        "lr": 0.001,
        "name": "shared",
        "betas": [
          0.9,
          0.999
        ],
        "eps": 1e-08,
        "weight_decay": 1e-05,
        "amsgrad": false,
        "maximize": false,
        "foreach": null,
        "capturable": false,
        "differentiable": false,
        "fused": null,
        "decoupled_weight_decay": true,
        "initial_lr": 0.001,
        "params": [
          0,
          1
        ]
      },
      {
        "lr": 0.001,
        "name": "scale_256",
        "betas": [
          0.9,
          0.999
        ],
        "eps": 1e-08,
        "weight_decay": 1e-05,
        "amsgrad": false,
        "maximize": false,
        "foreach": null,
        "capturable": false,
        "differentiable": false,
        "fused": null,
        "decoupled_weight_decay": true,
        "initial_lr": 0.001,
        "params": [
          2,
          3,
          4
        ]
      },
      {
        "lr": 0.001,
        "name": "scale_512",
        "betas": [
          0.9,
          0.999
        ],
        "eps": 1e-08,
        "weight_decay": 1e-05,
        "amsgrad": false,
        "maximize": false,
        "foreach": null,
        "capturable": false,
        "differentiable": false,
        "fused": null,
        "decoupled_weight_decay": true,
        "initial_lr": 0.001,
        "params": [
          5,
          6,
          7
        ]
      },
      {
        "lr": 0.001,
        "name": "scale_768",
        "betas": [
          0.9,
          0.999
        ],
        "eps": 1e-08,
        "weight_decay": 1e-05,
        "amsgrad": false,
        "maximize": false,
        "foreach": null,
        "capturable": false,
        "differentiable": false,
        "fused": null,
        "decoupled_weight_decay": true,
        "initial_lr": 0.001,
        "params": [
          8,
          9,
          10
        ]
      },
      {
        "lr": 0.001,
        "name": "scale_1024",
        "betas": [
          0.9,
          0.999
        ],
        "eps": 1e-08,
        "weight_decay": 1e-05,
        "amsgrad": false,
        "maximize": false,
        "foreach": null,
        "capturable": false,
        "differentiable": false,
        "fused": null,
        "decoupled_weight_decay": true,
        "initial_lr": 0.001,
        "params": [
          11,
          12,
          13
        ]
      },
      {
        "lr": 0.001,
        "name": "scale_1280",
        "betas": [
          0.9,
          0.999
        ],
        "eps": 1e-08,
        "weight_decay": 1e-05,
        "amsgrad": false,
        "maximize": false,
        "foreach": null,
        "capturable": false,
        "differentiable": false,
        "fused": null,
        "decoupled_weight_decay": true,
        "initial_lr": 0.001,
        "params": [
          14,
          15,
          16
        ]
      },
      {
        "lr": 0.001,
        "name": "scale_1536",
        "betas": [
          0.9,
          0.999
        ],
        "eps": 1e-08,
        "weight_decay": 1e-05,
        "amsgrad": false,
        "maximize": false,
        "foreach": null,
        "capturable": false,
        "differentiable": false,
        "fused": null,
        "decoupled_weight_decay": true,
        "initial_lr": 0.001,
        "params": [
          17,
          18,
          19
        ]
      },
      {
        "lr": 0.001,
        "name": "scale_1792",
        "betas": [
          0.9,
          0.999
        ],
        "eps": 1e-08,
        "weight_decay": 1e-05,
        "amsgrad": false,
        "maximize": false,
        "foreach": null,
        "capturable": false,
        "differentiable": false,
        "fused": null,
        "decoupled_weight_decay": true,
        "initial_lr": 0.001,
        "params": [
          20,
          21,
          22
        ]
      },
      {
        "lr": 0.001,
        "name": "scale_2048",
        "betas": [
          0.9,
          0.999
        ],
        "eps": 1e-08,
        "weight_decay": 1e-05,
        "amsgrad": false,
        "maximize": false,
        "foreach": null,
        "capturable": false,
        "differentiable": false,
        "fused": null,
        "decoupled_weight_decay": true,
        "initial_lr": 0.001,
        "params": [
          23,
          24,
          25
        ]
      },
      {
        "lr": 0.001,
        "name": "scale_2304",
        "betas": [
          0.9,
          0.999
        ],
        "eps": 1e-08,
        "weight_decay": 1e-05,
        "amsgrad": false,
        "maximize": false,
        "foreach": null,
        "capturable": false,
        "differentiable": false,
        "fused": null,
        "decoupled_weight_decay": true,
        "initial_lr": 0.001,
        "params": [
          26,
          27,
          28
        ]
      },
      {
        "lr": 0.001,
        "name": "scale_2560",
        "betas": [
          0.9,
          0.999
        ],
        "eps": 1e-08,
        "weight_decay": 1e-05,
        "amsgrad": false,
        "maximize": false,
        "foreach": null,
        "capturable": false,
        "differentiable": false,
        "fused": null,
        "decoupled_weight_decay": true,
        "initial_lr": 0.001,
        "params": [
          29,
          30,
          31
        ]
      },
      {
        "lr": 0.0005,
        "name": "fusion",
        "betas": [
          0.9,
          0.999
        ],
        "eps": 1e-08,
        "weight_decay": 1e-05,
        "amsgrad": false,
        "maximize": false,
        "foreach": null,
        "capturable": false,
        "differentiable": false,
        "fused": null,
        "decoupled_weight_decay": true,
        "initial_lr": 0.0005,
        "params": [
          32,
          33,
          34,
          35,
          36,
          37,
          38,
          39,
          40,
          41,
          42,
          43,
          44,
          45,
          46,
          47,
          48,
          49,
          50,
          51,
          52,
          53,
          54,
          55,
          56,
          57,
          58,
          59,
          60,
          61,
          62,
          63,
          64,
          65,
          66,
          67,
          68,
          69,
          70,
          71,
          72,
          73,
          74,
          75,
          76,
          77,
          78,
          79,
          80,
          81,
          82,
          83,
          84,
          85
        ]
      }
    ]
  },
  "scheduler_state_dict": {
    "T_0": 10,
    "T_i": 20,
    "T_mult": 2,
    "eta_min": 1e-06,
    "T_cur": 0,
    "base_lrs": [
      0.001,
      0.001,
      0.001,
      0.001,
      0.001,
      0.001,
      0.001,
      0.001,
      0.001,
      0.001,
      0.001,
      0.0005
    ],
    "last_epoch": 10,
    "_step_count": 0,
    "_is_initial": false,
    "_get_lr_called_within_step": false,
    "_last_lr": [
      0.001,
      0.001,
      0.001,
      0.001,
      0.001,
      0.001,
      0.001,
      0.001,
      0.001,
      0.001,
      0.001,
      0.0005
    ]
  },
  "metrics": {
    "final_val_acc": 83.056
  },
  "train_config": {
    "name": "david_training",
    "run_id": "20251012_065325",
    "dataset_name": "AbstractPhil/imagenet-clip-features-orderly",
    "model_variant": "clip_vit_l14",
    "num_classes": 1000,
    "preset": "clip_vit_l14_ultra_deep",
    "custom_config_path": null,
    "num_classes_override": null,
    "use_belly_override": null,
    "belly_expand_override": null,
    "progressive_training_override": true,
    "scale_warmup_epochs_override": null,
    "num_epochs": 10,
    "batch_size": 1024,
    "learning_rate": 0.001,
    "weight_decay": 1e-05,
    "warmup_epochs": 3,
    "use_rose_loss": true,
    "rose_initial_weight": 0.1,
    "rose_max_weight": 0.5,
    "rose_weight_schedule": "adaptive",
    "use_cayley_loss": false,
    "cayley_weight": 0.001,
    "scale_loss_balance": null,
    "use_mixed_precision": false,
    "gradient_clip": 10.0,
    "scheduler_type": "cosine_restarts",
    "min_lr": 1e-06,
    "freeze_strategy": "never",
    "freeze_threshold": 90.0,
    "unfreeze_on_plateau": true,
    "patience": 10,
    "track_gradients": true,
    "gradient_scale_threshold": 1e-05,
    "gradient_scale_multiplier": 10.0,
    "log_interval": 50,
    "val_interval": 1,
    "save_interval": 5,
    "log_fusion_weights": true,
    "log_loss_components": true,
    "save_format": "safetensors",
    "hf_repo": "AbstractPhil/gated-david",
    "upload_to_hub": true,
    "base_dir": "./david_training",
    "num_workers": 10,
    "pin_memory": true,
    "prefetch_factor": 4,
    "persistent_workers": true
  }
}